telstore 0.1.7 → 0.1.9

package/src/commands/list.js

@@ -1,11 +1,13 @@
-import { MANIFEST_TAG, parseManifestCaption } from '../caption.js'
+import { parseManifestCaption } from '../caption.js'
 import { chatName, describeChat } from '../chat.js'
 import {
   closeQuietly,
   connect as realConnect,
-  searchDocuments,
+  iterDocuments,
+  iterManifestSearch,
 } from '../client.js'
 import { configFile, defaultConfigDir, loadConfig } from '../config.js'
+import { MANIFEST_SUFFIX } from '../manifest.js'
 import { assertLoggedIn } from '../session.js'
 import { requireChat, resolveSettings } from '../settings.js'
 
@@ -30,15 +32,48 @@ function shorten(note) {
   return note.length > NOTE_WIDTH ? `${note.slice(0, NOTE_WIDTH - 1)}…` : note
 }
 
-// Telegram indexes the tag the manifest caption carries, so one search returns one hit
-// per backup instead of one per chunk. What comes back is still whatever the server
-// decided to match, which is why the caller filters on the file name afterwards.
-async function realSearchManifests(client, peer, limit) {
-  return await searchDocuments(client, peer, { search: MANIFEST_TAG, limit })
+// What list has to read through depends on how big the backups are, not how many there are:
+// it walks from the newest message down and stops the moment it has --limit manifests, so the
+// three thousandth backup in a chat costs nothing because it is never reached. What costs is
+// the chunks in between — one message each — which is why the ceiling is a budget per backup
+// asked for rather than one number for every chat.
+//
+// 60 documents per backup covers a backup of about 105GB at the default chunk size. A fixed
+// 1000 was both too tight and too loose at once: twenty backups of 100GB need 1140 documents
+// and got 1000 of them, while `--limit 5` never needed more than 300.
+export const DOCUMENTS_PER_BACKUP = 60
+
+// A search result is already a manifest, so it buys far more backups per document read. Its
+// budget only has to cover what matchesTerm throws away — measured 2026-09-08, a term like
+// "2026-09" comes back matching everything and is then cut down to the month asked for.
+export const RESULTS_PER_BACKUP = 20
+
+// --limit takes any whole number, so the budget needs an end of its own: without one,
+// `--limit 100000` would ask for six million documents and sixty thousand requests.
+export const MAX_LIST_DOCUMENTS = 10000
+
+export function documentBudget(limit, perBackup) {
+  return Math.min(limit * perBackup, MAX_LIST_DOCUMENTS)
 }
 
+// Stopping at the ceiling used to leave the reader standing there: "there may be older backups
+// further back" is true and offers nothing to do about it. --search reaches them without
+// reading the chunks in between, which is the whole reason it exists.
+const DEEPER_HINT =
+  '"npx telstore list --search <text>" reaches older ones without reading every chunk ' +
+  'in between.'
+
+// Telegram matches whole words and nothing shorter: measured 2026-09-08, "projex" found
+// projex.zip while "proj", "pro" and "pr" each found nothing at all. That is the one way a
+// search can come back empty over a backup that is plainly there, so the empty answer says
+// it, and points at the listing that never asks the index.
+const SEARCH_MISS_HELP =
+  'Telegram matches whole words: "projex" finds projex.zip, "proj" does not. ' +
+  'Run "npx telstore list" without --search to see every backup without going through ' +
+  'the search index.'
+
 function backupIdFromFileName(fileName) {
-  return fileName.replace(/\.manifest\.json$/, '')
+  return fileName.slice(0, -MANIFEST_SUFFIX.length)
 }
 
 function utcDay(unixSeconds) {
@@ -74,6 +109,76 @@ function toRow(message) {
   }
 }
 
+// A search term is a question about one run, and an empty one is not a question: answering
+// it with every backup would look exactly like a search that matched everything.
+function parseSearchTerm(raw) {
+  if (raw === undefined || raw === null) return null
+
+  const term = String(raw).trim()
+
+  if (term === '') {
+    throw new Error(
+      '--search is empty. Write the word to look for, or leave the flag off — "list" ' +
+        'without it shows every backup.',
+    )
+  }
+
+  return term
+}
+
+// The four fields a person remembers about a backup, and the whole of what --search compares
+// against. The note is matched entire rather than the 40 characters the table has room for:
+// a word that fell off the end of the column is still a word they typed. A card that cannot
+// be read back leaves only what the message itself knows.
+function searchableFields(message) {
+  const id = backupIdFromFileName(message.fileName)
+  const card = parseManifestCaption(message.caption)
+
+  if (!card) return [id, utcDay(message.date)]
+
+  return [id, card.name, card.note ?? '', card.createdAt.slice(0, 10)]
+}
+
+// Telegram decides what comes back; this decides what is true. The index answers a term the
+// way it wants to — measured 2026-09-08, "2026-09" returned every document in the chat — so
+// a hit is shown only if the term really is in one of the fields above. Without this pass a
+// search for a month would list backups from every other month, which is the plausible wrong
+// answer this project exists to refuse.
+function matchesTerm(message, term) {
+  return searchableFields(message).some((field) => field.toLowerCase().includes(term))
+}
+
+// A walk of one page is over in about the time it takes to notice — 165ms against a real
+// chat — and that is the usual case, so nothing is drawn for the first stretch: a line that
+// appears and is wiped in the same breath is a flicker, not information. Past that the read
+// is long enough that silence reads as the hang this project refuses everywhere else.
+//
+// \r only moves the cursor home, so every line is padded to the widest one drawn and the last
+// write wipes the row: the table that follows must never land on half a progress line.
+const NOTICE_QUIET_MS = 400
+const NOTICE_INTERVAL_MS = 200
+
+function createWalkNotice({ write, now, quietMs = NOTICE_QUIET_MS, intervalMs = NOTICE_INTERVAL_MS }) {
+  const startedAt = now()
+  let lastDrawnAt = 0
+  let widest = 0
+
+  return {
+    tick(text) {
+      if (now() - startedAt < quietMs) return
+      if (lastDrawnAt !== 0 && now() - lastDrawnAt < intervalMs) return
+
+      lastDrawnAt = now()
+      widest = Math.max(widest, text.length)
+      write(`\r${text.padEnd(widest)}`)
+    },
+    clear() {
+      if (widest === 0) return
+      write(`\r${' '.repeat(widest)}\r`)
+    },
+  }
+}
+
 function renderTable(rows) {
   // Most people never write a note, and a column of dashes tells them nothing they did not
   // already know while costing every other column the width it takes.
@@ -102,12 +207,21 @@ export async function runList(options = {}, deps = {}) {
     configDir = defaultConfigDir(),
     connect = realConnect,
     disconnect = (client) => client.destroy(),
-    searchManifests = realSearchManifests,
+    readDocuments = iterDocuments,
+    searchManifests = iterManifestSearch,
     log = (line) => console.log(line),
+    // The notice is drawn on stderr, and only onto a terminal: unlike an upload's progress
+    // bar, `list` is a command people pipe into grep, and a carriage return in a log file is
+    // rubbish. Null means draw nothing at all.
+    writeProgress = process.stderr.isTTY ? (text) => process.stderr.write(text) : null,
+    now = () => Date.now(),
   } = deps
 
   const config = await loadConfig(configDir)
   const { values: settings } = resolveSettings(options, config, { file: configFile(configDir) })
+  // Before the login gate: a bad term is the user's own typing, and telling them to log in
+  // first would send them off after the wrong thing.
+  const term = parseSearchTerm(options.search)
   // Ask about the login before the destination: telling someone who has never logged in
   // to pick a chat sends them off after the wrong thing.
   assertLoggedIn(config)
@@ -115,29 +229,99 @@ export async function runList(options = {}, deps = {}) {
 
   const client = await connect(config, { verbose: settings.verbose })
 
-  let found
+  // Walked rather than searched, unless a term was given. Telegram's text index can answer
+  // nothing at all about a chat that is full of backups — it did for a whole day in a channel
+  // that had just been created — and "No backups found" is a sentence someone acts on. The
+  // documents themselves were right every time they were asked for.
+  //
+  // --search is the one place worth paying the index for: a term matches a backup that may be
+  // ten thousand messages back, and walking to it would cost a request per hundred documents
+  // in between, every time, for as long as the chat keeps growing. So the search narrows and
+  // matchesTerm decides — the index is asked where to look, never what is true.
+  const found = []
+  let read = 0
+
+  const unit = term ? 'search results' : 'documents'
+  const budget = documentBudget(settings.limit, term ? RESULTS_PER_BACKUP : DOCUMENTS_PER_BACKUP)
+
+  const results = term
+    ? searchManifests(client, chat, term, { max: budget })
+    : readDocuments(client, chat, { max: budget })
+
+  const wanted = term === null ? null : term.toLowerCase()
+  const notice = writeProgress ? createWalkNotice({ write: writeProgress, now }) : null
+
   try {
-    found = await searchManifests(client, chat, settings.limit)
+    for await (const document of results) {
+      read += 1
+
+      notice?.tick(
+        `Reading ${chatName(chat)}… ${read} ${unit}, ${found.length} backup` +
+          `${found.length === 1 ? '' : 's'}`,
+      )
+
+      if (!document.fileName?.endsWith(MANIFEST_SUFFIX)) continue
+      if (wanted !== null && !matchesTerm(document, wanted)) continue
+
+      found.push(document)
+
+      // Everything past here is older than the twentieth newest backup, and nobody asked
+      // for it. In a chat of ten thousand chunks this is the difference between one
+      // request and ten.
+      if (found.length >= settings.limit) break
+    }
   } finally {
+    notice?.clear()
     await closeQuietly(client, disconnect)
   }
 
+  // The one thing either reader cannot see is what lies past its own ceiling, so anything it
+  // says about the whole chat has to stop at the edge of what it read.
+  const capped = found.length < settings.limit && read >= budget
+
   log(`Destination  ${describeChat(chat)}`)
+  if (term) log(`Search       ${JSON.stringify(term)}`)
   log('')
 
-  const rows = found
-    .filter((message) => message.fileName?.endsWith('.manifest.json'))
-    .map(toRow)
+  const rows = found.map(toRow)
 
   if (rows.length === 0) {
-    log(`No backups found in ${chatName(chat)}. Upload one with: npx telstore <file>`)
+    if (term) {
+      log(
+        capped
+          ? `No backups matching ${JSON.stringify(term)} in the newest ${budget} ` +
+              `${unit} from ${chatName(chat)}. There may be older ones further back.`
+          : `No backups matching ${JSON.stringify(term)} in ${chatName(chat)}.`,
+      )
+      log(SEARCH_MISS_HELP)
+      return rows
+    }
+
+    log(
+      capped
+        ? `No backups in the newest ${budget} ${unit} of ${chatName(chat)}. ` +
+            `There may be older ones further back. ${DEEPER_HINT}`
+        : `No backups found in ${chatName(chat)}. Upload one with: npx telstore <file>`,
+    )
     return rows
   }
 
   for (const line of renderTable(rows)) log(line)
 
   log('')
-  log(`${rows.length} backup${rows.length === 1 ? '' : 's'}. Restore with: npx telstore restore <backup-id>`)
+  log(
+    `${rows.length} backup${rows.length === 1 ? '' : 's'}` +
+      `${term ? ` matching ${JSON.stringify(term)}` : ''}. ` +
+      'Restore with: npx telstore restore <backup-id>',
+  )
+
+  if (capped) {
+    log(
+      `Read the newest ${budget} ${unit} in ${chatName(chat)} to find them — ` +
+        `there may be older ${term ? 'matches' : 'backups'} further back.` +
+        `${term ? '' : ` ${DEEPER_HINT}`}`,
+    )
+  }
 
   return rows
 }

package/src/commands/restore.js

@@ -11,9 +11,10 @@ import { askConfirm } from '../confirm.js'
 import { configFile, defaultConfigDir, loadConfig } from '../config.js'
 import { assertLoggedIn } from '../session.js'
 import { requireChat, resolveSettings } from '../settings.js'
-import { downloadToFile } from '../downloader.js'
+import { downloadToFile, hashRange } from '../downloader.js'
 import { parseManifest } from '../manifest.js'
 import { createProgress, formatBytes, formatDuration } from '../progress.js'
+import { clearRestore, pruneRestores, restoreKey, saveRestore } from '../state.js'
 
 // Anything past a minute of waiting needs saying out loud; below that the pause is shorter
 // than the time a user would spend wondering about it.
@@ -50,6 +51,29 @@ function safeOutName(name) {
   return base
 }
 
+// How many chunks at the front of a .partial already hold what the manifest says they
+// should. The evidence is the file, never a record: a record makes claims about a local
+// file anyone can edit between runs, and a claim that is wrong here renames a corrupt file
+// into place. Every chunk in the finished file was hashed against the manifest by the run
+// that renamed it, whether this run downloaded it or found it already there.
+async function scanPartial(handle, manifest, log) {
+  let done = 0
+
+  for (const chunk of manifest.chunks) {
+    const digest = await hashRange(handle.fd, chunk.i * manifest.chunkSize, chunk.size)
+
+    // Downloads run in order, so what is already present is a prefix. The first chunk that
+    // does not match is where this run starts, and reading past it would hash gigabytes
+    // nobody has written yet.
+    if (digest !== chunk.sha256) break
+
+    done += 1
+    log(`Chunk ${chunk.i + 1}/${manifest.chunks.length} already restored, skipping.`)
+  }
+
+  return done
+}
+
 export async function runRestore(backupId, options = {}, deps = {}) {
   const {
     connect = realConnect,
@@ -64,6 +88,7 @@ export async function runRestore(backupId, options = {}, deps = {}) {
     writeErr = (line) => process.stderr.write(line),
     log: writeLog = (line) => console.log(line),
     silent = false,
+    onBackupId = () => {},
   } = deps
 
   const config = await loadConfig(configDir)
@@ -112,6 +137,41 @@ export async function runRestore(backupId, options = {}, deps = {}) {
     const target = path.resolve(options.out ?? safeOutName(manifest.name))
     const partial = `${target}.partial`
 
+    const key = restoreKey(backupId, target)
+
+    // The record is a signpost for `status`, never evidence. The scan above proved every
+    // chunk it skipped against the manifest and would do so again if this file vanished, so
+    // a signpost that cannot be planted warns and gets out of the way. Deliberately the
+    // opposite of markChunkDone, where a failed write must be fatal because losing it
+    // strands chunks in a chat with nothing left pointing at them.
+    // A record that cannot be written warns once, not once per chunk: warn is the same
+    // stderr stream the progress bar owns with \r, and a warning for every one of ~30
+    // chunks on a real restore would tear through the bar as badly as the retry
+    // announcements above reason about at length.
+    let recordWarned = false
+
+    async function note(done) {
+      try {
+        await saveRestore(
+          key,
+          {
+            v: 1,
+            id: backupId,
+            target,
+            chat: String(chat),
+            size: manifest.size,
+            chunks: manifest.chunks.length,
+            done,
+          },
+          configDir,
+        )
+      } catch (err) {
+        if (recordWarned) return
+        recordWarned = true
+        warn(`\nWarning: could not record restore progress: ${err.message}\n`)
+      }
+    }
+
     // Only ENOENT means "no file yet". Treating a permission or I/O error as absence
     // would have telstore overwrite the user's file without asking.
     let exists = true
@@ -129,66 +189,120 @@ export async function runRestore(backupId, options = {}, deps = {}) {
     log(`Backup ${manifest.id}`)
     log(`File   ${target} (${formatBytes(manifest.size)}, ${manifest.chunks.length} chunks)\n`)
 
-    const handle = await fs.open(partial, 'w+')
+    let handle
+    let resuming = true
 
+    // r+ keeps whatever an earlier run left behind; w+ truncates it to zero, which is what
+    // made a kept .partial useless. Only ENOENT means "no file yet" — a permission error
+    // quietly becoming "start over" is how two hours of downloading disappear unexplained.
     try {
+      handle = await fs.open(partial, 'r+')
+    } catch (err) {
+      if (err.code !== 'ENOENT') throw err
+      handle = await fs.open(partial, 'w+')
+      resuming = false
+    }
+
+    // The message this id feeds says a .partial was kept, so naming the id is only honest
+    // once one actually exists — before this line, Ctrl-C would have nothing to carry on from.
+    onBackupId(backupId)
+
+    try {
+      // Extends a short .partial with zeros and cuts an over-long one, and touches no byte
+      // below manifest.size — so one path serves a fresh file and a resumed one alike.
       await handle.truncate(manifest.size)
 
-      // One bar for the whole restore. The label names the chunk in flight, but the bar, the
-      // byte counts, the speed and the ETA all describe the file, so the line runs 0% to 100%
-      // once instead of restarting at every chunk boundary — with 1800MB chunks, a per-chunk
-      // ETA answers a question nobody asked.
-      // warn is already the no-op when silent, and createProgress draws through nothing else.
-      const progress = createProgress({
-        total: manifest.size,
-        label: `Chunk 1/${manifest.chunks.length}`,
-        write: warn,
-      })
+      let done = 0
 
-      try {
-        for (const chunk of manifest.chunks) {
-          // Before getMessage, not after: the bar is then on screen from the first moment,
-          // and finish() below always has a line to close.
-          progress.setLabel(`Chunk ${chunk.i + 1}/${manifest.chunks.length}`)
+      if (resuming) {
+        // Hashing 1800MB takes about 9 seconds, so a large scan runs for minutes. Silence
+        // that long is the hang this project refuses everywhere: the heading lands before
+        // the first read and a line per chunk arrives as the scan advances.
+        log(`Checking what is already in ${partial}...`)
+        done = await scanPartial(handle, manifest, log)
+        if (done === 0) log(`Nothing in ${partial} matches this backup, starting over.`)
+        log('')
+      }
 
-          const message = await getMessage(client, chat, chunk.msgId)
+      await note(done)
 
-          if (!message) {
-            throw new Error(
-              `Missing chunk ${chunk.i + 1}/${manifest.chunks.length}: message ${chunk.msgId} is no longer in ${chat}. ` +
-                'This backup cannot be restored.',
-            )
-          }
+      // Housekeeping, the way runUpload prunes its own records. It removes signposts only:
+      // dropping one costs a line of `status` for a .partial that still resumes.
+      try {
+        await pruneRestores(configDir)
+      } catch (err) {
+        warn(`\nWarning: could not tidy old restore records: ${err.message}\n`)
+      }
 
-          const { sha256, size } = await downloadChunk(
-            client,
-            message,
-            handle,
-            chunk.i * manifest.chunkSize,
-            progress.advance,
-            {
-              retryOptions: { ...retryOptions, onRetry },
-              concurrency: settings.downloadConcurrency,
-            },
-          )
-
-          if (size !== chunk.size) {
-            throw new Error(
-              `Chunk ${chunk.i + 1} has ${size} bytes, the manifest records ${chunk.size} bytes — mismatch.`,
+      const pending = manifest.chunks.slice(done)
+
+      // A .partial holding every chunk is what a run that died between its last chunk and
+      // the rename leaves: no bar at all, rather than one springing into existence at 100%.
+      if (pending.length > 0) {
+        const present = manifest.chunks
+          .slice(0, done)
+          .reduce((sum, chunk) => sum + chunk.size, 0)
+
+        // One bar for the whole restore. The label names the chunk in flight, but the bar, the
+        // byte counts, the speed and the ETA all describe the file, so the line runs 0% to 100%
+        // once instead of restarting at every chunk boundary — with 1800MB chunks, a per-chunk
+        // ETA answers a question nobody asked. Chunks an earlier run left count towards the bar
+        // but not towards the speed, so an hour-old chunk cannot inflate the ETA of the rest.
+        // warn is already the no-op when silent, and createProgress draws through nothing else.
+        const progress = createProgress({
+          total: manifest.size,
+          done: present,
+          label: `Chunk ${pending[0].i + 1}/${manifest.chunks.length}`,
+          write: warn,
+        })
+
+        try {
+          for (const chunk of pending) {
+            // Before getMessage, not after: the bar is then on screen from the first moment,
+            // and finish() below always has a line to close.
+            progress.setLabel(`Chunk ${chunk.i + 1}/${manifest.chunks.length}`)
+
+            const message = await getMessage(client, chat, chunk.msgId)
+
+            if (!message) {
+              throw new Error(
+                `Missing chunk ${chunk.i + 1}/${manifest.chunks.length}: message ${chunk.msgId} is no longer in ${chat}. ` +
+                  'This backup cannot be restored.',
+              )
+            }
+
+            const { sha256, size } = await downloadChunk(
+              client,
+              message,
+              handle,
+              chunk.i * manifest.chunkSize,
+              progress.advance,
+              {
+                retryOptions: { ...retryOptions, onRetry },
+                concurrency: settings.downloadConcurrency,
+              },
             )
-          }
 
-          if (sha256 !== chunk.sha256) {
-            throw new Error(
-              `Chunk ${chunk.i + 1} has a sha256 that does not match the manifest. The download is kept at ${partial} for inspection.`,
-            )
+            if (size !== chunk.size) {
+              throw new Error(
+                `Chunk ${chunk.i + 1} has ${size} bytes, the manifest records ${chunk.size} bytes — mismatch.`,
+              )
+            }
+
+            if (sha256 !== chunk.sha256) {
+              throw new Error(
+                `Chunk ${chunk.i + 1} has a sha256 that does not match the manifest. The download is kept at ${partial} for inspection.`,
+              )
+            }
+
+            await note(chunk.i + 1)
           }
+        } finally {
+          // The bar owns a line that \r keeps returning to. Ending it here rather than after the
+          // loop means a chunk that fails mid-download still leaves the cursor on a fresh line,
+          // so "Error: ..." does not land on top of the bar.
+          progress.finish()
         }
-      } finally {
-        // The bar owns a line that \r keeps returning to. Ending it here rather than after the
-        // loop means a chunk that fails mid-download still leaves the cursor on a fresh line,
-        // so "Error: ..." does not land on top of the bar.
-        progress.finish()
       }
     } finally {
       await handle.close()
@@ -208,6 +322,13 @@ export async function runRestore(backupId, options = {}, deps = {}) {
 
     await fs.rename(partial, target)
 
+    // The restore is finished; the signpost has nothing left to point at.
+    try {
+      await clearRestore(key, configDir)
+    } catch (err) {
+      warn(`\nWarning: could not remove the restore record: ${err.message}\n`)
+    }
+
     log(`\nDone. Wrote ${formatBytes(manifest.size)} to ${target}`)
 
     return { path: target, size: manifest.size }
@@ -231,6 +352,8 @@ export async function runRestores(backupIds, options = {}, deps = {}) {
     writeErr = (line) => process.stderr.write(line),
     log: writeLog = (line) => console.log(line),
     silent = false,
+    onRestoreDone = () => {},
+    onBackupId = () => {},
   } = deps
 
   // One id must read exactly as it did before this existed: --out still works, the error still
@@ -289,12 +412,19 @@ export async function runRestores(backupIds, options = {}, deps = {}) {
       try {
         const { path: target, size } = await runRestore(backupId, options, perId)
         results.push({ id: backupId, path: target, size })
+        onRestoreDone({ id: backupId, path: target })
       } catch (err) {
         // A backup whose chunks are gone says nothing about the next one, and the summary at
         // the end would arrive an hour after the bar of the following id started scrolling
         // over it — so it is named here, and again down there, and carried out as exit code 1.
         results.push({ id: backupId, error: err.message })
         warn(`\n${backupId} failed: ${err.message}\n`)
+      } finally {
+        // Cleared whether this id finished, failed, or never got as far as opening a
+        // .partial: otherwise a Ctrl-C while the next id is still connecting, searching for
+        // its manifest, or blocked at the overwrite prompt would go on naming this one, as
+        // though there were a file to carry on from when there is none yet.
+        onBackupId(null)
       }
     }
   } finally {