telstore 0.1.8 → 0.1.10

package/src/commands/down.js

@@ -0,0 +1,311 @@
+import { promises as fs } from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+
+import { describeChat } from '../chat.js'
+import { configFile, defaultConfigDir, loadConfig } from '../config.js'
+import { askConfirm } from '../confirm.js'
+import { deleteCommand, shellArg } from '../shell.js'
+import { listRestores, listStates } from '../state.js'
+
+// Nothing here may reach src/client.js, directly or through a command that does: `down` is
+// run on a machine somebody is finished with, often one with no network and sometimes one
+// they no longer trust. It opens no socket, and test/bin.test.js measures that it stays
+// that way — a single import of status.js would put teleproto back in the path.
+
+const LABEL_WIDTH = 'Destination'.length + 2
+
+function row(label, value) {
+  return `  ${label.padEnd(LABEL_WIDTH)}${value}`
+}
+
+// status prints this same count about this same directory. It cannot be imported from there
+// without dragging teleproto in, so it is copied — and copied exactly, because two commands
+// disagreeing about what is unfinished on one machine is worse than either wording alone.
+function unfinishedCount(uploads, restores) {
+  const parts = []
+
+  if (uploads > 0) parts.push(`${uploads} upload${uploads === 1 ? '' : 's'}`)
+  if (restores > 0) parts.push(`${restores} restore${restores === 1 ? '' : 's'}`)
+
+  return parts.length === 0 ? 'none' : parts.join(', ')
+}
+
+// A recursive remove is the one mistake in this command nobody could apologise for, and the
+// path it walks is derived rather than typed — os.homedir() reading an empty HOME is all it
+// takes. These two are not reachable by any correct call, which is exactly why they are
+// worth refusing by name rather than trusting the caller.
+function refuseDangerousTarget(dir) {
+  if (dir === os.homedir() || path.dirname(dir) === dir) {
+    throw new Error(
+      `telstore refuses to remove ${dir}: that is a home directory or a filesystem root, ` +
+        'not a telstore config directory. Nothing was removed.',
+    )
+  }
+}
+
+// The config is read to describe it, never to trust it. loadConfig throws on a file that is
+// corrupt, that holds a `settings` which is not a group, or that holds both a sealed session
+// and a plain one — and its own advice for all three is to delete the file and log in again.
+// This is the command that does that, so it is the one command a broken config must not stop.
+async function describeConfig(configDir) {
+  try {
+    return { config: await loadConfig(configDir), error: null }
+  } catch (err) {
+    return { config: {}, error: err.message }
+  }
+}
+
+// The error itself is deliberately not quoted here. Every message loadConfig throws names the
+// config file, every path to it contains a dot, and a first-sentence split therefore cuts
+// "/home/sho/.telstore/config.json is not valid JSON" down to "/home/sho/" — a mangled path
+// presented as a diagnosis. The whole message is no better: its advice is "delete the file and
+// log in again", which is what this command is in the middle of doing.
+function sessionLine(config, error) {
+  if (error) return 'cannot be read — whatever is in that file, it goes with the rest'
+  if (config.sealed) return 'sealed — the api_id and api_hash are inside it, so they go too'
+  if (config.session) return 'stored here in plain text — api_id and api_hash go with it'
+
+  return 'not logged in'
+}
+
+// What an unfinished upload was of — either kind of record, which is why this is not the
+// `describeStreamSource` `status` has: that one answers only for a stream record, and one
+// name over two answers is how the two came to disagree. A stream record has no path — its
+// bytes came from a command's stdout — and carries the name the backup was given instead.
+// This listing exists so that nothing goes unnamed before a recursive remove, so a row
+// reading "undefined" is the exact failure it is here to prevent.
+//
+// Blank is the same failure wearing a string's clothes: a record holding `name: "   "` would
+// print `    (a command's output)` — a row that names nothing, in the one command whose job
+// is that nothing goes unnamed. Trimmed for that, exactly as `status` trims, and both fields
+// get the rule because a whitespace path is no more a name than a whitespace name is.
+function describeRecordSource(state) {
+  if (typeof state.path === 'string' && state.path.trim() !== '') return state.path
+  if (typeof state.name === 'string' && state.name.trim() !== '') {
+    return `${state.name} (a command's output)`
+  }
+
+  return 'a record that does not say what it was backing up'
+}
+
+// The last thing anything anywhere will say about those chunks. down.md prints a resume
+// command for a .partial "because this is the last time anything will mention that file", and
+// a stream record is the sharper case: it cannot be re-run onto — those bytes have gone past —
+// so once this record is gone nothing on this machine lists those message ids and no manifest
+// in the chat names them. Printing the command removes nothing and opens no socket; it is the
+// naming this whole listing exists for, done for something that lives on Telegram.
+//
+// The chat is named for the reason status names it, and the line itself is built by
+// `deleteCommand` in shell.js so that all four places that print it spell `--chat` the same
+// way. The check in front of it is this command's own decision and stays here: a record that
+// cannot say where its chunks went gets no command at all rather than the chatless one that
+// function would otherwise hand back, because here that would be a guess — `--chat` missing is
+// not `--chat` empty, and runDelete would take it as no destination and resolve one from
+// config. The id needs no guard — the listing above measures state.id.length for its own
+// column, so a record without one never reaches this line.
+function removeCommand(state) {
+  const chat = state.chat === null || state.chat === undefined ? '' : String(state.chat).trim()
+
+  if (chat === '') return null
+
+  return deleteCommand(state.id, chat)
+}
+
+// Only the ones actually on disk. A restore record survives a .partial that was deleted by
+// hand, and pointing at a file that is not there sends somebody looking for nothing.
+async function strandedPartials(restores) {
+  const found = []
+
+  for (const { record } of restores) {
+    const partial = `${record.target}.partial`
+
+    try {
+      await fs.stat(partial)
+      found.push({ partial, record })
+    } catch {
+      // Nothing there to tell them about.
+    }
+  }
+
+  return found
+}
+
+// Everything in the directory that telstore did not put there. The whole directory goes
+// either way — it is what was asked for — but delete's rule holds here too: nothing is
+// removed unnamed.
+//
+// `tmp` is telstore's too: a stream upload borrows one chunk of disk at a time under
+// ~/.telstore rather than /tmp, which is tmpfs on many distributions and would turn a chunk
+// size into a memory limit. Naming it here would be down reporting its own working directory
+// as a stranger's file — and if a run died mid-chunk it may hold up to one chunk, which the
+// recursive remove below takes with everything else.
+async function foreignEntries(configDir) {
+  const ours = new Set(['config.json', 'config.json.tmp', 'state', 'tmp'])
+
+  try {
+    return (await fs.readdir(configDir)).filter((name) => !ours.has(name))
+  } catch {
+    return []
+  }
+}
+
+export async function runDown(args = [], options = {}, deps = {}) {
+  const {
+    configDir = defaultConfigDir(),
+    confirm = askConfirm,
+    interactive = () => Boolean(process.stdin.isTTY),
+    log = (line) => console.log(line),
+  } = deps
+
+  // `telstore down telstore-20260905-7f3a91` is the plausible typo — somebody reaching for
+  // the command that removes one backup. Obeying it would wipe the machine instead.
+  if (args.length > 0) {
+    throw new Error(
+      `down takes no arguments, but got "${args.join(' ')}". It removes everything on this ` +
+        'machine or nothing. To remove one backup from Telegram, use ' +
+        '"npx telstore delete <backup-id>".',
+    )
+  }
+
+  refuseDangerousTarget(configDir)
+
+  try {
+    await fs.stat(configDir)
+  } catch (err) {
+    if (err.code !== 'ENOENT') throw err
+
+    log(`Nothing to remove: ${configDir} is not there.`)
+    return { removed: false, dir: configDir }
+  }
+
+  const { config, error } = await describeConfig(configDir)
+  const uploads = await listStates(configDir)
+  const restores = await listRestores(configDir)
+  const stranded = await strandedPartials(restores)
+  const foreign = await foreignEntries(configDir)
+  const chat = config.settings?.chat ?? null
+
+  log('This removes everything telstore keeps on this machine.')
+  log('')
+  log(row('Directory', configDir))
+  log(row('Session', sessionLine(config, error)))
+  if (chat !== null) log(row('Destination', describeChat(chat)))
+  log(row('Unfinished', unfinishedCount(uploads.length, restores.length)))
+  if (foreign.length > 0) log(row('Also there', foreign.join(', ')))
+
+  // The expensive half of what is about to go. An upload record is what lets a second run
+  // keep the same backupId and skip the chunks already sent; without it the same file goes
+  // up again as a new backup, and the chunks already in the chat stay there under an id
+  // nothing on this machine remembers. So the ids are said out loud while someone can read
+  // them. A restore record costs nothing to lose — the .partial resumes without it — and is
+  // dealt with on the way out instead.
+  if (uploads.length > 0) {
+    const width = Math.max(...uploads.map(({ state }) => state.id.length))
+
+    log('')
+    log('These uploads have not finished. Their records are the only thing that lets a second')
+    log('run carry on: without them the same file goes up again as a new backup, and the chunks')
+    log('already sent stay in the chat under these ids and nothing else:')
+    log('')
+    for (const { state } of uploads) {
+      log(`  ${state.id.padEnd(width)}  ${describeRecordSource(state)}`)
+    }
+
+    const streams = uploads.filter(({ state }) => state.kind === 'stream')
+
+    if (streams.length > 0) {
+      log('')
+      log('The ones marked as a command\'s output cannot be carried on at all: those bytes have')
+      log('gone past, and a second run cuts them differently. Their records are the only list of')
+      log('the chunks those runs left in the chat, and nothing replaces them — so this is the')
+      log('last chance to copy the commands that remove those chunks:')
+      log('')
+
+      for (const { state } of streams) {
+        const command = removeCommand(state)
+
+        if (command === null) {
+          log(`  ${state.id}`)
+          log('    this record does not say which chat its chunks went to, so there is no')
+          log('    command that could reach them')
+          continue
+        }
+
+        log(`  ${command}`)
+      }
+    }
+  }
+
+  log('')
+  log('Nothing on Telegram is touched: every backup stays where it is.')
+
+  if (!options.yes) {
+    if (!interactive()) {
+      throw new Error(
+        `Nothing was removed: there is no terminal to confirm in. Run again with --yes to ` +
+          `remove ${configDir} without being asked.`,
+      )
+    }
+
+    log('')
+
+    if (!(await confirm(`This cannot be undone. Remove ${configDir}? [y/N] `))) {
+      throw new Error('Cancelled on request.')
+    }
+  }
+
+  try {
+    await fs.rm(configDir, { recursive: true, force: true })
+  } catch (err) {
+    // fs.rm walks the tree, so a permission error halfway leaves some of it gone. Reporting
+    // that as a success is the one thing this project never does.
+    throw new Error(
+      `Could not remove ${configDir}: ${err.message}. Some of it may already be gone — ` +
+        'run "npx telstore down" again once the permissions allow it.',
+    )
+  }
+
+  log('')
+  log(`Done. Removed ${configDir}.`)
+
+  if (config.session || config.sealed || error) {
+    log(
+      'Note: this only deletes the local copy — the session is still alive on Telegram\'s ' +
+        'side. To revoke access for good, open Telegram → Settings → Devices (Active ' +
+        'sessions) and terminate that session.',
+    )
+  }
+
+  if (config.apiId || config.apiHash || config.sealed) {
+    log(
+      'api_id and api_hash are gone too, which logout would have kept: the next ' +
+        '"npx telstore login" asks for them again (my.telegram.org).',
+    )
+  }
+
+  log(
+    chat === null
+      ? 'Your backups are still on Telegram. Log in again and "npx telstore list" finds them.'
+      : `Your backups are still in ${describeChat(chat)} — log in again and ` +
+          '"npx telstore list" finds them.',
+  )
+
+  // The .partial is the user's data, and down removes what was asked for and nothing else.
+  // Unlike after a delete it can still be finished: the chunks and the manifest a resume
+  // checks against were never touched. What is gone is the record that listed it, so this
+  // is the last time anything will mention the file at all.
+  for (const { partial, record } of stranded) {
+    const chatFlag = record.chat ? ` --chat ${shellArg(record.chat)}` : ''
+
+    log('')
+    log(`${partial} is a half-finished restore of ${record.id}, and is left where it is.`)
+    log('It still resumes — log in again and run it from anywhere:')
+    log(
+      `  npx telstore restore ${shellArg(record.id)} --out ${shellArg(record.target)}${chatFlag}`,
+    )
+    log('Nothing else will remind you: the record that listed it went with the rest.')
+  }
+
+  return { removed: true, dir: configDir, uploads: uploads.length, restores: restores.length }
+}

package/src/commands/list.js

@@ -1,11 +1,14 @@
-import { MANIFEST_TAG, parseManifestCaption } from '../caption.js'
+import { parseManifestCaption } from '../caption.js'
 import { chatName, describeChat } from '../chat.js'
 import {
   closeQuietly,
   connect as realConnect,
-  searchDocuments,
+  iterDocuments,
+  iterManifestSearch,
 } from '../client.js'
 import { configFile, defaultConfigDir, loadConfig } from '../config.js'
+import { MANIFEST_SUFFIX } from '../manifest.js'
+import { createWalkNotice } from '../progress.js'
 import { assertLoggedIn } from '../session.js'
 import { requireChat, resolveSettings } from '../settings.js'
 
@@ -30,15 +33,48 @@ function shorten(note) {
   return note.length > NOTE_WIDTH ? `${note.slice(0, NOTE_WIDTH - 1)}…` : note
 }
 
-// Telegram indexes the tag the manifest caption carries, so one search returns one hit
-// per backup instead of one per chunk. What comes back is still whatever the server
-// decided to match, which is why the caller filters on the file name afterwards.
-async function realSearchManifests(client, peer, limit) {
-  return await searchDocuments(client, peer, { search: MANIFEST_TAG, limit })
+// What list has to read through depends on how big the backups are, not how many there are:
+// it walks from the newest message down and stops the moment it has --limit manifests, so the
+// three thousandth backup in a chat costs nothing because it is never reached. What costs is
+// the chunks in between — one message each — which is why the ceiling is a budget per backup
+// asked for rather than one number for every chat.
+//
+// 60 documents per backup covers a backup of about 105GB at the default chunk size. A fixed
+// 1000 was both too tight and too loose at once: twenty backups of 100GB need 1140 documents
+// and got 1000 of them, while `--limit 5` never needed more than 300.
+export const DOCUMENTS_PER_BACKUP = 60
+
+// A search result is already a manifest, so it buys far more backups per document read. Its
+// budget only has to cover what matchesTerm throws away — measured 2026-09-08, a term like
+// "2026-09" comes back matching everything and is then cut down to the month asked for.
+export const RESULTS_PER_BACKUP = 20
+
+// --limit takes any whole number, so the budget needs an end of its own: without one,
+// `--limit 100000` would ask for six million documents and sixty thousand requests.
+export const MAX_LIST_DOCUMENTS = 10000
+
+export function documentBudget(limit, perBackup) {
+  return Math.min(limit * perBackup, MAX_LIST_DOCUMENTS)
 }
 
+// Stopping at the ceiling used to leave the reader standing there: "there may be older backups
+// further back" is true and offers nothing to do about it. --search reaches them without
+// reading the chunks in between, which is the whole reason it exists.
+const DEEPER_HINT =
+  '"npx telstore list --search <text>" reaches older ones without reading every chunk ' +
+  'in between.'
+
+// Telegram matches whole words and nothing shorter: measured 2026-09-08, "projex" found
+// projex.zip while "proj", "pro" and "pr" each found nothing at all. That is the one way a
+// search can come back empty over a backup that is plainly there, so the empty answer says
+// it, and points at the listing that never asks the index.
+const SEARCH_MISS_HELP =
+  'Telegram matches whole words: "projex" finds projex.zip, "proj" does not. ' +
+  'Run "npx telstore list" without --search to see every backup without going through ' +
+  'the search index.'
+
 function backupIdFromFileName(fileName) {
-  return fileName.replace(/\.manifest\.json$/, '')
+  return fileName.slice(0, -MANIFEST_SUFFIX.length)
 }
 
 function utcDay(unixSeconds) {
@@ -74,6 +110,45 @@ function toRow(message) {
   }
 }
 
+// A search term is a question about one run, and an empty one is not a question: answering
+// it with every backup would look exactly like a search that matched everything.
+function parseSearchTerm(raw) {
+  if (raw === undefined || raw === null) return null
+
+  const term = String(raw).trim()
+
+  if (term === '') {
+    throw new Error(
+      '--search is empty. Write the word to look for, or leave the flag off — "list" ' +
+        'without it shows every backup.',
+    )
+  }
+
+  return term
+}
+
+// The four fields a person remembers about a backup, and the whole of what --search compares
+// against. The note is matched entire rather than the 40 characters the table has room for:
+// a word that fell off the end of the column is still a word they typed. A card that cannot
+// be read back leaves only what the message itself knows.
+function searchableFields(message) {
+  const id = backupIdFromFileName(message.fileName)
+  const card = parseManifestCaption(message.caption)
+
+  if (!card) return [id, utcDay(message.date)]
+
+  return [id, card.name, card.note ?? '', card.createdAt.slice(0, 10)]
+}
+
+// Telegram decides what comes back; this decides what is true. The index answers a term the
+// way it wants to — measured 2026-09-08, "2026-09" returned every document in the chat — so
+// a hit is shown only if the term really is in one of the fields above. Without this pass a
+// search for a month would list backups from every other month, which is the plausible wrong
+// answer this project exists to refuse.
+function matchesTerm(message, term) {
+  return searchableFields(message).some((field) => field.toLowerCase().includes(term))
+}
+
 function renderTable(rows) {
   // Most people never write a note, and a column of dashes tells them nothing they did not
   // already know while costing every other column the width it takes.
@@ -102,12 +177,21 @@ export async function runList(options = {}, deps = {}) {
     configDir = defaultConfigDir(),
     connect = realConnect,
     disconnect = (client) => client.destroy(),
-    searchManifests = realSearchManifests,
+    readDocuments = iterDocuments,
+    searchManifests = iterManifestSearch,
     log = (line) => console.log(line),
+    // The notice is drawn on stderr, and only onto a terminal: unlike an upload's progress
+    // bar, `list` is a command people pipe into grep, and a carriage return in a log file is
+    // rubbish. Null means draw nothing at all.
+    writeProgress = process.stderr.isTTY ? (text) => process.stderr.write(text) : null,
+    now = () => Date.now(),
   } = deps
 
   const config = await loadConfig(configDir)
   const { values: settings } = resolveSettings(options, config, { file: configFile(configDir) })
+  // Before the login gate: a bad term is the user's own typing, and telling them to log in
+  // first would send them off after the wrong thing.
+  const term = parseSearchTerm(options.search)
   // Ask about the login before the destination: telling someone who has never logged in
   // to pick a chat sends them off after the wrong thing.
   assertLoggedIn(config)
@@ -115,29 +199,99 @@ export async function runList(options = {}, deps = {}) {
 
   const client = await connect(config, { verbose: settings.verbose })
 
-  let found
+  // Walked rather than searched, unless a term was given. Telegram's text index can answer
+  // nothing at all about a chat that is full of backups — it did for a whole day in a channel
+  // that had just been created — and "No backups found" is a sentence someone acts on. The
+  // documents themselves were right every time they were asked for.
+  //
+  // --search is the one place worth paying the index for: a term matches a backup that may be
+  // ten thousand messages back, and walking to it would cost a request per hundred documents
+  // in between, every time, for as long as the chat keeps growing. So the search narrows and
+  // matchesTerm decides — the index is asked where to look, never what is true.
+  const found = []
+  let read = 0
+
+  const unit = term ? 'search results' : 'documents'
+  const budget = documentBudget(settings.limit, term ? RESULTS_PER_BACKUP : DOCUMENTS_PER_BACKUP)
+
+  const results = term
+    ? searchManifests(client, chat, term, { max: budget })
+    : readDocuments(client, chat, { max: budget })
+
+  const wanted = term === null ? null : term.toLowerCase()
+  const notice = writeProgress ? createWalkNotice({ write: writeProgress, now }) : null
+
   try {
-    found = await searchManifests(client, chat, settings.limit)
+    for await (const document of results) {
+      read += 1
+
+      notice?.tick(
+        `Reading ${chatName(chat)}… ${read} ${unit}, ${found.length} backup` +
+          `${found.length === 1 ? '' : 's'}`,
+      )
+
+      if (!document.fileName?.endsWith(MANIFEST_SUFFIX)) continue
+      if (wanted !== null && !matchesTerm(document, wanted)) continue
+
+      found.push(document)
+
+      // Everything past here is older than the twentieth newest backup, and nobody asked
+      // for it. In a chat of ten thousand chunks this is the difference between one
+      // request and ten.
+      if (found.length >= settings.limit) break
+    }
   } finally {
+    notice?.clear()
     await closeQuietly(client, disconnect)
   }
 
+  // The one thing either reader cannot see is what lies past its own ceiling, so anything it
+  // says about the whole chat has to stop at the edge of what it read.
+  const capped = found.length < settings.limit && read >= budget
+
   log(`Destination  ${describeChat(chat)}`)
+  if (term) log(`Search       ${JSON.stringify(term)}`)
   log('')
 
-  const rows = found
-    .filter((message) => message.fileName?.endsWith('.manifest.json'))
-    .map(toRow)
+  const rows = found.map(toRow)
 
   if (rows.length === 0) {
-    log(`No backups found in ${chatName(chat)}. Upload one with: npx telstore <file>`)
+    if (term) {
+      log(
+        capped
+          ? `No backups matching ${JSON.stringify(term)} in the newest ${budget} ` +
+              `${unit} from ${chatName(chat)}. There may be older ones further back.`
+          : `No backups matching ${JSON.stringify(term)} in ${chatName(chat)}.`,
+      )
+      log(SEARCH_MISS_HELP)
+      return rows
+    }
+
+    log(
+      capped
+        ? `No backups in the newest ${budget} ${unit} of ${chatName(chat)}. ` +
+            `There may be older ones further back. ${DEEPER_HINT}`
+        : `No backups found in ${chatName(chat)}. Upload one with: npx telstore <file>`,
+    )
     return rows
   }
 
   for (const line of renderTable(rows)) log(line)
 
   log('')
-  log(`${rows.length} backup${rows.length === 1 ? '' : 's'}. Restore with: npx telstore restore <backup-id>`)
+  log(
+    `${rows.length} backup${rows.length === 1 ? '' : 's'}` +
+      `${term ? ` matching ${JSON.stringify(term)}` : ''}. ` +
+      'Restore with: npx telstore restore <backup-id>',
+  )
+
+  if (capped) {
+    log(
+      `Read the newest ${budget} ${unit} in ${chatName(chat)} to find them — ` +
+        `there may be older ${term ? 'matches' : 'backups'} further back.` +
+        `${term ? '' : ` ${DEEPER_HINT}`}`,
+    )
+  }
 
   return rows
 }