telstore 0.1.0

package/src/commands/list.js

@@ -0,0 +1,119 @@
+import { MANIFEST_TAG, parseManifestCaption } from '../caption.js'
+import { chatName, describeChat } from '../chat.js'
+import {
+  assertLoggedIn,
+  closeQuietly,
+  connect as realConnect,
+  searchDocuments,
+} from '../client.js'
+import { configFile, defaultConfigDir, loadConfig } from '../config.js'
+import { requireChat, resolveSettings } from '../settings.js'
+
+const UNKNOWN = '—'
+const GAP = '  '
+
+const COLUMNS = [
+  { header: 'BACKUP ID', key: 'id' },
+  { header: 'FILE', key: 'name' },
+  { header: 'SIZE', key: 'size', right: true },
+  { header: 'CHUNKS', key: 'chunks', right: true },
+  { header: 'CREATED', key: 'created' },
+]
+
+// Telegram indexes the tag the manifest caption carries, so one search returns one hit
+// per backup instead of one per chunk. What comes back is still whatever the server
+// decided to match, which is why the caller filters on the file name afterwards.
+async function realSearchManifests(client, peer, limit) {
+  return await searchDocuments(client, peer, { search: MANIFEST_TAG, limit })
+}
+
+function backupIdFromFileName(fileName) {
+  return fileName.replace(/\.manifest\.json$/, '')
+}
+
+function utcDay(unixSeconds) {
+  return new Date(unixSeconds * 1000).toISOString().slice(0, 10)
+}
+
+// The card is text in a chat, which means a person can edit or predate it. The id is the
+// one field restore cannot be wrong about, so it always comes from the file name telark
+// wrote; the caption only decorates. A card that cannot be read back leaves the rest
+// unknown, because inventing it would describe a backup that does not exist.
+function toRow(message) {
+  const id = backupIdFromFileName(message.fileName)
+  const card = parseManifestCaption(message.caption)
+
+  if (!card) {
+    return { id, name: UNKNOWN, size: UNKNOWN, chunks: UNKNOWN, created: utcDay(message.date) }
+  }
+
+  return {
+    id,
+    name: card.name,
+    size: card.size,
+    chunks: String(card.chunks),
+    created: card.createdAt.slice(0, 10),
+  }
+}
+
+function renderTable(rows) {
+  const widths = COLUMNS.map((column) =>
+    Math.max(column.header.length, ...rows.map((row) => row[column.key].length)),
+  )
+
+  const line = (cells) =>
+    cells
+      .map((cell, i) => (COLUMNS[i].right ? cell.padStart(widths[i]) : cell.padEnd(widths[i])))
+      .join(GAP)
+      .trimEnd()
+
+  return [
+    line(COLUMNS.map((column) => column.header)),
+    ...rows.map((row) => line(COLUMNS.map((column) => row[column.key]))),
+  ]
+}
+
+export async function runList(options = {}, deps = {}) {
+  const {
+    configDir = defaultConfigDir(),
+    connect = realConnect,
+    disconnect = (client) => client.destroy(),
+    searchManifests = realSearchManifests,
+    log = (line) => console.log(line),
+  } = deps
+
+  const config = await loadConfig(configDir)
+  const { values: settings } = resolveSettings(options, config, { file: configFile(configDir) })
+  // Ask about the login before the destination: telling someone who has never logged in
+  // to pick a chat sends them off after the wrong thing.
+  assertLoggedIn(config)
+  const chat = requireChat(settings)
+
+  const client = await connect(config, { verbose: settings.verbose })
+
+  let found
+  try {
+    found = await searchManifests(client, chat, settings.limit)
+  } finally {
+    await closeQuietly(client, disconnect)
+  }
+
+  log(`Destination  ${describeChat(chat)}`)
+  log('')
+
+  const rows = found
+    .filter((message) => message.fileName?.endsWith('.manifest.json'))
+    .map(toRow)
+
+  if (rows.length === 0) {
+    log(`No backups found in ${chatName(chat)}. Upload one with: npx telark <file>`)
+    return rows
+  }
+
+  for (const line of renderTable(rows)) log(line)
+
+  log('')
+  log(`${rows.length} backup${rows.length === 1 ? '' : 's'}. Restore with: npx telark restore <backup-id>`)
+
+  return rows
+}

package/src/commands/login.js

@@ -0,0 +1,109 @@
+import readline from 'node:readline/promises'
+import { stdin, stdout } from 'node:process'
+
+import { TelegramClient } from 'telegram'
+import { StringSession } from 'telegram/sessions/index.js'
+
+import { loadConfig, saveConfig, defaultConfigDir } from '../config.js'
+import { normalizeChatTarget } from '../chat.js'
+import { createLogger } from '../client.js'
+import { resolveSettings } from '../settings.js'
+
+function createPrompts() {
+  const rl = readline.createInterface({ input: stdin, output: stdout })
+  return {
+    ask: (question) => rl.question(question),
+    close: () => rl.close(),
+  }
+}
+
+const LOGIN_ERROR_MESSAGES = {
+  PHONE_NUMBER_INVALID: 'invalid phone number',
+  PHONE_CODE_INVALID: 'wrong verification code',
+  PHONE_CODE_EXPIRED: 'verification code expired',
+  PASSWORD_HASH_INVALID: 'wrong two-step password',
+  FLOOD_WAIT: 'rate limited by Telegram, need to wait',
+}
+
+export function describeLoginError(err) {
+  const message = String(err?.message ?? err ?? '')
+  const known = Object.entries(LOGIN_ERROR_MESSAGES).find(([code]) => message.startsWith(code))
+
+  if (!known) return message
+
+  const [, description] = known
+  return `${description} (${message})`
+}
+
+// GramJS starts an update loop the moment a client connects, and that loop only stops when
+// destroy() marks the client destroyed. disconnect() alone leaves it pinging a socket that is
+// already closed: every ping fails with "Error: TIMEOUT" and asks the sender to reconnect,
+// printed straight over the destination question login asks after signing in. Every other
+// command shuts down the same way, and login is the seam tests need to reach it.
+const createTelegramClient = (apiId, apiHash, options) =>
+  new TelegramClient(new StringSession(''), apiId, apiHash, options)
+
+export async function runLogin({
+  configDir = defaultConfigDir(),
+  prompts = createPrompts(),
+  verbose = false,
+  createClient = createTelegramClient,
+  shutdown = (client) => client.destroy(),
+  log = (line) => console.log(line),
+} = {}) {
+  const config = await loadConfig(configDir)
+  const storedChat = config.settings?.chat
+  const loud = verbose || resolveSettings({}, config).values.verbose
+
+  log('You need your own api_id and api_hash. Get them at https://my.telegram.org → API development tools.\n')
+
+  const apiIdAnswer = (await prompts.ask(`api_id${config.apiId ? ` [${config.apiId}]` : ''}: `)).trim()
+
+  if (apiIdAnswer !== '' && !/^\d+$/.test(apiIdAnswer)) {
+    prompts.close()
+    throw new Error('api_id must be an integer.')
+  }
+
+  const apiId = apiIdAnswer === '' ? config.apiId : Number(apiIdAnswer)
+  const apiHash = (await prompts.ask(`api_hash${config.apiHash ? ' [keep current]' : ''}: `)) || config.apiHash
+
+  if (!apiId || !apiHash) {
+    prompts.close()
+    throw new Error('Missing api_id or api_hash.')
+  }
+
+  const client = createClient(apiId, apiHash, {
+    connectionRetries: 5,
+    baseLogger: createLogger(loud),
+  })
+
+  await client.start({
+    phoneNumber: () => prompts.ask('Phone number (e.g. +1...): '),
+    phoneCode: () => prompts.ask('Verification code Telegram just sent: '),
+    password: () => prompts.ask('Two-step password (leave blank if not enabled): '),
+    onError: (err) => console.error(`Login failed: ${describeLoginError(err)}`),
+  })
+
+  const me = await client.getMe()
+  const session = client.session.save()
+  await shutdown(client)
+
+  const chatAnswer = (
+    await prompts.ask(
+      `Which chat should backups go to? (@username, -100..., or me)${storedChat ? ` [${storedChat}]` : ''}, Enter to skip: `,
+    )
+  ).trim()
+
+  prompts.close()
+
+  const next = { ...config, apiId, apiHash, session }
+
+  if (chatAnswer !== '') {
+    next.settings = { ...config.settings, chat: String(normalizeChatTarget(chatAnswer)) }
+  }
+
+  await saveConfig(next, configDir)
+
+  log(`\nLogged in as ${me.username ? `@${me.username}` : me.firstName}.`)
+  log(`Config saved to ${configDir}/config.json`)
+}

package/src/commands/logout.js

@@ -0,0 +1,13 @@
+import { clearSession, defaultConfigDir } from '../config.js'
+
+export async function runLogout({ configDir = defaultConfigDir() } = {}) {
+  await clearSession(configDir)
+  console.log(
+    'Removed the session stored on this machine. api_id, api_hash and the destination are kept.',
+  )
+  console.log(
+    'Note: this only deletes the local copy — the session is still alive on Telegram\'s side. ' +
+      'To revoke access for good, open Telegram → Settings → Devices (Active sessions) ' +
+      'and terminate that session.',
+  )
+}

package/src/commands/restore.js

@@ -0,0 +1,215 @@
+import { promises as fs } from 'node:fs'
+import path from 'node:path'
+
+import {
+  closeQuietly,
+  connect as realConnect,
+  findManifestMessage,
+  readMessageBytes as realReadMessageBytes,
+} from '../client.js'
+import { askConfirm } from '../confirm.js'
+import { configFile, defaultConfigDir, loadConfig } from '../config.js'
+import { requireChat, resolveSettings } from '../settings.js'
+import { downloadToFile } from '../downloader.js'
+import { parseManifest } from '../manifest.js'
+import { createProgress, formatBytes, formatDuration } from '../progress.js'
+
+// Anything past a minute of waiting needs saying out loud; below that the pause is shorter
+// than the time a user would spend wondering about it.
+const LONG_WAIT_MS = 60_000
+
+// A transient error that resolves itself on the next try is not news, and one line per
+// occurrence buries the progress bar in a wall of text. Stay quiet until the third retry:
+// by then the trouble has outlived two backoffs and is worth saying out loud.
+const ANNOUNCE_AFTER_ATTEMPT = 3
+
+async function realGetMessage(client, peer, msgId) {
+  const [message] = await client.getMessages(peer, { ids: [msgId] })
+  return message ?? null
+}
+
+export async function realDownloadChunk(client, message, handle, offset, onProgress, retryOptions) {
+  return await downloadToFile(client, message, handle.fd, { offset, onProgress, retryOptions })
+}
+
+// manifest.name comes from data downloaded off Telegram — don't trust it when picking
+// a path ourselves. path.basename stops "../../x" but still returns "..", "." or "" for
+// a few pathological names: path.resolve('..') is the parent directory, so a multi-GB
+// .partial file would land outside the current directory and only blow up at rename.
+function safeOutName(name) {
+  const base = path.basename(String(name ?? ''))
+
+  if (base === '' || base === '.' || base === '..') {
+    throw new Error(
+      `The name in the manifest ("${name}") cannot be used as a file name. ` +
+        'Run again with --out <path> to choose where to write.',
+    )
+  }
+
+  return base
+}
+
+export async function runRestore(backupId, options = {}, deps = {}) {
+  const {
+    connect = realConnect,
+    disconnect = (client) => client.destroy(),
+    configDir = defaultConfigDir(),
+    searchManifest = findManifestMessage,
+    readMessageBytes = realReadMessageBytes,
+    getMessage = realGetMessage,
+    downloadChunk = realDownloadChunk,
+    confirm = askConfirm,
+    retryOptions = {},
+    writeErr = (line) => process.stderr.write(line),
+    log: writeLog = (line) => console.log(line),
+    silent = false,
+  } = deps
+
+  const config = await loadConfig(configDir)
+  const { values: settings } = resolveSettings(options, config, { file: configFile(configDir) })
+  const chat = requireChat(settings)
+  const log = silent ? () => {} : writeLog
+  const warn = silent ? () => {} : writeErr
+
+  // A restore keeps no progress file, so a part that comes back -503 is retried rather than
+  // thrown away — and a retry nobody is told about is indistinguishable from a hung transfer,
+  // because the progress bar simply stops moving while the wait runs.
+  function onRetry(err, attempt, delayMs, elapsedMs = 0) {
+    if (delayMs > LONG_WAIT_MS) {
+      warn(
+        `\nTelegram wants ${formatDuration(delayMs / 1000)} of waiting before the next part ` +
+          `(${err.message}). telark is waiting and will carry on by itself, leave it running.\n`,
+      )
+      return
+    }
+
+    // The exception to staying quiet: an attempt that took a minute to fail spent that
+    // minute with the bar frozen, which is exactly what a hang looks like. Those are worth
+    // a line the first time, whatever the attempt number.
+    if (attempt < ANNOUNCE_AFTER_ATTEMPT && elapsedMs < LONG_WAIT_MS) return
+
+    warn(
+      `\nTemporary error (${err.message}), retry ${attempt} in ` +
+        `${formatDuration(delayMs / 1000)}.\n`,
+    )
+  }
+
+  const client = await connect(config, { verbose: settings.verbose })
+
+  try {
+    const manifestMessage = await searchManifest(client, chat, backupId)
+
+    if (!manifestMessage) {
+      throw new Error(
+        `No manifest found for ${backupId} in ${chat}. ` +
+          'Check the backup id, or use --to to point at the right chat.',
+      )
+    }
+
+    const manifest = parseManifest(await readMessageBytes(client, manifestMessage))
+    // When the user passes --out, respect that path verbatim.
+    const target = path.resolve(options.out ?? safeOutName(manifest.name))
+    const partial = `${target}.partial`
+
+    // Only ENOENT means "no file yet". Treating a permission or I/O error as absence
+    // would have telark overwrite the user's file without asking.
+    let exists = true
+    try {
+      await fs.stat(target)
+    } catch (err) {
+      if (err.code !== 'ENOENT') throw err
+      exists = false
+    }
+
+    if (exists && !(await confirm(`${target} already exists. Overwrite? [y/N] `))) {
+      throw new Error('Cancelled on request.')
+    }
+
+    log(`Backup ${manifest.id}`)
+    log(`File   ${target} (${formatBytes(manifest.size)}, ${manifest.chunks.length} chunks)\n`)
+
+    const handle = await fs.open(partial, 'w+')
+
+    try {
+      await handle.truncate(manifest.size)
+
+      // One bar for the whole restore. The label names the chunk in flight, but the bar, the
+      // byte counts, the speed and the ETA all describe the file, so the line runs 0% to 100%
+      // once instead of restarting at every chunk boundary — with 1800MB chunks, a per-chunk
+      // ETA answers a question nobody asked.
+      // warn is already the no-op when silent, and createProgress draws through nothing else.
+      const progress = createProgress({
+        total: manifest.size,
+        label: `Chunk 1/${manifest.chunks.length}`,
+        write: warn,
+      })
+
+      try {
+        for (const chunk of manifest.chunks) {
+          // Before getMessage, not after: the bar is then on screen from the first moment,
+          // and finish() below always has a line to close.
+          progress.setLabel(`Chunk ${chunk.i + 1}/${manifest.chunks.length}`)
+
+          const message = await getMessage(client, chat, chunk.msgId)
+
+          if (!message) {
+            throw new Error(
+              `Missing chunk ${chunk.i + 1}/${manifest.chunks.length}: message ${chunk.msgId} is no longer in ${chat}. ` +
+                'This backup cannot be restored.',
+            )
+          }
+
+          const { sha256, size } = await downloadChunk(
+            client,
+            message,
+            handle,
+            chunk.i * manifest.chunkSize,
+            progress.advance,
+            { ...retryOptions, onRetry },
+          )
+
+          if (size !== chunk.size) {
+            throw new Error(
+              `Chunk ${chunk.i + 1} has ${size} bytes, the manifest records ${chunk.size} bytes — mismatch.`,
+            )
+          }
+
+          if (sha256 !== chunk.sha256) {
+            throw new Error(
+              `Chunk ${chunk.i + 1} has a sha256 that does not match the manifest. The download is kept at ${partial} for inspection.`,
+            )
+          }
+        }
+      } finally {
+        // The bar owns a line that \r keeps returning to. Ending it here rather than after the
+        // loop means a chunk that fails mid-download still leaves the cursor on a fresh line,
+        // so "Error: ..." does not land on top of the bar.
+        progress.finish()
+      }
+    } finally {
+      await handle.close()
+    }
+
+    // Last line of defence: if every chunk matched its sha256 and the file is still the
+    // wrong length, the layout went wrong somewhere. Better to fail than to rename a
+    // wrong file into the real one.
+    const written = await fs.stat(partial)
+
+    if (written.size !== manifest.size) {
+      throw new Error(
+        `The assembled file has ${written.size} bytes, the manifest records ${manifest.size} bytes — mismatch. ` +
+          `The download is kept at ${partial} for inspection.`,
+      )
+    }
+
+    await fs.rename(partial, target)
+
+    log(`\nDone. Wrote ${formatBytes(manifest.size)} to ${target}`)
+
+    return { path: target, size: manifest.size }
+  } finally {
+    await closeQuietly(client, disconnect, (err) =>
+      warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`),
+    )
+  }
+}

package/src/commands/status.js

@@ -0,0 +1,102 @@
+import path from 'node:path'
+
+import { countChunks } from '../chunking.js'
+import { describeChat } from '../chat.js'
+import { assertLoggedIn, closeQuietly, connect as realConnect } from '../client.js'
+import { configFile, defaultConfigDir, loadConfig } from '../config.js'
+import { formatBytes } from '../progress.js'
+import { resolveSettings } from '../settings.js'
+import { listStates } from '../state.js'
+
+const LABEL_WIDTH = 'Destination'.length + 2
+
+function row(label, value) {
+  return `${label.padEnd(LABEL_WIDTH)}${value}`
+}
+
+function describeAccount(me) {
+  const name = [me.firstName, me.lastName].filter(Boolean).join(' ')
+  const handle = me.username ? ` (@${me.username})` : ''
+
+  return `${name || me.username || me.phone || 'unknown'}${handle}`
+}
+
+// The account line is the only part that needs Telegram, and it is also the only part that
+// can fail. Whatever it costs, it must not take the rest of the report down with it: the
+// unfinished backups are exactly what someone runs status to see after a session expires.
+async function accountLine(config, verbose, deps) {
+  const { connect, disconnect } = deps
+
+  try {
+    assertLoggedIn(config)
+  } catch (err) {
+    return err.message
+  }
+
+  let client
+  try {
+    client = await connect(config, { verbose })
+  } catch (err) {
+    return err.message
+  }
+
+  try {
+    return describeAccount(await client.getMe())
+  } catch (err) {
+    return `could not be read: ${err.message}`
+  } finally {
+    await closeQuietly(client, disconnect)
+  }
+}
+
+export async function runStatus(options = {}, deps = {}) {
+  const {
+    configDir = defaultConfigDir(),
+    connect = realConnect,
+    disconnect = (client) => client.destroy(),
+    log = (line) => console.log(line),
+  } = deps
+
+  const config = await loadConfig(configDir)
+
+  // status is what someone runs *because* something is wrong, so nothing here may take the
+  // whole report down — the same reason accountLine catches its own failures. A stored
+  // setting that will not parse is loud everywhere else; here it is loud in the row it
+  // belongs to, with the account and the unfinished backups still printed around it.
+  let settings = null
+  let settingsError = null
+
+  try {
+    settings = resolveSettings(options, config, { file: configFile(configDir) }).values
+  } catch (err) {
+    settingsError = err.message
+  }
+
+  log(row('Account', await accountLine(config, settings?.verbose ?? false, { connect, disconnect })))
+  log(
+    row(
+      'Destination',
+      settingsError ??
+        (settings.chat === null
+          ? 'none set — run "npx telark config chat @my_backups" to set one'
+          : describeChat(settings.chat)),
+    ),
+  )
+
+  const states = await listStates(configDir)
+
+  if (states.length === 0) {
+    log(row('Unfinished', 'none'))
+    return
+  }
+
+  log(row('Unfinished', `${states.length} backup${states.length === 1 ? '' : 's'}`))
+
+  for (const state of states) {
+    const total = countChunks(state.size, state.chunkSize)
+    const done = Object.keys(state.done ?? {}).length
+
+    log(`  ${state.id}  ${path.basename(state.path)}  ${done}/${total} chunks  ${formatBytes(state.size)}`)
+    log(`  → ${describeChat(state.chat)}`)
+  }
+}