telstore 0.1.7 → 0.1.9

package/src/commands/status.js

@@ -1,3 +1,5 @@
+import { promises as fs } from 'node:fs'
+
 import { countChunks } from '../chunking.js'
 import { describeChat } from '../chat.js'
 import { closeQuietly, connect as realConnect } from '../client.js'
@@ -5,7 +7,7 @@ import { configFile, defaultConfigDir, loadConfig } from '../config.js'
 import { formatBytes } from '../progress.js'
 import { assertLoggedIn } from '../session.js'
 import { resolveSettings } from '../settings.js'
-import { canResume, listStates } from '../state.js'
+import { canResume, listRestores, listStates } from '../state.js'
 
 const LABEL_WIDTH = 'Destination'.length + 2
 
@@ -52,6 +54,15 @@ const NO_RESUME = {
   unreadable: 'the record does not name a file that can be read',
 }
 
+// Why the .partial cannot be resumed from, in the same spirit as NO_RESUME above: an
+// EACCES or an ELOOP is not the same fact as the file being gone, and telling the user
+// their multi-gigabyte download vanished when it is sitting there, unreadable, sends them
+// looking for the wrong problem.
+const NO_PARTIAL_RESUME = {
+  missing: 'the partial download is no longer there',
+  unreadable: 'the partial download cannot be read',
+}
+
 // A command is printed only when it will really resume. Printing one regardless would be
 // telling the user to run something that quietly starts a second backup and abandons every
 // chunk this one already sent — and those chunks are then findable only by this id, which
@@ -73,6 +84,44 @@ async function resumeLines(key, state, destination, done) {
   return lines
 }
 
+// The record holds an absolute target, and printing it is what makes the pasted line
+// correct. Without --out, runRestore resolves the manifest's own name against the current
+// directory — a name status does not have (it is in a manifest on Telegram, and status
+// reaches Telegram only for the account line, where it deliberately tolerates failure) and
+// a directory status cannot assume. A command pasted from elsewhere would resolve to
+// another path, miss the .partial and start over, which is the failure resume exists to end.
+function restoreCommand(record, destination) {
+  const matches = destination !== null && record.chat === String(destination)
+  const chat = matches ? '' : ` --chat ${shellArg(record.chat)}`
+
+  return `npx telstore restore ${shellArg(record.id)} --out ${shellArg(record.target)}${chat}`
+}
+
+// The .partial is the whole reason a resume is possible, so its absence is the one thing
+// worth checking before offering a command that would silently start over.
+async function restoreResumeLine(record, destination) {
+  try {
+    await fs.stat(`${record.target}.partial`)
+  } catch (err) {
+    const reason = err.code === 'ENOENT' ? 'missing' : 'unreadable'
+
+    return field('Resume', `not possible: ${NO_PARTIAL_RESUME[reason]}.`)
+  }
+
+  return field('Resume', restoreCommand(record, destination))
+}
+
+// Only the kinds actually present are named. "N backups" fits an upload and not a restore:
+// there the backup is finished and sitting in the chat, and it is the restore that stopped.
+function unfinishedCount(uploads, restores) {
+  const parts = []
+
+  if (uploads > 0) parts.push(`${uploads} upload${uploads === 1 ? '' : 's'}`)
+  if (restores > 0) parts.push(`${restores} restore${restores === 1 ? '' : 's'}`)
+
+  return parts.length === 0 ? 'none' : parts.join(', ')
+}
+
 function describeAccount(me) {
   const name = [me.firstName, me.lastName].filter(Boolean).join(' ')
   const handle = me.username ? ` (@${me.username})` : ''
@@ -153,24 +202,42 @@ export async function runStatus(options = {}, deps = {}) {
     ),
   )
 
-  const states = await listStates(configDir)
+  const uploads = await listStates(configDir)
+  const restores = await listRestores(configDir)
 
-  if (states.length === 0) {
-    log(row('Unfinished', 'none'))
-    return
-  }
+  log(row('Unfinished', unfinishedCount(uploads.length, restores.length)))
 
-  log(row('Unfinished', `${states.length} backup${states.length === 1 ? '' : 's'}`))
+  if (uploads.length === 0 && restores.length === 0) return
 
-  // The destination is what decides whether the resume command needs a --chat. A row that
+  // The destination is what decides whether a resume command needs a --chat. A row that
   // failed to parse leaves nothing to compare against, which is not the same as a match.
   const destination = settings?.chat ?? null
 
-  for (const { key, state } of states) {
+  // Newest first, by when the record last changed — which is when that transfer last made
+  // progress, and the same ordering pruneStates already means by "recent".
+  const entries = [
+    ...uploads.map((entry) => ({ ...entry, kind: 'upload' })),
+    ...restores.map((entry) => ({ ...entry, kind: 'restore' })),
+  ].sort((a, b) => b.mtimeMs - a.mtimeMs)
+
+  for (const entry of entries) {
+    log('')
+
+    if (entry.kind === 'restore') {
+      const { record } = entry
+
+      log(`  ${record.id}`)
+      log(field('File', `${record.target}  (${formatBytes(record.size)})`))
+      log(field('Chunks', `${record.done ?? 0} of ${record.chunks ?? '?'} restored`))
+      log(field('Chat', describeChat(record.chat)))
+      log(await restoreResumeLine(record, destination))
+      continue
+    }
+
+    const { key, state } = entry
     const total = countChunks(state.size, state.chunkSize)
     const done = Object.keys(state.done ?? {}).length
 
-    log('')
     log(`  ${state.id}`)
     log(field('File', `${state.path}  (${formatBytes(state.size)})`))
     log(field('Chunks', `${done} of ${total} uploaded`))

package/src/commands/verify.js

@@ -0,0 +1,298 @@
+import { chatName, describeChat } from '../chat.js'
+import {
+  MESSAGE_BATCH_SIZE,
+  closeQuietly,
+  connect as realConnect,
+  documentFileName,
+  documentSize,
+  findManifestMessage,
+  getDocuments as realGetDocuments,
+  readMessageBytes as realReadMessageBytes,
+} from '../client.js'
+import { configFile, defaultConfigDir, loadConfig } from '../config.js'
+import { chunkFileName, manifestFileName, parseManifest } from '../manifest.js'
+import { formatBytes, formatDuration } from '../progress.js'
+import { assertLoggedIn } from '../session.js'
+import { requireChat, resolveSettings } from '../settings.js'
+
+// parseManifest guarantees every number in a manifest, but not the file name — it is
+// decoration, and nothing verifies differently because of it. It is still text off a chat.
+function describeName(name) {
+  return typeof name === 'string' && name.trim() !== '' ? name : '—'
+}
+
+function plural(n, word) {
+  return `${n} ${word}${n === 1 ? '' : 's'}`
+}
+
+// What is wrong with one chunk, or null when nothing is. The first failing check wins: a
+// chunk is damaged or it is not, and listing three complaints about one message would make
+// "2 damaged" mean something other than two chunks.
+//
+// Every question here is one the chat can answer without sending a byte of the file. What
+// this cannot ask is whether the bytes inside are the bytes that went up — only downloading
+// them answers that, which is why the closing line says so rather than leaving it implied.
+function inspect(chunk, message, { backupId, total, chat }) {
+  const at = `Chunk ${chunk.i + 1}/${total}`
+
+  if (!message) {
+    return `${at} is gone: message ${chunk.msgId} is no longer in ${chatName(chat)}.`
+  }
+
+  const size = documentSize(message)
+
+  if (size === null) {
+    return `${at} is message ${chunk.msgId}, which has no file attached.`
+  }
+
+  const wanted = chunkFileName(backupId, chunk.i)
+  const fileName = documentFileName(message)
+
+  if (fileName !== wanted) {
+    return fileName === null
+      ? `${at} is message ${chunk.msgId}, whose file carries no name; the manifest expects ` +
+          `"${wanted}".`
+      : `${at} is message ${chunk.msgId}, which carries the file name ` +
+          `${JSON.stringify(fileName)} rather than "${wanted}".`
+  }
+
+  if (size !== chunk.size) {
+    return `${at} is ${size} bytes in the chat, the manifest records ${chunk.size}.`
+  }
+
+  return null
+}
+
+export async function runVerify(backupId, options = {}, deps = {}) {
+  const {
+    connect = realConnect,
+    disconnect = (client) => client.destroy(),
+    configDir = defaultConfigDir(),
+    searchManifest = findManifestMessage,
+    readMessageBytes = realReadMessageBytes,
+    getDocuments = realGetDocuments,
+    retryOptions = {},
+    writeErr = (line) => process.stderr.write(line),
+    log: writeLog = (line) => console.log(line),
+    silent = false,
+  } = deps
+
+  const config = await loadConfig(configDir)
+  const { values: settings } = resolveSettings(options, config, { file: configFile(configDir) })
+  // Ask about the login before the destination, as list does: telling someone who has never
+  // logged in to pick a chat sends them off after the wrong thing.
+  assertLoggedIn(config)
+  const chat = requireChat(settings)
+
+  const log = silent ? () => {} : writeLog
+  const warn = silent ? () => {} : writeErr
+
+  // Upload and restore stay quiet until the third retry so a handful of -503s do not bury
+  // the progress bar. There is no bar here to bury — this command sends a handful of small
+  // requests and prints one verdict — so a wait long enough to notice is announced at once.
+  function onRetry(err, attempt, delayMs) {
+    warn(
+      `\nTemporary error (${err.message}), retry ${attempt} in ` +
+        `${formatDuration(delayMs / 1000)}.\n`,
+    )
+  }
+
+  const client = await connect(config, { verbose: settings.verbose })
+
+  try {
+    const manifestMessage = await searchManifest(client, chat, backupId)
+
+    if (!manifestMessage) {
+      throw new Error(
+        `No backup ${backupId} found in ${chatName(chat)}. Check the id with ` +
+          '"npx telstore list", or use --chat to point at the right chat.',
+      )
+    }
+
+    // The full layout checks, not the lenient path delete takes. delete reads a manifest to
+    // destroy what it names, so a broken one is exactly what somebody is there to remove;
+    // verify reads it to answer whether restore would work, and restore would refuse this
+    // one. Saying so in parseManifest's own words keeps one description of one fault.
+    const manifest = parseManifest(await readMessageBytes(client, manifestMessage))
+
+    // The same refusal delete makes, for the same reason from the other side: the manifest
+    // was found by the file name telstore wrote, so a body naming another backup is a file
+    // that was renamed or replaced, and its message ids describe somebody else's chunks.
+    // Reporting those as this backup's health is the one wrong answer this command can give.
+    if (manifest.id !== undefined && manifest.id !== backupId) {
+      throw new Error(
+        `The manifest named ${manifestFileName(backupId)} describes backup ` +
+          `${JSON.stringify(manifest.id)}, not ${backupId}. Its message ids point at another ` +
+          `backup's chunks, so telstore cannot say whether ${backupId} is still there.`,
+      )
+    }
+
+    const total = manifest.chunks.length
+
+    log(`Backup ${backupId}`)
+    log(
+      `File   ${describeName(manifest.name)} ` +
+        `(${formatBytes(manifest.size)}, ${plural(total, 'chunk')})`,
+    )
+    log(`In     ${describeChat(chat)}`)
+    log('')
+
+    // A backup at the 10000-chunk ceiling is a hundred requests, sent one at a time. Silence
+    // for that long reads as a hang, which is the one thing no wait in this project may look
+    // like — and below a single request there is no progress worth a line.
+    const loud = total > MESSAGE_BATCH_SIZE
+
+    const found = await getDocuments(
+      client,
+      chat,
+      manifest.chunks.map((chunk) => chunk.msgId),
+      {
+        retryOptions: { ...retryOptions, onRetry },
+        onBatch: (done, all) => {
+          if (loud) warn(`\rChecking chunk messages ${done}/${all}…`)
+        },
+      },
+    )
+
+    if (loud) warn('\n')
+
+    const damaged = []
+
+    for (const chunk of manifest.chunks) {
+      const fault = inspect(chunk, found.get(chunk.msgId), { backupId, total, chat })
+
+      if (fault) {
+        damaged.push(fault)
+        log(fault)
+      }
+    }
+
+    if (damaged.length > 0) {
+      log('')
+      log(
+        `${plural(total, 'chunk')} checked, ${damaged.length} damaged. ` +
+          'This backup cannot be restored.',
+      )
+    } else {
+      log(
+        `${plural(total, 'chunk')} present, at the ` +
+          `${total === 1 ? 'size' : 'sizes'} the manifest records.`,
+      )
+      log('This does not download them, so it cannot prove their contents.')
+      log('')
+      log(`Restore with: npx telstore restore ${backupId}`)
+    }
+
+    return { id: backupId, name: manifest.name, chunks: total, damaged }
+  } finally {
+    await closeQuietly(client, disconnect)
+  }
+}
+
+// A backup is a failure whether telstore could not look it up or looked and found it broken.
+// The command worked either way — but "did the run find everything it was asked to check"
+// is the question the exit code answers, and both answers to that are no.
+function isFailure(result) {
+  return Boolean(result.error) || result.damaged.length > 0
+}
+
+// The same shape as runDeletes and runRestores, minus the question: verify removes nothing,
+// so there is nothing to authorise. What is knowable before the connection is refused up
+// front; what only the chat can answer is per id, named when it happens and again at the end.
+export async function runVerifies(backupIds, options = {}, deps = {}) {
+  const {
+    connect = realConnect,
+    disconnect = (client) => client.destroy(),
+    configDir = defaultConfigDir(),
+    writeErr = (line) => process.stderr.write(line),
+    log: writeLog = (line) => console.log(line),
+    silent = false,
+  } = deps
+
+  // One id keeps its own wording and its own thrown error. A summary about one backup only
+  // repeats the lines above it.
+  if (backupIds.length === 1) {
+    const result = await runVerify(backupIds[0], options, deps)
+    return { results: [result], failed: isFailure(result) ? 1 : 0 }
+  }
+
+  const duplicate = backupIds.find((id, index) => backupIds.indexOf(id) !== index)
+
+  if (duplicate) {
+    throw new Error(
+      `${duplicate} is named twice. Checking one backup twice asks the chat the same ` +
+        'question again — name it once.',
+    )
+  }
+
+  const config = await loadConfig(configDir)
+
+  assertLoggedIn(config)
+
+  const { values: settings } = resolveSettings(options, config, { file: configFile(configDir) })
+  const chat = requireChat(settings)
+
+  const log = silent ? () => {} : writeLog
+  const warn = silent ? () => {} : writeErr
+
+  let shared = null
+  const perId = {
+    ...deps,
+    connect: async (theirConfig, connectOptions) =>
+      (shared ??= await connect(theirConfig, connectOptions)),
+    disconnect: async () => {},
+  }
+
+  const results = []
+
+  try {
+    await perId.connect(config, { verbose: settings.verbose })
+
+    for (const [index, backupId] of backupIds.entries()) {
+      if (index > 0) log('')
+      log(`[${index + 1}/${backupIds.length}] ${backupId}`)
+
+      try {
+        results.push(await runVerify(backupId, options, perId))
+      } catch (err) {
+        // Unlike delete, an id nothing knows about does not stop the run: nothing here is
+        // destroyed, and the other ids are exactly the ones somebody is checking on.
+        results.push({ id: backupId, error: err.message, damaged: [] })
+        warn(`\n${backupId} failed: ${err.message}\n`)
+      }
+    }
+  } finally {
+    if (shared) {
+      await closeQuietly(shared, disconnect, (err) =>
+        warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`),
+      )
+    }
+  }
+
+  const failed = results.filter(isFailure).length
+
+  log('')
+  for (const line of summaryLines(results, failed)) log(line)
+
+  return { results, failed }
+}
+
+// Every id gets a line whether it checked out or not: one missing from this list would be a
+// backup nobody could tell the state of, which is the whole reason this command exists.
+function summaryLines(results, failed) {
+  const width = Math.max(...results.map((result) => result.id.length))
+
+  return [
+    `${results.length} backups: ${results.length - failed} verified, ${failed} failed.`,
+    '',
+    ...results.map((result) => {
+      const id = result.id.padEnd(width)
+
+      if (result.error) return `  ${id}  failed: ${result.error}`
+
+      return result.damaged.length > 0
+        ? `  ${id}  ${plural(result.damaged.length, 'chunk')} damaged`
+        : `  ${id}  ${plural(result.chunks, 'chunk')} present`
+    }),
+  ]
+}

package/src/downloader.js

@@ -43,7 +43,10 @@ async function readExactly(fd, length, position) {
 // the better check anyway: a slice written at the wrong offset, two slices overlapping, or
 // one silently skipped all show up here. It does not prove the bytes reached the platter —
 // this read may well be served from the page cache — it proves the assembly.
-async function hashRange(fd, offset, length) {
+//
+// Exported because a resumed restore asks the same question of a .partial left by an earlier
+// run. A second copy of it in restore.js is how two definitions of one check start to differ.
+export async function hashRange(fd, offset, length) {
   const hash = createHash('sha256')
 
   for (let at = 0; at < length; at += HASH_READ_SIZE) {

package/src/manifest.js

@@ -15,8 +15,13 @@ export function chunkFileName(id, i) {
   return `${id}.part${String(i + 1).padStart(4, '0')}`
 }
 
+// The suffix telstore has written on every manifest since version 1, and what `list` picks
+// a manifest out of a chat by. One definition, because a reader that disagrees with the
+// writer by one character finds nothing at all.
+export const MANIFEST_SUFFIX = '.manifest.json'
+
 export function manifestFileName(id) {
-  return `${id}.manifest.json`
+  return `${id}${MANIFEST_SUFFIX}`
 }
 
 export function buildManifest({

package/src/progress.js

@@ -1,6 +1,7 @@
 const UNITS = ['B', 'KB', 'MB', 'GB', 'TB']
 
 export function formatBytes(n) {
+  if (!Number.isFinite(n)) return '--'
   if (n < 1024) return `${n} B`
 
   let value = n