telstore 0.1.0

package/src/commands/upload.js

@@ -0,0 +1,327 @@
+import { promises as fs } from 'node:fs'
+import path from 'node:path'
+
+import { Api } from 'telegram'
+import { CustomFile } from 'telegram/client/uploads.js'
+
+import { PART_SIZE, planChunks } from '../chunking.js'
+import { chunkCaption, manifestCaption } from '../caption.js'
+import { describeChat } from '../chat.js'
+import { closeQuietly, connect as realConnect } from '../client.js'
+import { configFile, defaultConfigDir, loadConfig } from '../config.js'
+import { requireChat, resolveSettings } from '../settings.js'
+import {
+  buildManifest,
+  chunkFileName,
+  manifestFileName,
+  newBackupId,
+  serializeManifest,
+} from '../manifest.js'
+import { createProgress, formatBytes, formatDuration } from '../progress.js'
+import {
+  MAX_STATES,
+  clearState,
+  loadState,
+  markChunkDone,
+  pruneStates,
+  saveState,
+  stateFile,
+  stateKey,
+} from '../state.js'
+import { uploadRange } from '../uploader.js'
+
+// Above this threshold the wait must be spelled out, per spec §8.
+const LONG_WAIT_MS = 60_000
+
+// A transient error that resolves itself on the next try is not news, and one line per
+// occurrence buries the progress bar in a wall of text. Stay quiet until the third retry:
+// by then the trouble has outlived two backoffs and is worth saying out loud.
+const ANNOUNCE_AFTER_ATTEMPT = 3
+
+// Chunks and manifests differ only in where the bytes come from. Everything Telegram is
+// told about them — document, not preview; this exact file name — is decided once.
+async function sendDocument(client, peer, { file, fileName, caption }) {
+  return await client.sendFile(peer, {
+    file,
+    caption,
+    forceDocument: true,
+    attributes: [new Api.DocumentAttributeFilename({ fileName })],
+  })
+}
+
+async function realSendChunk(client, peer, { inputFile, fileName, caption }) {
+  return await sendDocument(client, peer, { file: inputFile, fileName, caption })
+}
+
+async function realSendManifest(client, peer, { bytes, fileName, caption }) {
+  return await sendDocument(client, peer, {
+    file: new CustomFile(fileName, bytes.length, '', bytes),
+    fileName,
+    caption,
+  })
+}
+
+export async function runUpload(filePath, options = {}, deps = {}) {
+  const {
+    connect = realConnect,
+    sendChunk = realSendChunk,
+    sendManifest = realSendManifest,
+    disconnect = (client) => client.destroy(),
+    configDir = defaultConfigDir(),
+    partSize = PART_SIZE,
+    retryOptions = {},
+    writeErr = (line) => process.stderr.write(line),
+    log: writeLog = (line) => console.log(line),
+    silent = false,
+    onBackupId = () => {},
+  } = deps
+
+  const absPath = path.resolve(filePath)
+
+  let stat
+  try {
+    stat = await fs.stat(absPath)
+  } catch (err) {
+    if (err.code === 'ENOENT') throw new Error(`File does not exist: ${absPath}`)
+    throw err
+  }
+
+  if (!stat.isFile()) {
+    throw new Error(`${absPath} is not a file.`)
+  }
+
+  const config = await loadConfig(configDir)
+  const { values: settings, source } = resolveSettings(options, config, {
+    file: configFile(configDir),
+  })
+  const chat = requireChat(settings)
+  const concurrency = settings.concurrency
+
+  const key = stateKey(absPath, stat.size, stat.mtimeMs)
+
+  let state = await loadState(key, configDir)
+
+  // The chunks already in the chat were cut at the size this backup started with, and
+  // nothing can re-cut them. Carrying on at a different size would abandon every one of
+  // them in the chat, where telark can no longer find them — so an unfinished backup
+  // keeps its own chunk size, and a flag that disagrees is refused rather than obeyed.
+  //
+  // Only a flag is a disagreement. A configured chunkSize says what to use when nobody asks
+  // for anything, and this run asked for nothing — so the backup quietly keeps its own size
+  // rather than being refused over a preference set weeks ago for other files.
+  if (state && source('chunkSize') === 'flag' && settings.chunkSize !== state.chunkSize) {
+    const file = stateFile(key, configDir)
+    throw new Error(
+      `This unfinished backup is cut into ${formatBytes(state.chunkSize)} chunks, but ` +
+        `--chunk-size asks for ${formatBytes(settings.chunkSize)} — the chunks already in ` +
+        `${state.chat} cannot be re-cut. Run again without --chunk-size to carry on, or delete ` +
+        `${file} and run again to start a new backup, which leaves the chunks already sent ` +
+        'sitting in the chat with nothing to point at them.',
+    )
+  }
+
+  const chunkSize = state ? state.chunkSize : settings.chunkSize
+
+  // A resumed upload takes its chunk size off disk, and nothing validated that file on the
+  // way in. planChunks will refuse an unusable one, but its message is about chunk sizes and
+  // would send the reader looking for a --chunk-size flag they never passed.
+  if (state && (!Number.isSafeInteger(chunkSize) || chunkSize < 1)) {
+    throw new Error(
+      `The record of this unfinished backup gives a chunk size of ${JSON.stringify(chunkSize)}, ` +
+        `which cannot be used. ${stateFile(key, configDir)} is damaged — delete it and run ` +
+        'again to start a new backup, which leaves the chunks already sent sitting in the ' +
+        'chat with nothing to point at them.',
+    )
+  }
+
+  const chunks = planChunks(stat.size, chunkSize)
+  const resuming = Boolean(state)
+
+  // Naming the way back rather than a flag to drop: the destination may have come from the
+  // command line or from the stored setting, and "run again without --to" is no help to
+  // someone who never typed one. Pointing at the chat itself is right either way.
+  if (resuming && state.chat !== String(chat)) {
+    const file = stateFile(key, configDir)
+    throw new Error(
+      `This unfinished backup is going to ${state.chat}, but the current command targets ${chat} — ` +
+        `a single backup cannot be split across two destinations. Run again with ` +
+        `--to ${state.chat} to carry on sending there, or delete ${file} and run again to ` +
+        `start a new backup in ${chat}.`,
+    )
+  }
+
+  if (!resuming) {
+    state = {
+      id: newBackupId(),
+      chat: String(chat),
+      path: absPath,
+      size: stat.size,
+      mtimeMs: stat.mtimeMs,
+      chunkSize,
+      done: {},
+    }
+    await saveState(key, state, configDir)
+
+    // Only a new backup adds to the directory, so this is the one place it can grow.
+    // The report goes out even when the caller asked for silence: this is not narration
+    // about a transfer, it is telark dropping the only record of someone else's chunks.
+    for (const gone of await pruneStates(configDir)) {
+      writeErr(
+        `\nDropped the record of unfinished backup ${gone.id}: telark keeps the ` +
+          `${MAX_STATES} most recent. The chunks it sent are still in ${gone.chat}, ` +
+          'searchable by that id, but that backup can no longer be resumed.\n',
+      )
+    }
+  }
+
+  onBackupId(state.id)
+
+  const log = silent ? () => {} : writeLog
+  const warn = silent ? () => {} : writeErr
+
+  // Retries and FLOOD_WAIT must be announced: a silent FLOOD_WAIT_3600 leaves the user
+  // staring at a frozen progress bar for an hour, assuming the process has hung.
+  function onRetry(err, attempt, delayMs, elapsedMs = 0) {
+    if (delayMs > LONG_WAIT_MS) {
+      warn(
+        `\nTelegram wants ${formatDuration(delayMs / 1000)} of waiting before the next send ` +
+          `(${err.message}). telark is waiting and will carry on by itself, leave it running.\n`,
+      )
+      return
+    }
+
+    // The exception to staying quiet: an attempt that took a minute to fail spent that
+    // minute with the bar frozen, which is exactly what a hang looks like. Those are worth
+    // a line the first time, whatever the attempt number.
+    if (attempt < ANNOUNCE_AFTER_ATTEMPT && elapsedMs < LONG_WAIT_MS) return
+
+    warn(
+      `\nTemporary error (${err.message}), retry ${attempt} in ` +
+        `${formatDuration(delayMs / 1000)}.\n`,
+    )
+  }
+
+  log(`Backup ${state.id}`)
+  log(`File   ${absPath} (${formatBytes(stat.size)}, ${chunks.length} chunks)`)
+  log(`To     ${describeChat(chat)}\n`)
+
+  const client = await connect(config, { verbose: settings.verbose })
+
+  try {
+    // Everything already in the chat is reported before the bar exists. These lines go to
+    // stdout while the bar is rewritten on stderr with \r, so printed from inside the loop
+    // they would land straight on the line the bar keeps returning to.
+    const pending = []
+
+    for (const chunk of chunks) {
+      if (state.done[String(chunk.i)]) {
+        log(`Chunk ${chunk.i + 1}/${chunks.length} already uploaded, skipping.`)
+        continue
+      }
+
+      pending.push(chunk)
+    }
+
+    // A run that only failed to send the manifest has every chunk done and nothing left to
+    // transfer: no bar at all, rather than one that springs into existence at 100%.
+    if (pending.length > 0) {
+      const remaining = pending.reduce((sum, chunk) => sum + chunk.length, 0)
+
+      // One bar for the whole upload: the label names the chunk in flight, everything else
+      // describes the file. Chunks a previous run sent count towards the bar but not towards
+      // the speed, so an hour-old chunk cannot inflate the ETA of the ones still to go.
+      // warn is already the no-op when silent, and createProgress draws through nothing else.
+      const progress = createProgress({
+        total: stat.size,
+        done: stat.size - remaining,
+        label: `Chunk ${pending[0].i + 1}/${chunks.length}`,
+        write: warn,
+      })
+
+      try {
+        for (const chunk of pending) {
+          progress.setLabel(`Chunk ${chunk.i + 1}/${chunks.length}`)
+
+          const fileName = chunkFileName(state.id, chunk.i)
+          const handle = await fs.open(absPath, 'r')
+
+          try {
+            const { inputFile, sha256 } = await uploadRange(client, handle.fd, {
+              offset: chunk.offset,
+              length: chunk.length,
+              fileName,
+              concurrency,
+              partSize,
+              onProgress: (bytes) => progress.advance(bytes),
+              retryOptions: { ...retryOptions, onRetry },
+            })
+
+            const message = await sendChunk(client, chat, {
+              inputFile,
+              fileName,
+              caption: chunkCaption({ id: state.id, number: chunk.i + 1, total: chunks.length }),
+            })
+
+            state = await markChunkDone(
+              key,
+              state,
+              chunk.i,
+              { msgId: message.id, size: chunk.length, sha256 },
+              configDir,
+            )
+          } finally {
+            await handle.close()
+          }
+        }
+      } finally {
+        // Same reason as restore: a send that fails must not leave "Error: ..." printed over
+        // the bar's own line.
+        progress.finish()
+      }
+    }
+
+    // The file can be overwritten mid-upload — with a 50GB file an hour passes between
+    // the first and last chunk. The manifest would then describe a hybrid file that never
+    // existed: restore still matches every sha256, but the data is garbage.
+    const after = await fs.stat(absPath)
+
+    if (after.size !== stat.size || after.mtimeMs !== stat.mtimeMs) {
+      throw new Error(
+        `${absPath} changed during the upload ` +
+          `(size ${stat.size} → ${after.size}, mtime ${stat.mtimeMs} → ${after.mtimeMs}). ` +
+          'This backup mixes old and new data and cannot be trusted — telark is not sending the manifest. ' +
+          'Wait until the file settles, then run again to create a new backup.',
+      )
+    }
+
+    const manifest = buildManifest({
+      id: state.id,
+      name: path.basename(absPath),
+      size: stat.size,
+      chunkSize,
+      chunks: chunks.map((chunk) => ({ i: chunk.i, ...state.done[String(chunk.i)] })),
+    })
+
+    await sendManifest(client, chat, {
+      bytes: serializeManifest(manifest),
+      fileName: manifestFileName(state.id),
+      caption: manifestCaption({
+        id: manifest.id,
+        name: manifest.name,
+        size: manifest.size,
+        chunks: manifest.chunks.length,
+        createdAt: manifest.createdAt,
+      }),
+    })
+
+    await clearState(key, configDir)
+
+    log(`\nDone. Restore with:\n  npx telark restore ${state.id}`)
+
+    return { id: state.id, chunks: chunks.length }
+  } finally {
+    await closeQuietly(client, disconnect, (err) =>
+      warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`),
+    )
+  }
+}

package/src/config.js

@@ -0,0 +1,86 @@
+import { promises as fs } from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+
+const FILE_NAME = 'config.json'
+
+export function defaultConfigDir() {
+  return path.join(os.homedir(), '.telark')
+}
+
+export function configFile(dir = defaultConfigDir()) {
+  return path.join(dir, FILE_NAME)
+}
+
+function isPlainObject(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value)
+}
+
+// `config.json` is hand-editable now that the `config` command invites people into it,
+// which puts it in the same category as a manifest or a state file: believed only after it
+// has been checked. A `settings` that is not an object would make every lookup below it
+// return undefined, and telark would then run happily on built-in defaults while the
+// user's own choices sat there ignored — so it is named and refused instead.
+export function checkConfigShape(raw, file) {
+  if (!isPlainObject(raw)) {
+    throw new Error(
+      `Corrupt config file: ${file} holds ${Array.isArray(raw) ? 'a list' : typeof raw}, ` +
+        'not a group of settings.',
+    )
+  }
+
+  if (raw.settings !== undefined && !isPlainObject(raw.settings)) {
+    throw new Error(
+      `"settings" in ${file} holds ${Array.isArray(raw.settings) ? 'a list' : typeof raw.settings}, ` +
+        'not a group of settings. telark will not guess what was meant — fix that entry, ' +
+        'or remove it to fall back to the defaults.',
+    )
+  }
+
+  return raw
+}
+
+export async function loadConfig(dir = defaultConfigDir()) {
+  const file = configFile(dir)
+
+  let raw
+  try {
+    raw = await fs.readFile(file, 'utf8')
+  } catch (err) {
+    if (err.code === 'ENOENT') return {}
+    throw err
+  }
+
+  let parsed
+  try {
+    parsed = JSON.parse(raw)
+  } catch (err) {
+    throw new Error(
+      `Corrupt config file: ${file} is not valid JSON (${err.message}). ` +
+        'Fix the syntax to keep your session, or delete the file and run "telark login" again.',
+    )
+  }
+
+  return checkConfigShape(parsed, file)
+}
+
+// Config and state are both small JSON files holding something that must survive a crash
+// mid-write: write beside the target, then rename, which is atomic on the same filesystem.
+export async function writeJsonAtomic(file, value) {
+  await fs.mkdir(path.dirname(file), { recursive: true, mode: 0o700 })
+
+  const tmp = `${file}.tmp`
+
+  await fs.writeFile(tmp, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 })
+  await fs.rename(tmp, file)
+}
+
+export async function saveConfig(config, dir = defaultConfigDir()) {
+  await writeJsonAtomic(configFile(dir), config)
+}
+
+export async function clearSession(dir = defaultConfigDir()) {
+  const config = await loadConfig(dir)
+  delete config.session
+  await saveConfig(config, dir)
+}

package/src/confirm.js

@@ -0,0 +1,11 @@
+import readline from 'node:readline/promises'
+import { stdin, stdout } from 'node:process'
+
+// Anything but a yes is a no. Both commands that ask are about to do something that cannot
+// be taken back, so a stray keystroke or an empty line has to mean stop.
+export async function askConfirm(question) {
+  const rl = readline.createInterface({ input: stdin, output: stdout })
+  const answer = await rl.question(question)
+  rl.close()
+  return /^y/i.test(answer.trim())
+}

package/src/downloader.js

@@ -0,0 +1,190 @@
+import { createHash } from 'node:crypto'
+import { read as readCallback, write as writeCallback } from 'node:fs'
+import { promisify } from 'node:util'
+
+import { returnBigInt } from 'telegram/Helpers.js'
+
+import { DEFAULT_CONCURRENCY, PART_SIZE, SLICE_SIZE } from './chunking.js'
+import { withRetry } from './retry.js'
+import { DEFAULT_STALL_MS, withStallTimeout } from './stall.js'
+
+const write = promisify(writeCallback)
+const read = promisify(readCallback)
+
+// Read back in far bigger bites than the 512KB the network hands us: this loop is pure disk.
+const HASH_READ_SIZE = 4 * 1024 * 1024
+
+async function writeExactly(fd, buffer, position) {
+  let written = 0
+
+  while (written < buffer.length) {
+    const { bytesWritten } = await write(fd, buffer, written, buffer.length - written, position + written)
+    written += bytesWritten
+  }
+}
+
+async function readExactly(fd, length, position) {
+  const buffer = Buffer.allocUnsafe(length)
+  let filled = 0
+
+  while (filled < length) {
+    const { bytesRead } = await read(fd, buffer, filled, length - filled, position + filled)
+    if (bytesRead === 0) {
+      throw new Error(`Short read: needed ${length} bytes at offset ${position} but the file ended.`)
+    }
+    filled += bytesRead
+  }
+
+  return buffer
+}
+
+// The digest is taken from the assembled range on disk rather than from the buffers as they
+// arrive. Once slices land out of order that is the only order left to hash in, and it is
+// the better check anyway: a slice written at the wrong offset, two slices overlapping, or
+// one silently skipped all show up here. It does not prove the bytes reached the platter —
+// this read may well be served from the page cache — it proves the assembly.
+async function hashRange(fd, offset, length) {
+  const hash = createHash('sha256')
+
+  for (let at = 0; at < length; at += HASH_READ_SIZE) {
+    hash.update(await readExactly(fd, Math.min(HASH_READ_SIZE, length - at), offset + at))
+  }
+
+  return hash.digest('hex')
+}
+
+export async function downloadToFile(
+  client,
+  message,
+  fd,
+  { offset, onProgress, retryOptions, concurrency = DEFAULT_CONCURRENCY, stallMs = DEFAULT_STALL_MS } = {},
+) {
+  const document = message?.media?.document
+
+  if (!document) {
+    throw new Error(`Message ${message?.id} has no file attached.`)
+  }
+
+  if (!Number.isFinite(offset)) {
+    throw new Error(`offset must be a finite number, got: ${offset}`)
+  }
+
+  // A pool of fewer than one worker does no work. In the download path that means
+  // Promise.all resolves at once and a chunk nobody fetched is reported as complete; in the
+  // upload path the batch loop never advances and the process spins in microtasks, which not
+  // even a test timeout can interrupt. Neither is something a caller should be able to ask
+  // for by accident.
+  if (!Number.isInteger(concurrency) || concurrency < 1) {
+    throw new Error(
+      `Worker count must be a whole number of at least one, got: ${JSON.stringify(concurrency)}.`,
+    )
+  }
+
+  // Telegram's own record of how long the document is. Taking the length from the caller
+  // instead would make runRestore's size check compare the manifest against itself.
+  const size = returnBigInt(document.size ?? 0).toJSNumber()
+
+  if (!Number.isFinite(size) || size <= 0) {
+    throw new Error(`Message ${message?.id} has a document with no usable size.`)
+  }
+
+  const sliceCount = Math.ceil(size / SLICE_SIZE)
+
+  // One slice is one short stream. `done` lives outside withRetry, so a retried slice picks
+  // up at its own watermark instead of fetching again what it already has.
+  //
+  // The two offsets are not the same number: `start + done` is a position inside the
+  // document, which is where the stream resumes, while `offset + start + done` is a position
+  // inside the file being assembled, which is where the bytes land.
+  async function downloadSlice(index) {
+    const start = index * SLICE_SIZE
+    const length = Math.min(SLICE_SIZE, size - start)
+    let done = 0
+
+    await withRetry(async () => {
+      // Iterated by hand rather than with `for await` so each part can be given a deadline:
+      // a stalled stream yields nothing and raises nothing, and only a race against a timer
+      // turns that silence into an error withRetry can act on. Nothing is lost by stepping
+      // outside `for await` — GramJS's download iterator exposes `next` alone, so breaking
+      // out of the loop never closed anything either.
+      const stream = client.iterDownload({
+        file: message.media,
+        offset: returnBigInt(start + done),
+        requestSize: PART_SIZE,
+      })
+      const parts = stream[Symbol.asyncIterator]()
+
+      for (;;) {
+        const { value: buffer, done: ended } = await withStallTimeout(
+          parts.next(),
+          stallMs,
+          () =>
+            `Telegram stopped sending slice ${index + 1}/${sliceCount} of message ` +
+            `${message?.id}: nothing arrived for ${Math.round(stallMs / 1000)}s after ` +
+            `${done} of ${length} bytes.`,
+        )
+
+        if (ended) break
+
+        // The stream runs to the end of the document; this slice stops at its own boundary.
+        const take = Math.min(buffer.length, length - done)
+
+        await writeExactly(fd, buffer.subarray(0, take), offset + start + done)
+        done += take
+        onProgress?.(take)
+
+        if (done >= length) break
+      }
+
+      // A stream that ends before the slice's own boundary is a short delivery, not a
+      // completed slice — without this, a hole here would surface only as a digest
+      // mismatch, telling the user their data is corrupt when it was simply cut short.
+      // Checked inside the retried callback so a transient short stream is retried, and a
+      // stream that yields nothing at all still counts as a failure worth retrying.
+      if (done < length) {
+        throw new Error(
+          `Slice ${index + 1}/${sliceCount} of message ${message?.id} ended after ${done} of ${length} bytes.`,
+        )
+      }
+    }, retryOptions)
+  }
+
+  let next = 0
+  let failed = false
+  let failure = null
+
+  const worker = async () => {
+    for (;;) {
+      // Stop handing out work the moment anything has failed: the chunk is lost either way,
+      // and every further request is bandwidth spent on a file about to be thrown away.
+      if (failed) return
+
+      const index = next
+      next += 1
+      if (index >= sliceCount) return
+
+      try {
+        await downloadSlice(index)
+      } catch (err) {
+        // Not `failure ??= err`: a falsy rejection reason would leave `failure` falsy and the
+        // throw below would read as success, turning a broken chunk into a silent one. The
+        // first error is the one kept — later ones are usually consequences of the shutdown
+        // rather than the cause.
+        if (!failed) {
+          failed = true
+          failure = err
+        }
+        return
+      }
+    }
+  }
+
+  // Every worker absorbs its own error above, so none of these promises rejects and none can
+  // become an unhandledRejection that hides the real one. This await is also what guarantees
+  // no write is still in flight when the function returns.
+  await Promise.all(Array.from({ length: Math.min(concurrency, sliceCount) }, worker))
+
+  if (failed) throw failure
+
+  return { sha256: await hashRange(fd, offset, size), size }
+}