telstore 0.1.0

package/src/manifest.js

@@ -0,0 +1,163 @@
+import { randomBytes } from 'node:crypto'
+
+import { countChunks } from './chunking.js'
+
+export const MANIFEST_VERSION = 1
+
+export function newBackupId(now = new Date(), randomHex = () => randomBytes(3).toString('hex')) {
+  const yyyy = now.getUTCFullYear()
+  const mm = String(now.getUTCMonth() + 1).padStart(2, '0')
+  const dd = String(now.getUTCDate()).padStart(2, '0')
+  return `telark-${yyyy}${mm}${dd}-${randomHex()}`
+}
+
+export function chunkFileName(id, i) {
+  return `${id}.part${String(i + 1).padStart(4, '0')}`
+}
+
+export function manifestFileName(id) {
+  return `${id}.manifest.json`
+}
+
+export function buildManifest({ id, name, size, chunkSize, chunks, createdAt = new Date().toISOString() }) {
+  return {
+    v: MANIFEST_VERSION,
+    id,
+    name,
+    size,
+    chunkSize,
+    createdAt,
+    chunks: [...chunks]
+      .sort((a, b) => a.i - b.i)
+      .map(({ i, msgId, size: chunkBytes, sha256 }) => ({ i, msgId, size: chunkBytes, sha256 })),
+  }
+}
+
+export function serializeManifest(manifest) {
+  return Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`, 'utf8')
+}
+
+// parseManifest's own front door, on its own so delete can read a manifest body without
+// the layout checks behind it.
+export function parseManifestJson(input) {
+  const text = Buffer.isBuffer(input) ? input.toString('utf8') : String(input)
+
+  try {
+    return JSON.parse(text)
+  } catch {
+    throw new Error('Cannot read manifest: content is not valid JSON.')
+  }
+}
+
+// Delete needs one thing from a manifest that restore does not, and none of the things
+// restore needs. parseManifest is the wrong gate for it: it validates the chunk *layout*,
+// because restore writes bytes at offsets computed from it — and a manifest that fails
+// those checks is exactly the broken backup somebody is trying to delete, so refusing to
+// read it here would leave the only way out through the Telegram app. It also never looks
+// at msgId, which is the only field delete actually uses.
+//
+// Every id is checked before a single message is removed. A msgId is handed to Telegram as
+// the name of something to destroy for good, and that is the one number nobody may guess
+// at — so a manifest that cannot say it exactly is refused whole, rather than half-deleted
+// and then left without the list that names the rest.
+export function manifestMessageIds(manifest) {
+  if (!Array.isArray(manifest?.chunks) || manifest.chunks.length === 0) {
+    throw new Error('Manifest has no chunk list, so it cannot say which messages to remove.')
+  }
+
+  return manifest.chunks.map((chunk, index) => {
+    const msgId = chunk?.msgId
+
+    if (!Number.isSafeInteger(msgId) || msgId < 1) {
+      throw new Error(
+        `Manifest gives ${JSON.stringify(msgId)} as the message id of chunk ${index + 1}, ` +
+          'which is not a message id. Deleting from this manifest could remove the wrong ' +
+          'messages, so telark is not deleting anything.',
+      )
+    }
+
+    return msgId
+  })
+}
+
+export function parseManifest(input) {
+  const manifest = parseManifestJson(input)
+
+  if (manifest.v !== MANIFEST_VERSION) {
+    throw new Error(
+      `Manifest uses version ${manifest.v}, this build of telark only understands version ${MANIFEST_VERSION}.`,
+    )
+  }
+
+  if (!Array.isArray(manifest.chunks) || manifest.chunks.length === 0) {
+    throw new Error('Manifest has no chunk list.')
+  }
+
+  // The manifest comes off a chat, so nothing in it is trusted. The checks below are
+  // arithmetic on these numbers, and arithmetic on a string or a null does not fail — it
+  // produces a comparison that rejects the manifest for the wrong reason. A string size used
+  // to be reported as "add up to 100, but the manifest records a file size of 100", which
+  // sends the reader hunting for a difference that is not there.
+  if (!Number.isSafeInteger(manifest.size) || manifest.size < 0) {
+    throw new Error(
+      `Manifest records a file size of ${JSON.stringify(manifest.size)}, ` +
+        'which is not a whole number of bytes.',
+    )
+  }
+
+  if (!Number.isSafeInteger(manifest.chunkSize) || manifest.chunkSize < 1) {
+    throw new Error(
+      `Manifest records a chunk size of ${JSON.stringify(manifest.chunkSize)}, ` +
+        'which is not a whole number of bytes above zero.',
+    )
+  }
+
+  manifest.chunks.forEach((chunk, index) => {
+    if (typeof chunk !== 'object' || chunk === null) {
+      throw new Error(
+        `Manifest entry for chunk ${index + 1} is not an object: ${JSON.stringify(chunk)}.`,
+      )
+    }
+
+    if (!Number.isSafeInteger(chunk.size) || chunk.size < 0) {
+      throw new Error(
+        `Manifest records ${JSON.stringify(chunk.size)} bytes for chunk ${index + 1}, ` +
+          'which is not a whole number of bytes.',
+      )
+    }
+
+    if (chunk.i !== index) {
+      throw new Error(`Manifest is missing chunk ${index}: the chunk list is not contiguous.`)
+    }
+  })
+
+  const expectedChunks = countChunks(manifest.size, manifest.chunkSize)
+  if (manifest.chunks.length !== expectedChunks) {
+    throw new Error(`Manifest is missing ${expectedChunks - manifest.chunks.length} chunk(s).`)
+  }
+
+  const total = manifest.chunks.reduce((sum, chunk) => sum + chunk.size, 0)
+  if (total !== manifest.size) {
+    throw new Error(
+      `Chunk sizes add up to ${total}, but the manifest records a file size of ${manifest.size}.`,
+    )
+  }
+
+  // Restore writes chunk i at exactly offset i * chunkSize, so the layout must be
+  // uniform: every chunk is chunkSize, except the last one which is the remainder.
+  // A correct total with individually wrong sizes yields a file with a hole or
+  // extra length while every per-chunk sha256 still matches — silently wrong data,
+  // precisely what telark must never produce.
+  manifest.chunks.forEach((chunk, index) => {
+    const expected = Math.min(manifest.chunkSize, manifest.size - index * manifest.chunkSize)
+    if (chunk.size !== expected) {
+      throw new Error(
+        `Manifest records ${chunk.size} bytes for chunk ${index + 1}, but a layout of ` +
+          `${manifest.chunkSize} bytes per chunk requires ${expected} bytes. ` +
+          'This manifest describes the wrong chunk positions; restoring it would produce a corrupt file.',
+      )
+    }
+  })
+
+  return manifest
+}

package/src/progress.js

@@ -0,0 +1,100 @@
+const UNITS = ['B', 'KB', 'MB', 'GB', 'TB']
+
+export function formatBytes(n) {
+  if (n < 1024) return `${n} B`
+
+  let value = n
+  let unit = 0
+
+  while (value >= 1024 && unit < UNITS.length - 1) {
+    value /= 1024
+    unit += 1
+  }
+
+  return `${value.toFixed(1)} ${UNITS[unit]}`
+}
+
+export function formatDuration(seconds) {
+  if (!Number.isFinite(seconds)) return '--'
+
+  const total = Math.round(seconds)
+  const h = Math.floor(total / 3600)
+  const m = Math.floor((total % 3600) / 60)
+  const s = total % 60
+
+  if (h > 0) return `${h}h${m}m`
+  if (m > 0) return `${m}m${s}s`
+  return `${s}s`
+}
+
+export function renderProgress({ done, total, elapsedMs, label, width = 24, transferred = done }) {
+  const ratio = total === 0 ? 1 : Math.min(done / total, 1)
+  const filled = Math.round(ratio * width)
+  const bar = `${'█'.repeat(filled)}${'░'.repeat(width - filled)}`
+
+  // Speed measures what this run moved, not what the file already has. A resumed upload
+  // starts with gigabytes behind it, and dividing those by a two-second-old run reports a
+  // fictional 3 GB/s and an ETA of almost nothing. The bar and the byte counts still speak
+  // for the whole file, because that is the question being asked.
+  const bytesPerSecond = elapsedMs > 0 ? transferred / (elapsedMs / 1000) : 0
+  // Clamp to 0: done can overshoot total (one extra tick) and a negative ETA is meaningless.
+  const remaining = bytesPerSecond > 0 ? Math.max(0, (total - done) / bytesPerSecond) : Infinity
+
+  const percent = String(Math.floor(ratio * 100)).padStart(3)
+  const speed = `${formatBytes(Math.round(bytesPerSecond))}/s`
+
+  return `${label} ${bar} ${percent}% ${formatBytes(done)}/${formatBytes(total)} ${speed} ETA ${formatDuration(remaining)}`
+}
+
+export function createProgress({
+  total,
+  label,
+  // Bytes a previous run already sent. They belong on the bar — the question is how much of
+  // the file is done, not how much of today's session — but not in the speed.
+  done: startedWith = 0,
+  write = (line) => process.stderr.write(line),
+  now = () => Date.now(),
+  minIntervalMs = 200,
+}) {
+  const startedAt = now()
+  let done = startedWith
+  let currentLabel = label
+  let lastDrawnAt = startedAt
+  let widestLine = 0
+
+  // \r only moves the cursor home, it does not erase. A redraw that is shorter than the one
+  // before it (a shrinking ETA, a speed that changes unit) would leave the previous tail on
+  // screen, so pad every line out to the widest one drawn so far.
+  function draw(suffix) {
+    const line = renderProgress({
+      done,
+      total,
+      elapsedMs: now() - startedAt,
+      label: currentLabel,
+      transferred: done - startedWith,
+    })
+    widestLine = Math.max(widestLine, line.length)
+    write(`\r${line.padEnd(widestLine)}${suffix}`)
+  }
+
+  return {
+    advance(bytes) {
+      done += bytes
+      if (now() - lastDrawnAt < minIntervalMs) return
+      lastDrawnAt = now()
+      draw('')
+    },
+    // One bar spans the whole transfer while the label names the chunk in flight, so the
+    // label changes mid-line and has to be drawn at once: throttled, it would keep showing
+    // the previous chunk's number for the first 200ms of the new one. It counts as a draw,
+    // because the redraw a moment later would say the same thing.
+    setLabel(next) {
+      currentLabel = next
+      lastDrawnAt = now()
+      draw('')
+    },
+    finish() {
+      draw('\n')
+    },
+  }
+}

package/src/retry.js

@@ -0,0 +1,57 @@
+const DEFAULT_ATTEMPTS = 8
+const DEFAULT_BASE_DELAY_MS = 1000
+
+// Past half a minute the doubling stops buying anything: the wait is already long enough
+// that the far side has either recovered or is not coming back on this attempt. Left
+// uncapped, eight attempts would end in a two-minute stare at a frozen bar.
+const MAX_BACKOFF_MS = 30_000
+
+function defaultSleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+function floodWaitSeconds(err) {
+  if (typeof err?.seconds === 'number' && String(err?.errorMessage ?? '').includes('FLOOD_WAIT')) {
+    return err.seconds
+  }
+  return null
+}
+
+export async function withRetry(fn, options = {}) {
+  const {
+    attempts = DEFAULT_ATTEMPTS,
+    baseDelayMs = DEFAULT_BASE_DELAY_MS,
+    sleep = defaultSleep,
+    now = Date.now,
+    onRetry,
+  } = options
+
+  let lastError
+
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    const startedAt = now()
+
+    try {
+      return await fn()
+    } catch (err) {
+      lastError = err
+
+      if (attempt === attempts) break
+
+      const flood = floodWaitSeconds(err)
+      const delayMs =
+        flood === null
+          ? Math.min(baseDelayMs * 2 ** (attempt - 1), MAX_BACKOFF_MS)
+          : flood * 1000
+
+      // How long the failed attempt itself took. A request that errors instantly costs the
+      // user nothing but a line of output; one that took a minute to give up left the
+      // progress bar frozen for that minute. Only the caller can weigh the two, so it is
+      // told which kind this was.
+      onRetry?.(err, attempt, delayMs, now() - startedAt)
+      await sleep(delayMs)
+    }
+  }
+
+  throw lastError
+}

package/src/settings.js

@@ -0,0 +1,190 @@
+import { normalizeChatTarget } from './chat.js'
+import {
+  DEFAULT_CHUNK_SIZE,
+  DEFAULT_CONCURRENCY,
+  MAX_CONCURRENCY,
+  parseSize,
+} from './chunking.js'
+import { formatBytes } from './progress.js'
+
+export const DEFAULT_LIMIT = 20
+
+// The three keys telark writes for itself. Naming them separately is what lets the
+// unknown-key error say "managed by login" instead of listing a session as something the
+// user forgot to spell correctly.
+const MANAGED_BY_LOGIN = new Set(['session', 'apiId', 'apiHash'])
+
+function wholeNumber(raw, where, minimum, maximum, explanation) {
+  if (typeof raw !== 'string' && typeof raw !== 'number') {
+    throw new Error(`Invalid ${where}: ${JSON.stringify(raw)}. ${explanation}`)
+  }
+
+  const value = Number(String(raw).trim())
+
+  if (!Number.isInteger(value) || value < minimum || (maximum !== null && value > maximum)) {
+    throw new Error(`Invalid ${where}: "${raw}". ${explanation}`)
+  }
+
+  return value
+}
+
+// One entry per setting: the flag that overrides it for a run, the built-in default, the
+// parser that both the flag and the stored value go through, and how the value is written
+// back out. `format` must round-trip — whatever it prints has to parse to the same value,
+// which is why chunkSize prints bytes and leaves "1.8 GB" to `describe`.
+export const SETTINGS = {
+  chat: {
+    flag: 'to',
+    default: null,
+    // A number here is a chat id, and a chat id is a whole number. 42.5 would otherwise
+    // slip through as the string "42.5" and only fail much later, at Telegram, as a chat
+    // that does not exist.
+    parse(raw, where) {
+      const usable = typeof raw === 'string' || (typeof raw === 'number' && Number.isInteger(raw))
+
+      if (!usable) {
+        throw new Error(
+          `Invalid ${where}: ${JSON.stringify(raw)}. A destination is @username, -100123..., or me.`,
+        )
+      }
+
+      return normalizeChatTarget(raw)
+    },
+    format: (value) => String(value),
+  },
+  chunkSize: {
+    flag: 'chunk-size',
+    default: DEFAULT_CHUNK_SIZE,
+    parse(raw, where) {
+      if (typeof raw !== 'string' && typeof raw !== 'number') {
+        throw new Error(
+          `Invalid ${where}: ${JSON.stringify(raw)}. Valid examples: 1800MB, 1.8GB, 524288.`,
+        )
+      }
+
+      try {
+        return parseSize(raw)
+      } catch (err) {
+        throw new Error(`Invalid ${where}: "${raw}". ${err.message}`)
+      }
+    },
+    // Bytes, not "1.8 GB": formatBytes rounds to one decimal, so the pretty form parses
+    // back to a different size and `config chunkSize` would print a value that, typed in
+    // again, cuts the file differently.
+    format: (value) => String(value),
+    describe: (value) => formatBytes(value),
+  },
+  concurrency: {
+    flag: 'concurrency',
+    default: DEFAULT_CONCURRENCY,
+    parse: (raw, where) =>
+      wholeNumber(
+        raw,
+        where,
+        1,
+        MAX_CONCURRENCY,
+        `Must be an integer from 1 to ${MAX_CONCURRENCY} — each slot holds a 512KB part in RAM ` +
+          'and Telegram answers with FLOOD_WAIT if too many requests go out at once.',
+      ),
+    format: (value) => String(value),
+  },
+  limit: {
+    flag: 'limit',
+    default: DEFAULT_LIMIT,
+    parse: (raw, where) =>
+      wholeNumber(raw, where, 1, null, 'Must be a whole number of backups, 1 or more.'),
+    format: (value) => String(value),
+  },
+  verbose: {
+    flag: 'verbose',
+    default: false,
+    parse(raw, where) {
+      if (typeof raw === 'boolean') return raw
+
+      const text = String(raw).trim().toLowerCase()
+
+      if (text === 'true') return true
+      if (text === 'false') return false
+
+      throw new Error(`Invalid ${where}: ${JSON.stringify(raw)}. Must be true or false.`)
+    },
+    format: (value) => String(value),
+  },
+}
+
+export const SETTING_KEYS = Object.keys(SETTINGS)
+
+// Someone who has been typing `--to` and `--chunk-size` for a week will type them at the
+// config command too. Accept the flag spelling as a way in, and canonicalise on the way to
+// disk so the file only ever holds one name per setting.
+const ALIASES = new Map()
+
+for (const [key, spec] of Object.entries(SETTINGS)) {
+  ALIASES.set(key.toLowerCase(), key)
+  ALIASES.set(spec.flag.toLowerCase(), key)
+}
+
+export function canonicalKey(input) {
+  return ALIASES.get(String(input).trim().toLowerCase()) ?? null
+}
+
+export function isManagedByLogin(input) {
+  return MANAGED_BY_LOGIN.has(String(input).trim())
+}
+
+// Where a value came from, spelled the way the user would recognise it. A stored value that
+// fails to parse must not be reported as a bad flag: nobody typed a flag, and telling them
+// to fix one sends them off after the wrong thing — the same mistake the old
+// "run again without --to" advice made.
+function origin(key, from, file) {
+  return from === 'flag' ? `--${SETTINGS[key].flag}` : `${key} in ${file}`
+}
+
+// Precedence is flag, then the stored setting, then the built-in default. `source` is not
+// decoration: an unfinished upload has to tell "you asked for this size" from "this is
+// merely your default", and only the source separates them.
+export function resolveSettings(options = {}, config = {}, { file = 'the config file' } = {}) {
+  const values = {}
+  const sources = {}
+  const stored = config.settings ?? {}
+
+  for (const [key, spec] of Object.entries(SETTINGS)) {
+    const flagValue = options[spec.flag]
+
+    if (flagValue !== undefined) {
+      values[key] = spec.parse(flagValue, origin(key, 'flag', file))
+      sources[key] = 'flag'
+    } else if (stored[key] !== undefined) {
+      values[key] = spec.parse(stored[key], origin(key, 'settings', file))
+      sources[key] = 'settings'
+    } else {
+      values[key] = spec.default
+      sources[key] = 'default'
+    }
+  }
+
+  return {
+    values,
+    // Throws rather than returning undefined for a typo: a misspelt key at the chunk-size
+    // call site would turn "refuse to resume at a different size" into "resume at a
+    // different size", silently, which is the one thing this project must never do.
+    source(key) {
+      if (!(key in sources)) {
+        throw new Error(`Unknown setting: ${key}. Known settings: ${SETTING_KEYS.join(', ')}.`)
+      }
+
+      return sources[key]
+    },
+  }
+}
+
+export function requireChat(values) {
+  if (values.chat === null || values.chat === undefined) {
+    throw new Error(
+      'No destination set — run "npx telark config chat @my_backups" to set one ' +
+        '("config chat me" for Saved Messages), or pass --to to choose one for this run.',
+    )
+  }
+
+  return values.chat
+}

package/src/stall.js

@@ -0,0 +1,63 @@
+// Sixty seconds of silence on one 512KB part is a dead connection, not a slow one: a link
+// that cannot deliver 8KB/s could not finish a multi-gigabyte restore anyway.
+export const DEFAULT_STALL_MS = 60_000
+
+// setTimeout stores its delay in a 32-bit signed integer. Node's answer to a larger one is
+// to warn and fire after a single tick, so a deadline meant to be more patient becomes the
+// least patient one there is.
+const MAX_TIMER_MS = 2 ** 31 - 1
+
+// A request that never comes back is not the same as one that fails, and only one of the
+// two is something withRetry can do anything about.
+//
+// GramJS can leave a request queued on a sender it has quietly given up on. Its own abort
+// path is unreachable: MTProtoSender rejects pending states only when
+// `_currentRetries > _reconnectRetries`, and `reconnectRetries` has no default to compare
+// against, so the test is never true (network/MTProtoSender.js:376). Nor does the reconnect
+// itself report failure — `connect()` exhausts its attempts and returns false rather than
+// throwing, so `_reconnect()` finishes as if it had worked and puts the request back on a
+// queue with no send loop left to drain it (network/MTProtoSender.js:148,795).
+//
+// The promise then simply sits there. Nothing throws, nothing prints, and once no handle is
+// left the process ends mid-transfer without a word — the one outcome this project forbids.
+export async function withStallTimeout(promise, ms, describe) {
+  if (!(ms > 0)) return await promise
+
+  if (ms > MAX_TIMER_MS) {
+    throw new Error(
+      `A stall deadline of ${ms}ms is out of range: a timer holds at most ${MAX_TIMER_MS}ms, ` +
+        'and a larger one fires after a single tick rather than waiting.',
+    )
+  }
+
+  let timer
+
+  try {
+    return await Promise.race([
+      promise,
+      new Promise((_, reject) => {
+        // Deliberately not unref'd. This timer is the only thing keeping the event loop
+        // alive while a request is outstanding, so a stall ends in this rejection instead
+        // of in Node running out of work and exiting silently.
+        timer = setTimeout(() => {
+          // A throw inside a timer callback is an uncaught exception, which ends the process
+          // rather than the request. Building the message must never be able to do that.
+          let message
+
+          try {
+            message = describe()
+          } catch (err) {
+            message = `A network wait went past ${ms}ms, and describing it failed: ${err.message}`
+          }
+
+          reject(new Error(message))
+        }, ms)
+      }),
+    ])
+  } finally {
+    // Promise.race has already attached a handler to `promise`, so a rejection arriving
+    // after we have stopped waiting is still handled and cannot surface as an
+    // unhandledRejection that hides the stall.
+    clearTimeout(timer)
+  }
+}