telstore 0.1.0 → 0.1.2

package/src/downloader.js

@@ -4,7 +4,7 @@ import { promisify } from 'node:util'
 
 import { returnBigInt } from 'telegram/Helpers.js'
 
-import { DEFAULT_CONCURRENCY, PART_SIZE, SLICE_SIZE } from './chunking.js'
+import { DEFAULT_DOWNLOAD_CONCURRENCY, PART_SIZE, SLICE_SIZE } from './chunking.js'
 import { withRetry } from './retry.js'
 import { DEFAULT_STALL_MS, withStallTimeout } from './stall.js'
 
@@ -57,7 +57,7 @@ export async function downloadToFile(
   client,
   message,
   fd,
-  { offset, onProgress, retryOptions, concurrency = DEFAULT_CONCURRENCY, stallMs = DEFAULT_STALL_MS } = {},
+  { offset, onProgress, retryOptions, concurrency = DEFAULT_DOWNLOAD_CONCURRENCY, stallMs = DEFAULT_STALL_MS } = {},
 ) {
   const document = message?.media?.document
 

package/src/manifest.js

@@ -8,7 +8,7 @@ export function newBackupId(now = new Date(), randomHex = () => randomBytes(3).t
   const yyyy = now.getUTCFullYear()
   const mm = String(now.getUTCMonth() + 1).padStart(2, '0')
   const dd = String(now.getUTCDate()).padStart(2, '0')
-  return `telark-${yyyy}${mm}${dd}-${randomHex()}`
+  return `telstore-${yyyy}${mm}${dd}-${randomHex()}`
 }
 
 export function chunkFileName(id, i) {
@@ -72,7 +72,7 @@ export function manifestMessageIds(manifest) {
       throw new Error(
         `Manifest gives ${JSON.stringify(msgId)} as the message id of chunk ${index + 1}, ` +
           'which is not a message id. Deleting from this manifest could remove the wrong ' +
-          'messages, so telark is not deleting anything.',
+          'messages, so telstore is not deleting anything.',
       )
     }
 
@@ -85,7 +85,7 @@ export function parseManifest(input) {
 
   if (manifest.v !== MANIFEST_VERSION) {
     throw new Error(
-      `Manifest uses version ${manifest.v}, this build of telark only understands version ${MANIFEST_VERSION}.`,
+      `Manifest uses version ${manifest.v}, this build of telstore only understands version ${MANIFEST_VERSION}.`,
     )
   }
 
@@ -147,7 +147,7 @@ export function parseManifest(input) {
   // uniform: every chunk is chunkSize, except the last one which is the remainder.
   // A correct total with individually wrong sizes yields a file with a hole or
   // extra length while every per-chunk sha256 still matches — silently wrong data,
-  // precisely what telark must never produce.
+  // precisely what telstore must never produce.
   manifest.chunks.forEach((chunk, index) => {
     const expected = Math.min(manifest.chunkSize, manifest.size - index * manifest.chunkSize)
     if (chunk.size !== expected) {

package/src/prompt.js

@@ -0,0 +1,81 @@
+import readline from 'node:readline/promises'
+import { stdin, stdout, stderr } from 'node:process'
+import { Writable } from 'node:stream'
+
+// readline echoes what it reads through its own output, so putting a curtain in front of that
+// output is what makes typing invisible.
+function veiledOutput(output) {
+  let hidden = false
+
+  return {
+    hide: () => {
+      hidden = true
+    },
+    show: () => {
+      hidden = false
+    },
+    // The question goes straight to the real output, past the curtain, so it stays on screen
+    // while the answer to it does not.
+    say: (text) => output.write(text),
+    stream: new Writable({
+      write(chunk, encoding, callback) {
+        if (!hidden) output.write(chunk, encoding)
+        callback()
+      },
+    }),
+  }
+}
+
+const NO_TERMINAL =
+  'There is no terminal here to type a secret into. Run this where you can type it.'
+
+// One readline for a whole conversation, some of whose answers must not stay on the screen.
+// It has to be one: two readlines over a single stdin do not take turns — the first keeps the
+// listener and everything typed after it lands nowhere, so the second reaches end-of-input
+// having read nothing and reports it as Ctrl-D. Verified against a real terminal, because a
+// fake stream takes turns perfectly well and would have called this fine.
+export function createPrompts({ input = stdin, output = stdout } = {}) {
+  const veil = veiledOutput(output)
+  // terminal follows stdin: true is what stops the tty driver from echoing on its own, which
+  // is what leaves the curtain as the only thing between the keyboard and the screen. Forcing
+  // it on a pipe would put readline into line editing over input with no terminal behind it.
+  const rl = readline.createInterface({ input, output: veil.stream, terminal: Boolean(input.isTTY) })
+
+  return {
+    ask: (question) => rl.question(question),
+    async askSecret(question) {
+      // A prompt written where nobody can see it, waiting on a stream that will never carry a
+      // typed answer, is a hang — the one failure this project refuses to produce anywhere.
+      // Reading it from the pipe instead would be worse: the secret would then have come from
+      // somewhere that kept a copy of it.
+      if (!input.isTTY) throw new Error(NO_TERMINAL)
+
+      veil.say(question)
+      veil.hide()
+
+      try {
+        return await rl.question('')
+      } finally {
+        veil.show()
+        veil.say('\n')
+      }
+    },
+    // Not optional: terminal mode put stdin in raw mode, and leaving it there hands the user
+    // back a shell that no longer echoes what they type.
+    close: () => rl.close(),
+  }
+}
+
+// The single-question case. Its question goes to stderr rather than stdout, because
+// "npx telstore token > token.txt" has to still show it, and stdout there carries one thing.
+export async function readSecret(question, { input = stdin, output = stderr } = {}) {
+  if (!input.isTTY) throw new Error(NO_TERMINAL)
+
+  const prompts = createPrompts({ input, output })
+
+  try {
+    return await prompts.askSecret(question)
+  } finally {
+    prompts.close()
+  }
+}

package/src/session.js

@@ -0,0 +1,24 @@
+import { readSecret as realReadSecret } from './prompt.js'
+import { decodeToken } from './token.js'
+
+// How a stored config becomes the three things GramJS needs. There are two shapes on disk —
+// the ordinary login, and the sealed blob `login --token` writes — and this is the one place
+// that knows the difference, so `connect` stays about Telegram and the commands stay about
+// their own narrative.
+export async function unlockConfig(config, { readSecret = realReadSecret } = {}) {
+  const { sealed } = config
+
+  if (!sealed) {
+    const { apiId, apiHash, session } = config
+
+    return { apiId, apiHash, session }
+  }
+
+  const passphrase = await readSecret('Passphrase for the stored session: ')
+  // Everything comes out of the blob and nothing from around it. The fields beside it on disk
+  // are editable by anyone who can reach the file, and an apiId taken from there would let an
+  // edit decide which account a passphrase unlocks.
+  const { apiId, apiHash, session } = await decodeToken(sealed, passphrase)
+
+  return { apiId, apiHash, session }
+}

package/src/settings.js

@@ -1,7 +1,8 @@
 import { normalizeChatTarget } from './chat.js'
 import {
   DEFAULT_CHUNK_SIZE,
-  DEFAULT_CONCURRENCY,
+  DEFAULT_DOWNLOAD_CONCURRENCY,
+  DEFAULT_UPLOAD_CONCURRENCY,
   MAX_CONCURRENCY,
   parseSize,
 } from './chunking.js'
@@ -9,7 +10,7 @@ import { formatBytes } from './progress.js'
 
 export const DEFAULT_LIMIT = 20
 
-// The three keys telark writes for itself. Naming them separately is what lets the
+// The three keys telstore writes for itself. Naming them separately is what lets the
 // unknown-key error say "managed by login" instead of listing a session as something the
 // user forgot to spell correctly.
 const MANAGED_BY_LOGIN = new Set(['session', 'apiId', 'apiHash'])
@@ -74,9 +75,9 @@ export const SETTINGS = {
     format: (value) => String(value),
     describe: (value) => formatBytes(value),
   },
-  concurrency: {
-    flag: 'concurrency',
-    default: DEFAULT_CONCURRENCY,
+  uploadConcurrency: {
+    flag: 'upload-concurrency',
+    default: DEFAULT_UPLOAD_CONCURRENCY,
     parse: (raw, where) =>
       wholeNumber(
         raw,
@@ -88,6 +89,20 @@ export const SETTINGS = {
       ),
     format: (value) => String(value),
   },
+  downloadConcurrency: {
+    flag: 'download-concurrency',
+    default: DEFAULT_DOWNLOAD_CONCURRENCY,
+    parse: (raw, where) =>
+      wholeNumber(
+        raw,
+        where,
+        1,
+        MAX_CONCURRENCY,
+        `Must be an integer from 1 to ${MAX_CONCURRENCY} — each slot runs its own download ` +
+          'stream, and Telegram answers with FLOOD_WAIT if too many requests go out at once.',
+      ),
+    format: (value) => String(value),
+  },
   limit: {
     flag: 'limit',
     default: DEFAULT_LIMIT,
@@ -181,10 +196,21 @@ export function resolveSettings(options = {}, config = {}, { file = 'the config
 export function requireChat(values) {
   if (values.chat === null || values.chat === undefined) {
     throw new Error(
-      'No destination set — run "npx telark config chat @my_backups" to set one ' +
+      'No destination set — run "npx telstore config chat @my_backups" to set one ' +
         '("config chat me" for Saved Messages), or pass --to to choose one for this run.',
     )
   }
 
   return values.chat
 }
+
+// The snapshot of somebody's settings that a session token carries to a machine which has no
+// config file of its own. A stray key would arrive there as a line in `config` telling the
+// reader to remove a setting they never wrote, so keys telstore does not know are left behind
+// rather than carried along. This lives here because this is the one place that knows which
+// settings exist.
+export function knownSettings(stored = {}) {
+  return Object.fromEntries(
+    SETTING_KEYS.filter((key) => stored[key] !== undefined).map((key) => [key, stored[key]]),
+  )
+}

package/src/state.js

@@ -96,6 +96,10 @@ export async function pruneStates(configDir = defaultConfigDir(), keep = MAX_STA
 // status needs every unfinished backup at once. A state file that cannot be read is skipped
 // rather than fatal, for the same reason loadState returns null: one corrupt file must not
 // hide the other backups still waiting to be finished.
+//
+// The key comes back alongside each record because canResume needs it, and the file name is
+// the only place it survives: the record's own path, size and mtime are exactly what a
+// rewritten file makes stale, so recomputing the key from them would always say yes.
 export async function listStates(configDir = defaultConfigDir()) {
   let names
   try {
@@ -110,8 +114,10 @@ export async function listStates(configDir = defaultConfigDir()) {
   for (const name of names) {
     if (!name.endsWith('.json')) continue
 
-    const state = await loadState(name.slice(0, -'.json'.length), configDir)
-    if (state) states.push(state)
+    const key = name.slice(0, -'.json'.length)
+    const state = await loadState(key, configDir)
+
+    if (state) states.push({ key, state })
   }
 
   return states
@@ -120,11 +126,11 @@ export async function listStates(configDir = defaultConfigDir()) {
 // delete needs the file a record came from, not just its contents — and the name of that
 // file is a hash of the path, size and mtime *inside* the record, so recomputing it would
 // be trusting an untrusted file to say where it lives. A hand-edited path yields a key that
-// names no file at all, clearState ignores a file that is not there, and telark reports a
+// names no file at all, clearState ignores a file that is not there, and telstore reports a
 // record dropped that is still sitting on disk. Matching the id inside each file is the one
 // way that cannot point at the wrong one.
 //
-// Every record claiming the id is returned rather than the first: two of them means telark
+// Every record claiming the id is returned rather than the first: two of them means telstore
 // cannot know which to drop, and that is the caller's decision to refuse, not ours to make
 // by picking one.
 export async function findStates(backupId, configDir = defaultConfigDir()) {
@@ -149,3 +155,28 @@ export async function findStates(backupId, configDir = defaultConfigDir()) {
 
   return found
 }
+
+// Whether a record can still be resumed, which is not a question about the record alone:
+// runUpload hashes the file it finds on disk and looks the result up, so a backup is
+// resumable exactly when that hash is still the key this record is filed under. Recomputing
+// through stateKey rather than comparing size and mtime by hand is the point — a second way
+// of asking is a second way to drift, and status would end up promising a resume that upload
+// turns into a brand new backup, stranding every chunk already sent.
+//
+// Never throws. status calls this for every record it prints, and one damaged path must not
+// take the rest of the report down with it.
+export async function canResume(key, state) {
+  let stat
+
+  try {
+    stat = await fs.stat(state.path)
+  } catch (err) {
+    if (err.code === 'ENOENT') return { ok: false, reason: 'missing' }
+    return { ok: false, reason: 'unreadable' }
+  }
+
+  if (!stat.isFile()) return { ok: false, reason: 'not-a-file' }
+  if (stateKey(state.path, stat.size, stat.mtimeMs) !== key) return { ok: false, reason: 'changed' }
+
+  return { ok: true }
+}

package/src/token.js

@@ -0,0 +1,221 @@
+import { createCipheriv, createDecipheriv, randomBytes, scrypt } from 'node:crypto'
+import { promisify } from 'node:util'
+
+const derive = promisify(scrypt)
+
+// A session token is this machine's Telegram login, written down so another machine can use
+// it. Two formats, not one format with a flag inside: a blob that looked encrypted and was
+// not would be telstore lying about what it handed over, and the prefix is the one part a
+// reader can check before knowing anything else about the bytes.
+export const TOKEN_PREFIX_SEALED = 'tls1.'
+export const TOKEN_PREFIX_PLAIN = 'tls0.'
+
+const SALT_BYTES = 16
+const IV_BYTES = 12
+const TAG_BYTES = 16
+const KEY_BYTES = 32
+const OVERHEAD = SALT_BYTES + IV_BYTES + TAG_BYTES
+
+// Pinned to the prefix above, never carried inside the token. Whoever holds a token can try
+// passphrases offline as fast as their hardware allows, and 64MB per attempt is what makes
+// that expensive; letting a token name its own parameters would let a stranger's N decide how
+// much memory this machine allocates, which is a denial of service that needs no passphrase
+// at all. maxmem is spelled out because Node's own default is 32MB and would refuse these
+// outright, as an error that reads like a bug in telstore.
+const SCRYPT = { N: 65536, r: 8, p: 1, maxmem: 128 * 1024 * 1024 }
+
+const MAKE_ONE = 'Make a new one with "npx telstore token" on the machine you are logged in on.'
+
+// Whitespace is the one damage repaired rather than refused: a chat client or a mail reader
+// wrapping the line is the likeliest thing to happen to a token, whitespace is never part of
+// one, and joining it back up cannot produce a *different* valid token, because the
+// authentication tag still has to match. Anything else is refused — repairing it would be
+// guessing at what somebody meant.
+function tidy(token) {
+  return String(token).replace(/\s+/g, '')
+}
+
+export function isSealedToken(text) {
+  return tidy(text).startsWith(TOKEN_PREFIX_SEALED)
+}
+
+function isPlainObject(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value)
+}
+
+// A token is untrusted input in the same category as a manifest or a state file. Opening one
+// proves whoever made it knew the passphrase, not that they made it correctly — and the
+// unprotected format proves nothing at all. An apiId of undefined does not throw here; it
+// reaches GramJS and fails much later as something that reads like a network problem.
+export function checkTokenBundle(bundle) {
+  if (!isPlainObject(bundle)) {
+    throw new Error(
+      `The session token holds ${Array.isArray(bundle) ? 'a list' : typeof bundle}, ` +
+        `not an account. ${MAKE_ONE}`,
+    )
+  }
+
+  if (!Number.isSafeInteger(bundle.apiId) || bundle.apiId < 1) {
+    throw new Error(
+      `The session token gives ${JSON.stringify(bundle.apiId)} as the api_id, which is not ` +
+        `one. ${MAKE_ONE}`,
+    )
+  }
+
+  for (const field of ['apiHash', 'session']) {
+    if (typeof bundle[field] !== 'string' || bundle[field] === '') {
+      throw new Error(`The session token is missing its ${field}. ${MAKE_ONE}`)
+    }
+  }
+
+  // The twin of checkConfigShape, refused for the twin's reason: every lookup below a
+  // settings that is not an object returns undefined, and telstore would then run on its
+  // built-in defaults while the choices carried in the token sat there ignored.
+  if (bundle.settings !== undefined && !isPlainObject(bundle.settings)) {
+    throw new Error(
+      `"settings" in the session token holds ` +
+        `${Array.isArray(bundle.settings) ? 'a list' : typeof bundle.settings}, not a group ` +
+        `of settings. ${MAKE_ONE}`,
+    )
+  }
+
+  return bundle
+}
+
+// Both spellings of the same accented passphrase have to derive the same key: macOS hands
+// back one and Linux the other, and the difference would surface as "wrong passphrase" for a
+// passphrase that is right.
+function keyFrom(passphrase, salt) {
+  return derive(String(passphrase).normalize('NFC'), salt, KEY_BYTES, SCRYPT)
+}
+
+export async function encodeToken(bundle, passphrase) {
+  checkTokenBundle(bundle)
+
+  const json = JSON.stringify(bundle)
+
+  // An empty passphrase is not a weak secret, it is the absence of one. Saying so in the
+  // prefix is what keeps the format honest: the alternative is a token that opens for anyone
+  // who reads the channel it travelled through while looking exactly like a protected one.
+  if (String(passphrase) === '') {
+    return TOKEN_PREFIX_PLAIN + Buffer.from(json, 'utf8').toString('base64url')
+  }
+
+  const salt = randomBytes(SALT_BYTES)
+  const iv = randomBytes(IV_BYTES)
+  const cipher = createCipheriv('aes-256-gcm', await keyFrom(passphrase, salt), iv)
+  const body = Buffer.concat([cipher.update(json, 'utf8'), cipher.final()])
+
+  return (
+    TOKEN_PREFIX_SEALED +
+    Buffer.concat([salt, iv, cipher.getAuthTag(), body]).toString('base64url')
+  )
+}
+
+function splitToken(token) {
+  const text = tidy(token)
+
+  for (const prefix of [TOKEN_PREFIX_SEALED, TOKEN_PREFIX_PLAIN]) {
+    if (text.startsWith(prefix)) return { sealed: prefix === TOKEN_PREFIX_SEALED, encoded: text.slice(prefix.length) }
+  }
+
+  const version = /^([a-z]+\d+)\./.exec(text)
+
+  if (version) {
+    throw new Error(
+      `This session token is in format "${version[1]}", which this telstore does not know ` +
+        `(it reads "${TOKEN_PREFIX_SEALED.slice(0, -1)}" and ` +
+        `"${TOKEN_PREFIX_PLAIN.slice(0, -1)}"). Upgrade telstore, or make a new token with ` +
+        'the version you have.',
+    )
+  }
+
+  throw new Error(
+    `That does not look like a session token: one starts with "${TOKEN_PREFIX_SEALED}" or ` +
+      `"${TOKEN_PREFIX_PLAIN}". ${MAKE_ONE}`,
+  )
+}
+
+function bodyBytes(encoded) {
+  const trimmed = encoded.replace(/=+$/, '')
+  const bytes = Buffer.from(trimmed, 'base64url')
+
+  // Buffer.from silently drops every character outside the alphabet, so a token with a stray
+  // quote in it decodes to *something* — which would then fail authentication and be
+  // reported as a wrong passphrase. Re-encoding and comparing is what turns that into the
+  // sentence that actually helps.
+  if (bytes.toString('base64url') !== trimmed) {
+    throw new Error(
+      'The session token has characters that do not belong to one — it was probably ' +
+        'truncated or altered on the way here. Copy it again, whole.',
+    )
+  }
+
+  return bytes
+}
+
+export async function decodeToken(token, passphrase) {
+  const { sealed, encoded } = splitToken(token)
+  const bytes = bodyBytes(encoded)
+
+  if (!sealed) {
+    return checkTokenBundle(readJson(bytes.toString('utf8')))
+  }
+
+  // Asked here rather than at the top: a token nobody can read is a different problem from a
+  // passphrase nobody supplied, and telling someone to type a passphrase for a token that was
+  // damaged in transit sends them after the wrong thing.
+  if (String(passphrase) === '') {
+    throw new Error(
+      'This session token is protected by a passphrase, and none was given. Run the command ' +
+        'again where you can type it.',
+    )
+  }
+
+  if (bytes.length <= OVERHEAD) {
+    throw new Error(
+      `The session token is too short to be one: it carries ${bytes.length} bytes and the ` +
+        `salt, nonce and tag alone need ${OVERHEAD}. Copy it again, whole.`,
+    )
+  }
+
+  const decipher = createDecipheriv(
+    'aes-256-gcm',
+    await keyFrom(passphrase, bytes.subarray(0, SALT_BYTES)),
+    bytes.subarray(SALT_BYTES, SALT_BYTES + IV_BYTES),
+  )
+
+  decipher.setAuthTag(bytes.subarray(SALT_BYTES + IV_BYTES, OVERHEAD))
+
+  let json
+  try {
+    json = Buffer.concat([
+      decipher.update(bytes.subarray(OVERHEAD)),
+      decipher.final(),
+    ]).toString('utf8')
+  } catch {
+    // AES-GCM cannot tell a wrong key from altered bytes: both are one failed tag check, and
+    // guessing which it was would send half the readers after the wrong thing. Name both, put
+    // the likelier one first, and say out loud that telstore is not guessing.
+    throw new Error(
+      'Could not open the session token: either the passphrase is wrong or the token was ' +
+        'altered on the way here. Encryption cannot tell those two apart, so telstore will ' +
+        'not guess — check the passphrase first, then copy the token again, whole.',
+    )
+  }
+
+  return checkTokenBundle(readJson(json))
+}
+
+// Past the tag check the plaintext is ours, so bad JSON here is not somebody else's mistake:
+// it means the token was written by a telstore that disagreed with this one about the format.
+function readJson(json) {
+  try {
+    return JSON.parse(json)
+  } catch (err) {
+    throw new Error(
+      `The session token opened, but what is inside it is not what telstore writes ` +
+        `(${err.message}). ${MAKE_ONE}`,
+    )
+  }
+}

package/src/uploader.js

@@ -5,7 +5,7 @@ import { promisify } from 'node:util'
 import { Api } from 'telegram'
 import { readBigIntFromBuffer } from 'telegram/Helpers.js'
 
-import { DEFAULT_CONCURRENCY, PART_SIZE, MAX_PARTS } from './chunking.js'
+import { DEFAULT_UPLOAD_CONCURRENCY, PART_SIZE, MAX_PARTS } from './chunking.js'
 import { withRetry } from './retry.js'
 import { DEFAULT_STALL_MS, withStallTimeout } from './stall.js'
 
@@ -13,7 +13,7 @@ const read = promisify(readCallback)
 
 // Telegram splits its upload API by file size: only files above 10MB may use the
 // "big" family. GramJS picks the same threshold (LARGE_FILE_THRESHOLD in
-// node_modules/telegram/client/uploads.js), and telark follows it.
+// node_modules/telegram/client/uploads.js), and telstore follows it.
 export const LARGE_FILE_THRESHOLD = 10 * 1024 * 1024
 
 async function readExactly(fd, length, position) {
@@ -39,7 +39,7 @@ export async function uploadRange(client, fd, options) {
     offset,
     length,
     fileName,
-    concurrency = DEFAULT_CONCURRENCY,
+    concurrency = DEFAULT_UPLOAD_CONCURRENCY,
     partSize = PART_SIZE,
     onProgress,
     retryOptions,