telstore 0.1.9 → 0.1.11

package/src/password.js

@@ -0,0 +1,101 @@
+import { stderr, stdin } from 'node:process'
+
+import { parseHint, terminalSafe } from './caption.js'
+import { openManifest } from './cipher.js'
+import { createPrompts, readSecret } from './prompt.js'
+
+export const PASSWORD_ATTEMPTS = 3
+
+// A password that came from an environment variable, a flag or a file came from somewhere that
+// kept a copy of it, which is the rule src/token.js keeps for a passphrase. Unattended encrypted
+// backups stay on the `--` pipeline with a key-based tool.
+const NO_TERMINAL_UPLOAD =
+  '--encrypt needs a terminal to type the password in, and there is none here. Run the upload ' +
+  'where you can type it; telstore does not read a password from anywhere else.'
+
+// One readline for the whole exchange, as docs/design/terminal-prompts.md requires: two over one
+// stdin do not take turns, and the second question would read nothing. On stderr, so a stdout
+// someone redirected still carries only what telstore reports.
+export async function askNewPassword({ input = stdin, output = stderr } = {}) {
+  if (!input.isTTY) throw new Error(NO_TERMINAL_UPLOAD)
+
+  const prompts = createPrompts({ input, output })
+
+  try {
+    const password = await prompts.askSecret('Password: ')
+
+    if (password === '') {
+      throw new Error('The password is empty. An encrypted backup needs one — run again and type it.')
+    }
+
+    const again = await prompts.askSecret('Password again: ')
+
+    if (again !== password) {
+      throw new Error(
+        'The two passwords are different, so telstore does not know which one you meant. ' +
+          'Nothing was sent — run again.',
+      )
+    }
+
+    const hint = parseHint(await prompts.ask('Hint (optional, shown in the chat as plain text): '), password)
+
+    return { password, hint }
+  } finally {
+    prompts.close()
+  }
+}
+
+export function askPassword(question) {
+  return readSecret(question)
+}
+
+// Opens an encrypted manifest or says why it cannot. Passwords this run has already seen work
+// are tried first and silently, so a batch of backups under one password asks once. A failed
+// tag cannot tell a wrong password from an altered manifest, so the last refusal names both,
+// likelier first, as src/token.js does for a token.
+export async function unlockManifest(
+  manifest,
+  { askPassword: ask, interactive = () => Boolean(stdin.isTTY), known = [], say = () => {} },
+) {
+  for (const password of known) {
+    const opened = await openManifest(manifest, password)
+    if (opened) return { ...opened, password }
+  }
+
+  if (!interactive()) {
+    throw new Error(
+      `${manifest.id} is encrypted, and there is no terminal here to type its password in. ` +
+        'Run this where you can type it; telstore does not read a password from anywhere else.',
+    )
+  }
+
+  say(`Backup ${manifest.id} is encrypted.`)
+
+  // Printed before any key exists, so the seal that covers it cannot have been checked yet: at
+  // this moment it is as trustworthy as the chat it came from, and is made safe to print as such.
+  const hint = manifest.enc.hint ? terminalSafe(manifest.enc.hint) : ''
+
+  if (hint) say(`Hint   ${hint}`)
+
+  for (let attempt = 1; attempt <= PASSWORD_ATTEMPTS; attempt += 1) {
+    const password = await ask('Password: ')
+    const opened = password === '' ? null : await openManifest(manifest, password)
+
+    if (opened) {
+      known.push(password)
+      return { ...opened, password }
+    }
+
+    if (attempt < PASSWORD_ATTEMPTS) say('That password does not open it. Try again.')
+  }
+
+  throw new Error(
+    `Could not open ${manifest.id} after ${PASSWORD_ATTEMPTS} attempts: either the password is ` +
+      'wrong or the manifest was altered. Encryption cannot tell those two apart, so telstore ' +
+      'will not guess — check the password first.' +
+      (hint
+        ? ' The hint shown comes from the chat and is only checked once the password opens the ' +
+          'backup, so a hint that does not help may itself have been altered.'
+        : ''),
+  )
+}

package/src/progress.js

@@ -47,6 +47,57 @@ export function renderProgress({ done, total, elapsedMs, label, width = 24, tran
   return `${label} ${bar} ${percent}% ${formatBytes(done)}/${formatBytes(total)} ${speed} ETA ${formatDuration(remaining)}`
 }
 
+// A percentage of an unknown total is an invented number, and an ETA from one is worse: it
+// would count down to a finish nobody can predict.
+export function renderStreamProgress({ done, elapsedMs, label }) {
+  const bytesPerSecond = elapsedMs > 0 ? done / (elapsedMs / 1000) : 0
+
+  return `${label} ${formatBytes(done)} sent ${formatBytes(Math.round(bytesPerSecond))}/s`
+}
+
+export function createStreamProgress({
+  label,
+  write = (line) => process.stderr.write(line),
+  now = () => Date.now(),
+  minIntervalMs = 200,
+}) {
+  const startedAt = now()
+  let done = 0
+  let currentLabel = label
+  let lastDrawnAt = startedAt
+  let widestLine = 0
+
+  // Same \r discipline as createProgress: a redraw shorter than the one before it would
+  // leave the previous line's tail on screen, so pad every line out to the widest drawn so far.
+  function draw(suffix) {
+    const line = renderStreamProgress({ done, elapsedMs: now() - startedAt, label: currentLabel })
+    widestLine = Math.max(widestLine, line.length)
+    write(`\r${line.padEnd(widestLine)}${suffix}`)
+  }
+
+  return {
+    advance(bytes) {
+      done += bytes
+      if (now() - lastDrawnAt < minIntervalMs) return
+      lastDrawnAt = now()
+      draw('')
+    },
+    setLabel(next) {
+      currentLabel = next
+      lastDrawnAt = now()
+      draw('')
+    },
+    finish() {
+      draw('\n')
+    },
+  }
+}
+
+// A number turned into words a person reads: "1 chunk" for one, "3 chunks" for the rest.
+export function plural(n, word) {
+  return `${n} ${word}${n === 1 ? '' : 's'}`
+}
+
 export function createProgress({
   total,
   label,
@@ -99,3 +150,38 @@ export function createProgress({
     },
   }
 }
+
+// A walk of one page is over in about the time it takes to notice — 165ms against a real
+// chat — and that is the usual case, so nothing is drawn for the first stretch: a line that
+// appears and is wiped in the same breath is a flicker, not information. Past that the read
+// is long enough that silence reads as the hang this project refuses everywhere else.
+//
+// \r only moves the cursor home, so every line is padded to the widest one drawn and the last
+// write wipes the row: whatever the command prints next must never land on half a notice.
+//
+// Here rather than beside either caller: `list` walks a chat to find backups and `delete`
+// walks it to find chunks nothing on this machine names, and two copies of "when is a read
+// long enough to say something about" is how they start disagreeing about it.
+const NOTICE_QUIET_MS = 400
+const NOTICE_INTERVAL_MS = 200
+
+export function createWalkNotice({ write, now, quietMs = NOTICE_QUIET_MS, intervalMs = NOTICE_INTERVAL_MS }) {
+  const startedAt = now()
+  let lastDrawnAt = 0
+  let widest = 0
+
+  return {
+    tick(text) {
+      if (now() - startedAt < quietMs) return
+      if (lastDrawnAt !== 0 && now() - lastDrawnAt < intervalMs) return
+
+      lastDrawnAt = now()
+      widest = Math.max(widest, text.length)
+      write(`\r${text.padEnd(widest)}`)
+    },
+    clear() {
+      if (widest === 0) return
+      write(`\r${' '.repeat(widest)}\r`)
+    },
+  }
+}

package/src/shell.js

@@ -0,0 +1,33 @@
+// A command meant to be pasted has to survive the shell that receives it: anything a shell
+// would take apart comes back quoted, and a path with a space in it is the ordinary case,
+// not an exotic one. `status` and `down` both print resume commands, and two copies of this
+// rule is how they start disagreeing about which paths are safe to print bare.
+const BARE_ARG = /^[A-Za-z0-9_@%+:,./-]+$/
+
+export function shellArg(text) {
+  const value = String(text)
+
+  return BARE_ARG.test(value) ? value : `'${value.replaceAll("'", `'\\''`)}'`
+}
+
+// The one line in telstore that destroys data when it is wrong. `delete` resolves its own
+// destination from config and then fires the recorded message ids at whatever peer that turns
+// out to be, so a command pasted next week — or one built here under a `--chat` this run was
+// given — would remove whatever happens to carry those ids in the chat it resolves. Naming a
+// chat that turns out to be the default costs a few characters; leaving it out when it is not
+// costs somebody else's messages, and nothing undoes that. Four commands print this string
+// (`upload-stream`'s rollback, `cli`'s second Ctrl-C, `status`, `down`) and they had four
+// copies of the rule, which is how three of them stayed right and one drifted.
+//
+// The chatless branch is not a convenience: it is for the single caller that genuinely does
+// not know where the chunks went. `bin/telstore.js` holds the chat the run handed it, and a
+// Ctrl-C arriving before the run ever reported one leaves the id — the only part of the
+// message with any value — rather than printing `--chat undefined`, which `delete` would read
+// as no destination at all while looking like one. A caller that would rather print nothing
+// than a command missing its chat decides that for itself before calling: `down` does, and
+// says why beside its own check.
+export function deleteCommand(id, chat) {
+  const where = chat === null || chat === undefined ? '' : String(chat).trim()
+
+  return `npx telstore delete ${shellArg(id)}${where === '' ? '' : ` --chat ${shellArg(where)}`}`
+}

package/src/spawn.js

@@ -0,0 +1,38 @@
+import { spawn } from 'node:child_process'
+
+// stderr is inherited, never captured: when a producer command fails it explains itself in
+// its own words, on the stream the user is already watching. telstore adds the exit code
+// and what it did about it, and does not paraphrase.
+//
+// `exited` can be built before the caller has any chance to await it — the caller reads
+// `stdout` first, and only awaits `exited` once the stream ends. A rejected promise with no
+// handler attached yet makes Node report an unhandledRejection, so a no-op `.catch` is
+// attached here immediately. That does not consume the rejection: the `exited` this function
+// returns is the same promise, and `await`ing it later still resolves or rejects exactly as
+// it would have.
+export function spawnProducer(argv, { stdio = ['ignore', 'pipe', 'inherit'] } = {}) {
+  const [command, ...args] = argv
+  const child = spawn(command, args, { stdio })
+
+  const exited = new Promise((resolve, reject) => {
+    child.on('error', (err) => {
+      reject(
+        new Error(
+          err.code === 'ENOENT'
+            ? `Cannot run ${command}: no such command on this machine.`
+            : `Cannot run ${command}: ${err.message}`,
+        ),
+      )
+    })
+
+    child.on('close', (code, signal) => resolve({ code, signal }))
+  })
+  exited.catch(() => {})
+
+  return {
+    stdout: child.stdout,
+    stdin: child.stdin,
+    exited,
+    kill: (signal = 'SIGTERM') => child.kill(signal),
+  }
+}

package/src/state.js

@@ -8,14 +8,97 @@ export function stateDir(configDir = defaultConfigDir()) {
   return path.join(configDir, 'state')
 }
 
+// The other thing telstore keeps on this machine, and the only one that is not a record: a
+// stream upload borrows one chunk of disk at a time here while it sends it. Under
+// ~/.telstore rather than os.tmpdir() because /tmp is tmpfs on many Linux distributions, and
+// "borrow one chunk of disk" would silently mean "borrow 1800MB of RAM" — a memory limit
+// dressed up as a chunk size.
+//
+// It lives beside stateDir because it answers the same question — what has this machine got
+// of telstore's on it — and because `status` has to be able to ask without importing the
+// upload command, which would drag a second upload loop and teleproto in with it.
+export function tempDirFor(configDir = defaultConfigDir()) {
+  return path.join(configDir, 'tmp')
+}
+
+// What is in there now. A run removes its own chunk file on every ending it gets to run code
+// for, so a file here is either a run happening at this moment or a run that was stopped
+// where it stood — a SIGKILL, a crash, a machine losing power. Nothing here removes them:
+// from outside the run that owns one, those two cases look exactly the same, and deleting
+// the chunk a live upload is filling is the confident wrong thing this project refuses
+// everywhere else. Naming them is the whole job.
+//
+// A stat that fails yields an unknown size rather than a dropped row, the same care
+// listStates takes with mtimes: the file really can vanish between the readdir and the stat —
+// that is what a run finishing normally does — and status is the command someone runs
+// *because* something is wrong.
+export async function listTempChunks(configDir = defaultConfigDir()) {
+  const dir = tempDirFor(configDir)
+  let names
+
+  try {
+    names = await fs.readdir(dir)
+  } catch (err) {
+    // A machine that has never made a backup from a command has no such directory, and that
+    // is not a fault to report. Anything else is: a directory telstore cannot read may be
+    // holding a whole chunk, and answering "nothing there" would be the silent wrong answer
+    // this listing exists to prevent. The caller decides what to do with it.
+    if (err.code === 'ENOENT') return []
+
+    throw err
+  }
+
+  const found = []
+
+  for (const name of names.sort()) {
+    const file = path.join(dir, name)
+    let size = null
+
+    try {
+      const stat = await fs.stat(file)
+
+      if (!stat.isFile()) continue
+
+      size = stat.size
+    } catch {
+      // Gone or unreadable between the readdir and here. Still a name worth printing: the
+      // point of the listing is that nothing telstore left behind goes unmentioned.
+    }
+
+    found.push({ name, file, size })
+  }
+
+  return found
+}
+
 export function stateKey(absPath, size, mtimeMs) {
   return createHash('sha1').update(`${absPath}:${size}:${mtimeMs}`).digest('hex')
 }
 
+// Where an encrypted upload's record is filed, and the reason it is not filed under stateKey is
+// the reason the manifest went to version 2: an older telstore. That build computes stateKey
+// exactly as this one does and never reads `enc`, so given an unfinished encrypted record under
+// that key it would resume it without --encrypt, send the remaining chunks in plain, write a
+// version 1 manifest with no ivs in it — and a later restore would match every sha256 and print
+// Done over a file that is half ciphertext. A lookup that misses is the only refusal an older
+// build can be made to give. Same 40-hex shape, so every listing, prune and status report that
+// already handles upload records handles this one without learning a new name.
+export function encryptedStateKey(absPath, size, mtimeMs) {
+  return createHash('sha1').update(`enc:${absPath}:${size}:${mtimeMs}`).digest('hex')
+}
+
 export function stateFile(key, configDir = defaultConfigDir()) {
   return path.join(stateDir(configDir), `${key}.json`)
 }
 
+// stateKey hashes path:size:mtime, and a stream has none of the three. What holds still is
+// the backup id, and hashing it keeps the file name in the same 40-hex shape the directory
+// already sorts, prunes and filters on — a stream record is an upload record, not a third
+// kind, so it shares that namespace rather than getting a prefix of its own.
+export function streamKey(backupId) {
+  return createHash('sha1').update(`stream:${backupId}`).digest('hex')
+}
+
 // A restore's record is filed beside the uploads and must never compete with them for a
 // prune slot. Losing an upload record strands chunks in a chat where only the id can still
 // find them, which is why pruneStates reads each file back to name what it drops; losing a
@@ -211,13 +294,21 @@ export async function findStates(backupId, configDir = defaultConfigDir()) {
 // Whether a record can still be resumed, which is not a question about the record alone:
 // runUpload hashes the file it finds on disk and looks the result up, so a backup is
 // resumable exactly when that hash is still the key this record is filed under. Recomputing
-// through stateKey rather than comparing size and mtime by hand is the point — a second way
+// through stateKey (encryptedStateKey for an encrypted record) rather than comparing size and
+// mtime by hand is the point — a second way
 // of asking is a second way to drift, and status would end up promising a resume that upload
 // turns into a brand new backup, stranding every chunk already sent.
 //
 // Never throws. status calls this for every record it prints, and one damaged path must not
 // take the rest of the report down with it.
 export async function canResume(key, state) {
+  // A stream cannot be resumed by anyone, so this is not a question about a file. Answering
+  // it by stat-ing state.path would report "missing" for a record that never had a path,
+  // and status would then offer a resume command that starts a brand new backup. This has
+  // to run before the stat below, not after it fails: a stream record's key could still
+  // happen to match a real file on disk, and that file is not what makes it unresumable.
+  if (state.kind === 'stream') return { ok: false, reason: 'stream' }
+
   let stat
 
   try {
@@ -228,7 +319,16 @@ export async function canResume(key, state) {
   }
 
   if (!stat.isFile()) return { ok: false, reason: 'not-a-file' }
-  if (stateKey(state.path, stat.size, stat.mtimeMs) !== key) return { ok: false, reason: 'changed' }
+
+  // Two keys now, and the record says which one it belongs under: runUpload files an encrypted
+  // record under encryptedStateKey and a plain one under stateKey. A record sitting under the
+  // other kind's key for a file that has not changed is not a changed file — runUpload refuses
+  // it as damaged rather than resume it, and the report has to say the same thing.
+  const own = (state.enc ? encryptedStateKey : stateKey)(state.path, stat.size, stat.mtimeMs)
+  const other = (state.enc ? stateKey : encryptedStateKey)(state.path, stat.size, stat.mtimeMs)
+
+  if (key === other) return { ok: false, reason: 'damaged' }
+  if (key !== own) return { ok: false, reason: 'changed' }
 
   return { ok: true }
 }