telstore 0.1.6 → 0.1.8

package/src/commands/upload.js

@@ -5,10 +5,13 @@ import { Api } from 'teleproto'
 import { CustomFile } from 'teleproto/client/uploads.js'
 
 import { PART_SIZE, planChunks } from '../chunking.js'
-import { chunkCaption, manifestCaption } from '../caption.js'
+import { chunkCaption, manifestCaption, parseNote } from '../caption.js'
 import { describeChat } from '../chat.js'
 import { closeQuietly, connect as realConnect } from '../client.js'
+import { askConfirm } from '../confirm.js'
 import { configFile, defaultConfigDir, loadConfig } from '../config.js'
+import { expandSources } from '../sources.js'
+import { assertLoggedIn } from '../session.js'
 import { requireChat, resolveSettings } from '../settings.js'
 import {
   buildManifest,
@@ -61,6 +64,49 @@ async function realSendManifest(client, peer, { bytes, fileName, caption }) {
   })
 }
 
+// `--note quarterly accounts` is two arguments by the time node sees it: the note is
+// "quarterly", and "accounts" is a file telstore has been asked to upload. Nothing here can
+// put them back together — the shell dropped the quotes before the process started — so the
+// one useful thing left is to name the mistake this probably was.
+//
+// Two things have to be true before it is said, and neither is enough alone. The note must
+// still be one word — one that kept its spaces is one the shell was told to keep whole, so
+// this cannot have happened to it — and a file must have been named *after* the flag, which
+// is where an unquoted note's remaining words land. Without both, the file name is simply
+// wrong, and sending someone off after a quoting bug that is not there costs them the typo
+// waiting at the front of the same sentence.
+function splitNoteHint(note, filesAfterNote) {
+  if (!note || !filesAfterNote || /\s/.test(note)) return ''
+
+  return (
+    ` This run also carries --note ${JSON.stringify(note)}: a note with spaces in it has to be ` +
+    'quoted, or the shell hands every word after the first to telstore as another file. ' +
+    'Write it as --note "the whole note".'
+  )
+}
+
+// What telstore will read the bytes from. A batch checks every path through this before it
+// sends anything, so "File does not exist" reads the same whether it came from the one file
+// asked for or from the fourth of six — one definition, one wording.
+async function statSource(absPath, { note = null, filesAfterNote = false } = {}) {
+  let stat
+
+  try {
+    stat = await fs.stat(absPath)
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      throw new Error(`File does not exist: ${absPath}.${splitNoteHint(note, filesAfterNote)}`)
+    }
+    throw err
+  }
+
+  if (!stat.isFile()) {
+    throw new Error(`${absPath} is not a file.`)
+  }
+
+  return stat
+}
+
 export async function runUpload(filePath, options = {}, deps = {}) {
   const {
     connect = realConnect,
@@ -74,21 +120,20 @@ export async function runUpload(filePath, options = {}, deps = {}) {
     log: writeLog = (line) => console.log(line),
     silent = false,
     onBackupId = () => {},
+    // Where on the command line the note sat, as `route` saw it. Nothing else can know, and
+    // a run that never says leaves the advice unsaid rather than guessed at.
+    filesAfterNote = false,
   } = deps
 
   const absPath = path.resolve(filePath)
 
-  let stat
-  try {
-    stat = await fs.stat(absPath)
-  } catch (err) {
-    if (err.code === 'ENOENT') throw new Error(`File does not exist: ${absPath}`)
-    throw err
-  }
+  // Before the file is even looked at: the note is the only thing this command sends that a
+  // person typed by hand, and it is written into the manifest, which goes out last. A note
+  // Telegram would refuse has to stop the run here, not after an hour of chunks whose only
+  // list can no longer be sent.
+  const note = parseNote(options.note)
 
-  if (!stat.isFile()) {
-    throw new Error(`${absPath} is not a file.`)
-  }
+  const stat = await statSource(absPath, { note, filesAfterNote })
 
   const config = await loadConfig(configDir)
   const { values: settings, source } = resolveSettings(options, config, {
@@ -138,14 +183,14 @@ export async function runUpload(filePath, options = {}, deps = {}) {
   const resuming = Boolean(state)
 
   // Naming the way back rather than a flag to drop: the destination may have come from the
-  // command line or from the stored setting, and "run again without --to" is no help to
+  // command line or from the stored setting, and "run again without --chat" is no help to
   // someone who never typed one. Pointing at the chat itself is right either way.
   if (resuming && state.chat !== String(chat)) {
     const file = stateFile(key, configDir)
     throw new Error(
       `This unfinished backup is going to ${state.chat}, but the current command targets ${chat} — ` +
         `a single backup cannot be split across two destinations. Run again with ` +
-        `--to ${state.chat} to carry on sending there, or delete ${file} and run again to ` +
+        `--chat ${state.chat} to carry on sending there, or delete ${file} and run again to ` +
         `start a new backup in ${chat}.`,
     )
   }
@@ -299,6 +344,7 @@ export async function runUpload(filePath, options = {}, deps = {}) {
       name: path.basename(absPath),
       size: stat.size,
       chunkSize,
+      note,
       chunks: chunks.map((chunk) => ({ i: chunk.i, ...state.done[String(chunk.i)] })),
     })
 
@@ -311,6 +357,7 @@ export async function runUpload(filePath, options = {}, deps = {}) {
         size: manifest.size,
         chunks: manifest.chunks.length,
         createdAt: manifest.createdAt,
+        note: manifest.note ?? null,
       }),
     })
 
@@ -325,3 +372,198 @@ export async function runUpload(filePath, options = {}, deps = {}) {
     )
   }
 }
+
+// `telstore a b c` is three backups, not one: each file keeps its own id, its own manifest and
+// its own resumable record, so a batch is exactly what running the command three times would
+// have produced — minus two logins. The connection is the one thing worth sharing, and it is
+// shared through the same deps seam the tests drive, so runUpload stays the only caller of
+// connect and nothing here has to know what a client is.
+export async function runUploads(filePaths, options = {}, deps = {}) {
+  const {
+    connect = realConnect,
+    disconnect = (client) => client.destroy(),
+    configDir = defaultConfigDir(),
+    writeErr = (line) => process.stderr.write(line),
+    log: writeLog = (line) => console.log(line),
+    silent = false,
+    onFileDone = () => {},
+    confirm = askConfirm,
+    interactive = () => Boolean(process.stdin.isTTY),
+    filesAfterNote = false,
+  } = deps
+
+  const log = silent ? () => {} : writeLog
+  const warn = silent ? () => {} : writeErr
+
+  // One note covers the whole run, so a note telstore cannot send is one mistake, not one per
+  // file. Left to runUpload it would be caught per file and swallowed into the summary as a
+  // failed row, after the files ahead of it had already gone out labelled with nothing.
+  const note = parseNote(options.note)
+
+  // What was typed and what will be sent are two different lists once a folder or a pattern is
+  // allowed: resolve them here, before anything else has an opinion about them.
+  const { paths: found, skipped } = await expandSources(filePaths)
+  const paths = found.map((filePath) => path.resolve(filePath))
+
+  // Something inside a named folder that is not going to be uploaded is still something the
+  // user pointed at, so it is said out loud rather than quietly missing from the list.
+  const folders = skipped.filter((entry) => entry.reason === 'directory')
+
+  if (folders.length > 0) {
+    warn(
+      `\ntelstore reads one level down, so these folders were left alone: ` +
+        `${folders.map((entry) => path.basename(entry.path)).join(', ')}. ` +
+        'Name one of them to upload what is inside it.\n',
+    )
+  }
+
+  for (const entry of skipped) {
+    if (entry.reason !== 'directory') warn(`\n${entry.path} was skipped: ${entry.reason}.\n`)
+  }
+
+  // A single file is the common case and must read exactly as it did before this existed:
+  // no batch heading, no question, no summary, and an error that reaches the caller rather
+  // than a report.
+  if (paths.length === 1) {
+    const { id, chunks } = await runUpload(paths[0], options, deps)
+    return { results: [{ path: paths[0], id, chunks }], failed: 0 }
+  }
+
+  // Everything that can be known before the first byte goes out is settled here. A typo in the
+  // fourth name must not surface an hour into the third file, and a destination nobody set is
+  // one problem, not one per file — the report at the end is for what only the transfer can
+  // discover.
+  const duplicate = paths.find((absPath, index) => paths.indexOf(absPath) !== index)
+
+  if (duplicate) {
+    throw new Error(
+      `${duplicate} is named twice — a folder or a pattern can pick up a file that was ` +
+        'named on its own as well. Uploading one file twice would make two backups of the ' +
+        'same bytes, each with its own id: name it once, or run telstore again afterwards if ' +
+        'a second copy is really what you want.',
+    )
+  }
+
+  const config = await loadConfig(configDir)
+
+  // The login is checked here rather than left to the first connect: reported from inside the
+  // loop it would arrive once per file, each one having already written a state record for a
+  // backup that never sent a byte.
+  assertLoggedIn(config)
+
+  const { values: settings } = resolveSettings(options, config, { file: configFile(configDir) })
+  const chat = requireChat(settings)
+
+  const sizes = []
+
+  for (const absPath of paths) sizes.push((await statSource(absPath, { note, filesAfterNote })).size)
+
+  // The last thing before the first byte. A folder or a pattern hands telstore a list nobody
+  // has read yet, and even a hand-typed one is worth seeing added up: this is the moment where
+  // "23 files, 180 GB, to @family_photos" is still a question rather than an afternoon.
+  if (!options.yes) {
+    if (!interactive()) {
+      throw new Error(
+        `${paths.length} files to upload, and no terminal to confirm that in. Run again with ` +
+          '--yes to upload them without being asked.',
+      )
+    }
+
+    for (const line of listingLines(paths, sizes, chat)) log(line)
+
+    if (!(await confirm(`Upload these ${paths.length} files? [y/N] `))) {
+      throw new Error('Cancelled on request.')
+    }
+  }
+
+  let shared = null
+  const perFile = {
+    ...deps,
+    connect: async (theirConfig, connectOptions) =>
+      (shared ??= await connect(theirConfig, connectOptions)),
+    disconnect: async () => {},
+  }
+
+  const results = []
+
+  try {
+    for (const [index, absPath] of paths.entries()) {
+      if (index > 0) log('')
+      log(`[${index + 1}/${paths.length}] ${path.basename(absPath)}`)
+
+      let result
+      try {
+        const { id, chunks } = await runUpload(absPath, options, perFile)
+        result = { path: absPath, id, chunks }
+      } catch (err) {
+        // One file's trouble is that file's trouble. Stopping here would leave the files
+        // named after it untouched and unmentioned, which is the batch equivalent of the
+        // silent drop this command exists to end — so it is recorded and named in the
+        // summary, and the exit code carries it out to the shell.
+        //
+        // It is also said out loud here and now. Waiting for the summary would leave the bar
+        // of the next file scrolling for an hour over a failure nobody had been told about.
+        result = { path: absPath, error: err.message }
+        warn(`\n${path.basename(absPath)} failed: ${err.message}\n`)
+      }
+
+      results.push(result)
+      onFileDone(result)
+    }
+  } finally {
+    if (shared) {
+      await closeQuietly(shared, disconnect, (err) =>
+        warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`),
+      )
+    }
+  }
+
+  const failed = results.filter((result) => result.error).length
+
+  log('')
+  for (const line of summaryLines(results, failed)) log(line)
+
+  return { results, failed }
+}
+
+// What the batch is about to do, in the shape the summary will report it afterwards: the same
+// names, the same order, so the two lists can be read against each other.
+function listingLines(paths, sizes, chat) {
+  const names = paths.map((absPath) => path.basename(absPath))
+  const amounts = sizes.map(formatBytes)
+  const width = Math.max(...names.map((name) => name.length))
+  const amountWidth = Math.max(...amounts.map((amount) => amount.length))
+  const total = sizes.reduce((sum, size) => sum + size, 0)
+
+  return [
+    `${paths.length} files, ${formatBytes(total)}, to ${describeChat(chat)}`,
+    '',
+    ...names.map((name, i) => `  ${name.padEnd(width)}  ${amounts[i].padStart(amountWidth)}`),
+    '',
+  ]
+}
+
+// The one place a batch says how it went. Every file gets a line whether it worked or not:
+// a name missing from this list would be a file nobody could tell the fate of.
+function summaryLines(results, failed) {
+  const width = Math.max(...results.map((result) => path.basename(result.path).length))
+  const uploaded = results.length - failed
+  const lines = [`${results.length} files: ${uploaded} uploaded, ${failed} failed.`, '']
+
+  for (const result of results) {
+    const name = path.basename(result.path).padEnd(width)
+
+    if (result.error) {
+      lines.push(`  ${name}  failed: ${result.error}`)
+      continue
+    }
+
+    lines.push(`  ${name}  ${result.id}  (${result.chunks} chunk${result.chunks === 1 ? '' : 's'})`)
+  }
+
+  if (uploaded > 0) {
+    lines.push('', 'Restore with: npx telstore restore <backup-id>')
+  }
+
+  return lines
+}

package/src/downloader.js

@@ -43,7 +43,10 @@ async function readExactly(fd, length, position) {
 // the better check anyway: a slice written at the wrong offset, two slices overlapping, or
 // one silently skipped all show up here. It does not prove the bytes reached the platter —
 // this read may well be served from the page cache — it proves the assembly.
-async function hashRange(fd, offset, length) {
+//
+// Exported because a resumed restore asks the same question of a .partial left by an earlier
+// run. A second copy of it in restore.js is how two definitions of one check start to differ.
+export async function hashRange(fd, offset, length) {
   const hash = createHash('sha256')
 
   for (let at = 0; at < length; at += HASH_READ_SIZE) {

package/src/manifest.js

@@ -19,7 +19,15 @@ export function manifestFileName(id) {
   return `${id}.manifest.json`
 }
 
-export function buildManifest({ id, name, size, chunkSize, chunks, createdAt = new Date().toISOString() }) {
+export function buildManifest({
+  id,
+  name,
+  size,
+  chunkSize,
+  chunks,
+  createdAt = new Date().toISOString(),
+  note = null,
+}) {
   return {
     v: MANIFEST_VERSION,
     id,
@@ -27,6 +35,9 @@ export function buildManifest({ id, name, size, chunkSize, chunks, createdAt = n
     size,
     chunkSize,
     createdAt,
+    // Absent rather than null when there is none: a manifest without a note has to be the
+    // same file telstore wrote before the flag existed, down to the bytes.
+    ...(note ? { note } : {}),
     chunks: [...chunks]
       .sort((a, b) => a.i - b.i)
       .map(({ i, msgId, size: chunkBytes, sha256 }) => ({ i, msgId, size: chunkBytes, sha256 })),
@@ -105,6 +116,15 @@ export function parseManifest(input) {
     )
   }
 
+  // The note is decoration — nothing restores differently because of it — but a manifest is
+  // a file a person can edit and send back, and a field holding something other than what it
+  // claims to be is the point where telstore stops reading rather than guesses.
+  if (manifest.note !== undefined && typeof manifest.note !== 'string') {
+    throw new Error(
+      `Manifest records a note of ${JSON.stringify(manifest.note)}, which is not text.`,
+    )
+  }
+
   if (!Number.isSafeInteger(manifest.chunkSize) || manifest.chunkSize < 1) {
     throw new Error(
       `Manifest records a chunk size of ${JSON.stringify(manifest.chunkSize)}, ` +

package/src/progress.js

@@ -1,6 +1,7 @@
 const UNITS = ['B', 'KB', 'MB', 'GB', 'TB']
 
 export function formatBytes(n) {
+  if (!Number.isFinite(n)) return '--'
   if (n < 1024) return `${n} B`
 
   let value = n

package/src/settings.js

@@ -35,7 +35,7 @@ function wholeNumber(raw, where, minimum, maximum, explanation) {
 // which is why chunkSize prints bytes and leaves "1.8 GB" to `describe`.
 export const SETTINGS = {
   chat: {
-    flag: 'to',
+    flag: 'chat',
     default: null,
     // A number here is a chat id, and a chat id is a whole number. 42.5 would otherwise
     // slip through as the string "42.5" and only fail much later, at Telegram, as a chat
@@ -129,9 +129,9 @@ export const SETTINGS = {
 
 export const SETTING_KEYS = Object.keys(SETTINGS)
 
-// Someone who has been typing `--to` and `--chunk-size` for a week will type them at the
-// config command too. Accept the flag spelling as a way in, and canonicalise on the way to
-// disk so the file only ever holds one name per setting.
+// Someone who has been typing `--chunk-size` and `--upload-concurrency` for a week will type
+// them at the config command too. Accept the flag spelling as a way in, and canonicalise on
+// the way to disk so the file only ever holds one name per setting.
 const ALIASES = new Map()
 
 for (const [key, spec] of Object.entries(SETTINGS)) {
@@ -150,7 +150,7 @@ export function isManagedByLogin(input) {
 // Where a value came from, spelled the way the user would recognise it. A stored value that
 // fails to parse must not be reported as a bad flag: nobody typed a flag, and telling them
 // to fix one sends them off after the wrong thing — the same mistake the old
-// "run again without --to" advice made.
+// "run again without --chat" advice made.
 function origin(key, from, file) {
   return from === 'flag' ? `--${SETTINGS[key].flag}` : `${key} in ${file}`
 }
@@ -197,7 +197,7 @@ export function requireChat(values) {
   if (values.chat === null || values.chat === undefined) {
     throw new Error(
       'No destination set — run "npx telstore config chat @my_backups" to set one ' +
-        '("config chat me" for Saved Messages), or pass --to to choose one for this run.',
+        '("config chat me" for Saved Messages), or pass --chat to choose one for this run.',
     )
   }
 

package/src/sources.js

@@ -0,0 +1,163 @@
+import { promises as fs } from 'node:fs'
+import path from 'node:path'
+
+// What a command line names and what telstore uploads are not the same list: a folder stands
+// for the files inside it, and a pattern for the names that match. Both are resolved here,
+// before anything connects, so the run is decided against the disk rather than discovered one
+// file at a time.
+//
+// The shell expands `*` long before node sees it, and that is the expansion telstore prefers —
+// this one only runs on a pattern that arrived intact: quoted, or left alone by a shell that
+// found nothing to expand it to. Whichever did the work, the rule is the same one glob has
+// always had, so a quoted pattern is not a different feature with different results.
+
+// One level, because a folder is a place someone put files, not a tree telstore may walk on
+// its own: a recursive sweep of a home directory is thousands of backups nobody asked for.
+const HIDDEN = /^\./
+
+function isPattern(text) {
+  return /[*?]/.test(text)
+}
+
+// `*` and `?` mean what they mean in a shell; everything else is a literal, including the
+// regex metacharacters a file name is perfectly allowed to contain.
+function patternToRegExp(pattern) {
+  const body = pattern
+    .replace(/[.+^${}()|[\]\\]/g, '\\$&')
+    .replace(/[*?]/g, (char) => (char === '*' ? '[^/]*' : '[^/]'))
+
+  return new RegExp(`^${body}$`)
+}
+
+// A directory entry is only a source if it is a file — following symlinks, since a link to a
+// file is a perfectly ordinary thing to keep in a folder of backups. Anything that cannot be
+// read at all is named rather than dropped.
+async function classify(full) {
+  try {
+    const stat = await fs.stat(full)
+
+    if (stat.isFile()) return { kind: 'file' }
+    if (stat.isDirectory()) return { kind: 'skip', reason: 'directory' }
+
+    return { kind: 'skip', reason: 'not a file' }
+  } catch (err) {
+    return { kind: 'skip', reason: `cannot be read (${err.code ?? err.message})` }
+  }
+}
+
+async function readEntries(dir) {
+  return (await fs.readdir(dir)).sort()
+}
+
+async function expandDirectory(dir, skipped) {
+  const paths = []
+  const folders = []
+
+  for (const name of await readEntries(dir)) {
+    if (HIDDEN.test(name)) continue
+
+    const full = path.join(dir, name)
+    const { kind, reason } = await classify(full)
+
+    if (kind === 'file') {
+      paths.push(full)
+      continue
+    }
+
+    if (reason === 'directory') folders.push(name)
+    skipped.push({ path: full, reason })
+  }
+
+  if (paths.length === 0) {
+    throw new Error(
+      `${dir} has no files to upload — telstore reads one level down and leaves hidden files ` +
+        'alone.' +
+        (folders.length > 0
+          ? ` telstore does not walk into the folders inside it (${folders.join(', ')}) — ` +
+            'name one of those instead.'
+          : ''),
+    )
+  }
+
+  return paths
+}
+
+async function expandPattern(pattern) {
+  const dir = path.dirname(pattern)
+  const base = path.basename(pattern)
+
+  if (isPattern(dir)) {
+    throw new Error(
+      `Wildcards only work in the last part of a path, and ${pattern} has one earlier: ` +
+        'telstore does not walk folders looking for matches.',
+    )
+  }
+
+  let entries
+
+  try {
+    entries = await readEntries(dir)
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      throw new Error(`No file matches ${pattern}: ${dir} does not exist.`)
+    }
+    throw err
+  }
+
+  const matches = patternToRegExp(base)
+  const paths = []
+
+  for (const name of entries) {
+    // A pattern that does not start with a dot does not go looking for hidden files, the same
+    // rule every shell uses — otherwise `telstore '*'` in a home directory sends .ssh upwards.
+    if (HIDDEN.test(name) && !HIDDEN.test(base)) continue
+    if (!matches.test(name)) continue
+
+    const full = path.join(dir, name)
+    if ((await classify(full)).kind === 'file') paths.push(full)
+  }
+
+  if (paths.length === 0) {
+    throw new Error(
+      `No file matches ${pattern} in ${dir}. Note that the shell usually expands patterns ` +
+        'itself, so a quoted one reaches telstore exactly as typed.',
+    )
+  }
+
+  return paths
+}
+
+/**
+ * Turns the positionals of an upload into the list of files it will actually send.
+ * Returns `{ paths, skipped }`; a name that is neither a folder nor a pattern is handed on
+ * untouched, missing ones included, so "File does not exist" keeps coming from one place.
+ */
+export async function expandSources(args) {
+  const paths = []
+  const skipped = []
+
+  for (const arg of args) {
+    if (isPattern(arg)) {
+      paths.push(...(await expandPattern(arg)))
+      continue
+    }
+
+    let stat = null
+
+    try {
+      stat = await fs.stat(arg)
+    } catch {
+      paths.push(arg)
+      continue
+    }
+
+    if (stat.isDirectory()) {
+      paths.push(...(await expandDirectory(arg, skipped)))
+      continue
+    }
+
+    paths.push(arg)
+  }
+
+  return { paths, skipped }
+}