telstore 0.1.0

package/src/state.js

@@ -0,0 +1,151 @@
+import { createHash } from 'node:crypto'
+import { promises as fs } from 'node:fs'
+import path from 'node:path'
+
+import { defaultConfigDir, writeJsonAtomic } from './config.js'
+
+export function stateDir(configDir = defaultConfigDir()) {
+  return path.join(configDir, 'state')
+}
+
+export function stateKey(absPath, size, mtimeMs) {
+  return createHash('sha1').update(`${absPath}:${size}:${mtimeMs}`).digest('hex')
+}
+
+export function stateFile(key, configDir = defaultConfigDir()) {
+  return path.join(stateDir(configDir), `${key}.json`)
+}
+
+export async function loadState(key, configDir = defaultConfigDir()) {
+  try {
+    return JSON.parse(await fs.readFile(stateFile(key, configDir), 'utf8'))
+  } catch (err) {
+    if (err.code === 'ENOENT') return null
+    if (err instanceof SyntaxError) return null
+    throw err
+  }
+}
+
+export async function saveState(key, state, configDir = defaultConfigDir()) {
+  await writeJsonAtomic(stateFile(key, configDir), state)
+}
+
+export async function markChunkDone(key, state, i, entry, configDir = defaultConfigDir()) {
+  const updated = { ...state, done: { ...state.done, [String(i)]: entry } }
+  await saveState(key, updated, configDir)
+  return updated
+}
+
+export async function clearState(key, configDir = defaultConfigDir()) {
+  try {
+    await fs.unlink(stateFile(key, configDir))
+  } catch (err) {
+    if (err.code !== 'ENOENT') throw err
+  }
+}
+
+// A state file is only useful while its backup can still be resumed, and nothing ever
+// removes one whose file was edited since: the key includes mtime, so that state can never
+// match again. Left alone the directory only grows, and with it the report `status` prints.
+export const MAX_STATES = 20
+
+// The newest states are the ones worth keeping, and a state file is rewritten every time a
+// chunk lands, so its mtime is when this backup last made progress. Returns the states that
+// were dropped: the caller says their ids out loud, because after this the id is the only
+// way left to find those chunks in the chat.
+export async function pruneStates(configDir = defaultConfigDir(), keep = MAX_STATES) {
+  let names
+  try {
+    names = await fs.readdir(stateDir(configDir))
+  } catch (err) {
+    if (err.code === 'ENOENT') return []
+    throw err
+  }
+
+  const files = []
+
+  for (const name of names) {
+    if (!name.endsWith('.json')) continue
+
+    const file = path.join(stateDir(configDir), name)
+
+    try {
+      const stat = await fs.stat(file)
+      files.push({ key: name.slice(0, -'.json'.length), file, mtimeMs: stat.mtimeMs })
+    } catch (err) {
+      // Gone between readdir and stat: nothing left to prune.
+      if (err.code !== 'ENOENT') throw err
+    }
+  }
+
+  files.sort((a, b) => b.mtimeMs - a.mtimeMs)
+
+  const dropped = []
+
+  for (const { key, file } of files.slice(keep)) {
+    // Read before unlink: a file that cannot be read back, or that carries no id, is
+    // still pruned — it just cannot be named, and a report naming nothing helps no one.
+    const state = await loadState(key, configDir)
+    await fs.unlink(file)
+    if (state?.id) dropped.push(state)
+  }
+
+  return dropped
+}
+
+// status needs every unfinished backup at once. A state file that cannot be read is skipped
+// rather than fatal, for the same reason loadState returns null: one corrupt file must not
+// hide the other backups still waiting to be finished.
+export async function listStates(configDir = defaultConfigDir()) {
+  let names
+  try {
+    names = await fs.readdir(stateDir(configDir))
+  } catch (err) {
+    if (err.code === 'ENOENT') return []
+    throw err
+  }
+
+  const states = []
+
+  for (const name of names) {
+    if (!name.endsWith('.json')) continue
+
+    const state = await loadState(name.slice(0, -'.json'.length), configDir)
+    if (state) states.push(state)
+  }
+
+  return states
+}
+
+// delete needs the file a record came from, not just its contents — and the name of that
+// file is a hash of the path, size and mtime *inside* the record, so recomputing it would
+// be trusting an untrusted file to say where it lives. A hand-edited path yields a key that
+// names no file at all, clearState ignores a file that is not there, and telark reports a
+// record dropped that is still sitting on disk. Matching the id inside each file is the one
+// way that cannot point at the wrong one.
+//
+// Every record claiming the id is returned rather than the first: two of them means telark
+// cannot know which to drop, and that is the caller's decision to refuse, not ours to make
+// by picking one.
+export async function findStates(backupId, configDir = defaultConfigDir()) {
+  let names
+  try {
+    names = await fs.readdir(stateDir(configDir))
+  } catch (err) {
+    if (err.code === 'ENOENT') return []
+    throw err
+  }
+
+  const found = []
+
+  for (const name of names) {
+    if (!name.endsWith('.json')) continue
+
+    const key = name.slice(0, -'.json'.length)
+    const state = await loadState(key, configDir)
+
+    if (state?.id === backupId) found.push({ key, file: stateFile(key, configDir), state })
+  }
+
+  return found
+}

package/src/uploader.js

@@ -0,0 +1,155 @@
+import { createHash, randomBytes } from 'node:crypto'
+import { read as readCallback } from 'node:fs'
+import { promisify } from 'node:util'
+
+import { Api } from 'telegram'
+import { readBigIntFromBuffer } from 'telegram/Helpers.js'
+
+import { DEFAULT_CONCURRENCY, PART_SIZE, MAX_PARTS } from './chunking.js'
+import { withRetry } from './retry.js'
+import { DEFAULT_STALL_MS, withStallTimeout } from './stall.js'
+
+const read = promisify(readCallback)
+
+// Telegram splits its upload API by file size: only files above 10MB may use the
+// "big" family. GramJS picks the same threshold (LARGE_FILE_THRESHOLD in
+// node_modules/telegram/client/uploads.js), and telark follows it.
+export const LARGE_FILE_THRESHOLD = 10 * 1024 * 1024
+
+async function readExactly(fd, length, position) {
+  // allocUnsafe skips zero-filling 512KB per part — about 0.4s of memset per 1800MB chunk,
+  // on the same thread that drives the in-flight requests. Safe only because the loop below
+  // either fills every byte or throws: no uninitialised byte can reach a request or the hash.
+  const buffer = Buffer.allocUnsafe(length)
+  let filled = 0
+
+  while (filled < length) {
+    const { bytesRead } = await read(fd, buffer, filled, length - filled, position + filled)
+    if (bytesRead === 0) {
+      throw new Error(`Short read: needed ${length} bytes at offset ${position} but the file ended.`)
+    }
+    filled += bytesRead
+  }
+
+  return buffer
+}
+
+export async function uploadRange(client, fd, options) {
+  const {
+    offset,
+    length,
+    fileName,
+    concurrency = DEFAULT_CONCURRENCY,
+    partSize = PART_SIZE,
+    onProgress,
+    retryOptions,
+    stallMs = DEFAULT_STALL_MS,
+  } = options
+
+  // A pool of fewer than one worker does no work. In the download path that means
+  // Promise.all resolves at once and a chunk nobody fetched is reported as complete; in the
+  // upload path the batch loop never advances and the process spins in microtasks, which not
+  // even a test timeout can interrupt. Neither is something a caller should be able to ask
+  // for by accident.
+  if (!Number.isInteger(concurrency) || concurrency < 1) {
+    throw new Error(
+      `Worker count must be a whole number of at least one, got: ${JSON.stringify(concurrency)}.`,
+    )
+  }
+
+  const totalParts = Math.ceil(length / partSize)
+
+  if (totalParts > MAX_PARTS) {
+    throw new Error(
+      `This byte range needs ${totalParts} parts, but Telegram accepts at most 4000 parts per file.`,
+    )
+  }
+
+  const isLarge = length > LARGE_FILE_THRESHOLD
+  const fileId = readBigIntFromBuffer(randomBytes(8), true, true)
+  const hash = createHash('sha256')
+
+  function partRequest(part, bytes) {
+    if (isLarge) {
+      return new Api.upload.SaveBigFilePart({
+        fileId,
+        filePart: part,
+        fileTotalParts: totalParts,
+        bytes,
+      })
+    }
+
+    return new Api.upload.SaveFilePart({ fileId, filePart: part, bytes })
+  }
+
+  for (let start = 0; start < totalParts; start += concurrency) {
+    const end = Math.min(start + concurrency, totalParts)
+    const sending = []
+    let sendError = null
+    let sendFailed = false
+    let readError = null
+
+    // Read sequentially so the hash sees parts in order, but send in parallel.
+    try {
+      for (let part = start; part < end; part += 1) {
+        const partOffset = part * partSize
+        const partLength = Math.min(partSize, length - partOffset)
+        const bytes = await readExactly(fd, partLength, offset + partOffset)
+
+        hash.update(bytes)
+
+        // Attach the handler when the promise is created rather than waiting for the
+        // Promise.all at the end of the batch: if readExactly throws on a later part,
+        // an already-pushed promise with no handler becomes an unhandledRejection and
+        // hides the real error.
+        sending.push(
+          withRetry(
+            () =>
+              // Same exposure as the download path: a request left on a sender GramJS has
+              // stopped draining never settles, so without a deadline this await would hold
+              // the batch open forever and the upload would end without a word.
+              withStallTimeout(
+                client.invoke(partRequest(part, bytes)),
+                stallMs,
+                () =>
+                  `Telegram stopped acknowledging part ${part + 1}/${totalParts} of ` +
+                  `${fileName}: nothing back for ${Math.round(stallMs / 1000)}s.`,
+              ),
+            retryOptions,
+          ).then(
+            () => onProgress?.(partLength),
+            (err) => {
+              // Not `sendError ??= err`: if the rejection reason is falsy
+              // (undefined/null), that assignment still sets sendError to a falsy
+              // value and the `if (sendError)` below reads as "no error" — swallowing
+              // a failed send and turning it into a fake success.
+              if (!sendFailed) {
+                sendFailed = true
+                sendError = err
+              }
+            },
+          ),
+        )
+      }
+    } catch (err) {
+      readError = err
+    }
+
+    // Every promise in `sending` already absorbed its own error above, so they all
+    // resolve; this await only guarantees no request is still in flight when we throw.
+    await Promise.all(sending)
+
+    if (readError) throw readError
+    if (sendFailed) throw sendError
+  }
+
+  const inputFile = isLarge
+    ? new Api.InputFileBig({ id: fileId, parts: totalParts, name: fileName })
+    : new Api.InputFile({ id: fileId, parts: totalParts, name: fileName, md5Checksum: '' })
+
+  return {
+    inputFile,
+    sha256: hash.digest('hex'),
+    parts: totalParts,
+  }
+}