start-command 0.39.0 → 0.39.2

package/src/lib/log-sanitizer.js

@@ -1,267 +1,350 @@
-/** Bounded byte-wise sanitization for logs prepared for publication. */
+/**
+ * Prepares logs and issue text for publication with
+ * @link-foundation/sensitive-data-sanitizer, the sanitizer shared by
+ * start-command, gh-upload-log and hive-mind (issue #250).
+ *
+ * The library masks credentials only (--credentials-only), plus the values of
+ * credential-named environment variables (--env-secrets) and the tokens in the
+ * local gh hosts.yml (--gh-config). It streams the log from stdin, keeps
+ * invalid UTF-8 bytes and writes a new private file atomically (--output).
+ *
+ * One rule stays here: the library masks an unquoted or single-quoted
+ * `Authorization: <scheme> <value>` to the end of the line, which drops the
+ * rest of a logged one-line JSON prompt (issue #249). The pre-pass below masks
+ * only the scheme and its token first; the library keeps values that already
+ * start with [REDACTED]. Remove it with sensitive-data-sanitizer#45.
+ */
+
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
+const { spawnSync } = require('child_process');
 
 const BLOCK_SIZE = 64 * 1024;
 const MARKER = '[REDACTED]';
-const SECRET_ENV_NAME =
-  /(?:^|_)(?:TOKEN|(?:API|ACCESS|PRIVATE)?KEY|SECRET|PASSWORD|PASSWD|CREDENTIALS?|AUTH|PAT)(?:_|$)/i;
+const PACKAGE = '@link-foundation/sensitive-data-sanitizer';
+const COMMAND = 'sensitive-data-sanitizer';
+const OVERRIDE = 'START_SENSITIVE_DATA_SANITIZER';
+const OPTIONS = ['--credentials-only', '--env-secrets', '--gh-config'];
 
-// Prefixes intentionally match even truncated tokens. Unbounded values retain a
-// continuation state, so a token or header larger than a block cannot leak its
-// suffix. Latin-1 provides a reversible byte mapping, including invalid UTF-8.
-const RULES = [
-  {
-    pattern:
-      /\b[A-Za-z0-9_-]{2,512}\.[A-Za-z0-9_-]{2,512}\.[A-Za-z0-9_-]{0,512}/g,
-    validate: (value) => {
-      try {
-        return (
-          typeof JSON.parse(
-            Buffer.from(value.split('.')[0], 'base64url').toString()
-          ).alg === 'string'
-        );
-      } catch {
-        return false;
-      }
-    },
-    continuation: /[A-Za-z0-9_.-]/,
-  },
-  {
-    pattern: /[A-Za-z0-9_-]{512,}[A-Za-z0-9_.-]*/g,
-    continuation: /[A-Za-z0-9_.-]/,
-  },
-  {
-    pattern: /(?:gh[pousr]_|github_pat_|sk-)[A-Za-z0-9_-]*/g,
-    continuation: /[A-Za-z0-9_-]/,
-  },
-  { pattern: /(?:AKIA|ASIA)[A-Z0-9]{16}/g },
-  { pattern: /eyJ[A-Za-z0-9_.-]*/g, continuation: /[A-Za-z0-9_.-]/ },
-  {
-    pattern: /(?:Proxy-)?Authorization[ \t]{0,32}:[^\r\n]*/gi,
-    continuation: /[^\r\n]/,
-  },
-  {
-    pattern:
-      /(?:AWS_SECRET_ACCESS_KEY|AWS_SESSION_TOKEN)[ \t]{0,32}[=:][ \t]*["']?[^\s"';,]*/g,
-    continuation: /[^\s"';,]/,
-  },
-  {
-    pattern: /(?:https?|git|ssh):\/\/[^\s:/@]{1,256}:[^\s@]*/g,
-    continuation: /[^\s@]/,
-  },
-];
+// The longest header and scheme prefix is 122 bytes.
+const HOLDBACK = 160;
+const HEADER =
+  /((?:Proxy-)?Authorization[ \t]{0,32}:[ \t]{0,32})(?:Bearer|Basic|Digest|token)[ \t]{1,32}/gi;
+// Shell expressions, placeholders and masked values are not credentials.
+const NOT_A_VALUE = '$<[{';
 
-class StreamingSanitizer {
-  constructor(env = process.env) {
-    this.secrets = [
-      ...new Set(
-        Object.entries(env)
-          .filter(
-            ([name, value]) =>
-              SECRET_ENV_NAME.test(name) &&
-              typeof value === 'string' &&
-              value.length
-          )
-          .map(([, value]) => Buffer.from(value).toString('latin1'))
-      ),
-    ].sort((a, b) => b.length - a.length);
-    if (this.secrets.some((value) => value.length > BLOCK_SIZE)) {
-      throw new Error(
-        'Known environment secret is too large for bounded sanitization.'
-      );
-    }
-    this.holdback = this.secrets.reduce(
-      (size, value) => Math.max(size, value.length),
-      2048
-    );
-    this.pending = '';
-    this.continuation = null;
-    this.redactions = 0;
+function isValueByte(code) {
+  return (
+    code > 0x20 &&
+    code !== 0x22 &&
+    code !== 0x27 &&
+    code !== 0x2c &&
+    code !== 0x5c
+  );
+}
+
+function valueEnd(text, start) {
+  let end = start;
+  while (end < text.length && isValueByte(text.charCodeAt(end))) {
+    end++;
   }
+  return end;
+}
 
-  get pendingBytes() {
-    return this.pending.length;
+/**
+ * Masks `Authorization: <scheme> <token>` as `Authorization: [REDACTED]`.
+ * It works on bytes (latin1 maps each byte to one character), so invalid
+ * UTF-8 passes through unchanged, and holds at most HOLDBACK pending bytes.
+ */
+class AuthorizationHeaderMask {
+  constructor() {
+    this.pending = '';
+    this.inValue = false;
+    this.redactions = 0;
   }
 
-  findMatches(input) {
-    const matches = [];
-    for (const rule of RULES) {
-      rule.pattern.lastIndex = 0;
-      for (
-        let match = rule.pattern.exec(input);
-        match;
-        match = rule.pattern.exec(input)
-      ) {
-        if (!rule.validate || rule.validate(match[0])) {
-          matches.push({
-            index: match.index,
-            end: match.index + match[0].length,
-            continuation: rule.continuation,
-          });
-        }
+  push(block, final = false) {
+    const text = this.pending + block.toString('latin1');
+    this.pending = '';
+    let output = '';
+    let offset = 0;
+    if (this.inValue) {
+      offset = valueEnd(text, 0);
+      if (offset === text.length && !final) {
+        return Buffer.alloc(0);
       }
+      this.inValue = false;
     }
-    for (const secret of this.secrets) {
-      for (
-        let index = input.indexOf(secret);
-        index >= 0;
-        index = input.indexOf(secret, index + 1)
+    HEADER.lastIndex = offset;
+    let match;
+    while ((match = HEADER.exec(text))) {
+      const valueStart = match.index + match[0].length;
+      if (valueStart === text.length && !final) {
+        output += text.slice(offset, match.index);
+        this.pending = text.slice(match.index);
+        return Buffer.from(output, 'latin1');
+      }
+      if (
+        valueStart === text.length ||
+        !isValueByte(text.charCodeAt(valueStart)) ||
+        NOT_A_VALUE.includes(text[valueStart])
       ) {
-        matches.push({ index, end: index + secret.length });
+        continue;
       }
-    }
-    matches.sort((a, b) => a.index - b.index || b.end - a.end);
-    const merged = [];
-    for (const match of matches) {
-      const previous = merged[merged.length - 1];
-      if (previous && match.index <= previous.end) {
-        if (match.end > previous.end) {
-          previous.end = match.end;
-          previous.continuation = match.continuation;
-        } else if (match.end === previous.end && match.continuation) {
-          // Nested tokens must not narrow a containing header/URL's suffix.
-          const retained = previous.continuation;
-          const additional = match.continuation;
-          previous.continuation = retained
-            ? { test: (byte) => retained.test(byte) || additional.test(byte) }
-            : additional;
-        }
-      } else {
-        merged.push({ ...match });
+      output += text.slice(offset, match.index + match[1].length) + MARKER;
+      this.redactions++;
+      offset = valueEnd(text, valueStart);
+      if (offset === text.length && !final) {
+        this.inValue = true;
+        return Buffer.from(output, 'latin1');
       }
+      HEADER.lastIndex = offset;
     }
-    return merged;
+    const safeEnd = final
+      ? text.length
+      : Math.max(offset, text.length - HOLDBACK);
+    output += text.slice(offset, safeEnd);
+    this.pending = text.slice(safeEnd);
+    return Buffer.from(output, 'latin1');
   }
+}
 
-  push(block, final = false) {
-    let input = this.pending + block.toString('latin1');
-    this.pending = '';
-    if (this.continuation) {
-      let end = 0;
-      while (end < input.length && this.continuation.test(input[end])) {
-        end++;
-      }
-      input = input.slice(end);
-      if (!input.length && !final) {
-        return Buffer.alloc(0);
-      }
-      this.continuation = null;
+/** Value-free failure: `code` is a fixed ERR_* name, never log content. */
+class SanitizationError extends Error {
+  constructor(code, subject = 'Log') {
+    super(
+      `${subject} sanitization failed (${code}); ${subject === 'Log' ? 'upload' : 'publication'} blocked.`
+    );
+    this.code = code;
+  }
+}
+
+function isFile(filePath) {
+  try {
+    return fs.statSync(filePath).isFile();
+  } catch {
+    return false;
+  }
+}
+
+function findBundledCli() {
+  // Walk up node_modules directories the way require() does; the package
+  // exports only ESM, so require.resolve() cannot locate its files.
+  const relative = path.join(PACKAGE, 'bin', `${COMMAND}.js`);
+  let directory = __dirname;
+  for (;;) {
+    const candidate = path.join(directory, 'node_modules', relative);
+    if (isFile(candidate)) {
+      return candidate;
     }
-    const safeEnd = final
-      ? input.length
-      : Math.max(0, input.length - this.holdback);
-    let offset = 0;
-    const output = [];
-    for (const match of this.findMatches(input)) {
-      if (match.index >= safeEnd) {
-        break;
+    const parent = path.dirname(directory);
+    if (parent === directory) {
+      return null;
+    }
+    directory = parent;
+  }
+}
+
+/**
+ * Resolve the library CLI: START_SENSITIVE_DATA_SANITIZER, then the bundled
+ * dependency run by the current runtime, then sensitive-data-sanitizer on PATH.
+ * @returns {{command: string, args: string[]}|null}
+ */
+function resolveSanitizerCommand(env = process.env) {
+  if (env[OVERRIDE]) {
+    return { command: env[OVERRIDE], args: [] };
+  }
+  const bundled = findBundledCli();
+  if (bundled) {
+    return { command: process.execPath, args: [bundled] };
+  }
+  const { resolveCommand } = require('./log-uploader');
+  const command = resolveCommand(COMMAND);
+  return command ? { command, args: [] } : null;
+}
+
+function runSanitizer(args, options, subject) {
+  const env = options.env || process.env;
+  const resolved = resolveSanitizerCommand(env);
+  if (!resolved) {
+    throw new SanitizationError('ERR_UNAVAILABLE', subject);
+  }
+  const result = spawnSync(resolved.command, [...resolved.args, ...args], {
+    env,
+    input: options.input,
+    stdio: options.stdio || ['pipe', 'pipe', 'pipe'],
+    maxBuffer: 64 * 1024 * 1024,
+    shell:
+      process.platform === 'win32' && /\.(cmd|bat)$/i.test(resolved.command),
+    windowsHide: true,
+  });
+  if (result.error) {
+    throw new SanitizationError('ERR_UNAVAILABLE', subject);
+  }
+  if (result.status !== 0) {
+    // The library prints value-free messages; only their ERR_* code is kept.
+    const code = String(result.stderr || '').match(/\bERR_[A-Z_]+\b/);
+    throw new SanitizationError(code ? code[0] : 'ERR_SANITIZER', subject);
+  }
+  return result;
+}
+
+/**
+ * Sanitize a short text, such as the failing command quoted in an issue.
+ * @returns {string}
+ */
+function sanitizeText(text, options = {}) {
+  const input = new AuthorizationHeaderMask().push(Buffer.from(text), true);
+  const result = runSanitizer(
+    ['redact', '-', ...OPTIONS],
+    { ...options, input },
+    'Text'
+  );
+  return result.stdout.toString();
+}
+
+function countMarkers(filePath) {
+  const descriptor = fs.openSync(filePath, 'r');
+  const buffer = Buffer.alloc(BLOCK_SIZE);
+  let count = 0;
+  let carry = '';
+  try {
+    for (;;) {
+      const length = fs.readSync(descriptor, buffer, 0, BLOCK_SIZE, null);
+      const text = carry + buffer.toString('latin1', 0, length);
+      let index = text.indexOf(MARKER);
+      let next = 0;
+      while (index !== -1) {
+        count++;
+        next = index + MARKER.length;
+        index = text.indexOf(MARKER, next);
       }
-      output.push(input.slice(offset, match.index), MARKER);
-      this.redactions++;
-      offset = match.end;
-      if (offset === input.length && match.continuation && !final) {
-        this.continuation = match.continuation;
-        break;
+      if (length === 0) {
+        return count;
       }
+      carry = text.slice(Math.max(next, text.length - MARKER.length + 1));
     }
-    const end = Math.max(offset, safeEnd);
-    output.push(input.slice(offset, end));
-    this.pending = input.slice(end);
-    return Buffer.from(output.join(''), 'latin1');
+  } finally {
+    fs.closeSync(descriptor);
   }
 }
 
-/** Returns a private copy plus cleanup; every preparation error blocks upload. */
-function sanitizeLogToTemp(sourcePath, options = {}) {
-  let directory;
-  let source;
-  let destination;
-  let ready = false;
-  let stage = 'environment';
+function createPrivateFile(filePath) {
+  const descriptor = fs.openSync(filePath, 'wx', 0o600);
+  if (process.platform !== 'win32') {
+    fs.fchmodSync(descriptor, 0o600);
+  }
+  return descriptor;
+}
+
+function maskAuthorizationHeaders(sourcePath, stagePath) {
+  const source = fs.openSync(sourcePath, 'r');
+  let stage;
   try {
-    const sanitizer = new StreamingSanitizer(options.env || process.env);
-    stage = 'source';
-    source = fs.openSync(sourcePath, 'r');
     if (!fs.fstatSync(source).isFile()) {
-      throw new Error('Source must be a regular file.');
+      throw new SanitizationError('ERR_SOURCE');
     }
-    stage = 'private-file';
-    directory = fs.mkdtempSync(path.join(os.tmpdir(), 'start-sanitized-'));
-    fs.chmodSync(directory, 0o700);
-    const target = path.join(directory, 'execution.log');
-    destination = fs.openSync(target, 'wx', 0o600);
-    fs.fchmodSync(destination, 0o600);
+    stage = createPrivateFile(stagePath);
+    const mask = new AuthorizationHeaderMask();
     const buffer = Buffer.alloc(BLOCK_SIZE);
     let bytes = 0;
-    const write = (output) => {
-      stage = 'write';
-      let offset = 0;
-      while (offset < output.length) {
-        const written = fs.writeSync(
-          destination,
-          output,
-          offset,
-          output.length - offset
-        );
-        if (!written) {
-          throw new Error('No progress writing sanitized log.');
-        }
-        offset += written;
-      }
-    };
     for (;;) {
-      stage = 'read';
-      const length = fs.readSync(source, buffer, 0, buffer.length, null);
-      if (!length) {
+      const length = fs.readSync(source, buffer, 0, BLOCK_SIZE, null);
+      const output = mask.push(buffer.subarray(0, length), length === 0);
+      if (output.length) {
+        fs.writeSync(stage, output);
+      }
+      if (length === 0) {
         break;
       }
       bytes += length;
-      stage = 'scan';
-      write(sanitizer.push(buffer.subarray(0, length)));
     }
-    write(sanitizer.push(Buffer.alloc(0), true));
-    stage = 'sync';
-    fs.fsyncSync(destination);
-    fs.closeSync(destination);
-    destination = undefined;
+    fs.fsyncSync(stage);
+    return bytes;
+  } finally {
     fs.closeSync(source);
-    source = undefined;
+    if (stage !== undefined) {
+      fs.closeSync(stage);
+    }
+  }
+}
+
+/**
+ * Create a private sanitized copy of a log. The source is never modified;
+ * any failure removes the copy and throws a value-free SanitizationError.
+ * @returns {{path: string, cleanup: Function}}
+ */
+function sanitizeLogToTemp(sourcePath, options = {}) {
+  let directory;
+  let code = 'ERR_PRIVATE_FILE';
+  const cleanup = () => {
+    if (directory) {
+      fs.rmSync(directory, { recursive: true, force: true });
+    }
+  };
+  try {
+    directory = fs.mkdtempSync(path.join(os.tmpdir(), 'start-sanitized-'));
+    if (process.platform !== 'win32') {
+      fs.chmodSync(directory, 0o700);
+    }
+    const stagePath = path.join(directory, 'authorization-masked.log');
+    const target = path.join(directory, 'execution.log');
+    code = 'ERR_SOURCE';
+    const bytes = maskAuthorizationHeaders(sourcePath, stagePath);
+    code = 'ERR_PRIVATE_FILE';
+    const input = fs.openSync(stagePath, 'r');
+    try {
+      const size = Math.max(1, fs.fstatSync(input).size);
+      runSanitizer(
+        [
+          'redact',
+          '-',
+          '--stream',
+          ...OPTIONS,
+          '--max-bytes',
+          String(size),
+          '--output',
+          target,
+        ],
+        { env: options.env, stdio: [input, 'ignore', 'pipe'] }
+      );
+    } finally {
+      fs.closeSync(input);
+    }
+    code = 'ERR_OUTPUT';
+    const status = fs.lstatSync(target);
+    if (
+      !status.isFile() ||
+      (process.platform !== 'win32' && (status.mode & 0o077) !== 0)
+    ) {
+      throw new SanitizationError('ERR_OUTPUT');
+    }
     if (options.verbose) {
+      const redactions = countMarkers(target) - countMarkers(sourcePath);
       console.log(
-        `Prepared private sanitized log: ${bytes} bytes scanned, ${sanitizer.redactions} redactions.`
+        `Prepared private sanitized log: ${bytes} bytes scanned, ${Math.max(0, redactions)} redactions.`
       );
     }
-    ready = true;
-    return {
-      path: target,
-      cleanup: () => fs.rmSync(directory, { recursive: true, force: true }),
-    };
-  } catch {
+    fs.rmSync(stagePath, { force: true });
+    return { path: target, cleanup };
+  } catch (error) {
+    cleanup();
+    // Static messages keep paths and system error details out of the output.
+    const failure =
+      error instanceof SanitizationError ? error : new SanitizationError(code);
     if (options.verbose) {
-      console.log(`Log sanitization failed at stage: ${stage}.`);
-    }
-    // Do not echo error messages: paths and exception details can carry secrets.
-    throw new Error('Log sanitization failed; upload blocked.');
-  } finally {
-    for (const fd of [destination, source]) {
-      if (fd !== undefined) {
-        try {
-          fs.closeSync(fd);
-        } catch {
-          /* Best effort after failure. */
-        }
-      }
-    }
-    if (directory && !ready) {
-      try {
-        fs.rmSync(directory, { recursive: true, force: true });
-      } catch {
-        /* Upload remains blocked. */
-      }
+      console.error(failure.message);
     }
+    throw failure;
   }
 }
 
-module.exports = { StreamingSanitizer, sanitizeLogToTemp };
+module.exports = {
+  AuthorizationHeaderMask,
+  BLOCK_SIZE,
+  SanitizationError,
+  resolveSanitizerCommand,
+  sanitizeLogToTemp,
+  sanitizeText,
+};

package/src/lib/log-uploader.js

@@ -6,7 +6,7 @@ const fs = require('fs');
 const os = require('os');
 const path = require('path');
 const { spawnSync } = require('child_process');
-const { sanitizeLogToTemp } = require('./log-sanitizer');
+const { SanitizationError, sanitizeLogToTemp } = require('./log-sanitizer');
 
 function isExecutable(filePath) {
   try {
@@ -174,11 +174,14 @@ function uploadLogPath(logPath, options = {}) {
       };
     }
     return { success: true, exitCode: 0 };
-  } catch {
+  } catch (error) {
     return {
       success: false,
       exitCode: 1,
-      error: 'Log preparation or upload failed; upload blocked.',
+      error:
+        error instanceof SanitizationError
+          ? error.message
+          : 'Log preparation or upload failed; upload blocked.',
     };
   } finally {
     if (prepared) {