start-command 0.39.1 → 0.40.0

package/src/lib/log-sanitizer.js

@@ -1,267 +1,229 @@
-/** Bounded byte-wise sanitization for logs prepared for publication. */
+/**
+ * Prepares logs and issue text for publication with
+ * @link-foundation/sensitive-data-sanitizer, the sanitizer shared by
+ * start-command, gh-upload-log and hive-mind (issue #250).
+ *
+ * The library masks credentials only (--credentials-only), plus the values of
+ * credential-named environment variables (--env-secrets) and the tokens in the
+ * local gh hosts.yml (--gh-config). It streams the log from stdin, keeps
+ * invalid UTF-8 bytes and writes a new private file atomically (--output).
+ *
+ * Since 0.12.1 the library masks only the scheme and token of an
+ * `Authorization: <scheme> <value>` header, so a logged one-line JSON prompt
+ * keeps the rest of its line (issue #249, sensitive-data-sanitizer#45); the
+ * pre-pass that did this here is gone (issue #253).
+ */
+
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
+const { spawnSync } = require('child_process');
 
 const BLOCK_SIZE = 64 * 1024;
 const MARKER = '[REDACTED]';
-const SECRET_ENV_NAME =
-  /(?:^|_)(?:TOKEN|(?:API|ACCESS|PRIVATE)?KEY|SECRET|PASSWORD|PASSWD|CREDENTIALS?|AUTH|PAT)(?:_|$)/i;
-
-// Prefixes intentionally match even truncated tokens. Unbounded values retain a
-// continuation state, so a token or header larger than a block cannot leak its
-// suffix. Latin-1 provides a reversible byte mapping, including invalid UTF-8.
-const RULES = [
-  {
-    pattern:
-      /\b[A-Za-z0-9_-]{2,512}\.[A-Za-z0-9_-]{2,512}\.[A-Za-z0-9_-]{0,512}/g,
-    validate: (value) => {
-      try {
-        return (
-          typeof JSON.parse(
-            Buffer.from(value.split('.')[0], 'base64url').toString()
-          ).alg === 'string'
-        );
-      } catch {
-        return false;
-      }
-    },
-    continuation: /[A-Za-z0-9_.-]/,
-  },
-  {
-    pattern: /[A-Za-z0-9_-]{512,}[A-Za-z0-9_.-]*/g,
-    continuation: /[A-Za-z0-9_.-]/,
-  },
-  {
-    pattern: /(?:gh[pousr]_|github_pat_|sk-)[A-Za-z0-9_-]*/g,
-    continuation: /[A-Za-z0-9_-]/,
-  },
-  { pattern: /(?:AKIA|ASIA)[A-Z0-9]{16}/g },
-  { pattern: /eyJ[A-Za-z0-9_.-]*/g, continuation: /[A-Za-z0-9_.-]/ },
-  {
-    pattern: /(?:Proxy-)?Authorization[ \t]{0,32}:[^\r\n]*/gi,
-    continuation: /[^\r\n]/,
-  },
-  {
-    pattern:
-      /(?:AWS_SECRET_ACCESS_KEY|AWS_SESSION_TOKEN)[ \t]{0,32}[=:][ \t]*["']?[^\s"';,]*/g,
-    continuation: /[^\s"';,]/,
-  },
-  {
-    pattern: /(?:https?|git|ssh):\/\/[^\s:/@]{1,256}:[^\s@]*/g,
-    continuation: /[^\s@]/,
-  },
-];
+const PACKAGE = '@link-foundation/sensitive-data-sanitizer';
+const COMMAND = 'sensitive-data-sanitizer';
+const OVERRIDE = 'START_SENSITIVE_DATA_SANITIZER';
+const OPTIONS = ['--credentials-only', '--env-secrets', '--gh-config'];
 
-class StreamingSanitizer {
-  constructor(env = process.env) {
-    this.secrets = [
-      ...new Set(
-        Object.entries(env)
-          .filter(
-            ([name, value]) =>
-              SECRET_ENV_NAME.test(name) &&
-              typeof value === 'string' &&
-              value.length
-          )
-          .map(([, value]) => Buffer.from(value).toString('latin1'))
-      ),
-    ].sort((a, b) => b.length - a.length);
-    if (this.secrets.some((value) => value.length > BLOCK_SIZE)) {
-      throw new Error(
-        'Known environment secret is too large for bounded sanitization.'
-      );
-    }
-    this.holdback = this.secrets.reduce(
-      (size, value) => Math.max(size, value.length),
-      2048
+/** Value-free failure: `code` is a fixed ERR_* name, never log content. */
+class SanitizationError extends Error {
+  constructor(code, subject = 'Log') {
+    super(
+      `${subject} sanitization failed (${code}); ${subject === 'Log' ? 'upload' : 'publication'} blocked.`
     );
-    this.pending = '';
-    this.continuation = null;
-    this.redactions = 0;
+    this.code = code;
   }
+}
 
-  get pendingBytes() {
-    return this.pending.length;
+function isFile(filePath) {
+  try {
+    return fs.statSync(filePath).isFile();
+  } catch {
+    return false;
   }
+}
 
-  findMatches(input) {
-    const matches = [];
-    for (const rule of RULES) {
-      rule.pattern.lastIndex = 0;
-      for (
-        let match = rule.pattern.exec(input);
-        match;
-        match = rule.pattern.exec(input)
-      ) {
-        if (!rule.validate || rule.validate(match[0])) {
-          matches.push({
-            index: match.index,
-            end: match.index + match[0].length,
-            continuation: rule.continuation,
-          });
-        }
-      }
-    }
-    for (const secret of this.secrets) {
-      for (
-        let index = input.indexOf(secret);
-        index >= 0;
-        index = input.indexOf(secret, index + 1)
-      ) {
-        matches.push({ index, end: index + secret.length });
-      }
+function findBundledCli() {
+  // Walk up node_modules directories the way require() does; the package
+  // exports only ESM, so require.resolve() cannot locate its files.
+  const relative = path.join(PACKAGE, 'bin', `${COMMAND}.js`);
+  let directory = __dirname;
+  for (;;) {
+    const candidate = path.join(directory, 'node_modules', relative);
+    if (isFile(candidate)) {
+      return candidate;
     }
-    matches.sort((a, b) => a.index - b.index || b.end - a.end);
-    const merged = [];
-    for (const match of matches) {
-      const previous = merged[merged.length - 1];
-      if (previous && match.index <= previous.end) {
-        if (match.end > previous.end) {
-          previous.end = match.end;
-          previous.continuation = match.continuation;
-        } else if (match.end === previous.end && match.continuation) {
-          // Nested tokens must not narrow a containing header/URL's suffix.
-          const retained = previous.continuation;
-          const additional = match.continuation;
-          previous.continuation = retained
-            ? { test: (byte) => retained.test(byte) || additional.test(byte) }
-            : additional;
-        }
-      } else {
-        merged.push({ ...match });
-      }
+    const parent = path.dirname(directory);
+    if (parent === directory) {
+      return null;
     }
-    return merged;
+    directory = parent;
   }
+}
 
-  push(block, final = false) {
-    let input = this.pending + block.toString('latin1');
-    this.pending = '';
-    if (this.continuation) {
-      let end = 0;
-      while (end < input.length && this.continuation.test(input[end])) {
-        end++;
-      }
-      input = input.slice(end);
-      if (!input.length && !final) {
-        return Buffer.alloc(0);
-      }
-      this.continuation = null;
-    }
-    const safeEnd = final
-      ? input.length
-      : Math.max(0, input.length - this.holdback);
-    let offset = 0;
-    const output = [];
-    for (const match of this.findMatches(input)) {
-      if (match.index >= safeEnd) {
-        break;
+/**
+ * Resolve the library CLI: START_SENSITIVE_DATA_SANITIZER, then the bundled
+ * dependency run by the current runtime, then sensitive-data-sanitizer on PATH.
+ * @returns {{command: string, args: string[]}|null}
+ */
+function resolveSanitizerCommand(env = process.env) {
+  if (env[OVERRIDE]) {
+    return { command: env[OVERRIDE], args: [] };
+  }
+  const bundled = findBundledCli();
+  if (bundled) {
+    return { command: process.execPath, args: [bundled] };
+  }
+  const { resolveCommand } = require('./log-uploader');
+  const command = resolveCommand(COMMAND);
+  return command ? { command, args: [] } : null;
+}
+
+function runSanitizer(args, options, subject) {
+  const env = options.env || process.env;
+  const resolved = resolveSanitizerCommand(env);
+  if (!resolved) {
+    throw new SanitizationError('ERR_UNAVAILABLE', subject);
+  }
+  const result = spawnSync(resolved.command, [...resolved.args, ...args], {
+    env,
+    input: options.input,
+    stdio: options.stdio || ['pipe', 'pipe', 'pipe'],
+    maxBuffer: 64 * 1024 * 1024,
+    shell:
+      process.platform === 'win32' && /\.(cmd|bat)$/i.test(resolved.command),
+    windowsHide: true,
+  });
+  if (result.error) {
+    throw new SanitizationError('ERR_UNAVAILABLE', subject);
+  }
+  if (result.status !== 0) {
+    // The library prints value-free messages; only their ERR_* code is kept.
+    const code = String(result.stderr || '').match(/\bERR_[A-Z_]+\b/);
+    throw new SanitizationError(code ? code[0] : 'ERR_SANITIZER', subject);
+  }
+  return result;
+}
+
+/**
+ * Sanitize a short text, such as the failing command quoted in an issue.
+ * @returns {string}
+ */
+function sanitizeText(text, options = {}) {
+  const result = runSanitizer(
+    ['redact', '-', ...OPTIONS],
+    { ...options, input: text },
+    'Text'
+  );
+  return result.stdout.toString();
+}
+
+function countMarkers(filePath) {
+  const descriptor = fs.openSync(filePath, 'r');
+  const buffer = Buffer.alloc(BLOCK_SIZE);
+  let count = 0;
+  let carry = '';
+  try {
+    for (;;) {
+      const length = fs.readSync(descriptor, buffer, 0, BLOCK_SIZE, null);
+      const text = carry + buffer.toString('latin1', 0, length);
+      let index = text.indexOf(MARKER);
+      let next = 0;
+      while (index !== -1) {
+        count++;
+        next = index + MARKER.length;
+        index = text.indexOf(MARKER, next);
       }
-      output.push(input.slice(offset, match.index), MARKER);
-      this.redactions++;
-      offset = match.end;
-      if (offset === input.length && match.continuation && !final) {
-        this.continuation = match.continuation;
-        break;
+      if (length === 0) {
+        return count;
       }
+      carry = text.slice(Math.max(next, text.length - MARKER.length + 1));
     }
-    const end = Math.max(offset, safeEnd);
-    output.push(input.slice(offset, end));
-    this.pending = input.slice(end);
-    return Buffer.from(output.join(''), 'latin1');
+  } finally {
+    fs.closeSync(descriptor);
   }
 }
 
-/** Returns a private copy plus cleanup; every preparation error blocks upload. */
+/**
+ * Create a private sanitized copy of a log. The source is never modified;
+ * any failure removes the copy and throws a value-free SanitizationError.
+ * @returns {{path: string, cleanup: Function}}
+ */
 function sanitizeLogToTemp(sourcePath, options = {}) {
   let directory;
-  let source;
-  let destination;
-  let ready = false;
-  let stage = 'environment';
-  try {
-    const sanitizer = new StreamingSanitizer(options.env || process.env);
-    stage = 'source';
-    source = fs.openSync(sourcePath, 'r');
-    if (!fs.fstatSync(source).isFile()) {
-      throw new Error('Source must be a regular file.');
+  let code = 'ERR_PRIVATE_FILE';
+  const cleanup = () => {
+    if (directory) {
+      fs.rmSync(directory, { recursive: true, force: true });
     }
-    stage = 'private-file';
+  };
+  try {
     directory = fs.mkdtempSync(path.join(os.tmpdir(), 'start-sanitized-'));
-    fs.chmodSync(directory, 0o700);
+    if (process.platform !== 'win32') {
+      fs.chmodSync(directory, 0o700);
+    }
     const target = path.join(directory, 'execution.log');
-    destination = fs.openSync(target, 'wx', 0o600);
-    fs.fchmodSync(destination, 0o600);
-    const buffer = Buffer.alloc(BLOCK_SIZE);
-    let bytes = 0;
-    const write = (output) => {
-      stage = 'write';
-      let offset = 0;
-      while (offset < output.length) {
-        const written = fs.writeSync(
-          destination,
-          output,
-          offset,
-          output.length - offset
-        );
-        if (!written) {
-          throw new Error('No progress writing sanitized log.');
-        }
-        offset += written;
-      }
-    };
-    for (;;) {
-      stage = 'read';
-      const length = fs.readSync(source, buffer, 0, buffer.length, null);
-      if (!length) {
-        break;
+    code = 'ERR_SOURCE';
+    const input = fs.openSync(sourcePath, 'r');
+    let bytes;
+    try {
+      const source = fs.fstatSync(input);
+      if (!source.isFile()) {
+        throw new SanitizationError('ERR_SOURCE');
       }
-      bytes += length;
-      stage = 'scan';
-      write(sanitizer.push(buffer.subarray(0, length)));
+      bytes = source.size;
+      const size = Math.max(1, bytes);
+      code = 'ERR_PRIVATE_FILE';
+      runSanitizer(
+        [
+          'redact',
+          '-',
+          '--stream',
+          // A NUL byte in a terminal transcript must not block the upload.
+          '--allow-binary',
+          ...OPTIONS,
+          '--max-bytes',
+          String(size),
+          '--output',
+          target,
+        ],
+        { env: options.env, stdio: [input, 'ignore', 'pipe'] }
+      );
+    } finally {
+      fs.closeSync(input);
+    }
+    code = 'ERR_OUTPUT';
+    const status = fs.lstatSync(target);
+    if (
+      !status.isFile() ||
+      (process.platform !== 'win32' && (status.mode & 0o077) !== 0)
+    ) {
+      throw new SanitizationError('ERR_OUTPUT');
     }
-    write(sanitizer.push(Buffer.alloc(0), true));
-    stage = 'sync';
-    fs.fsyncSync(destination);
-    fs.closeSync(destination);
-    destination = undefined;
-    fs.closeSync(source);
-    source = undefined;
     if (options.verbose) {
+      const redactions = countMarkers(target) - countMarkers(sourcePath);
       console.log(
-        `Prepared private sanitized log: ${bytes} bytes scanned, ${sanitizer.redactions} redactions.`
+        `Prepared private sanitized log: ${bytes} bytes scanned, ${Math.max(0, redactions)} redactions.`
       );
     }
-    ready = true;
-    return {
-      path: target,
-      cleanup: () => fs.rmSync(directory, { recursive: true, force: true }),
-    };
-  } catch {
+    return { path: target, cleanup };
+  } catch (error) {
+    cleanup();
+    // Static messages keep paths and system error details out of the output.
+    const failure =
+      error instanceof SanitizationError ? error : new SanitizationError(code);
     if (options.verbose) {
-      console.log(`Log sanitization failed at stage: ${stage}.`);
-    }
-    // Do not echo error messages: paths and exception details can carry secrets.
-    throw new Error('Log sanitization failed; upload blocked.');
-  } finally {
-    for (const fd of [destination, source]) {
-      if (fd !== undefined) {
-        try {
-          fs.closeSync(fd);
-        } catch {
-          /* Best effort after failure. */
-        }
-      }
-    }
-    if (directory && !ready) {
-      try {
-        fs.rmSync(directory, { recursive: true, force: true });
-      } catch {
-        /* Upload remains blocked. */
-      }
+      console.error(failure.message);
     }
+    throw failure;
   }
 }
 
-module.exports = { StreamingSanitizer, sanitizeLogToTemp };
+module.exports = {
+  BLOCK_SIZE,
+  SanitizationError,
+  resolveSanitizerCommand,
+  sanitizeLogToTemp,
+  sanitizeText,
+};

package/src/lib/log-uploader.js

@@ -6,7 +6,7 @@ const fs = require('fs');
 const os = require('os');
 const path = require('path');
 const { spawnSync } = require('child_process');
-const { sanitizeLogToTemp } = require('./log-sanitizer');
+const { SanitizationError, sanitizeLogToTemp } = require('./log-sanitizer');
 
 function isExecutable(filePath) {
   try {
@@ -174,11 +174,14 @@ function uploadLogPath(logPath, options = {}) {
       };
     }
     return { success: true, exitCode: 0 };
-  } catch {
+  } catch (error) {
     return {
       success: false,
       exitCode: 1,
-      error: 'Log preparation or upload failed; upload blocked.',
+      error:
+        error instanceof SanitizationError
+          ? error.message
+          : 'Log preparation or upload failed; upload blocked.',
     };
   } finally {
     if (prepared) {

package/test/ci-workflow-invariants.js

@@ -504,6 +504,24 @@ describe('CI workflow invariants', () => {
     assert.match(lint.body, /cargo metadata --locked/);
   });
 
+  it('gates pull requests on every dependency being its latest release', () => {
+    // Obsolete dependencies, sensitive-data-sanitizer among them, stayed in
+    // use unnoticed (issue #253).
+    const jsJobs = parseJobs(readWorkflow('js.yml'));
+    const freshness = jsJobs.find((job) => job.name === 'dependency-freshness');
+    assert.ok(freshness, 'js.yml has no dependency-freshness job');
+    assert.match(
+      freshness.body,
+      /scripts\/check-dependency-freshness\.mjs --lockfiles\n/
+    );
+    const status = jsJobs.find((job) => job.name === 'pipeline-status');
+    assert.ok(parseNeeds(status.body).includes('dependency-freshness'));
+    const lint = parseJobs(readWorkflow('rust.yml')).find(
+      (job) => job.name === 'lint'
+    );
+    assert.match(lint.body, /cargo update --dry-run/);
+  });
+
   it('reports a failed Rust test as a failed test, not low coverage', () => {
     // Without pipefail a failing test reached the threshold check as an empty
     // value: "Coverage % is below the 50% threshold" (issue #234).

package/test/command-stream-log.js

@@ -0,0 +1,85 @@
+/**
+ * Issue #253: command-stream 2.x returns completed `stdout` and `stderr` as
+ * readable stream snapshots instead of strings. `--use-command-stream` must
+ * still write the command's output to its log file.
+ */
+const { describe, it, expect } = require('bun:test');
+const { spawnSync } = require('child_process');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { pathToFileURL } = require('url');
+
+const CLI = path.join(__dirname, '../src/bin/cli.js');
+
+describe('--use-command-stream log (#253)', () => {
+  it('loads execution without compatibility and terminal modules', () => {
+    const wrapper = pathToFileURL(
+      path.join(__dirname, '../src/lib/command-stream.js')
+    ).href;
+    const result = spawnSync(
+      'node',
+      [
+        '--input-type=module',
+        '-e',
+        `import { registerHooks } from 'node:module';
+         import assert from 'node:assert/strict';
+         const extras = [];
+         registerHooks({ load(url, context, next) {
+           if (/command-stream\\/src\\/(shelljs|terminal|rust-repl)/.test(url)) extras.push(url);
+           return next(url, context);
+         }});
+         const { default: wrapper } = await import(${JSON.stringify(wrapper)});
+         const core = await wrapper.getCommandStream();
+         assert.equal(typeof core.$, 'function');
+         assert.equal(typeof core.ProcessRunner, 'function');
+         assert.deepEqual(extras, []);`,
+      ],
+      { encoding: 'utf8', timeout: 30000 }
+    );
+    expect(result.error).toBeUndefined();
+    expect(result.stderr).toBe('');
+    expect(result.status).toBe(0);
+  });
+
+  it.skipIf(process.platform === 'win32')(
+    'writes stdout and stderr to the log file',
+    () => {
+      const root = fs.mkdtempSync(path.join(os.tmpdir(), 'start-253-'));
+      try {
+        const result = spawnSync(
+          process.execPath,
+          [
+            CLI,
+            '--use-command-stream',
+            '--',
+            'sh',
+            '-c',
+            'echo out-253; echo err-253 >&2',
+          ],
+          {
+            env: {
+              ...process.env,
+              START_APP_FOLDER: path.join(root, 'app'),
+              START_LOG_DIR: path.join(root, 'logs'),
+              START_DISABLE_AUTO_ISSUE: '1',
+              START_DISABLE_LOG_UPLOAD: '1',
+            },
+            encoding: 'utf8',
+            timeout: 30000,
+          }
+        );
+        expect(result.status).toBe(0);
+        expect(result.stderr).not.toContain('Could not append log file');
+        const logDir = path.join(root, 'logs', 'direct');
+        const [logName] = fs.readdirSync(logDir);
+        const log = fs.readFileSync(path.join(logDir, logName), 'utf8');
+        expect(log).toContain('out-253\n');
+        expect(log).toContain('err-253\n');
+      } finally {
+        fs.rmSync(root, { recursive: true, force: true });
+      }
+    },
+    40000
+  );
+});