scip-query 0.1.0

package/src/reindex/detect.ts

@@ -0,0 +1,58 @@
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import type { SupportedLanguage } from '../types.js';
+
+interface LanguageMarker {
+  language: SupportedLanguage;
+  files: string[];
+}
+
+/**
+ * Marker files that indicate a language is present in a project.
+ * Ordered roughly by specificity — more specific markers first.
+ */
+const LANGUAGE_MARKERS: LanguageMarker[] = [
+  { language: 'typescript', files: ['tsconfig.json', 'tsconfig.base.json'] },
+  { language: 'rust', files: ['Cargo.toml'] },
+  { language: 'go', files: ['go.mod'] },
+  { language: 'java', files: ['pom.xml', 'build.gradle', 'build.gradle.kts'] },
+  { language: 'kotlin', files: ['build.gradle.kts'] },
+  { language: 'scala', files: ['build.sbt'] },
+  { language: 'python', files: ['pyproject.toml', 'setup.py', 'setup.cfg', 'Pipfile'] },
+  { language: 'ruby', files: ['Gemfile'] },
+  { language: 'csharp', files: ['*.csproj', '*.sln'] },
+  { language: 'dart', files: ['pubspec.yaml'] },
+  { language: 'php', files: ['composer.json'] },
+  { language: 'javascript', files: ['package.json'] }, // Last — very common, low specificity
+];
+
+/**
+ * Detect which languages are present in a project directory
+ * by checking for marker files.
+ */
+export function detectLanguages(projectRoot: string): SupportedLanguage[] {
+  const detected: SupportedLanguage[] = [];
+
+  for (const marker of LANGUAGE_MARKERS) {
+    for (const file of marker.files) {
+      if (file.includes('*')) {
+        // Glob patterns — skip for now, basic detection only
+        continue;
+      }
+      if (existsSync(join(projectRoot, file))) {
+        if (!detected.includes(marker.language)) {
+          detected.push(marker.language);
+        }
+        break;
+      }
+    }
+  }
+
+  // If we found TypeScript, don't also report JavaScript (it's implied)
+  if (detected.includes('typescript')) {
+    const jsIdx = detected.indexOf('javascript');
+    if (jsIdx !== -1) detected.splice(jsIdx, 1);
+  }
+
+  return detected;
+}

package/src/reindex/index.ts

@@ -0,0 +1,153 @@
+import { execFileSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import type { SupportedLanguage } from '../types.js';
+import { detectLanguages } from './detect.js';
+import { getIndexerConfig } from './indexers.js';
+import { isBinaryAvailable, isIndexerInstalled, tryInstallIndexer, tryInstallScipCli } from './install.js';
+
+export interface ReindexOptions {
+  projectRoot: string;
+  /** Override language detection — index only these languages */
+  languages?: SupportedLanguage[];
+  /** Path for the SCIP protobuf output (default: <projectRoot>/index.scip) */
+  outputScip?: string;
+  /** Path for the SQLite output (default: <projectRoot>/index.db) */
+  outputDb?: string;
+  /** Max Node.js heap size in MB (default: 8192) */
+  maxHeapMb?: number;
+  /** Callback for status updates */
+  onStatus?: (message: string) => void;
+  /** Extra flags for pnpm-workspace-aware TS indexing */
+  pnpmWorkspaces?: boolean;
+  /** Skip auto-install prompts */
+  skipAutoInstall?: boolean;
+}
+
+export interface ReindexResult {
+  languages: SupportedLanguage[];
+  indexPath: string;
+  dbPath: string;
+  durationMs: number;
+}
+
+/**
+ * Reindex a project: detect languages, run the appropriate SCIP indexer(s),
+ * and convert the output to SQLite.
+ */
+export async function reindex(opts: ReindexOptions): Promise<ReindexResult> {
+  const {
+    projectRoot,
+    maxHeapMb = 8192,
+    onStatus = console.log,
+    skipAutoInstall = false,
+  } = opts;
+
+  const outputScip = opts.outputScip ?? join(projectRoot, 'index.scip');
+  const outputDb = opts.outputDb ?? join(projectRoot, 'index.db');
+  const start = Date.now();
+
+  // Detect or use provided languages
+  const languages = opts.languages ?? detectLanguages(projectRoot);
+  if (languages.length === 0) {
+    throw new Error(
+      'No supported languages detected in this project. ' +
+      'Looked for: tsconfig.json, Cargo.toml, go.mod, pyproject.toml, etc.',
+    );
+  }
+
+  onStatus(`Detected languages: ${languages.join(', ')}`);
+
+  // Check that the scip CLI is available, auto-install if needed
+  if (!isBinaryAvailable('scip')) {
+    if (skipAutoInstall) {
+      throw new Error(
+        'The scip CLI is required but not found on PATH.\n' +
+        'Install from: https://github.com/sourcegraph/scip/releases',
+      );
+    }
+    onStatus('scip CLI not found on PATH. Attempting auto-install...');
+    if (!tryInstallScipCli(onStatus)) {
+      throw new Error(
+        'The scip CLI is required but could not be installed.\n' +
+        'Install manually from: https://github.com/sourcegraph/scip/releases',
+      );
+    }
+  }
+
+  const env = {
+    ...process.env,
+    NODE_OPTIONS: `--max-old-space-size=${maxHeapMb}`,
+  };
+
+  // Index each language
+  for (const lang of languages) {
+    const config = getIndexerConfig(lang);
+
+    // Check if indexer is installed, auto-install if needed
+    if (!isIndexerInstalled(config)) {
+      if (skipAutoInstall) {
+        throw new Error(
+          `${config.indexerBinary} is required to index ${lang} but not found on PATH.\n` +
+          (config.installUrl ? `Install from: ${config.installUrl}` : `Make sure ${config.indexerBinary} is installed and available on PATH.`),
+        );
+      }
+      onStatus(`${config.indexerBinary} not found. Attempting auto-install...`);
+      if (!tryInstallIndexer(config, onStatus)) {
+        throw new Error(
+          `${config.indexerBinary} is required to index ${lang} but could not be installed.\n` +
+          (config.installUrl ? `Install manually from: ${config.installUrl}` : `Make sure ${config.indexerBinary} is installed and available on PATH.`),
+        );
+      }
+    }
+
+    onStatus(`Indexing ${lang} with ${config.indexerBinary}...`);
+
+    const { binary, args } = config.indexArgs({
+      projectRoot,
+      outputPath: outputScip,
+      pnpmWorkspaces: opts.pnpmWorkspaces,
+    });
+
+    try {
+      execFileSync(binary, args, {
+        cwd: projectRoot,
+        env,
+        stdio: 'pipe',
+        maxBuffer: 50 * 1024 * 1024,
+      });
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      throw new Error(
+        `Failed to index ${lang} with ${config.indexerBinary}: ${msg}\n` +
+        `Make sure ${config.indexerBinary} is installed and available on PATH.`,
+      );
+    }
+  }
+
+  // Convert SCIP protobuf to SQLite
+  onStatus('Converting to SQLite...');
+  if (!existsSync(outputScip)) {
+    throw new Error(`SCIP index not found at ${outputScip} after indexing`);
+  }
+
+  try {
+    execFileSync('scip', ['expt-convert', '--output', outputDb, outputScip], {
+      env,
+      stdio: 'pipe',
+      maxBuffer: 50 * 1024 * 1024,
+    });
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to convert SCIP index to SQLite: ${msg}`);
+  }
+
+  const durationMs = Date.now() - start;
+  onStatus(`Done in ${(durationMs / 1000).toFixed(1)}s`);
+
+  return { languages, indexPath: outputScip, dbPath: outputDb, durationMs };
+}
+
+export { detectLanguages } from './detect.js';
+export { getIndexerConfig, INDEXER_CONFIGS } from './indexers.js';
+export { isBinaryAvailable, isIndexerInstalled, tryInstallIndexer, tryInstallScipCli } from './install.js';

package/src/reindex/indexers.ts

@@ -0,0 +1,220 @@
+import type { SupportedLanguage, IndexerConfig } from '../types.js';
+
+/**
+ * Indexer configurations for each supported language.
+ * Each entry describes how to produce a SCIP index for that language.
+ *
+ * The `scip` CLI binary is required for all languages (to convert protobuf -> SQLite).
+ * Each language also needs its own SCIP indexer installed.
+ */
+export const INDEXER_CONFIGS: Record<SupportedLanguage, IndexerConfig> = {
+  typescript: {
+    language: 'typescript',
+    indexerBinary: 'scip-typescript',
+    checkCommand: 'npx scip-typescript --version',
+    indexArgs: ({ outputPath, pnpmWorkspaces }) => {
+      const args = ['scip-typescript', 'index', '--infer-tsconfig', '--output', outputPath, '--no-progress-bar'];
+      if (pnpmWorkspaces) args.splice(2, 0, '--pnpm-workspaces');
+      return { binary: 'npx', args };
+    },
+    markerFiles: ['tsconfig.json'],
+    installMethods: [
+      { label: 'npm', prerequisite: 'npm', binary: 'npm', args: ['install', '-g', '@sourcegraph/scip-typescript'] },
+    ],
+    installUrl: 'https://github.com/sourcegraph/scip-typescript',
+  },
+
+  javascript: {
+    language: 'javascript',
+    indexerBinary: 'scip-typescript',
+    checkCommand: 'npx scip-typescript --version',
+    indexArgs: ({ outputPath }) => ({
+      binary: 'npx',
+      args: ['scip-typescript', 'index', '--infer-tsconfig', '--output', outputPath, '--no-progress-bar'],
+    }),
+    markerFiles: ['package.json'],
+    installMethods: [
+      { label: 'npm', prerequisite: 'npm', binary: 'npm', args: ['install', '-g', '@sourcegraph/scip-typescript'] },
+    ],
+    installUrl: 'https://github.com/sourcegraph/scip-typescript',
+  },
+
+  java: {
+    language: 'java',
+    indexerBinary: 'scip-java',
+    checkCommand: 'scip-java --version',
+    indexArgs: ({ outputPath }) => ({
+      binary: 'scip-java',
+      args: ['index', '--output', outputPath],
+    }),
+    markerFiles: ['pom.xml', 'build.gradle'],
+    installMethods: [
+      { label: 'coursier', prerequisite: 'cs', binary: 'cs', args: ['install', 'scip-java'] },
+    ],
+    installUrl: 'https://github.com/sourcegraph/scip-java/releases',
+  },
+
+  scala: {
+    language: 'scala',
+    indexerBinary: 'scip-java',
+    checkCommand: 'scip-java --version',
+    indexArgs: ({ outputPath }) => ({
+      binary: 'scip-java',
+      args: ['index', '--output', outputPath],
+    }),
+    markerFiles: ['build.sbt'],
+    installMethods: [
+      { label: 'coursier', prerequisite: 'cs', binary: 'cs', args: ['install', 'scip-java'] },
+    ],
+    installUrl: 'https://github.com/sourcegraph/scip-java/releases',
+  },
+
+  kotlin: {
+    language: 'kotlin',
+    indexerBinary: 'scip-java',
+    checkCommand: 'scip-java --version',
+    indexArgs: ({ outputPath }) => ({
+      binary: 'scip-java',
+      args: ['index', '--output', outputPath],
+    }),
+    markerFiles: ['build.gradle.kts'],
+    installMethods: [
+      { label: 'coursier', prerequisite: 'cs', binary: 'cs', args: ['install', 'scip-java'] },
+    ],
+    installUrl: 'https://github.com/sourcegraph/scip-java/releases',
+  },
+
+  rust: {
+    language: 'rust',
+    indexerBinary: 'rust-analyzer',
+    checkCommand: 'rust-analyzer --version',
+    indexArgs: ({ outputPath }) => ({
+      binary: 'rust-analyzer',
+      args: ['scip', '.', '--output', outputPath],
+    }),
+    markerFiles: ['Cargo.toml'],
+    installMethods: [
+      { label: 'rustup', prerequisite: 'rustup', binary: 'rustup', args: ['component', 'add', 'rust-analyzer'] },
+    ],
+    installUrl: 'https://github.com/rust-lang/rust-analyzer',
+  },
+
+  python: {
+    language: 'python',
+    indexerBinary: 'scip-python',
+    checkCommand: 'scip-python --version',
+    indexArgs: ({ outputPath }) => ({
+      binary: 'scip-python',
+      args: ['index', '--output', outputPath, '--project-name', 'project'],
+    }),
+    markerFiles: ['pyproject.toml', 'setup.py'],
+    installMethods: [
+      { label: 'npm', prerequisite: 'npm', binary: 'npm', args: ['install', '-g', 'scip-python-plus'] },
+    ],
+    installUrl: 'https://github.com/PlunderStruck/scip-python',
+  },
+
+  ruby: {
+    language: 'ruby',
+    indexerBinary: 'scip-ruby',
+    checkCommand: 'scip-ruby --version',
+    indexArgs: ({ outputPath }) => ({
+      binary: 'scip-ruby',
+      args: ['--output', outputPath],
+    }),
+    markerFiles: ['Gemfile'],
+    installMethods: [],
+    installUrl: 'https://github.com/sourcegraph/scip-ruby/releases',
+  },
+
+  go: {
+    language: 'go',
+    indexerBinary: 'scip-go',
+    checkCommand: 'scip-go --version',
+    indexArgs: ({ outputPath }) => ({
+      binary: 'scip-go',
+      args: ['--output', outputPath],
+    }),
+    markerFiles: ['go.mod'],
+    installMethods: [
+      { label: 'go install', prerequisite: 'go', binary: 'go', args: ['install', 'github.com/sourcegraph/scip-go@latest'] },
+    ],
+    installUrl: 'https://github.com/sourcegraph/scip-go',
+  },
+
+  cpp: {
+    language: 'cpp',
+    indexerBinary: 'scip-clang',
+    checkCommand: 'scip-clang --version',
+    indexArgs: ({ outputPath }) => ({
+      binary: 'scip-clang',
+      args: ['--output', outputPath],
+    }),
+    markerFiles: ['CMakeLists.txt', 'Makefile'],
+    installMethods: [],
+    installUrl: 'https://github.com/sourcegraph/scip-clang/releases',
+  },
+
+  c: {
+    language: 'c',
+    indexerBinary: 'scip-clang',
+    checkCommand: 'scip-clang --version',
+    indexArgs: ({ outputPath }) => ({
+      binary: 'scip-clang',
+      args: ['--output', outputPath],
+    }),
+    markerFiles: ['CMakeLists.txt', 'Makefile'],
+    installMethods: [],
+    installUrl: 'https://github.com/sourcegraph/scip-clang/releases',
+  },
+
+  csharp: {
+    language: 'csharp',
+    indexerBinary: 'scip-dotnet',
+    checkCommand: 'scip-dotnet --version',
+    indexArgs: ({ outputPath }) => ({
+      binary: 'scip-dotnet',
+      args: ['index', '--output', outputPath],
+    }),
+    markerFiles: [],
+    installMethods: [
+      { label: 'dotnet', prerequisite: 'dotnet', binary: 'dotnet', args: ['tool', 'install', '--global', 'scip-dotnet'] },
+    ],
+    installUrl: 'https://github.com/sourcegraph/scip-dotnet/releases',
+  },
+
+  dart: {
+    language: 'dart',
+    indexerBinary: 'scip-dart',
+    checkCommand: 'scip-dart --version',
+    indexArgs: ({ outputPath }) => ({
+      binary: 'scip-dart',
+      args: ['index', '--output', outputPath],
+    }),
+    markerFiles: ['pubspec.yaml'],
+    installMethods: [
+      { label: 'dart pub', prerequisite: 'dart', binary: 'dart', args: ['pub', 'global', 'activate', 'scip_dart'] },
+    ],
+    installUrl: 'https://github.com/Workiva/scip-dart/releases',
+  },
+
+  php: {
+    language: 'php',
+    indexerBinary: 'scip-php',
+    checkCommand: 'scip-php --version',
+    indexArgs: ({ outputPath }) => ({
+      binary: 'scip-php',
+      args: ['index', '--output', outputPath],
+    }),
+    markerFiles: ['composer.json'],
+    installMethods: [
+      { label: 'composer', prerequisite: 'composer', binary: 'composer', args: ['global', 'require', 'davidrjenni/scip-php'] },
+    ],
+    installUrl: 'https://github.com/davidrjenni/scip-php/releases',
+  },
+};
+
+/** Get the indexer config for a language */
+export function getIndexerConfig(language: SupportedLanguage): IndexerConfig {
+  return INDEXER_CONFIGS[language];
+}

package/src/reindex/install.ts

@@ -0,0 +1,125 @@
+import { execFileSync } from 'node:child_process';
+import { platform } from 'node:os';
+import type { IndexerConfig } from '../types.js';
+
+const IS_WINDOWS = platform() === 'win32';
+
+/**
+ * Check if a binary is available on PATH.
+ */
+export function isBinaryAvailable(name: string): boolean {
+  const cmd = IS_WINDOWS ? 'where' : 'which';
+  try {
+    execFileSync(cmd, [name], { stdio: 'pipe' });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Check if an indexer's binary is available on PATH.
+ */
+export function isIndexerInstalled(config: IndexerConfig): boolean {
+  return isBinaryAvailable(config.indexerBinary);
+}
+
+/**
+ * Attempt to auto-install an indexer using its configured install methods.
+ * Tries each method in order, checking prerequisites first.
+ * Returns true if installation succeeded.
+ */
+export function tryInstallIndexer(
+  config: IndexerConfig,
+  onStatus: (msg: string) => void,
+): boolean {
+  const methods = config.installMethods;
+  if (!methods?.length) {
+    onStatus(`No auto-install method available for ${config.indexerBinary}.`);
+    if (config.installUrl) {
+      onStatus(`Install manually from: ${config.installUrl}`);
+    }
+    return false;
+  }
+
+  for (const method of methods) {
+    if (!isBinaryAvailable(method.prerequisite)) {
+      continue;
+    }
+
+    onStatus(`Installing ${config.indexerBinary} via ${method.label}...`);
+    try {
+      execFileSync(method.binary, method.args, {
+        stdio: 'inherit',
+        timeout: 300_000,
+        env: process.env,
+      });
+
+      if (isIndexerInstalled(config)) {
+        onStatus(`Successfully installed ${config.indexerBinary} via ${method.label}`);
+        return true;
+      }
+      onStatus(`${method.label} command completed but ${config.indexerBinary} not found on PATH`);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      onStatus(`${method.label} install failed: ${msg}`);
+    }
+  }
+
+  onStatus(`Could not auto-install ${config.indexerBinary}.`);
+  if (config.installUrl) {
+    onStatus(`Install manually from: ${config.installUrl}`);
+  }
+  return false;
+}
+
+/**
+ * Attempt to auto-install the `scip` CLI binary.
+ * Tries brew (macOS), then go install, then prints manual instructions.
+ * Returns true if installation succeeded.
+ */
+export function tryInstallScipCli(
+  onStatus: (msg: string) => void,
+): boolean {
+  // macOS: try Homebrew first
+  if (platform() === 'darwin' && isBinaryAvailable('brew')) {
+    onStatus('Installing scip CLI via Homebrew...');
+    try {
+      execFileSync('brew', ['install', 'sourcegraph/scip/scip'], {
+        stdio: 'inherit',
+        timeout: 300_000,
+        env: process.env,
+      });
+      if (isBinaryAvailable('scip')) {
+        onStatus('Successfully installed scip CLI via Homebrew');
+        return true;
+      }
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      onStatus(`Homebrew install failed: ${msg}`);
+    }
+  }
+
+  // Any platform: try go install
+  if (isBinaryAvailable('go')) {
+    onStatus('Installing scip CLI via go install...');
+    try {
+      execFileSync('go', ['install', 'github.com/sourcegraph/scip/cmd/scip@latest'], {
+        stdio: 'inherit',
+        timeout: 300_000,
+        env: process.env,
+      });
+      if (isBinaryAvailable('scip')) {
+        onStatus('Successfully installed scip CLI via go install');
+        return true;
+      }
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      onStatus(`go install failed: ${msg}`);
+    }
+  }
+
+  onStatus('Could not auto-install scip CLI.');
+  onStatus('Install manually from: https://github.com/sourcegraph/scip/releases');
+  return false;
+}

package/src/reindex-worker.ts

@@ -0,0 +1,35 @@
+/**
+ * Child process worker for background reindexing.
+ * Invoked by the watcher via fork(). Reads config from env vars.
+ */
+import { reindex } from './reindex/index.js';
+import type { SupportedLanguage } from './types.js';
+
+const projectRoot = process.env['SCIP_REINDEX_PROJECT_ROOT'];
+const outputScip = process.env['SCIP_REINDEX_OUTPUT_SCIP'];
+const outputDb = process.env['SCIP_REINDEX_OUTPUT_DB'];
+const languagesRaw = process.env['SCIP_REINDEX_LANGUAGES'];
+const pnpmWorkspaces = process.env['SCIP_REINDEX_PNPM_WORKSPACES'] === '1';
+
+if (!projectRoot || !outputScip || !outputDb) {
+  console.error('reindex-worker: missing required env vars');
+  process.exit(1);
+}
+
+const languages = languagesRaw
+  ? (languagesRaw.split(',').filter(Boolean) as SupportedLanguage[])
+  : undefined;
+
+reindex({
+  projectRoot,
+  outputScip,
+  outputDb,
+  languages: languages?.length ? languages : undefined,
+  pnpmWorkspaces,
+  onStatus: (msg) => process.stderr.write(`[reindex] ${msg}\n`),
+}).then(() => {
+  process.exit(0);
+}).catch((err) => {
+  console.error(`reindex-worker error: ${err}`);
+  process.exit(1);
+});