skannr 0.1.1

package/src/scanner.ts

@@ -0,0 +1,201 @@
+/**
+ * File scanner for universal codebases.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+export const DEFAULT_EXCLUDES = [
+  '**/node_modules/**',
+  '**/.git/**',
+  '**/dist/**',
+  '**/build/**',
+  '**/coverage/**',
+  '**/__pycache__/**',
+  '**/*.min.js',
+  '**/*.bundle.js',
+  '**/vendor/**',
+  '**/.next/**',
+  '**/.nuxt/**',
+  '**/target/**',
+  '**/*.pyc',
+  '**/.cache/**',
+];
+
+const DEFAULT_IGNORED_DIRS = new Set([
+  'node_modules',
+  '.git',
+  'dist',
+  'build',
+  'coverage',
+  '__pycache__',
+  'vendor',
+  '.next',
+  '.nuxt',
+  'target',
+  '.cache',
+  '.idea',
+  '.vscode',
+]);
+
+const SRC_CANDIDATES = ['src', 'lib', 'packages', 'apps', 'modules'];
+
+export interface ScanOptions {
+  extensions?: string[];
+  exclude?: string[];
+  moduleKeys?: string[];
+  moduleDefinitions?: Record<string, string[]>;
+}
+
+export function discoverModules(root: string): string[] {
+  const srcCandidates = ['src', 'lib', 'packages', 'apps', 'modules'];
+
+  for (const candidate of srcCandidates) {
+    const candidatePath = path.join(root, candidate);
+    if (fs.existsSync(candidatePath) && fs.statSync(candidatePath).isDirectory()) {
+      const subdirs = fs.readdirSync(candidatePath)
+        .filter((name) => {
+          const fullPath = path.join(candidatePath, name);
+          return fs.statSync(fullPath).isDirectory() && !name.startsWith('.');
+        });
+      if (subdirs.length > 0) return subdirs;
+    }
+  }
+
+  const ignored = new Set(['node_modules', '.git', 'dist', 'build', 'coverage', '.cache', '__pycache__', 'vendor']);
+  return fs.readdirSync(root)
+    .filter((name) => {
+      const fullPath = path.join(root, name);
+      return fs.statSync(fullPath).isDirectory() && !name.startsWith('.') && !ignored.has(name);
+    });
+}
+
+export function scanFiles(rootDir: string, options: ScanOptions = {}): string[] {
+  const results: string[] = [];
+  const excludes = options.exclude ?? DEFAULT_EXCLUDES;
+  const extensions = options.extensions?.map((ext) => ext.toLowerCase());
+  const scanRoots = resolveScanRoots(rootDir, options);
+
+  const scan = (dir: string): void => {
+    let entries: fs.Dirent[];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      if (shouldExclude(fullPath, rootDir, excludes)) {
+        continue;
+      }
+
+      if (entry.isDirectory()) {
+        if (!DEFAULT_IGNORED_DIRS.has(entry.name)) {
+          scan(fullPath);
+        }
+      } else if (entry.isFile()) {
+        if (!extensions || extensions.length === 0) {
+          results.push(fullPath);
+          continue;
+        }
+        const ext = path.extname(entry.name).toLowerCase();
+        if (extensions.includes(ext)) {
+          results.push(fullPath);
+        }
+      }
+    }
+  };
+
+  for (const scanRoot of scanRoots) {
+    if (fs.existsSync(scanRoot) && fs.statSync(scanRoot).isDirectory()) {
+      scan(scanRoot);
+    }
+  }
+
+  return Array.from(new Set(results));
+}
+
+export function scanTypeScriptFiles(rootDir: string): string[] {
+  return scanFiles(rootDir, { extensions: ['.ts', '.tsx'] });
+}
+
+export function scanRocketChatFiles(rootDir: string, moduleKeys?: string[]): string[] {
+  return scanFiles(rootDir, {
+    extensions: ['.ts', '.tsx'],
+    moduleKeys,
+  });
+}
+
+export function readFileContent(filePath: string): string | null {
+  try {
+    return fs.readFileSync(filePath, 'utf-8');
+  } catch {
+    return null;
+  }
+}
+
+function resolveScanRoots(rootDir: string, options: ScanOptions): string[] {
+  const moduleKeys = options.moduleKeys;
+  const moduleDefinitions = options.moduleDefinitions;
+  if (!moduleKeys || moduleKeys.length === 0) {
+    return [rootDir];
+  }
+
+  const resolved: string[] = [];
+  for (const key of moduleKeys) {
+    if (moduleDefinitions?.[key]) {
+      for (const rel of moduleDefinitions[key]) {
+        const full = path.resolve(rootDir, rel);
+        if (fs.existsSync(full) && fs.statSync(full).isDirectory()) {
+          resolved.push(full);
+        }
+      }
+      continue;
+    }
+
+    const direct = path.join(rootDir, key);
+    if (fs.existsSync(direct) && fs.statSync(direct).isDirectory()) {
+      resolved.push(direct);
+      continue;
+    }
+
+    for (const base of SRC_CANDIDATES) {
+      const candidate = path.join(rootDir, base, key);
+      if (fs.existsSync(candidate) && fs.statSync(candidate).isDirectory()) {
+        resolved.push(candidate);
+      }
+    }
+  }
+
+  if (resolved.length === 0 && moduleKeys.length > 0) {
+    throw new Error(`No matching modules found for keys: ${moduleKeys.join(', ')}`);
+  }
+
+  return resolved;
+}
+
+function shouldExclude(fullPath: string, rootDir: string, excludes: string[]): boolean {
+  const rel = path.relative(rootDir, fullPath).split(path.sep).join('/');
+  const hasSegment = (segment: string): boolean =>
+    rel === segment || rel.startsWith(`${segment}/`) || rel.includes(`/${segment}/`);
+
+  for (const pattern of excludes) {
+    if (pattern.includes('node_modules') && hasSegment('node_modules')) return true;
+    if (pattern.includes('.git') && hasSegment('.git')) return true;
+    if (pattern.includes('/dist/') && hasSegment('dist')) return true;
+    if (pattern.includes('/build/') && hasSegment('build')) return true;
+    if (pattern.includes('/coverage/') && hasSegment('coverage')) return true;
+    if (pattern.includes('__pycache__') && hasSegment('__pycache__')) return true;
+    if (pattern.includes('/vendor/') && hasSegment('vendor')) return true;
+    if (pattern.includes('/.next/') && hasSegment('.next')) return true;
+    if (pattern.includes('/.nuxt/') && hasSegment('.nuxt')) return true;
+    if (pattern.includes('/target/') && hasSegment('target')) return true;
+    if (pattern.includes('/.cache/') && hasSegment('.cache')) return true;
+    if (pattern.endsWith('*.min.js') && rel.endsWith('.min.js')) return true;
+    if (pattern.endsWith('*.bundle.js') && rel.endsWith('.bundle.js')) return true;
+    if (pattern.endsWith('*.pyc') && rel.endsWith('.pyc')) return true;
+  }
+
+  return false;
+}

package/src/skeletonizer.ts

@@ -0,0 +1,16 @@
+/**
+ * Language-aware skeleton generation.
+ */
+
+import * as fs from 'fs';
+import { getAdapter } from './languages/registry';
+
+export function buildSkeletonForFile(filePath: string): string {
+  try {
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const adapter = getAdapter(filePath);
+    return adapter.generateSkeleton(content, filePath);
+  } catch (error) {
+    return `/* Failed to parse file: ${error instanceof Error ? error.message : 'Unknown error'} */`;
+  }
+}

package/src/tokenizer.ts

@@ -0,0 +1,20 @@
+/**
+ * Simple tokenizer for counting approximate tokens in code
+ */
+
+/**
+ * Count approximate tokens in a string by splitting on whitespace and punctuation
+ * This is a simple heuristic for estimating token count
+ */
+export function countTokens(text: string): number {
+  if (!text || text.trim().length === 0) {
+    return 0;
+  }
+
+  // Split on whitespace, punctuation, and common code delimiters
+  const tokens = text
+    .split(/[\s\n\r\t.,;:!?(){}[\]<>'"/\\|`~@#$%^&*+=-]+/)
+    .filter(token => token.length > 0);
+
+  return tokens.length;
+}

package/src/types.ts

@@ -0,0 +1,71 @@
+/**
+ * Shared type definitions for skannr
+ */
+
+/**
+ * Options for analyzing a project
+ */
+export interface AnalyzeOptions {
+  /** Root directory of the project */
+  root: string;
+  /** Natural language question about the codebase */
+  question: string;
+  /** Number of top files to return */
+  limit?: number;
+  /** Generate symbol mapping for on-demand retrieval */
+  generateMapping?: boolean;
+  /** Output path for mapping file (default: <root>/code-analyzer.mapping.json) */
+  mappingOutputPath?: string;
+  /** Use enhanced ranking algorithm with dependency analysis */
+  enhancedRanking?: boolean;
+  /** Optional filter for module keys (e.g. ['auth', 'api']) */
+  moduleKeys?: string[];
+  /** Optional module definitions from config (module -> directory list) */
+  moduleDefinitions?: Record<string, string[]>;
+  /** Optional exclusion globs */
+  exclude?: string[];
+  /** Optional extension allow-list */
+  extensions?: string[];
+  /** Language strategy */
+  lang?: 'typescript' | 'javascript' | 'python' | 'auto';
+}
+
+/**
+ * Ranked file with relevance score
+ */
+export interface RankedFile {
+  /** Absolute path to the file */
+  path: string;
+  /** Relevance score (0-1) */
+  score: number;
+}
+
+/**
+ * Analysis result for a single file
+ */
+export interface FileAnalysis {
+  /** Relative path to the file */
+  path: string;
+  /** Relevance score (0-1) */
+  score: number;
+  /** Generated skeleton code */
+  skeleton: string;
+  /** Approximate token count of original file */
+  originalTokenCount: number;
+  /** Approximate token count of skeleton */
+  skeletonTokenCount: number;
+}
+
+/**
+ * Complete analysis result
+ */
+export interface AnalysisResult {
+  /** The original question */
+  question: string;
+  /** The project root directory */
+  root: string;
+  /** Number of files requested */
+  limit: number;
+  /** Analyzed files with skeletons */
+  files: FileAnalysis[];
+}

package/tests/agent.tools.test.ts

@@ -0,0 +1,81 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { CodeAnalysisAgent } from '../src/agent';
+import { buildSkeletonWithMapping, saveMappingToFile, type SymbolMapping } from '../src/mapper';
+
+describe('CodeAnalysisAgent local tool wrappers', () => {
+  let tempDir: string;
+  let sampleFile: string;
+  let mappingPath: string;
+  let agent: CodeAnalysisAgent;
+
+  beforeAll(() => {
+    tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'uca-agent-test-'));
+    sampleFile = path.join(tempDir, 'sample.ts');
+    mappingPath = path.join(tempDir, 'code-analyzer.mapping.json');
+
+    fs.writeFileSync(
+      sampleFile,
+      [
+        'export class AuthService {',
+        '  login(user: string): boolean {',
+        '    return user.length > 0;',
+        '  }',
+        '}',
+        '',
+        'export function sendMessage(room: string, text: string): string {',
+        '  return `${room}:${text}`;',
+        '}',
+      ].join('\n'),
+      'utf-8'
+    );
+
+    const mappingResult = buildSkeletonWithMapping(sampleFile, tempDir);
+    const mapping: SymbolMapping = {
+      generatedAt: new Date().toISOString(),
+      rootPath: tempDir,
+      files: {
+        'sample.ts': {
+          originalPath: sampleFile,
+          symbols: mappingResult.symbols,
+        },
+      },
+    };
+
+    saveMappingToFile(mapping, mappingPath);
+
+    agent = new CodeAnalysisAgent('test-api-key');
+    agent.loadMapping(mappingPath);
+  });
+
+  afterAll(() => {
+    if (fs.existsSync(tempDir)) {
+      fs.rmSync(tempDir, { recursive: true, force: true });
+    }
+  });
+
+  it('searchSymbols should find matching symbols', () => {
+    const result = agent.searchSymbols('login');
+    expect(result.error).toBeUndefined();
+    expect(result.count).toBeGreaterThan(0);
+
+    const found = result.results.some((item: any) => item.symbolId === 'AuthService.login');
+    expect(found).toBe(true);
+  });
+
+  it('getSymbolDetails should return implementation with location', () => {
+    const result = agent.getSymbolDetails('sendMessage');
+    expect(result.error).toBeUndefined();
+    expect(result.content).toContain('return `${room}:${text}`;');
+    expect(result.location.file).toBe('sample.ts');
+  });
+
+  it('analyzeFileDependencies should return symbol list for file', () => {
+    const result = agent.analyzeFileDependencies('sample.ts');
+    expect(result.error).toBeUndefined();
+    expect(result.file).toBe('sample.ts');
+    expect(Array.isArray(result.symbols)).toBe(true);
+    expect(result.symbols.length).toBeGreaterThan(0);
+  });
+});

package/tests/benchmark.test.ts

@@ -0,0 +1,31 @@
+import { benchmarkProject } from '../src/benchmark';
+import * as path from 'path';
+
+describe('Benchmark Tests', () => {
+  it('BENCHMARK_SCENARIOS module keys are strings', () => {
+    const { BENCHMARK_SCENARIOS } = require('../src/benchmark');
+    BENCHMARK_SCENARIOS.forEach((scenario: { moduleKey: string }) => {
+      expect(typeof scenario.moduleKey).toBe('string');
+      expect(scenario.moduleKey.length).toBeGreaterThan(0);
+    });
+  });
+
+  it('Scenario path resolution is deterministic and scoped', async () => {
+    const fixtureRoot = path.join(__dirname, 'fixtures');
+    const moduleKey = 'src';
+
+    const benchmarkResult = await benchmarkProject(fixtureRoot, moduleKey, 'Test Project', 'authentication flow');
+
+    const expectedPath = path.join(fixtureRoot, 'src');
+    expect(benchmarkResult.projectRoot).toBe(expectedPath);
+
+    expect(benchmarkResult.top1KeywordCoverage).toBeGreaterThanOrEqual(0);
+    expect(benchmarkResult.top1KeywordCoverage).toBeLessThanOrEqual(1);
+    expect(benchmarkResult.avgTopKKeywordCoverage).toBeGreaterThanOrEqual(0);
+    expect(benchmarkResult.avgTopKKeywordCoverage).toBeLessThanOrEqual(1);
+    expect(benchmarkResult.pathIntentMatchRate).toBeGreaterThanOrEqual(0);
+    expect(benchmarkResult.pathIntentMatchRate).toBeLessThanOrEqual(1);
+    expect(benchmarkResult.directoryDiversity).toBeGreaterThanOrEqual(0);
+    expect(benchmarkResult.directoryDiversity).toBeLessThanOrEqual(1);
+  });
+});

package/tests/fixtures/sample.py

@@ -0,0 +1,17 @@
+import os
+
+
+class UserStore:
+    def __init__(self, root: str) -> None:
+        self.root = root
+
+    def path_for(self, user_id: str) -> str:
+        return os.path.join(self.root, user_id)
+
+
+def normalize_username(value: str) -> str:
+    return value.strip().lower()
+
+
+def can_login(enabled: bool, attempts: int) -> bool:
+    return enabled and attempts < 5

package/tests/fixtures/sample.ts

@@ -0,0 +1,13 @@
+import { readFileSync } from 'fs';
+
+export class SampleService {
+  constructor(private readonly source: string) {}
+
+  load(): string {
+    return readFileSync(this.source, 'utf-8');
+  }
+}
+
+export function formatMessage(name: string): string {
+  return `hello ${name}`;
+}

package/tests/fixtures/src/api/routes.ts

@@ -0,0 +1 @@
+export const routes = ['/health', '/users'];

package/tests/fixtures/src/auth/permission.ts

@@ -0,0 +1,3 @@
+export function hasPermission(role: string, action: string): boolean {
+  return role === 'admin' || action === 'read';
+}

package/tests/ranker-enhanced.test.ts

@@ -0,0 +1,68 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { rankFilesEnhanced } from '../src/ranker-enhanced';
+
+describe('rankFilesEnhanced hybrid retrieval', () => {
+  let tempDir: string;
+
+  beforeEach(() => {
+    tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ranker-enhanced-'));
+  });
+
+  afterEach(() => {
+    if (fs.existsSync(tempDir)) {
+      fs.rmSync(tempDir, { recursive: true, force: true });
+    }
+  });
+
+  it('prioritizes files aligned with auth intent', () => {
+    const authFile = path.join(tempDir, 'auth-permissions.ts');
+    const genericFile = path.join(tempDir, 'utility.ts');
+
+    fs.writeFileSync(
+      authFile,
+      [
+        'export class AuthorizationService {',
+        '  checkPermission(user: string, action: string): boolean {',
+        '    return !!user && !!action;',
+        '  }',
+        '}',
+      ].join('\n'),
+      'utf-8'
+    );
+
+    fs.writeFileSync(
+      genericFile,
+      [
+        'export function sum(a: number, b: number): number {',
+        '  return a + b;',
+        '}',
+      ].join('\n'),
+      'utf-8'
+    );
+
+    const ranked = rankFilesEnhanced(
+      [authFile, genericFile],
+      'how are permissions validated for users?',
+      2,
+      tempDir
+    );
+
+    expect(ranked.length).toBe(2);
+    expect(path.basename(ranked[0].path)).toBe('auth-permissions.ts');
+  });
+
+  it('returns deterministic ordering for equal scores', () => {
+    const aFile = path.join(tempDir, 'a.ts');
+    const bFile = path.join(tempDir, 'b.ts');
+
+    fs.writeFileSync(aFile, 'export const a = 1;', 'utf-8');
+    fs.writeFileSync(bFile, 'export const b = 2;', 'utf-8');
+
+    const ranked = rankFilesEnhanced([bFile, aFile], 'zzzz-unmatched-term', 2, tempDir);
+
+    expect(ranked.length).toBe(2);
+    expect(ranked[0].path <= ranked[1].path).toBe(true);
+  });
+});

package/tests/ranker.test.ts

@@ -0,0 +1,75 @@
+/**
+ * Tests for file ranking functionality
+ */
+
+import { calculateRelevanceScore, rankFiles } from '../src/ranker';
+
+describe('calculateRelevanceScore', () => {
+  it('should return 0 for null content', () => {
+    const score = calculateRelevanceScore('/path/to/file.ts', null, 'test question');
+    expect(score).toBe(0);
+  });
+
+  it('should return higher score for filename matches', () => {
+    const filePath = '/src/authentication/auth-service.ts';
+    const content = 'export class Service {}';
+    const score = calculateRelevanceScore(filePath, content, 'authentication');
+    expect(score).toBeGreaterThanOrEqual(0.3);
+  });
+
+  it('should return higher score for content matches', () => {
+    const filePath = '/src/service.ts';
+    const content = `
+      export class AuthService {
+        async login() { }
+        async logout() { }
+        authenticate() { }
+      }
+    `;
+    const score = calculateRelevanceScore(filePath, content, 'authentication login');
+    expect(score).toBeGreaterThan(0);
+  });
+
+  it('should return neutral score for empty question', () => {
+    const score = calculateRelevanceScore('/path/to/file.ts', 'content', '');
+    expect(score).toBe(0.5);
+  });
+
+  it('should handle multiple word questions', () => {
+    const filePath = '/src/user/profile.ts';
+    const content = 'export class UserProfile {}';
+    const score = calculateRelevanceScore(filePath, content, 'user profile settings');
+    expect(score).toBeGreaterThan(0);
+  });
+});
+
+describe('rankFiles', () => {
+  it('should return empty array for no files', () => {
+    const ranked = rankFiles([], 'test question', 10);
+    expect(ranked).toEqual([]);
+  });
+
+  it('should limit results to requested number', () => {
+    // Create mock files - note: rankFiles reads from filesystem
+    // For a real test, we'd need actual files, but this tests the logic
+    const files = [
+      '/test/auth.ts',
+      '/test/user.ts',
+      '/test/service.ts',
+    ];
+
+    // Mock would be needed here for a full test
+    // This is a placeholder to show the structure
+    const ranked = rankFiles(files, 'test', 2);
+    expect(ranked.length).toBeLessThanOrEqual(2);
+  });
+
+  it('should sort by score descending', () => {
+    const files = ['/test/file1.ts', '/test/file2.ts'];
+    const ranked = rankFiles(files, 'test', 10);
+
+    for (let i = 1; i < ranked.length; i++) {
+      expect(ranked[i - 1].score).toBeGreaterThanOrEqual(ranked[i].score);
+    }
+  });
+});

package/tests/scanner.scope.test.ts

@@ -0,0 +1,41 @@
+import * as path from 'path';
+import { discoverModules, scanFiles } from '../src/scanner';
+
+const rootDir = path.join(__dirname, 'fixtures');
+
+describe('generic scanner', () => {
+  it('discovers modules from common source folders', () => {
+    const modules = discoverModules(rootDir);
+    expect(modules).toContain('auth');
+    expect(modules).toContain('api');
+  });
+
+  it('scans only requested module keys', () => {
+    const files = scanFiles(rootDir, {
+      moduleKeys: ['auth'],
+      moduleDefinitions: {
+        auth: ['src/auth'],
+      },
+      extensions: ['.ts'],
+    });
+    expect(files.length).toBeGreaterThan(0);
+    files.forEach((file) => {
+      expect(file.startsWith(path.join(rootDir, 'src/auth'))).toBe(true);
+    });
+  });
+
+  it('returns empty array on invalid root', () => {
+    const invalidRoot = path.join(__dirname, 'missing-fixtures');
+    const files = scanFiles(invalidRoot, { extensions: ['.ts'] });
+    expect(files).toEqual([]);
+  });
+
+  it('excludes common ignored folders', () => {
+    const files = scanFiles(rootDir, { extensions: ['.ts'] });
+    const excludedDir = path.join(rootDir, 'node_modules');
+
+    files.forEach((file) => {
+      expect(file.startsWith(excludedDir)).toBe(false);
+    });
+  });
+});

package/tests/setup-fixtures.js

@@ -0,0 +1,29 @@
+const fs = require('fs');
+const path = require('path');
+
+// The fake repo root expected by the tests
+const root = path.join(__dirname, 'fixtures');
+
+const dirs = [
+  'src/auth',
+  'src/api',
+  'node_modules',
+];
+
+const files = [
+  'src/auth/permission.ts',
+  'src/api/routes.ts',
+  'node_modules/ignored.ts',
+];
+
+console.log(`Creating test fixtures in: ${root}`);
+
+dirs.forEach((dir) => {
+  fs.mkdirSync(path.join(root, dir), { recursive: true });
+});
+
+files.forEach((file) => {
+  fs.writeFileSync(path.join(root, file), '// test file content');
+});
+
+console.log('✅ Fixtures created successfully.');