readme-gen-analyzer 1.0.0

package/src/analyzers/route.extractor.ts

@@ -0,0 +1,54 @@
+import { Project, SyntaxKind, CallExpression, PropertyAccessExpression } from 'ts-morph';
+import { PolyglotExtractors } from './polyglot.extractors';
+
+export interface Route {
+  method: string;
+  path: string;
+  file: string;
+  snippet?: string;
+}
+
+export class RouteExtractor {
+  private static METHODS = ['get', 'post', 'put', 'delete', 'patch', 'use'];
+
+  public static extract(files: Record<string, string>): Route[] {
+    const project = new Project({ useInMemoryFileSystem: true });
+    const routes: Route[] = [];
+
+    for (const [filePath, content] of Object.entries(files)) {
+      if (!filePath.endsWith('.ts') && !filePath.endsWith('.js') && !filePath.endsWith('.tsx') && !filePath.endsWith('.jsx')) {
+        continue;
+      }
+
+      const sourceFile = project.createSourceFile(filePath, content);
+      
+      // Look for call expressions like app.get('/', ...) or router.post('/', ...)
+      const callExpressions = sourceFile.getDescendantsOfKind(SyntaxKind.CallExpression);
+
+      for (const call of callExpressions) {
+        const expression = call.getExpression();
+        
+        if (expression.getKind() === SyntaxKind.PropertyAccessExpression) {
+          const pae = expression as PropertyAccessExpression;
+          const methodName = pae.getName().toLowerCase();
+          
+          if (this.METHODS.includes(methodName)) {
+            const args = call.getArguments();
+            if (args.length > 0 && args[0].getKind() === SyntaxKind.StringLiteral) {
+              const routePath = args[0].getText().replace(/['"]/g, '');
+              
+              routes.push({
+                method: methodName.toUpperCase(),
+                path: routePath,
+                file: filePath,
+                snippet: call.getText().substring(0, 200) // snippet limited to 200 chars
+              });
+            }
+          }
+        }
+      }
+    }
+
+    return [...routes, ...PolyglotExtractors.extractRoutes(files)];
+  }
+}

package/src/analyzers/schema.analyzer.ts

@@ -0,0 +1,50 @@
+import { Project, SyntaxKind, ObjectLiteralExpression } from 'ts-morph';
+
+export class SchemaAnalyzer {
+  public static analyze(files: Record<string, string>) {
+    const schemas: any[] = [];
+    const project = new Project({ useInMemoryFileSystem: true });
+
+    for (const [filePath, content] of Object.entries(files)) {
+      // 1. Prisma
+      if (filePath.endsWith('.prisma')) {
+        const models = [...content.matchAll(/model\s+([A-Za-z0-9_-]+)\s+\{([\s\S]+?)\}/g)];
+        models.forEach(m => {
+          const fields = m[2].trim().split('\n')
+            .map(l => l.trim())
+            .filter(l => l && !l.startsWith('//') && !l.startsWith('@@'))
+            .map(l => l.split(/\s+/)[0]);
+          schemas.push({ model: m[1], fields, file: filePath });
+        });
+        continue;
+      }
+
+      // 2. Mongoose (ST)
+      if (filePath.match(/\.(ts|js|tsx|jsx)$/)) {
+        const sourceFile = project.createSourceFile(filePath, content);
+        const newExpressions = sourceFile.getDescendantsOfKind(SyntaxKind.NewExpression);
+        
+        for (const newExpr of newExpressions) {
+          if (newExpr.getExpression().getText().includes('Schema')) {
+            const args = newExpr.getArguments();
+            if (args.length > 0 && Node.isObjectLiteralExpression(args[0])) {
+                const fields = (args[0] as ObjectLiteralExpression).getProperties().map(p => p.getText().split(':')[0].trim());
+                // Try to find model name from the same file
+                const modelName = content.match(/mongoose\.model\(['"]([^'"]+)['"]/)?.[1] || "UnknownModel";
+                schemas.push({ model: modelName, fields, file: filePath });
+            }
+          }
+        }
+      }
+    }
+
+    return schemas.length > 0 ? schemas : undefined;
+  }
+}
+
+// Helper needed because Node is not imported in this scope correctly for isObjectLiteral
+const Node = {
+    isObjectLiteralExpression: (node: any): node is ObjectLiteralExpression => {
+        return node.getKind() === SyntaxKind.ObjectLiteralExpression;
+    }
+}

package/src/analyzers/semantic.refiner.ts

@@ -0,0 +1,163 @@
+import { ProjectAnalysis } from '../types';
+
+export class SemanticRefiner {
+  /**
+   * Transforms raw analysis into a high-quality, semantically rich structure.
+   */
+  public static refine(analysis: any): any {
+    const { dependencies = [], devDependencies = [], tree = [], framework, routes = [], envVars = [] } = analysis;
+    const allDeps = [...dependencies, ...devDependencies];
+
+    return {
+      projectName: analysis.name || 'Untitled Project',
+      description: analysis.description || 'No description provided.',
+      
+      techStack: {
+        backend: this.detectBackend(allDeps, framework),
+        frontend: this.detectFrontend(allDeps, framework),
+        database: this.detectDatabase(allDeps),
+        auth: this.detectAuth(allDeps),
+        tooling: this.detectTooling(allDeps),
+        ai: this.detectAI(allDeps)
+      },
+
+      architecture: this.detectArchitecture(analysis),
+      
+      features: this.enrichFeatures(analysis),
+      
+      api: {
+        routes: routes.map((r: any) => ({
+          method: r.method,
+          path: r.path,
+          purpose: this.inferRoutePurpose(r)
+        })),
+        totalEndpoints: routes.length
+      },
+
+      flows: this.inferFlows(analysis)
+    };
+  }
+
+  private static detectBackend(deps: string[], framework?: any): string[] {
+    const map: Record<string, string> = {
+      'express': 'Express.js',
+      'fastify': 'Fastify',
+      'nestjs': 'NestJS',
+      'koa': 'Koa',
+      'hono': 'Hono',
+      'apollo-server': 'GraphQL (Apollo)',
+      'socket.io': 'Real-time (Socket.io)'
+    };
+    const found = deps.filter(d => map[d]).map(d => map[d]);
+    if (framework?.name === 'Express') found.push('Express.js');
+    return Array.from(new Set(found));
+  }
+
+  private static detectFrontend(deps: string[], framework?: any): string[] {
+    const map: Record<string, string> = {
+      'react': 'React',
+      'vue': 'Vue',
+      'svelte': 'Svelte',
+      'next': 'Next.js',
+      'nuxt': 'Nuxt.js',
+      'tailwindcss': 'Tailwind CSS',
+      'framer-motion': 'Animations (Framer Motion)',
+      'lucide-react': 'Icons (Lucide)'
+    };
+    const found = deps.filter(d => map[d]).map(d => map[d]);
+    return Array.from(new Set(found));
+  }
+
+  private static detectDatabase(deps: string[]): string[] {
+    const map: Record<string, string> = {
+      'prisma': 'Prisma ORM',
+      'mongoose': 'MongoDB (Mongoose)',
+      'sequelize': 'Sequelize ORM',
+      'typeorm': 'TypeORM',
+      'redis': 'Redis (Caching)',
+      'pg': 'PostgreSQL'
+    };
+    return deps.filter(d => map[d]).map(d => map[d]);
+  }
+
+  private static detectAuth(deps: string[]): string[] {
+    const map: Record<string, string> = {
+      'passport': 'Passport.js',
+      'next-auth': 'NextAuth.js',
+      'jsonwebtoken': 'JWT Authentication',
+      'bcrypt': 'Password Hashing (bcrypt)',
+      'firebase': 'Firebase Auth'
+    };
+    return deps.filter(d => map[d]).map(d => map[d]);
+  }
+
+  private static detectTooling(deps: string[]): string[] {
+    const map: Record<string, string> = {
+      'typescript': 'TypeScript',
+      'jest': 'Jest (Testing)',
+      'vitest': 'Vitest (Testing)',
+      'eslint': 'ESLint (Linting)',
+      'prettier': 'Prettier (Formatting)',
+      'turbo': 'Turborepo (Build system)'
+    };
+    return deps.filter(d => map[d]).map(d => map[d]);
+  }
+
+  private static detectAI(deps: string[]): string[] {
+    const map: Record<string, string> = {
+      'openai': 'OpenAI SDK',
+      'langchain': 'LangChain (LLM Orchestration)',
+      'groq': 'Groq AI',
+      '@google/generative-ai': 'Google Gemini AI'
+    };
+    return deps.filter(d => map[d]).map(d => map[d]);
+  }
+
+  private static detectArchitecture(analysis: any): { type: string, summary: string } {
+    if (analysis.isMonorepo) {
+      return {
+        type: 'Monorepo',
+        summary: 'A multi-package repository managing distinct apps and shared packages using a build system like Turborepo.'
+      };
+    }
+    return {
+      type: 'Monolith',
+      summary: 'A single-package modular architecture centered around a core application structure.'
+    };
+  }
+
+  private static enrichFeatures(analysis: any): string[] {
+    const rawFeatures = analysis.features || [];
+    const featureMap: Record<string, string> = {
+      'Authentication': 'Secure User Authentication & Session Management',
+      'API Endpoints': 'RESTful API Interface for external communication',
+      'Database Integration': 'Persistent Data Storage with Schema Models',
+      'Environment Configuration': 'Robust configuration using environment variables',
+      'Real-time': 'Bi-directional real-time communication'
+    };
+    return rawFeatures.map((f: string) => featureMap[f] || f);
+  }
+
+  private static inferRoutePurpose(route: any): string {
+    const path = route.path.toLowerCase();
+    if (path.includes('auth') || path.includes('login') || path.includes('signup')) return 'Identity & Access Management';
+    if (path.includes('user') || path.includes('profile')) return 'User Profile Management';
+    if (path.includes('message') || path.includes('chat')) return 'Messaging logic';
+    if (path.includes('upload') || path.includes('cloudinary')) return 'Media handling';
+    return 'General business logic';
+  }
+
+  private static inferFlows(analysis: any): string[] {
+    const flows: string[] = [];
+    if (analysis.dependencies?.includes('mongoose')) {
+      flows.push('Request -> Controller -> Mongoose Model -> MongoDB');
+    }
+    if (analysis.dependencies?.includes('jsonwebtoken')) {
+      flows.push('Client -> JWT Middleware -> Protected Route -> Response');
+    }
+    if (analysis.tree?.some((f: string) => f.includes('Frontend') || f.includes('web'))) {
+      flows.push('Frontend (React) -> Axios -> API Endpoints (Backend)');
+    }
+    return flows;
+  }
+}

package/src/analyzers/structure.analyzer.ts

@@ -0,0 +1,156 @@
+import ignore from 'ignore';
+
+export interface FileEntry {
+  path: string;
+  name: string;
+}
+
+export class StructureAnalyzer {
+  private static ENTRY_BASE_NAMES = new Set([
+    'index.ts',
+    'index.js',
+    'main.py',
+    'app.ts',
+    'app.js',
+    'server.ts',
+    'server.js',
+    '__main__.py',
+    'manage.py',
+    'wsgi.py',
+    'asgi.py',
+    'app.py',
+  ]);
+
+  public static async analyze(files: string[], gitignoreContent: string): Promise<{
+    entryPoints: string[];
+    keyDirectories: string[];
+    importantFiles: string[];
+    hasDocker: boolean;
+    isMonorepo: boolean;
+    tree: string[];
+  }> {
+    const ig = ignore().add(gitignoreContent || '');
+    ig.add(['node_modules', '.git', '.turbo', 'dist', 'build', '.next', '.vscode']);
+
+    const normalizedFiles = files.map(f => f.replace(/\\/g, '/'));
+    const filteredFiles = normalizedFiles.filter(f => !ig.ignores(f));
+    const entryPoints = filteredFiles.filter(f => this.isEntryPoint(f));
+    
+    // Key directories detection
+    const directories = new Set<string>();
+    const keyPatterns = [
+      'src',
+      'lib',
+      'app',
+      'tests',
+      'docs',
+      'config',
+      'packages',
+      'apps',
+      'controllers',
+      'routes',
+      'services',
+      'cmd',
+      'internal',
+      'pkg',
+      'api',
+      'handlers',
+      'middleware',
+    ];
+    
+    filteredFiles.forEach(f => {
+      const parts = f.split('/');
+      parts.forEach(part => {
+        if (keyPatterns.includes(part.toLowerCase())) {
+          directories.add(part);
+        }
+      });
+    });
+    const keyDirs = Array.from(directories);
+
+    const hasDocker = filteredFiles.some(f => f.toLowerCase().includes('dockerfile') || f.toLowerCase().includes('docker-compose'));
+
+    // Monorepo detection
+    const isMonorepo = filteredFiles.some(f => 
+      f === 'pnpm-workspace.yaml' || 
+      f === 'lerna.json' || 
+      f === 'turbo.json' ||
+      f === 'go.work' ||
+      (f.endsWith('package.json') && filteredFiles.some(file => file.startsWith('packages/') || file.startsWith('apps/'))) ||
+      (f.endsWith('go.mod') && filteredFiles.some(file => file.startsWith('cmd/') || file.startsWith('internal/')))
+    );
+
+    // Scored files
+    const scoredFiles = filteredFiles.map(f => ({
+      path: f,
+      score: this.calculateImportance(f, entryPoints)
+    })).sort((a, b) => b.score - a.score);
+
+    return {
+      entryPoints,
+      keyDirectories: keyDirs,
+      importantFiles: scoredFiles.slice(0, 40).map(f => f.path),
+      hasDocker,
+      isMonorepo,
+      tree: this.generateTreeSnippet(filteredFiles, 5) // Increased depth to 5
+    };
+  }
+
+  private static calculateImportance(filePath: string, entryPoints: string[]): number {
+    let score = 0;
+    const name = filePath.split('/').pop() || '';
+    const lowerPath = filePath.toLowerCase();
+
+    if (entryPoints.includes(filePath)) score += 100; // Entry points are critical
+    
+    // Core architecture folders
+    if (lowerPath.includes('controller')) score += 60;
+    if (lowerPath.includes('route')) score += 60;
+    if (lowerPath.includes('api/')) score += 50; 
+    if (lowerPath.includes('service')) score += 40;
+    if (lowerPath.endsWith('.py') && (lowerPath.includes('handler') || lowerPath.includes('/api/'))) score += 45;
+    if (lowerPath.endsWith('.go') && (lowerPath.includes('handler') || lowerPath.includes('/http'))) score += 45;
+    if (lowerPath.includes('/cmd/')) score += 35;
+    if (lowerPath.includes('/internal/')) score += 25;
+    
+    // Monorepo specific deep scoring
+    if (lowerPath.includes('apps/') && (lowerPath.includes('/src/') || lowerPath.includes('/app/') || lowerPath.includes('/pages/'))) score += 30;
+    if (lowerPath.includes('packages/') && lowerPath.includes('/src/')) score += 20;
+
+    // File-based Route Detection (Next.js / Nuxt)
+    if (lowerPath.includes('pages/api/') || lowerPath.includes('app/api/')) score += 60;
+
+    // Infrastructure
+    if (['package.json', 'go.mod', 'requirements.txt', '.env.example', 'pyproject.toml', 'turbo.json'].includes(name)) score += 60;
+    if (lowerPath.includes('docker')) score += 20;
+    
+    // Negative weights
+    if (lowerPath.includes('test')) score -= 40;
+    if (lowerPath.includes('spec')) score -= 40;
+    if (lowerPath.includes('mock')) score -= 50;
+    if (lowerPath.includes('util')) score += 5; // Utils are okay but less critical than controllers
+
+    return score;
+  }
+
+  private static generateTreeSnippet(files: string[], maxDepth: number): string[] {
+    const tree: string[] = [];
+    files.forEach(f => {
+      const parts = f.split('/');
+      if (parts.length <= maxDepth) {
+        tree.push(f);
+      }
+    });
+    return tree.slice(0, 500); // Increased count to 500
+  }
+
+  private static isEntryPoint(filePath: string): boolean {
+    const parts = filePath.split('/');
+    const base = parts.pop() || '';
+    if (this.ENTRY_BASE_NAMES.has(base)) return true;
+    if (base === 'main.go') return true;
+    if (parts[parts.length - 1] === 'cmd' && base === 'main.go') return true;
+    return false;
+  }
+}
+

package/src/analyzers/trace.analyzer.ts

@@ -0,0 +1,75 @@
+import { Project, SyntaxKind } from 'ts-morph';
+import path from 'path';
+
+export class TraceAnalyzer {
+  public static analyze(entryPoints: string[], files: Record<string, string>) {
+    const project = new Project({ useInMemoryFileSystem: true });
+    const graph: Map<string, string[]> = new Map();
+    const scores: Map<string, number> = new Map();
+
+    // Populate project and scores
+    for (const [filePath, content] of Object.entries(files)) {
+      if (filePath.match(/\.(ts|js|tsx|jsx)$/)) {
+        project.createSourceFile(filePath, content);
+        scores.set(filePath, 0);
+      }
+    }
+
+    const visited = new Set<string>();
+    const stack = [...entryPoints];
+
+    while (stack.length > 0) {
+      const current = stack.pop()!;
+      if (visited.has(current)) continue;
+      visited.add(current);
+
+      const sourceFile = project.getSourceFile(current);
+      if (!sourceFile) continue;
+
+      const imports = sourceFile.getImportDeclarations();
+      const currentDeps: string[] = [];
+
+      imports.forEach(imp => {
+        const moduleSpecifier = imp.getModuleSpecifierValue();
+        if (moduleSpecifier.startsWith('.')) {
+          // Resolve relative path
+          const dir = path.dirname(current);
+          const resolved = this.resolvePath(dir, moduleSpecifier, Object.keys(files));
+          if (resolved) {
+            currentDeps.push(resolved);
+            scores.set(resolved, (scores.get(resolved) || 0) + 1);
+            stack.push(resolved);
+          }
+        }
+      });
+      graph.set(current, currentDeps);
+    }
+
+    // Top 10 most imported files
+    const topFiles = Array.from(scores.entries())
+      .sort((a, b) => b[1] - a[1])
+      .filter(f => f[1] > 0)
+      .slice(0, 10)
+      .map(f => f[0]);
+
+    return {
+      topFiles,
+      entryPoints,
+      graph: Object.fromEntries(graph)
+    };
+  }
+
+  private static resolvePath(dir: string, specifier: string, allFiles: string[]): string | null {
+    const fullPath = path.posix.join(dir, specifier);
+    const extensions = ['', '.ts', '.js', '.tsx', '.jsx', '/index.ts', '/index.js'];
+    
+    for (const ext of extensions) {
+      const candidate = `${fullPath}${ext}`;
+      if (allFiles.includes(candidate)) return candidate;
+      // Also check with leading slash if needed
+      const absoluteCandidate = candidate.startsWith('/') ? candidate : `/${candidate}`;
+      if (allFiles.includes(absoluteCandidate)) return absoluteCandidate;
+    }
+    return null;
+  }
+}

package/src/index.ts

@@ -0,0 +1,29 @@
+export * from './analyzers/structure.analyzer';
+export * from './analyzers/dependency.analyzer';
+export * from './analyzers/route.extractor';
+export * from './analyzers/env.extractor';
+export * from './analyzers/package.parser';
+export * from './analyzers/ast-feature.detector';
+export * from './analyzers/definition.extractor';
+export * from './analyzers/schema.analyzer';
+export * from './analyzers/trace.analyzer';
+export * from './analyzers/example.analyzer';
+export * from './analyzers/devops.analyzer';
+export * from './analyzers/polyglot.extractors';
+export * from './analyzers/semantic.refiner';
+export * from './types';
+export * from './utils/scanner';
+export * from './utils/scriptsMarkdown';
+
+// Production-grade semantic README pipeline
+export * from './internal/analysis/chunker';
+export * from './internal/analysis/evidence';
+export * from './internal/analysis/techStack';
+export * from './internal/llm/llmClient';
+export * from './internal/pipeline/types';
+export * from './internal/pipeline/stages';
+export * from './internal/pipeline/merge';
+export * from './internal/pipeline/readme';
+export * from './internal/pipeline/runPipeline';
+export * from './internal/pipeline/quality';
+export * from './internal/pipeline/persona';

package/src/internal/analysis/chunker.ts

@@ -0,0 +1,103 @@
+export interface ChunkerOptions {
+  /**
+   * ~6k tokens max per call. We approximate 1 token ~= 4 chars.
+   * Default 24k chars.
+   */
+  maxCharsPerChunk?: number;
+  /**
+   * Avoid tiny tail chunks.
+   */
+  minCharsPerChunk?: number;
+  /**
+   * Prefix each chunk with a header that helps grounding.
+   */
+  chunkHeader?: string;
+}
+
+export interface TextChunk {
+  id: string;
+  text: string;
+  approxTokens: number;
+}
+
+function approxTokens(text: string): number {
+  return Math.max(1, Math.ceil(text.length / 4));
+}
+
+/**
+ * Intelligently chunk already-extracted evidence blocks (not raw code generation).
+ * Keeps blocks intact when possible; falls back to line-splitting for oversized blocks.
+ */
+export function chunkEvidenceBlocks(blocks: string[], options: ChunkerOptions = {}): TextChunk[] {
+  const maxChars = options.maxCharsPerChunk ?? 24_000;
+  const minChars = options.minCharsPerChunk ?? Math.floor(maxChars * 0.4);
+  const header = options.chunkHeader ? `${options.chunkHeader.trim()}\n\n` : '';
+
+  const normalizedBlocks = blocks
+    .map((b) => b.trim())
+    .filter(Boolean);
+
+  const chunks: string[] = [];
+  let current = header;
+
+  const pushCurrent = () => {
+    const text = current.trim();
+    if (text.length > 0) chunks.push(text + '\n');
+    current = header;
+  };
+
+  const pushOversizedBlock = (block: string) => {
+    // Split oversized blocks by lines, preserving ordering and some context.
+    const lines = block.split('\n');
+    let buf = header;
+    for (const line of lines) {
+      const next = (buf ? buf + '\n' : '') + line;
+      if (next.length > maxChars) {
+        const trimmed = buf.trim();
+        if (trimmed.length > 0) chunks.push(trimmed + '\n');
+        buf = header + line;
+      } else {
+        buf = next;
+      }
+    }
+    const trimmed = buf.trim();
+    if (trimmed.length > 0) chunks.push(trimmed + '\n');
+  };
+
+  for (const block of normalizedBlocks) {
+    if (block.length > maxChars) {
+      // Flush anything accumulated so far, then split this block.
+      if (current.trim().length > 0) pushCurrent();
+      pushOversizedBlock(block);
+      continue;
+    }
+
+    const candidate = current.trim().length === 0 ? header + block : `${current.trim()}\n\n${block}`;
+    if (candidate.length <= maxChars) {
+      current = candidate;
+      continue;
+    }
+
+    // Candidate would overflow.
+    pushCurrent();
+    current = header + block;
+  }
+
+  if (current.trim().length > 0) pushCurrent();
+
+  // Merge last tiny chunk into previous if possible.
+  if (chunks.length >= 2) {
+    const last = chunks[chunks.length - 1]!;
+    const prev = chunks[chunks.length - 2]!;
+    if (last.length < minChars && (prev.length + 2 + last.length) <= maxChars) {
+      chunks.splice(chunks.length - 2, 2, `${prev.trim()}\n\n${last.trim()}\n`);
+    }
+  }
+
+  return chunks.map((text, idx) => ({
+    id: `chunk_${idx + 1}`,
+    text,
+    approxTokens: approxTokens(text),
+  }));
+}
+