reverse-engine 0.3.1 → 0.4.0

package/dist/cli/index.js

@@ -7,6 +7,7 @@ import { execFileSync } from 'child_process';
 import { dirname, join, resolve } from 'path';
 import { createRequire } from 'module';
 import { analyze } from '../analyzer/index.js';
+import { crawl } from '../crawler/index.js';
 import { generateReport } from '../docgen/index.js';
 import { generateTests } from '../testgen/index.js';
 // ─── 기본값 ───
@@ -135,22 +136,96 @@ program
     console.log(chalk.green('✓'), `완료! (${files.length}개 파일)`);
     files.forEach(p => console.log(`  → ${chalk.cyan(p)}`));
 });
+// ─── crawl ───
+program
+    .command('crawl')
+    .argument('[url]', '크롤링 대상 URL')
+    .option('--max-depth <n>', '최대 탐색 깊이', '5')
+    .option('--max-pages <n>', '최대 페이지 수', '100')
+    .option('--no-screenshot', '스크린샷 비활성화')
+    .option('--no-headless', '브라우저 표시 (디버깅용)')
+    .option('--auth-cookie <cookie>', '인증 쿠키 (name=value;name2=value2)')
+    .option('--auth-bearer <token>', 'Bearer 토큰')
+    .option('--login-url <url>', '로그인 페이지 URL')
+    .option('--login-id <id>', '로그인 ID 필드값')
+    .option('--login-pw <pw>', '로그인 PW 필드값')
+    .option('--wait <ms>', '페이지 로드 후 대기시간(ms)', '1500')
+    .option('-o, --output <dir>', '출력 디렉토리')
+    .description('실행 중인 서비스를 브라우저로 크롤링하여 화면/API 수집')
+    .action(async (url, opts) => {
+    if (!url) {
+        console.log(chalk.red('✗'), 'URL을 입력하세요: reverse-engine crawl http://localhost:3000');
+        process.exit(1);
+    }
+    const outputDir = opts.output || DEFAULT_OUTPUT;
+    console.log(chalk.green('▶'), '크롤링 시작:', chalk.cyan(url));
+    console.log(`  최대 깊이: ${opts.maxDepth} | 최대 페이지: ${opts.maxPages}`);
+    // 인증 옵션 구성
+    const auth = {};
+    if (opts.authCookie)
+        auth.cookie = opts.authCookie;
+    if (opts.authBearer)
+        auth.bearer = opts.authBearer;
+    if (opts.loginUrl) {
+        auth.loginUrl = opts.loginUrl;
+        auth.credentials = {};
+        if (opts.loginId)
+            auth.credentials.email = opts.loginId;
+        if (opts.loginPw)
+            auth.credentials.password = opts.loginPw;
+    }
+    const result = await crawl({
+        url,
+        maxDepth: parseInt(opts.maxDepth),
+        maxPages: parseInt(opts.maxPages),
+        screenshot: opts.screenshot !== false,
+        headless: opts.headless !== false,
+        outputDir,
+        waitTime: parseInt(opts.wait),
+        auth: Object.keys(auth).length > 0 ? auth : undefined,
+    });
+    console.log(chalk.green('✓'), `크롤링 완료!`);
+    console.log(`  페이지: ${result.pages.length}개`);
+    console.log(`  API 호출: ${result.pages.reduce((n, p) => n + p.apiCalls.length, 0)}개`);
+    await mkdir(outputDir, { recursive: true });
+    const crawlPath = join(outputDir, 'crawl-result.json');
+    await writeFile(crawlPath, JSON.stringify(result, null, 2));
+    console.log(`  → ${chalk.cyan(crawlPath)}`);
+});
 // ─── full ───
 program
     .command('full')
     .argument('[path]', '소스코드 경로 (생략하면 현재 디렉토리)')
+    .option('--url <url>', '실행 중인 서비스 URL (크롤링 추가)')
     .option('--framework <name>', '프레임워크', 'auto')
+    .option('--no-headless', '크롤링 시 브라우저 표시')
+    .option('--auth-cookie <cookie>', '크롤링 인증 쿠키')
     .option('-o, --output <dir>', '출력 디렉토리 (기본: <프로젝트>/.reverse-engine)')
-    .description('전체 파이프라인 (analyze → report → test)')
+    .description('전체 파이프라인 (crawl → analyze → report → test)')
     .action(async (path, opts) => {
     const sourcePath = path ? resolve(path) : detectProjectRoot();
     const outputDir = resolveOutput(opts.output, sourcePath);
     console.log(chalk.green('\n◆'), 'ReversEngine', nativeBin ? chalk.dim('⚡') : '', '\n');
+    if (opts.url)
+        console.log(`  URL:  ${chalk.cyan(opts.url)}`);
     console.log(`  소스: ${chalk.cyan(sourcePath)}`);
     console.log(`  출력: ${chalk.cyan(outputDir)}\n`);
     await mkdir(outputDir, { recursive: true });
     const analysisPath = join(outputDir, 'analysis.json');
-    // Step 1: 분석
+    // Step 0: 크롤링 (URL이 있는 경우)
+    if (opts.url) {
+        console.log(chalk.gray('━'.repeat(50)));
+        console.log(chalk.green('▶'), '크롤링:', chalk.cyan(opts.url));
+        const crawlResult = await crawl({
+            url: opts.url,
+            outputDir,
+            headless: opts.headless !== false,
+            auth: opts.authCookie ? { cookie: opts.authCookie } : undefined,
+        });
+        console.log(chalk.green('✓'), `크롤링: 페이지 ${crawlResult.pages.length} | API ${crawlResult.pages.reduce((n, p) => n + p.apiCalls.length, 0)}`);
+        await writeFile(join(outputDir, 'crawl-result.json'), JSON.stringify(crawlResult, null, 2));
+    }
+    // Step 1: 코드 분석
     console.log(chalk.gray('━'.repeat(50)));
     if (nativeBin) {
         runNative(['analyze', sourcePath, '--framework', opts.framework || 'auto']);

package/dist/crawler/index.d.ts

@@ -0,0 +1,20 @@
+import type { CrawlResult } from '../types.js';
+export interface CrawlOptions {
+    url: string;
+    maxDepth?: number;
+    maxPages?: number;
+    screenshot?: boolean;
+    outputDir?: string;
+    auth?: AuthOptions;
+    headless?: boolean;
+    waitTime?: number;
+    ignorePatterns?: string[];
+}
+export interface AuthOptions {
+    cookie?: string;
+    bearer?: string;
+    loginUrl?: string;
+    credentials?: Record<string, string>;
+    submitSelector?: string;
+}
+export declare function crawl(options: CrawlOptions): Promise<CrawlResult>;

package/dist/crawler/index.js

@@ -0,0 +1,201 @@
+import { chromium } from 'playwright';
+import { mkdir } from 'fs/promises';
+import { join } from 'path';
+export async function crawl(options) {
+    const { url, maxDepth = 5, maxPages = 100, screenshot = true, outputDir = '.reverse-engine', headless = true, waitTime = 1500, ignorePatterns = ['/logout', '/signout', '/auth/logout'], } = options;
+    const screenshotDir = join(outputDir, 'screenshots');
+    if (screenshot)
+        await mkdir(screenshotDir, { recursive: true });
+    const browser = await chromium.launch({ headless });
+    const context = await browser.newContext({
+        viewport: { width: 1920, height: 1080 },
+        locale: 'ko-KR',
+    });
+    // 인증 설정
+    if (options.auth) {
+        await setupAuth(context, url, options.auth);
+    }
+    const result = {
+        targetUrl: url,
+        pages: [],
+        timestamp: new Date().toISOString(),
+    };
+    const visited = new Set();
+    const queue = [{ url, depth: 0 }];
+    const baseHost = new URL(url).hostname;
+    while (queue.length > 0 && result.pages.length < maxPages) {
+        const current = queue.shift();
+        const normalizedUrl = normalizeUrl(current.url);
+        if (visited.has(normalizedUrl) || current.depth > maxDepth)
+            continue;
+        if (ignorePatterns.some(p => normalizedUrl.includes(p)))
+            continue;
+        visited.add(normalizedUrl);
+        const page = await context.newPage();
+        try {
+            const pageInfo = await scanPage(page, current.url, waitTime);
+            // 스크린샷
+            if (screenshot) {
+                const safeName = normalizedUrl.replace(/[^a-zA-Z0-9가-힣]/g, '_').slice(0, 80);
+                const ssPath = join(screenshotDir, `${safeName}.png`);
+                await page.screenshot({ path: ssPath, fullPage: true });
+                pageInfo.screenshotPath = ssPath;
+            }
+            // API 호출 수집 (이미 scanPage에서 네트워크 인터셉트)
+            result.pages.push(pageInfo);
+            // 새 URL 큐에 추가
+            for (const link of pageInfo.elements.links) {
+                try {
+                    const linkUrl = new URL(link.href, current.url);
+                    if (linkUrl.hostname === baseHost && !visited.has(normalizeUrl(linkUrl.href))) {
+                        queue.push({ url: linkUrl.href, depth: current.depth + 1 });
+                    }
+                }
+                catch { /* invalid URL */ }
+            }
+            // 버튼 클릭으로 발견되는 URL도 추가
+            for (const nav of pageInfo.navigatesTo) {
+                try {
+                    const navUrl = new URL(nav, current.url);
+                    if (navUrl.hostname === baseHost && !visited.has(normalizeUrl(navUrl.href))) {
+                        queue.push({ url: navUrl.href, depth: current.depth + 1 });
+                    }
+                }
+                catch { /* invalid URL */ }
+            }
+        }
+        catch (err) {
+            // 페이지 로드 실패 → 건너뜀
+        }
+        finally {
+            await page.close();
+        }
+    }
+    await browser.close();
+    return result;
+}
+/** 개별 페이지 스캔: DOM 요소 + 네트워크 인터셉트 */
+async function scanPage(page, url, waitTime) {
+    const apiCalls = [];
+    // 네트워크 인터셉트 — API 호출 캡처
+    page.on('response', async (response) => {
+        const request = response.request();
+        const resUrl = request.url();
+        const resourceType = request.resourceType();
+        if ((resourceType === 'xhr' || resourceType === 'fetch') && !isStaticResource(resUrl)) {
+            apiCalls.push({
+                method: request.method(),
+                url: resUrl,
+                responseStatus: response.status(),
+                triggeredBy: null,
+            });
+        }
+    });
+    // 페이지 로드
+    await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
+    await page.waitForTimeout(waitTime);
+    const title = await page.title();
+    // DOM 스캔
+    const elements = await page.evaluate(() => {
+        // 링크
+        const links = Array.from(document.querySelectorAll('a[href]')).map((el, i) => {
+            const a = el;
+            return {
+                text: a.textContent?.trim().slice(0, 100) || '',
+                href: a.href,
+                selector: a.id ? `#${a.id}` : `a:nth-of-type(${i + 1})`,
+            };
+        });
+        // 버튼
+        const buttons = Array.from(document.querySelectorAll('button, [role="button"], input[type="submit"]')).map((el, i) => {
+            const btn = el;
+            return {
+                text: btn.textContent?.trim().slice(0, 100) || btn.value || '',
+                selector: btn.id ? `#${btn.id}` : `button:nth-of-type(${i + 1})`,
+                navigatesTo: null,
+            };
+        });
+        // 폼
+        const forms = Array.from(document.querySelectorAll('form')).map((el) => {
+            const form = el;
+            const fields = Array.from(form.querySelectorAll('input, select, textarea')).map((f) => {
+                const field = f;
+                return {
+                    name: field.name || field.id || '',
+                    fieldType: field.type || field.tagName.toLowerCase(),
+                    required: field.required,
+                };
+            });
+            return {
+                id: form.id || null,
+                action: form.action || null,
+                method: form.method?.toUpperCase() || 'GET',
+                fields,
+            };
+        });
+        return { links, buttons, forms };
+    });
+    // 네비게이션 대상 URL 수집
+    const navigatesTo = [
+        ...new Set(elements.links.map(l => l.href).filter(Boolean)),
+    ];
+    // 로그인 페이지 감지
+    const authRequired = await page.evaluate(() => {
+        const html = document.body?.innerHTML?.toLowerCase() || '';
+        return html.includes('login') || html.includes('sign in') || html.includes('로그인');
+    });
+    return {
+        url,
+        title,
+        screenshotPath: null,
+        elements,
+        apiCalls,
+        navigatesTo,
+        authRequired,
+    };
+}
+/** 인증 설정 */
+async function setupAuth(context, baseUrl, auth) {
+    const host = new URL(baseUrl).hostname;
+    if (auth.cookie) {
+        const cookies = auth.cookie.split(';').map(c => {
+            const [name, ...rest] = c.trim().split('=');
+            return { name, value: rest.join('='), domain: host, path: '/' };
+        });
+        await context.addCookies(cookies);
+    }
+    if (auth.bearer) {
+        await context.setExtraHTTPHeaders({
+            Authorization: `Bearer ${auth.bearer}`,
+        });
+    }
+    if (auth.loginUrl && auth.credentials) {
+        const page = await context.newPage();
+        await page.goto(auth.loginUrl, { waitUntil: 'networkidle' });
+        for (const [field, value] of Object.entries(auth.credentials)) {
+            await page.fill(`[name="${field}"], #${field}, input[type="${field}"]`, value).catch(() => { });
+        }
+        const submitSelector = auth.submitSelector || 'button[type="submit"], button:has-text("로그인"), button:has-text("Login")';
+        await page.click(submitSelector).catch(() => { });
+        await page.waitForNavigation({ waitUntil: 'networkidle' }).catch(() => { });
+        await page.close();
+    }
+}
+function normalizeUrl(url) {
+    try {
+        const u = new URL(url);
+        u.hash = '';
+        u.search = '';
+        let path = u.pathname;
+        if (path.endsWith('/') && path.length > 1)
+            path = path.slice(0, -1);
+        u.pathname = path;
+        return u.href;
+    }
+    catch {
+        return url;
+    }
+}
+function isStaticResource(url) {
+    return /\.(js|css|png|jpg|jpeg|gif|svg|woff|woff2|ttf|ico|map)(\?|$)/i.test(url);
+}

package/dist/index.d.ts

@@ -1,4 +1,5 @@
 export { analyze } from './analyzer/index.js';
+export { crawl } from './crawler/index.js';
 export { generateReport } from './docgen/index.js';
 export { generateTests } from './testgen/index.js';
 export type * from './types.js';

package/dist/index.js

@@ -1,3 +1,4 @@
 export { analyze } from './analyzer/index.js';
+export { crawl } from './crawler/index.js';
 export { generateReport } from './docgen/index.js';
 export { generateTests } from './testgen/index.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "reverse-engine",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "웹 서비스 역분석 자동화 도구 - 소스코드 분석, 문서 생성, 테스트 자동화",
   "keywords": [
     "reverse-engineering",