playwright-archaeologist 0.1.1

package/dist/index.js

@@ -0,0 +1,789 @@
+import {
+  ArchaeologistError,
+  AuthError,
+  BundleError,
+  CollectorError,
+  ConfigError,
+  CrawlConfigSchema,
+  CrawlError,
+  DiffConfigSchema,
+  DiffError,
+  Frontier,
+  Logger,
+  NavigationError,
+  ViewportSchema,
+  buildFlowGraph,
+  captureScreenshots,
+  createBundle,
+  createNetworkLogger,
+  detectCycles,
+  diffBundles,
+  dig,
+  escapeAttribute,
+  escapeHtml,
+  escapeJsonInHtml,
+  generateDiffReportHtml,
+  generateMermaidDefinition,
+  generateOpenApiSpec,
+  generateReportHtml,
+  groupEndpoints,
+  logger,
+  normalizeEntryUrl,
+  parseViewport,
+  probeForms,
+  writeOpenApiSpec
+} from "./chunk-F5WCXM7I.js";
+import {
+  scanPage
+} from "./chunk-7ZQGW5OV.js";
+import "./chunk-RWPEKZOW.js";
+
+// src/auth/auth-handler.ts
+import { resolve } from "path";
+import { readFile, writeFile, access } from "fs/promises";
+import { constants } from "fs";
+
+// src/security/auth-validator.ts
+import { readFileSync } from "fs";
+import { pathToFileURL } from "url";
+var DANGEROUS_MODULES = [
+  "child_process",
+  "fs",
+  "fs/promises",
+  "net",
+  "dgram",
+  "cluster",
+  "worker_threads",
+  "vm",
+  "v8",
+  "perf_hooks",
+  "dns",
+  "tls",
+  "http",
+  "https"
+];
+var DANGEROUS_PATTERNS = [
+  {
+    pattern: /\beval\s*\(/g,
+    description: "eval() usage detected -- potential code injection"
+  },
+  {
+    pattern: /\bnew\s+Function\s*\(/g,
+    description: "Function constructor detected -- potential code injection"
+  },
+  {
+    pattern: /\bprocess\.exit\b/g,
+    description: "process.exit() detected -- could terminate the tool unexpectedly"
+  }
+];
+function buildModulePatterns() {
+  const results = [];
+  for (const mod of DANGEROUS_MODULES) {
+    const escaped = mod.replace(/\//g, "\\/");
+    results.push({
+      pattern: new RegExp(`\\bimport\\b[^;]*['"](?:node:)?${escaped}['"]`, "g"),
+      moduleName: mod
+    });
+    results.push({
+      pattern: new RegExp(`\\bimport\\s*\\(\\s*['"](?:node:)?${escaped}['"]\\s*\\)`, "g"),
+      moduleName: mod
+    });
+    results.push({
+      pattern: new RegExp(`\\brequire\\s*\\(\\s*['"](?:node:)?${escaped}['"]\\s*\\)`, "g"),
+      moduleName: mod
+    });
+  }
+  return results;
+}
+var MODULE_PATTERNS = buildModulePatterns();
+function validateAuthScript(scriptPath) {
+  const warnings = [];
+  let content;
+  try {
+    content = readFileSync(scriptPath, "utf-8");
+  } catch (err) {
+    const code = err?.code;
+    if (code === "ENOENT") {
+      throw new Error(`Auth script not found: ${scriptPath}`);
+    }
+    throw new Error(`Failed to read auth script '${scriptPath}': ${err?.message ?? String(err)}`);
+  }
+  for (const { pattern, moduleName } of MODULE_PATTERNS) {
+    pattern.lastIndex = 0;
+    if (pattern.test(content)) {
+      warnings.push(`Dangerous import of '${moduleName}' detected`);
+    }
+  }
+  for (const { pattern, description } of DANGEROUS_PATTERNS) {
+    pattern.lastIndex = 0;
+    if (pattern.test(content)) {
+      warnings.push(description);
+    }
+  }
+  return {
+    valid: warnings.length === 0,
+    warnings
+  };
+}
+
+// src/auth/auth-handler.ts
+var LOGIN_PATH_PATTERNS = [
+  /\/login\b/i,
+  /\/signin\b/i,
+  /\/sign-in\b/i,
+  /\/auth\b/i,
+  /\/sso\b/i
+];
+async function fileExists(filePath) {
+  try {
+    await access(filePath, constants.R_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function isLoginUrl(url) {
+  try {
+    const parsed = new URL(url);
+    return LOGIN_PATH_PATTERNS.some((re) => re.test(parsed.pathname));
+  } catch {
+    return false;
+  }
+}
+async function getCookieCount(context) {
+  const cookies = await context.cookies();
+  return cookies.length;
+}
+async function loadStorageState(context, statePath) {
+  const raw = await readFile(statePath, "utf-8");
+  const state = JSON.parse(raw);
+  if (!state || typeof state !== "object") {
+    throw new Error(`Invalid storage state file: ${statePath}`);
+  }
+  if (Array.isArray(state.cookies) && state.cookies.length > 0) {
+    await context.addCookies(state.cookies);
+  }
+  if (Array.isArray(state.origins)) {
+    for (const origin of state.origins) {
+      if (!origin.origin || !Array.isArray(origin.localStorage) || origin.localStorage.length === 0) {
+        continue;
+      }
+      const items = origin.localStorage;
+      await context.addInitScript(
+        (data) => {
+          if (window.location.origin === data.origin) {
+            for (const item of data.items) {
+              try {
+                localStorage.setItem(item.name, item.value);
+              } catch {
+              }
+            }
+          }
+        },
+        { origin: origin.origin, items }
+      );
+    }
+  }
+}
+async function saveStorageState(context, statePath) {
+  const state = await context.storageState();
+  await writeFile(statePath, JSON.stringify(state, null, 2), "utf-8");
+}
+async function loadCookies(context, cookiesPath) {
+  const raw = await readFile(cookiesPath, "utf-8");
+  let cookies;
+  try {
+    cookies = JSON.parse(raw);
+  } catch {
+    throw new Error(
+      `Invalid JSON in cookies file: ${cookiesPath}`
+    );
+  }
+  if (!Array.isArray(cookies)) {
+    throw new Error(
+      `Cookies file must contain a JSON array: ${cookiesPath}`
+    );
+  }
+  if (cookies.length === 0) {
+    return 0;
+  }
+  await context.addCookies(cookies);
+  return cookies.length;
+}
+async function runAuthScript(page, scriptPath) {
+  const resolvedPath = resolve(scriptPath);
+  const validation = validateAuthScript(resolvedPath);
+  if (!validation.valid) {
+    throw new Error(
+      `Auth script failed validation: ${validation.warnings.join("; ")}`
+    );
+  }
+  if (!await fileExists(resolvedPath)) {
+    throw new Error(`Auth script not found: ${resolvedPath}`);
+  }
+  let mod;
+  try {
+    mod = await import(resolvedPath);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to import auth script '${resolvedPath}': ${message}`);
+  }
+  const authFn = mod.default;
+  if (typeof authFn !== "function") {
+    throw new Error(
+      `Auth script '${resolvedPath}' must export a default async function`
+    );
+  }
+  await authFn(page);
+}
+async function detectAuthFailure(page) {
+  const url = page.url();
+  if (isLoginUrl(url)) {
+    return {
+      failed: true,
+      reason: `Still on login page: ${url}`
+    };
+  }
+  try {
+    const response = await page.evaluate(() => {
+      const entries = performance.getEntriesByType("navigation");
+      if (entries.length > 0) {
+        return { responseStatus: entries[0].responseStatus ?? null };
+      }
+      return { responseStatus: null };
+    });
+    if (response.responseStatus === 401 || response.responseStatus === 403) {
+      return {
+        failed: true,
+        reason: `Received HTTP ${response.responseStatus} response`
+      };
+    }
+  } catch {
+  }
+  return { failed: false };
+}
+async function executeAuth(page, context, options) {
+  if (!options.storageStatePath && !options.cookiesFile && !options.authScript) {
+    return {
+      success: false,
+      method: "script",
+      error: "No authentication method provided",
+      cookieCount: 0
+    };
+  }
+  if (options.storageStatePath) {
+    const stateExists = await fileExists(options.storageStatePath);
+    if (stateExists) {
+      try {
+        await loadStorageState(context, options.storageStatePath);
+        const cookieCount = await getCookieCount(context);
+        return {
+          success: true,
+          method: "storageState",
+          finalUrl: page.url(),
+          cookieCount
+        };
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return {
+          success: false,
+          method: "storageState",
+          error: `Failed to load storage state: ${message}`,
+          cookieCount: 0
+        };
+      }
+    }
+  }
+  if (options.cookiesFile) {
+    try {
+      const injectedCount = await loadCookies(context, options.cookiesFile);
+      const cookieCount = await getCookieCount(context);
+      return {
+        success: true,
+        method: "cookies",
+        finalUrl: page.url(),
+        cookieCount: cookieCount > 0 ? cookieCount : injectedCount
+      };
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      return {
+        success: false,
+        method: "cookies",
+        error: `Failed to inject cookies: ${message}`,
+        cookieCount: 0
+      };
+    }
+  }
+  if (options.authScript) {
+    try {
+      await runAuthScript(page, options.authScript);
+      const failureCheck = await detectAuthFailure(page);
+      if (failureCheck.failed) {
+        return {
+          success: false,
+          method: "script",
+          error: failureCheck.reason,
+          finalUrl: page.url(),
+          cookieCount: await getCookieCount(context)
+        };
+      }
+      if (options.storageStatePath) {
+        try {
+          await saveStorageState(context, options.storageStatePath);
+        } catch {
+        }
+      }
+      const cookieCount = await getCookieCount(context);
+      return {
+        success: true,
+        method: "script",
+        finalUrl: page.url(),
+        cookieCount
+      };
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      return {
+        success: false,
+        method: "script",
+        error: `Auth script failed: ${message}`,
+        finalUrl: page.url(),
+        cookieCount: 0
+      };
+    }
+  }
+  return {
+    success: false,
+    method: "script",
+    error: "No authentication method could be applied",
+    cookieCount: 0
+  };
+}
+async function refreshAuth(page, context, options) {
+  await context.clearCookies();
+  return executeAuth(page, context, options);
+}
+
+// src/crawl/context-pool.ts
+var ContextPool = class {
+  browser;
+  concurrency;
+  recycleAfter;
+  contextOptions;
+  storageState;
+  /** Contexts currently in use by callers */
+  acquired = /* @__PURE__ */ new Map();
+  /** Contexts sitting idle, ready to be acquired */
+  idle = [];
+  /** Waiters blocked on acquire() when pool is full */
+  waiters = [];
+  /** Whether closeAll() has been called */
+  closed = false;
+  constructor(browser, options) {
+    if (!browser) {
+      throw new Error("Browser instance is required");
+    }
+    if (options.concurrency < 1) {
+      throw new Error("Concurrency must be >= 1");
+    }
+    if (options.recycleAfter < 1) {
+      throw new Error("recycleAfter must be >= 1");
+    }
+    this.browser = browser;
+    this.concurrency = options.concurrency;
+    this.recycleAfter = options.recycleAfter;
+    this.contextOptions = options.contextOptions;
+    this.storageState = options.storageState;
+  }
+  /**
+   * Acquire a browser context from the pool.
+   * Blocks if the pool is at full capacity until a context is released.
+   */
+  async acquire() {
+    if (this.closed) {
+      throw new Error("ContextPool is closed");
+    }
+    while (this.acquired.size + this.idle.length >= this.concurrency && this.idle.length === 0) {
+      await new Promise((resolve2) => {
+        this.waiters.push(resolve2);
+      });
+      if (this.closed) {
+        throw new Error("ContextPool is closed");
+      }
+    }
+    if (this.idle.length > 0) {
+      const entry2 = this.idle.pop();
+      this.acquired.set(entry2.context, entry2);
+      return entry2.context;
+    }
+    const entry = await this.createEntry();
+    this.acquired.set(entry.context, entry);
+    return entry.context;
+  }
+  /**
+   * Release a context back to the pool. Increments the page count
+   * and recycles the context if it has exceeded recycleAfter.
+   */
+  async release(context) {
+    const entry = this.acquired.get(context);
+    if (!entry) {
+      throw new Error("Context is not owned by this pool or was already released");
+    }
+    this.acquired.delete(context);
+    entry.pageCount += 1;
+    if (this.closed) {
+      await context.close();
+      return;
+    }
+    if (entry.pageCount >= this.recycleAfter) {
+      await context.close();
+    } else {
+      this.idle.push(entry);
+    }
+    this.drainWaiters();
+  }
+  /**
+   * Close all contexts (both idle and acquired) and reject future operations.
+   */
+  async closeAll() {
+    this.closed = true;
+    for (const resolve2 of this.waiters) {
+      resolve2();
+    }
+    this.waiters.length = 0;
+    const closeOps = [];
+    for (const entry of this.idle) {
+      closeOps.push(entry.context.close());
+    }
+    this.idle.length = 0;
+    for (const [, entry] of this.acquired) {
+      closeOps.push(entry.context.close());
+    }
+    this.acquired.clear();
+    await Promise.all(closeOps);
+  }
+  /** Current number of active (acquired) contexts */
+  get activeCount() {
+    return this.acquired.size;
+  }
+  /** Current number of available (idle) contexts */
+  get availableCount() {
+    return this.idle.length;
+  }
+  // ---------------------------------------------------------------------------
+  // Private
+  // ---------------------------------------------------------------------------
+  async createEntry() {
+    const opts = { ...this.contextOptions };
+    if (this.storageState !== void 0) {
+      opts.storageState = this.storageState;
+    }
+    const context = await this.browser.newContext(opts);
+    return { context, pageCount: 0 };
+  }
+  drainWaiters() {
+    if (this.waiters.length > 0) {
+      const next = this.waiters.shift();
+      next();
+    }
+  }
+};
+
+// src/crawl/checkpoint.ts
+import { readFile as readFile2, writeFile as writeFile2, rename, unlink } from "fs/promises";
+import { join } from "path";
+import { z } from "zod";
+var CHECKPOINT_FILENAME = ".checkpoint.json";
+var CHECKPOINT_TMP_FILENAME = ".checkpoint.json.tmp";
+var DEFAULT_AUTO_CHECKPOINT_INTERVAL_MS = 3e4;
+var CheckpointStateSchema = z.object({
+  version: z.literal(1),
+  startedAt: z.string(),
+  checkpointedAt: z.string(),
+  frontier: z.array(
+    z.object({
+      url: z.string(),
+      depth: z.number(),
+      referrer: z.string().optional()
+    })
+  ),
+  visited: z.array(z.string()),
+  skipped: z.array(z.string()),
+  artifactsDir: z.string(),
+  pagesVisited: z.number(),
+  errors: z.array(
+    z.object({
+      timestamp: z.string(),
+      url: z.string(),
+      code: z.string(),
+      message: z.string(),
+      collector: z.enum(["page-scanner", "form-prober", "network-logger", "screenshot-capturer"]).optional(),
+      status: z.enum(["timeout", "network_error", "http_error", "no_response", "aborted", "redirect_loop"]).optional(),
+      httpStatus: z.number().optional(),
+      securityReason: z.enum([
+        "private_ip",
+        "blocked_protocol",
+        "dns_rebinding",
+        "metadata_endpoint",
+        "redirect_to_private"
+      ]).optional()
+    })
+  ),
+  configHash: z.string()
+});
+async function writeCheckpoint(state, outputDir) {
+  if (!outputDir) {
+    throw new Error("outputDir must be a non-empty string");
+  }
+  state.checkpointedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const tmpPath = join(outputDir, CHECKPOINT_TMP_FILENAME);
+  const finalPath = join(outputDir, CHECKPOINT_FILENAME);
+  const json = JSON.stringify(state, null, 2);
+  await writeFile2(tmpPath, json, "utf-8");
+  await rename(tmpPath, finalPath);
+}
+async function readCheckpoint(outputDir) {
+  if (!outputDir) {
+    throw new Error("outputDir must be a non-empty string");
+  }
+  const filePath = join(outputDir, CHECKPOINT_FILENAME);
+  let raw;
+  try {
+    raw = await readFile2(filePath, "utf-8");
+  } catch (err) {
+    if (isNodeError(err) && err.code === "ENOENT") {
+      return null;
+    }
+    throw err;
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    throw new Error(`Checkpoint file is not valid JSON: ${filePath}`);
+  }
+  const result = CheckpointStateSchema.safeParse(parsed);
+  if (!result.success) {
+    throw new Error(
+      `Invalid checkpoint data in ${filePath}: ${result.error.message}`
+    );
+  }
+  return result.data;
+}
+async function deleteCheckpoint(outputDir) {
+  if (!outputDir) {
+    throw new Error("outputDir must be a non-empty string");
+  }
+  const filesToRemove = [
+    join(outputDir, CHECKPOINT_FILENAME),
+    join(outputDir, CHECKPOINT_TMP_FILENAME)
+  ];
+  await Promise.all(
+    filesToRemove.map(async (filePath) => {
+      try {
+        await unlink(filePath);
+      } catch (err) {
+        if (isNodeError(err) && err.code === "ENOENT") {
+          return;
+        }
+        throw err;
+      }
+    })
+  );
+}
+function createCheckpointState(params) {
+  if (!params.configHash) {
+    throw new Error("configHash must be a non-empty string");
+  }
+  if (!params.artifactsDir) {
+    throw new Error("artifactsDir must be a non-empty string");
+  }
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  return {
+    version: 1,
+    startedAt: now,
+    checkpointedAt: now,
+    frontier: [],
+    visited: [],
+    skipped: [],
+    artifactsDir: params.artifactsDir,
+    pagesVisited: 0,
+    errors: [],
+    configHash: params.configHash
+  };
+}
+function setupAutoCheckpoint(getState, outputDir, intervalMs = DEFAULT_AUTO_CHECKPOINT_INTERVAL_MS) {
+  if (!outputDir) {
+    throw new Error("outputDir must be a non-empty string");
+  }
+  if (intervalMs <= 0) {
+    throw new Error("intervalMs must be a positive number");
+  }
+  const timer = setInterval(() => {
+    const state = getState();
+    writeCheckpoint(state, outputDir).catch(() => {
+    });
+  }, intervalMs);
+  if (typeof timer === "object" && "unref" in timer) {
+    timer.unref();
+  }
+  return {
+    stop: () => {
+      clearInterval(timer);
+    }
+  };
+}
+function isNodeError(err) {
+  return err instanceof Error && "code" in err;
+}
+
+// src/utils/progress.ts
+var ROLLING_WINDOW_SIZE = 10;
+var ProgressTracker = class {
+  pagesVisited = 0;
+  pagesTotal;
+  errorsCount = 0;
+  startTime;
+  currentUrls = /* @__PURE__ */ new Set();
+  /** Rolling window of timestamps (ms) when pages were recorded as visited */
+  visitTimestamps = [];
+  constructor(estimatedTotal) {
+    this.pagesTotal = estimatedTotal ?? 0;
+    this.startTime = Date.now();
+  }
+  /** Record a page visit. */
+  recordVisit(url) {
+    this.pagesVisited += 1;
+    this.visitTimestamps.push(Date.now());
+    if (this.visitTimestamps.length > ROLLING_WINDOW_SIZE) {
+      this.visitTimestamps.shift();
+    }
+  }
+  /** Record an error. */
+  recordError() {
+    this.errorsCount += 1;
+  }
+  /** Mark a URL as currently being crawled. */
+  startPage(url) {
+    this.currentUrls.add(url);
+  }
+  /** Mark a URL as done crawling. */
+  endPage(url) {
+    this.currentUrls.delete(url);
+  }
+  /** Update the estimated total page count. */
+  updateTotal(total) {
+    if (total < 0) {
+      throw new Error("Estimated total must be >= 0");
+    }
+    this.pagesTotal = total;
+  }
+  /** Get the current progress state snapshot. */
+  getState() {
+    const now = Date.now();
+    const elapsedMs = now - this.startTime;
+    const pps = this.calculatePagesPerSecond();
+    const remaining = this.calculateRemainingMs(pps);
+    return {
+      pagesVisited: this.pagesVisited,
+      pagesTotal: this.pagesTotal,
+      currentUrls: Array.from(this.currentUrls),
+      errorsCount: this.errorsCount,
+      startTime: this.startTime,
+      elapsedMs,
+      estimatedRemainingMs: remaining,
+      pagesPerSecond: pps
+    };
+  }
+  /**
+   * Format the current progress as a terminal-friendly string.
+   *
+   * Example: `[12/50] 2.3 p/s | ETA: 16s | Errors: 0 | Crawling: /about, /api/users`
+   */
+  format() {
+    const state = this.getState();
+    const pps = state.pagesPerSecond.toFixed(1);
+    const eta = state.estimatedRemainingMs > 0 ? `${Math.ceil(state.estimatedRemainingMs / 1e3)}s` : "--";
+    const crawling = state.currentUrls.length > 0 ? state.currentUrls.join(", ") : "idle";
+    return `[${state.pagesVisited}/${state.pagesTotal}] ${pps} p/s | ETA: ${eta} | Errors: ${state.errorsCount} | Crawling: ${crawling}`;
+  }
+  // ---------------------------------------------------------------------------
+  // Private
+  // ---------------------------------------------------------------------------
+  /**
+   * Calculate pages per second from the rolling window of visit timestamps.
+   * Uses the time span between the oldest and newest entry in the window.
+   */
+  calculatePagesPerSecond() {
+    const timestamps = this.visitTimestamps;
+    if (timestamps.length < 2) {
+      return 0;
+    }
+    const oldest = timestamps[0];
+    const newest = timestamps[timestamps.length - 1];
+    const spanMs = newest - oldest;
+    if (spanMs <= 0) {
+      return 0;
+    }
+    return (timestamps.length - 1) / spanMs * 1e3;
+  }
+  /**
+   * Estimate remaining milliseconds: (remaining pages) / pagesPerSecond * 1000
+   */
+  calculateRemainingMs(pps) {
+    if (pps <= 0 || this.pagesTotal <= 0) {
+      return 0;
+    }
+    const remaining = this.pagesTotal - this.pagesVisited;
+    if (remaining <= 0) {
+      return 0;
+    }
+    return remaining / pps * 1e3;
+  }
+};
+export {
+  ArchaeologistError,
+  AuthError,
+  BundleError,
+  CollectorError,
+  ConfigError,
+  ContextPool,
+  CrawlConfigSchema,
+  CrawlError,
+  DiffConfigSchema,
+  DiffError,
+  Frontier,
+  Logger,
+  NavigationError,
+  ProgressTracker,
+  ViewportSchema,
+  buildFlowGraph,
+  captureScreenshots,
+  createBundle,
+  createCheckpointState,
+  createNetworkLogger,
+  deleteCheckpoint,
+  detectAuthFailure,
+  detectCycles,
+  diffBundles,
+  dig,
+  escapeAttribute,
+  escapeHtml,
+  escapeJsonInHtml,
+  executeAuth,
+  generateDiffReportHtml,
+  generateMermaidDefinition,
+  generateOpenApiSpec,
+  generateReportHtml,
+  groupEndpoints,
+  logger,
+  normalizeEntryUrl,
+  parseViewport,
+  probeForms,
+  readCheckpoint,
+  refreshAuth,
+  scanPage,
+  setupAutoCheckpoint,
+  writeCheckpoint,
+  writeOpenApiSpec
+};
+//# sourceMappingURL=index.js.map