sentinel-scanner 1.0.1 → 1.1.0-alpha.2

package/src/commands/spider.ts

@@ -0,0 +1,193 @@
+import fs from "node:fs";
+import path from "node:path";
+import type { ArgumentsCamelCase, CommandModule } from "yargs";
+import SpiderScanner from "../modules/spider";
+import { createLogger } from "../utils";
+
+export type SpiderScannerCLIOptions = {
+	url: string;
+	depth?: number;
+	output?: string;
+	concurrency?: number;
+	timeout?: number;
+	retries?: number;
+};
+
+const cliLogger = createLogger("CLI");
+
+export const spiderCommand: CommandModule = {
+	command: "spider",
+	describe:
+		"Crawl a website and get an array of URLs which are internal to the website",
+	builder: (yargs) => {
+		return yargs
+			.option("url", {
+				alias: "u",
+				type: "string",
+				description: "The URL of the website to scan",
+				demandOption: true,
+				coerce: (url) => {
+					try {
+						new URL(url);
+
+						return url;
+					} catch (error) {
+						throw new Error(`Invalid URL: ${url}`);
+					}
+				},
+			})
+			.option("depth", {
+				alias: "d",
+				type: "number",
+				description: "The maximum depth to crawl",
+				default: 250,
+				coerce: (depth) => {
+					if (depth < 0) {
+						throw new Error("Depth must be a positive number");
+					}
+
+					if (depth > 250) {
+						throw new Error("Depth must be less than 250");
+					}
+
+					return depth;
+				},
+			})
+			.option("output", {
+				alias: "o",
+				type: "string",
+				description:
+					"The output file to write the results to. Must be a JSON file",
+				coerce: (output) => {
+					try {
+						// Should throw an error if the path is invalid
+						// Should Be A JSON File
+						const resolvedPath = path.resolve(output);
+						const parsedPath = path.parse(resolvedPath);
+
+						if (parsedPath.ext !== ".json") {
+							throw new Error("Output file must be a JSON file");
+						}
+
+						if (fs.existsSync(resolvedPath)) {
+							throw new Error("Output file already exists");
+						}
+
+						return resolvedPath;
+					} catch (error) {
+						throw new Error(`Invalid output file: ${output}`);
+					}
+				},
+				default: getDefaultFilePath(),
+			})
+			.option("concurrency", {
+				alias: "c",
+				type: "number",
+				description: "The number of concurrent requests to make",
+				default: 10,
+				coerce: (concurrency) => {
+					if (concurrency < 1) {
+						throw new Error("Concurrency must be a positive number");
+					}
+
+					if (concurrency > 20) {
+						throw new Error("Concurrency must be less than 20");
+					}
+
+					return concurrency;
+				},
+			})
+			.option("timeout", {
+				alias: "t",
+				type: "number",
+				description: "The timeout for each request in milliseconds",
+				default: 5000,
+				coerce: (timeout) => {
+					if (timeout < 0) {
+						throw new Error("Timeout must be a positive number");
+					}
+
+					if (timeout > 25_000) {
+						throw new Error("Timeout must be less than 25,000");
+					}
+
+					return timeout;
+				},
+			})
+			.option("retries", {
+				alias: "r",
+				type: "number",
+				description: "The number of retries for each request",
+				default: 3,
+				coerce: (retries) => {
+					if (retries < 0) {
+						throw new Error("Retries must be a positive number");
+					}
+
+					if (retries > 10) {
+						throw new Error("Retries must be less than 10");
+					}
+
+					return retries;
+				},
+			});
+	},
+	handler: async (args) => {
+		try {
+			const argData = args as ArgumentsCamelCase<SpiderScannerCLIOptions>;
+
+			const scanner = new SpiderScanner(argData.url, {
+				depth: argData.depth ?? 250,
+				concurrency: argData.concurrency ?? 10,
+				timeout: argData.timeout ?? 5000,
+				retries: argData.retries ?? 3,
+			});
+
+			cliLogger.info("Starting to crawl website");
+
+			const results = await scanner.crawl();
+
+			if (argData.output) {
+				fs.writeFileSync(argData.output, JSON.stringify(results, null, 2));
+				cliLogger.info(`Results written to ${argData.output}`);
+			} else {
+				const resolvedPath = getDefaultFilePath();
+				fs.writeFileSync(resolvedPath, JSON.stringify(results, null, 2));
+				cliLogger.info(`Results written to ${resolvedPath}`);
+			}
+		} catch (error) {
+			if (error instanceof Error) {
+				cliLogger.error(error.message);
+			}
+			cliLogger.error("Failed to run spider command");
+			process.exit(1);
+		}
+	},
+};
+
+const getDefaultFilePath = () => {
+	try {
+		const resolvedDir = path.resolve("sentinel_output");
+		// Check If Directory Exists
+		if (!fs.existsSync(resolvedDir)) {
+			fs.mkdirSync(resolvedDir);
+		}
+
+		const resolvedPath = path.resolve(
+			`sentinel_output/spider_${Date.now()}.json`,
+		);
+		// Check If File Exists
+		if (fs.existsSync(resolvedPath)) {
+			throw new Error("Output file already exists");
+		}
+		const parsedPath = path.parse(resolvedPath);
+
+		if (parsedPath.ext !== ".json") {
+			throw new Error("Output file must be a JSON file");
+		}
+
+		return resolvedPath;
+	} catch (error) {
+		throw new Error("Invalid output file");
+	}
+};

package/src/index.ts

@@ -1,27 +1,3 @@
-#!/usr/bin/env node
+import SpiderScanner, { type SpiderScannerOptions } from "./modules/spider";
 
-import { Command } from "commander";
-// @ts-ignore: For TypeScript compatibility when importing JSON files
-import packageData from "../package.json";
-
-// Create a new Command object
-const program = new Command();
-
-// Set version, name, and description from the package.json
-program
-	.version(packageData.version)
-	.name(packageData.name)
-	.description(packageData.description);
-
-// Add a help command explicitly if needed
-program.helpOption("-h, --help", "Display help for command");
-
-// Parse command-line arguments
-program.parse(process.argv);
-
-const options = program.opts();
-
-// If no arguments are provided, display help
-if (Object.keys(options).length === 0) {
-	program.help();
-}
+export { SpiderScanner, type SpiderScannerOptions };

package/src/modules/spider/index.ts

@@ -0,0 +1,212 @@
+import fetch from "isomorphic-fetch";
+import jsdom from "jsdom";
+import UserAgent from "user-agents";
+import { createLogger } from "../../utils";
+
+export interface SpiderScannerOptions {
+	depth?: number;
+	concurrency?: number;
+	retries?: number;
+	timeout?: number;
+}
+
+export default class SpiderScanner {
+	private header: Record<string, string> = {
+		"User-Agent": new UserAgent().toString(),
+	};
+	private url: URL;
+	private logger = createLogger("SpiderScanner");
+
+	private depth: number;
+	private concurrency: number;
+	private retries: number;
+	private timeout: number;
+
+	constructor(url: string, options: SpiderScannerOptions = {}) {
+		const {
+			depth = 250,
+			concurrency = 5,
+			retries = 3,
+			timeout = 5000,
+		} = options;
+		this.depth = depth;
+		this.concurrency = concurrency;
+		this.retries = retries;
+		this.timeout = timeout;
+
+		try {
+			this.url = new URL(url);
+			this.logger.info(
+				`Initialized with URL: ${url}, User-Agent: ${this.header["User-Agent"]}`,
+			);
+		} catch (error) {
+			if (error instanceof TypeError) {
+				this.logger.error("Invalid URL");
+				throw new Error("Invalid URL");
+			}
+			this.logger.error(`Unexpected error in constructor: ${error}`);
+			throw error;
+		}
+	}
+
+	private normalizeDomain(domain: string): string {
+		return domain.startsWith("www.") ? domain.slice(4) : domain;
+	}
+
+	private convertRelativeUrlToAbsolute(url: string): string {
+		return new URL(url, this.url.toString()).toString();
+	}
+
+	private isInternalLink(url: string): boolean {
+		try {
+			const parsedUrl = new URL(url, this.url.href);
+			if (!["http:", "https:"].includes(parsedUrl.protocol)) {
+				return false;
+			}
+			const baseDomain = this.normalizeDomain(this.url.hostname);
+			const parsedDomain = this.normalizeDomain(parsedUrl.hostname);
+			return parsedDomain === baseDomain;
+		} catch (error) {
+			this.logger.warn(`Error parsing URL: ${url} - ${error}`);
+			return false;
+		}
+	}
+
+	private async fetchWithRetries(
+		url: string,
+		retries: number,
+	): Promise<string | null> {
+		for (let attempt = 1; attempt <= retries; attempt++) {
+			const controller = new AbortController();
+			const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+
+			try {
+				this.logger.debug(`Fetching URL (Attempt ${attempt}): ${url}`);
+				const randomUserAgent = new UserAgent().toString();
+				this.logger.info(`Changing User-Agent to: ${randomUserAgent}`);
+				this.header["User-Agent"] = randomUserAgent;
+				const response = await fetch(url, {
+					headers: this.header,
+					signal: controller.signal,
+					redirect: "follow",
+				});
+
+				clearTimeout(timeoutId);
+
+				if (response.ok) {
+					this.logger.info(`Successfully fetched URL: ${url}`);
+					return await response.text();
+				}
+
+				this.logger.warn(`Failed to fetch URL (${response.status}): ${url}`);
+			} catch (error) {
+				if ((error as Error).name === "AbortError") {
+					this.logger.warn(`Fetch timed out: ${url}`);
+				} else {
+					this.logger.error(`Error fetching URL: ${url} - ${error}`);
+				}
+			}
+		}
+		return null;
+	}
+
+	private extractLinks(html: string): string[] {
+		const { JSDOM } = jsdom;
+		const dom = new JSDOM(html);
+		const links = Array.from(dom.window.document.querySelectorAll("a"));
+		const hrefs = links.map((link) => link.href);
+		const internalLinks = hrefs.filter((href) => this.isInternalLink(href));
+		this.logger.debug(
+			`Extracted ${internalLinks.length} internal links from HTML content`,
+		);
+		return internalLinks.map((link) => this.convertRelativeUrlToAbsolute(link));
+	}
+
+	public async crawl(): Promise<Array<string>> {
+		const visited = new Set<string>();
+		const queue = new Set<string>([this.url.href]);
+		const resultLinks = new Set<string>();
+
+		// Assets to ignore
+		const assetExtensions = [
+			".css",
+			".js",
+			".png",
+			".jpg",
+			".jpeg",
+			".gif",
+			".svg",
+			".ico",
+			".webp",
+			".mp4",
+			".mp3",
+			".wav",
+			".avi",
+			".mov",
+			".webm",
+			".pdf",
+			".doc",
+			".docx",
+			".xls",
+			".xlsx",
+			".ppt",
+			".pptx",
+			".zip",
+			".rar",
+			".tar",
+			".gz",
+		];
+
+		const fetchAndExtract = async (currentUrl: string) => {
+			if (visited.has(currentUrl)) {
+				this.logger.debug(`Skipping already visited URL: ${currentUrl}`);
+				return;
+			}
+			visited.add(currentUrl);
+			this.logger.info(`Visiting URL: ${currentUrl}`);
+
+			const html = await this.fetchWithRetries(currentUrl, this.retries);
+			if (!html) return;
+
+			const links = this.extractLinks(html);
+
+			// Filter out asset links
+			for (const link of links) {
+				if (assetExtensions.some((ext) => link.endsWith(ext))) {
+					this.logger.debug(`Ignoring asset link: ${link}`);
+					continue;
+				}
+				this.logger.debug(`Found link: ${link}`);
+			}
+
+			for (const link of links) {
+				if (!visited.has(link) && queue.size < this.depth) {
+					queue.add(link);
+					this.logger.debug(`Added to queue: ${link}`);
+				}
+			}
+			resultLinks.add(currentUrl);
+		};
+
+		const processBatch = async () => {
+			const batch = Array.from(queue).slice(0, this.concurrency);
+			for (const url of batch) {
+				queue.delete(url);
+			}
+			await Promise.allSettled(batch.map((url) => fetchAndExtract(url)));
+		};
+
+		this.logger.info(
+			`Starting crawl with depth: ${this.depth}, concurrency: ${this.concurrency}`,
+		);
+		while (queue.size > 0 && visited.size < this.depth) {
+			await processBatch();
+		}
+
+		this.logger.info(
+			`Crawling completed. Total pages visited: ${resultLinks.size}`,
+		);
+
+		return Array.from(resultLinks);
+	}
+}

package/src/utils/index.ts

@@ -0,0 +1,29 @@
+import winston from "winston";
+
+export const createLogger = (label: string) =>
+	winston.createLogger({
+		levels: {
+			error: 0,
+			warn: 1,
+			info: 2,
+			http: 3,
+			verbose: 4,
+			debug: 5,
+			silly: 6,
+		},
+		format: winston.format.combine(
+			winston.format.label({ label }),
+			winston.format.colorize(),
+			winston.format.timestamp({
+				format: () => {
+					return new Date().toLocaleString("en-US");
+				},
+			}),
+			winston.format.align(),
+			winston.format.printf(
+				(info) =>
+					`\x1b[34m(${info.label})\x1b[0m \x1b[33m${info.timestamp}\x1b[0m [${info.level}]: ${info.message}`,
+			),
+		),
+		transports: [new winston.transports.Console()],
+	});

package/tsconfig.json

@@ -2,13 +2,13 @@
   "include": ["./src/**/*.ts"],
   "compilerOptions": {
     "lib": ["es2023"],
-    "module": "nodenext",
+    "module": "CommonJS",
     "target": "es2022",
-    "moduleResolution": "nodenext",
+    "moduleResolution": "node",
+    "allowSyntheticDefaultImports": true,
 
     "rootDir": "./src",
     "outDir": "build",
-    "resolvePackageJsonImports": true,
 
     "strict": true,
     "noUncheckedIndexedAccess": true,
@@ -18,7 +18,7 @@
     "forceConsistentCasingInFileNames": true,
     "declaration": true,
     "resolveJsonModule": true,
-    "emitDeclarationOnly": true,
-    "allowImportingTsExtensions": true
-  }
+    "allowImportingTsExtensions": false
+  },
+  "exclude": ["src/__tests__/**/*"]
 }