sentinel-scanner 2.4.1 → 2.5.0

package/src/commands/spider.ts

@@ -1,17 +1,9 @@
 import fs from "node:fs";
 import path from "node:path";
 import type { ArgumentsCamelCase, CommandModule } from "yargs";
-import SpiderScanner from "../modules/spider/index.js";
-import { createLogger } from "../utils/index.js";
-
-export type SpiderScannerCLIOptions = {
-	url: string;
-	depth?: number;
-	output?: string;
-	concurrency?: number;
-	timeout?: number;
-	retries?: number;
-};
+import { Spider } from "../spider/index.ts";
+import type { SpiderConstructorOptions } from "../spider/types/index.ts";
+import { createLogger } from "../utils/index.ts";
 
 const cliLogger = createLogger("CLI");
 
@@ -21,145 +13,88 @@ export const spiderCommand: CommandModule = {
 		"Crawl a website and get an array of URLs which are internal to the website",
 	builder: (yargs) => {
 		return yargs
-			.option("url", {
-				alias: "u",
+			.option("seed", {
+				alias: "s",
+				describe: "The seed URL to start crawling",
 				type: "string",
-				description: "The URL of the website to scan",
 				demandOption: true,
-				coerce: (url) => {
+				coerce: (arg) => {
 					try {
-						new URL(url);
+						new URL(arg);
 
-						return url;
+						return arg;
 					} catch (error) {
-						throw new Error(`Invalid URL: ${url}`);
+						cliLogger.error(error instanceof Error ? error.message : error);
+						process.exit(1);
 					}
 				},
 			})
-			.option("depth", {
+			.option("maxDepth", {
 				alias: "d",
+				describe: "The maximum depth to crawl",
 				type: "number",
-				description: "The maximum depth to crawl",
 				default: 250,
-				coerce: (depth) => {
-					if (depth < 0) {
-						throw new Error("Depth must be a positive number");
-					}
-
-					if (depth > 250) {
-						throw new Error("Depth must be less than 250");
-					}
-
-					return depth;
-				},
 			})
-			.option("output", {
-				alias: "o",
-				type: "string",
-				description:
-					"The output file to write the results to. Must be a JSON file",
-				coerce: (output) => {
-					try {
-						// Should throw an error if the path is invalid
-						// Should Be A JSON File
-						const resolvedPath = path.resolve(output);
-						const parsedPath = path.parse(resolvedPath);
-
-						if (parsedPath.ext !== ".json") {
-							throw new Error("Output file must be a JSON file");
-						}
-
-						if (fs.existsSync(resolvedPath)) {
-							throw new Error("Output file already exists");
-						}
-
-						return resolvedPath;
-					} catch (error) {
-						throw new Error(`Invalid output file: ${output}`);
-					}
-				},
-				default: getDefaultFilePath(),
+			.option("maxRetries", {
+				alias: "r",
+				describe: "The maximum retries for a failed request",
+				type: "number",
+				default: 3,
 			})
 			.option("concurrency", {
 				alias: "c",
+				describe: "The number of concurrent requests",
 				type: "number",
-				description: "The number of concurrent requests to make",
-				default: 10,
-				coerce: (concurrency) => {
-					if (concurrency < 1) {
-						throw new Error("Concurrency must be a positive number");
-					}
-
-					if (concurrency > 20) {
-						throw new Error("Concurrency must be less than 20");
-					}
-
-					return concurrency;
-				},
+				default: 30,
+			})
+			.option("ignoreExternalLinks", {
+				alias: "i",
+				describe: "Ignore external links",
+				type: "boolean",
+				default: true,
 			})
 			.option("timeout", {
 				alias: "t",
+				describe: "Request timeout in milliseconds",
 				type: "number",
-				description: "The timeout for each request in milliseconds",
-				default: 5000,
-				coerce: (timeout) => {
-					if (timeout < 0) {
-						throw new Error("Timeout must be a positive number");
-					}
-
-					if (timeout > 25_000) {
-						throw new Error("Timeout must be less than 25,000");
-					}
-
-					return timeout;
-				},
+				default: 8000,
 			})
-			.option("retries", {
-				alias: "r",
-				type: "number",
-				description: "The number of retries for each request",
-				default: 3,
-				coerce: (retries) => {
-					if (retries < 0) {
-						throw new Error("Retries must be a positive number");
-					}
-
-					if (retries > 10) {
-						throw new Error("Retries must be less than 10");
-					}
-
-					return retries;
-				},
+			.option("output", {
+				alias: "o",
+				describe: "Output file path",
+				type: "string",
+				default: getDefaultFilePath(),
 			});
 	},
-	handler: async (args) => {
+	handler: async (yargs) => {
+		const args = yargs as ArgumentsCamelCase<{
+			seed: string;
+			maxDepth: number;
+			maxRetries: number;
+			concurrency: number;
+			ignoreExternalLinks: boolean;
+			timeout: number;
+			output: string;
+		}>;
+		const opts: SpiderConstructorOptions = {
+			seed: args.seed,
+			maxDepth: args.maxDepth || 250,
+			maxRetries: args.maxRetries || 3,
+			concurrency: args.concurrency || 30,
+			ignoreExternalLinks:
+				args.ignoreExternalLinks === undefined
+					? true
+					: args.ignoreExternalLinks,
+			timeout: args.timeout || 8000,
+		};
+
+		const scanner = new Spider(opts);
 		try {
-			const argData = args as ArgumentsCamelCase<SpiderScannerCLIOptions>;
-
-			const scanner = new SpiderScanner(argData.url, {
-				depth: argData.depth ?? 250,
-				concurrency: argData.concurrency ?? 10,
-				timeout: argData.timeout ?? 5000,
-				retries: argData.retries ?? 3,
-			});
-
-			cliLogger.info("Starting to crawl website");
-
-			const results = await scanner.crawl();
-
-			if (argData.output) {
-				fs.writeFileSync(argData.output, JSON.stringify(results, null, 2));
-				cliLogger.info(`Results written to ${argData.output}`);
-			} else {
-				const resolvedPath = getDefaultFilePath();
-				fs.writeFileSync(resolvedPath, JSON.stringify(results, null, 2));
-				cliLogger.info(`Results written to ${resolvedPath}`);
-			}
+			const results = await scanner.scan();
+			fs.writeFileSync(args.output, JSON.stringify(results, null, 2));
+			cliLogger.info(`Results saved to ${args.output}`);
 		} catch (error) {
-			if (error instanceof Error) {
-				cliLogger.error(error.message);
-			}
-			cliLogger.error("Failed to run spider command");
+			cliLogger.error(error instanceof Error ? error.message : error);
 			process.exit(1);
 		}
 	},
@@ -187,7 +122,7 @@ const getDefaultFilePath = () => {
 		}
 
 		return resolvedPath;
-	} catch (error) {
+	} catch (_) {
 		throw new Error("Invalid output file");
 	}
 };

package/src/index.ts

@@ -1,27 +1,24 @@
-import HeaderScanner, {
-	type HeadersData,
-	type HeaderScannerOptions,
-} from "./modules/headers/index.js";
-import PortsScanner, { type PortsScannerOpts } from "./modules/ports/index.js";
-import SpiderScanner, {
-	type SpiderScannerOptions,
-} from "./modules/spider/index.js";
-import SqliScanner, {
-	type SqliConstructorOpts,
-	type SQLErrors,
-	type SupportedDatabases,
-} from "./modules/sqli/index.js";
-import XSSScanner, { type XSSConstructorOpts } from "./modules/xss/index.js";
-import { Vulnerability } from "./utils/types.js";
+#!/usr/bin/env node --no-warnings
 
-export { SpiderScanner, type SpiderScannerOptions };
-export { XSSScanner, type XSSConstructorOpts };
-export { Vulnerability };
-export { HeaderScanner, type HeadersData, type HeaderScannerOptions };
-export {
-	SqliScanner,
-	type SqliConstructorOpts,
-	type SQLErrors,
-	type SupportedDatabases,
-};
-export { PortsScanner, type PortsScannerOpts };
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { spiderCommand } from "./commands/spider.ts";
+import { getPackageData } from "./utils/index.ts";
+
+const { name, version } = getPackageData();
+
+const commandHandler = yargs(hideBin(process.argv));
+
+commandHandler.demandCommand();
+commandHandler.version(version);
+commandHandler.scriptName(name);
+commandHandler.usage("Usage: $0 <command> [options]");
+commandHandler.help().alias("help", "h");
+commandHandler.version().alias("version", "v");
+commandHandler.strict();
+commandHandler.showHelpOnFail(true);
+
+commandHandler.command(spiderCommand);
+
+commandHandler.version().alias("version", "v");
+commandHandler.parse();

package/src/spider/index.ts

@@ -0,0 +1,345 @@
+import { parse } from "node-html-parser";
+import {
+	chunkArray,
+	createLogger,
+	safeStringify,
+	withRetries,
+} from "../utils/index.ts";
+import type { SpiderConstructorOptions, SpiderResults } from "./types/index.ts";
+import { SpiderConstructorOptionsSchema } from "./types/schema.ts";
+
+/**
+ * The Spider class is used to scan a web application by crawling through the URLs and extracting information.
+ * The Spider class uses a breadth-first search algorithm to crawl through the URLs.
+ */
+export class Spider {
+	/**
+	 * The logger instance for the Spider class.
+	 * We use this to log messages to the console.
+	 */
+	private logger = createLogger("Spider");
+	/**
+	 * The options provided to the Spider constructor.
+	 * These options are used to configure the behavior of the Spider.
+	 *
+	 * @see SpiderConstructorOptionsSchema
+	 */
+	private options: SpiderConstructorOptions;
+
+	constructor(opts: SpiderConstructorOptions) {
+		/**
+		 * Validate the options provided to the Spider constructor.
+		 */
+		const result = SpiderConstructorOptionsSchema.safeParse(opts);
+
+		if (result.error !== undefined || !result.data) {
+			/**
+			 * If the options are invalid, we should throw an error and exit the process.
+			 */
+			this.logger.error("Invalid options provided to the Spider constructor.");
+			throw new Error(
+				`Invalid options provided to the Spider constructor: ${safeStringify(
+					result.error,
+				)}`,
+			);
+		}
+
+		/**
+		 * If the options are valid, we can proceed with the initialization of the Spider.
+		 */
+		this.options = SpiderConstructorOptionsSchema.parse(opts);
+
+		/**
+		 * Log the options provided to the Spider constructor.
+		 */
+		this.logger.info(
+			`Spider created with options: ${safeStringify(this.options)}`,
+		);
+	}
+
+	private isInternalUrl(url: string): boolean {
+		/**
+		 * Check if the URL starts with the seed URL.
+		 * If it does, then it is an internal URL.
+		 * Otherwise, it is an external URL.
+		 */
+		return new URL(url).origin === new URL(this.options.seed).origin;
+	}
+
+	/**
+	 * Fetches the page at the given URL.
+	 * @param url - The URL of the page to fetch.
+	 * @returns A promise that resolves to the fetched page content as a string.
+	 */
+	private async fetchPage(url: string): Promise<string | null> {
+		const fetchUrl = (url: string) => {
+			this.logger.info(`Fetching URL: ${url}`);
+			/**
+			 * We return a promise that resolves when the first of the following promises resolves.
+			 * This allows us to handle cases where the request takes too long to complete.
+			 */
+			return Promise.race([
+				/**
+				 * We use the `fetch` API to fetch the page at the given URL.
+				 */
+				fetch(url, {
+					/**
+					 * We set the `redirect` option to "follow" to follow redirects.
+					 *
+					 * @see https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch#parameters
+					 */
+					redirect: "follow",
+				})
+					/**
+					 * We extract the text content of the response.
+					 * This will be the HTML content of the page.
+					 */
+					.then((res) => res.text()),
+				/**
+				 * We create a promise that resolves to null after the specified timeout.
+				 * This handles cases where the request takes too long to complete.
+				 */
+				new Promise<string | null>((resolve) =>
+					setTimeout(() => resolve(null), this.options.timeout),
+				),
+			]);
+		};
+
+		/**
+		 * Fetch the page at the given URL.
+		 * We use the `withRetries` utility function to retry the fetch operation
+		 * in case of a failure.
+		 */
+		return await withRetries(fetchUrl, [url], this.options.maxRetries);
+	}
+
+	private normalizeUrl(baseUrl: string, href: string): string | null {
+		try {
+			if (href.startsWith("http://") || href.startsWith("https://")) {
+				return new URL(href).toString();
+			}
+
+			if (href.startsWith("/")) {
+				return new URL(href, baseUrl).toString();
+			}
+
+			const url = new URL(href, baseUrl);
+
+			return url.toString();
+		} catch (error) {
+			/**
+			 * If an error occurs while normalizing the URL, log the error and return null.
+			 */
+			this.logger.error(`Error normalizing URL: ${href}`);
+			this.logger.error(error);
+			return null;
+		}
+	}
+
+	/**
+	 * Extracts URLs from the given HTML content using a URL regex and a base URL.
+	 *
+	 * @param html - The HTML content from which to extract URLs.
+	 * @param baseUrl - The base URL used to normalize the extracted URLs.
+	 * @returns An array of extracted URLs.
+	 */
+	private extractUrls(html: string, baseUrl: string) {
+		const extracted = new Set<string>();
+
+		/**
+		 * Parse the HTML content using the `parse` function from the `node-html-parser` package.
+		 */
+		const root = parse(html);
+
+		/**
+		 * Find all the anchor elements in the HTML content.
+		 */
+		const anchors = root
+			.querySelectorAll("a")
+			.concat(root.querySelectorAll("link"))
+			.concat(root.querySelectorAll("area"))
+			.concat(root.querySelectorAll("base"));
+
+		/**
+		 * Iterate over the anchor elements.
+		 */
+		for (const anchor of anchors) {
+			/**
+			 * Extract the `href` attribute from the anchor element.
+			 */
+			const href = anchor.getAttribute("href");
+
+			/**
+			 * If the `href` attribute is not present, skip to the next anchor element.
+			 */
+			if (!href) {
+				continue;
+			}
+
+			/**
+			 * Normalize the extracted URL using the base URL.
+			 */
+			const normalized = this.normalizeUrl(baseUrl, href);
+
+			if (normalized) {
+				if (
+					this.options.ignoreExternalLinks &&
+					!this.isInternalUrl(normalized)
+				) {
+					this.logger.info(`Ignoring external URL: ${normalized}`);
+					continue;
+				}
+
+				extracted.add(normalized);
+			}
+		}
+
+		/**
+		 * Return the array of extracted URLs.
+		 */
+		return Array.from(extracted);
+	}
+
+	/**
+	 * Scans the web application by crawling through the URLs and extracting information.
+	 * Returns the spider results containing the seed URL and the visited URLs.
+	 *
+	 * @returns A promise that resolves to the spider results.
+	 * @see SpiderResults
+	 */
+	public async scan(): Promise<SpiderResults> {
+		this.logger.info("Starting scan...");
+		/**
+		 * Create a set to keep track of visited URLs.
+		 * This set will be used to avoid visiting the same URL multiple times.
+		 * Initially, the set is empty.
+		 */
+		const visited = new Set<string>();
+		/**
+		 * Create a queue of URLs to visit.
+		 * Initially, the queue contains only the seed URL.
+		 */
+		const queue = new Set<string>([this.options.seed]);
+
+		/**
+		 * Process a URL.
+		 * This function fetches the content of the URL, extracts URLs from the content, and adds the extracted URLs to the queue.
+		 * It also adds the current URL to the set of visited URLs.
+		 *
+		 * @param url - The URL to process.
+		 * @returns A promise that resolves to an array of extracted URLs.
+		 */
+		const processUrl = async (url: string) => {
+			this.logger.info(`Processing URL: ${url}`);
+			/**
+			 * Fetch the page at the given URL.
+			 */
+			const pageContent = await this.fetchPage(url);
+
+			/**
+			 * Extract URLs from the fetched page content.
+			 * and log the number of URLs extracted.
+			 */
+			if (!pageContent) {
+				this.logger.warn(`Failed to fetch URL: ${url}`);
+				return [];
+			}
+
+			const extractedUrls = this.extractUrls(pageContent, url);
+			this.logger.info(`Extracted ${extractedUrls.length} URLs`);
+
+			/**
+			 * Add the current URL to the set of visited URLs.
+			 */
+			visited.add(url);
+
+			/**
+			 * Return the extracted URLs.
+			 */
+			return extractedUrls;
+		};
+
+		/**
+		 * Process a batch of URLs.
+		 * This function fetches the content of the URLs in the batch,
+		 * extracts URLs from the content, and adds the extracted URLs to the queue.
+		 * It also removes the processed URLs from the queue.
+		 *
+		 * @param batch - The batch of URLs to process.
+		 * @returns A promise that resolves when the batch is processed.
+		 */
+		const processBatch = async (batch: string[]) => {
+			/**
+			 * Process the URLs in the current batch.
+			 */
+			const promises = batch.map(processUrl);
+			/**
+			 * Wait for all the promises to resolve.
+			 */
+			const results = await Promise.all(promises);
+			/**
+			 * Flatten the results to get a single array of URLs.
+			 * Then log the number of URLs processed.
+			 */
+			const urls = results.flat();
+			this.logger.info(`Processed ${batch.length} URLs`);
+
+			/**
+			 * Add the extracted URLs to the queue.
+			 */
+			for (const url of urls) {
+				this.logger.info(`Adding URL to queue: ${url}`);
+				if (!visited.has(url)) {
+					this.logger.info(`URL not visited: ${url}`);
+					queue.add(url);
+					visited.add(url);
+				}
+			}
+
+			/**
+			 * Remove the processed URLs from the queue.
+			 */
+			for (const url of batch) {
+				queue.delete(url);
+			}
+		};
+
+		/**
+		 * Initialize the current depth to 0.
+		 */
+		let currentDepth = 0;
+
+		while (queue.size > 0 && currentDepth < this.options.maxDepth) {
+			this.logger.info(`Processing depth: ${currentDepth}`);
+			/**
+			 * Split the queue into batches of URLs.
+			 */
+			const batches = chunkArray(Array.from(queue), this.options.concurrency);
+			/**
+			 * Iterate over the batches of URLs.
+			 */
+			for (const batch of batches) {
+				/**
+				 * Process the current batch of URLs.
+				 */
+				await withRetries(processBatch, [batch], this.options.maxRetries);
+			}
+
+			/**
+			 * Increment the current depth.
+			 */
+			currentDepth++;
+			this.logger.silly(`Processed depth: ${currentDepth}`);
+		}
+
+		/**
+		 * Return The Spider Results
+		 *
+		 * @see SpiderResults
+		 */
+		return {
+			seed: this.options.seed,
+			urls: Array.from(visited),
+		};
+	}
+}

package/src/spider/types/index.ts

@@ -0,0 +1,21 @@
+import type { z } from "zod";
+import type {
+	SpiderConstructorOptionsSchema,
+	SpiderResultSchema,
+} from "./schema.ts";
+
+/**
+ * Represents the options for constructing a Spider object.
+ *
+ * @see SpiderConstructorOptionsSchema
+ */
+export type SpiderConstructorOptions = z.infer<
+	typeof SpiderConstructorOptionsSchema
+>;
+
+/**
+ * Represents the result of a Spider object.
+ *
+ * @see SpiderResultSchema
+ */
+export type SpiderResults = z.infer<typeof SpiderResultSchema>;

package/src/spider/types/schema.ts

@@ -0,0 +1,54 @@
+import { z } from "zod";
+
+/**
+ * Options for constructing a Spider instance.
+ */
+export const SpiderConstructorOptionsSchema = z
+	.object({
+		/**
+		 * The seed URL for the spider to start crawling from.
+		 */
+		seed: z.string().url(),
+
+		/**
+		 * The maximum depth of crawling. Defaults to 250.
+		 */
+		maxDepth: z.number().int().positive().max(250).default(250),
+
+		/**
+		 * The concurrency level for crawling. Defaults to 10.
+		 */
+		concurrency: z.number().int().positive().max(30).default(30),
+
+		/**
+		 * Whether to ignore external links. Defaults to true.
+		 */
+		ignoreExternalLinks: z.boolean().default(true),
+		/**
+		 * The maximum number of retries for failed requests. Defaults to 3.
+		 */
+		maxRetries: z.number().int().positive().max(10).default(3),
+
+		/**
+		 * The timeout for requests in milliseconds. Defaults to 5000.
+		 */
+		timeout: z.number().int().positive().max(60_000).default(5000),
+	})
+	/**
+	 * Ensure that default values are applied when the options are not provided.
+	 */
+	.strict();
+
+/**
+ * Represents the result of a spider operation.
+ */
+export const SpiderResultSchema = z.object({
+	/**
+	 * The seed URL used for the spider operation.
+	 */
+	seed: z.string(),
+	/**
+	 * An array of URLs found during the spider operation.
+	 */
+	urls: z.array(z.string()),
+});