web-to-markdown 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,222 @@
+interface ConvertOptions {
+    /** Force headless browser rendering (for SPAs). */
+    readonly browser: boolean;
+    /** Convert full HTML without content extraction. */
+    readonly raw: boolean;
+    /** Include YAML frontmatter with metadata. */
+    readonly frontmatter: boolean;
+    /** Strip images from output. */
+    readonly noImages: boolean;
+    /** Timeout in milliseconds for page loading. */
+    readonly timeout: number;
+    /** Output file path (undefined means stdout). */
+    readonly output?: string;
+}
+interface FetchResult {
+    /** The raw HTML string of the page. */
+    readonly html: string;
+    /** The final URL after any redirects. */
+    readonly finalUrl: string;
+}
+interface PageMetadata {
+    readonly title: string | null;
+    readonly byline: string | null;
+    readonly excerpt: string | null;
+    readonly siteName: string | null;
+    readonly publishedTime: string | null;
+    readonly lang: string | null;
+}
+interface ExtractResult {
+    /** Cleaned HTML content (article body). */
+    readonly content: string;
+    /** Metadata about the page. */
+    readonly metadata: PageMetadata;
+}
+interface ConvertResult {
+    /** The Markdown output string. */
+    readonly markdown: string;
+    /** Metadata extracted from the page. */
+    readonly metadata: PageMetadata;
+    /** Non-fatal warnings generated during conversion. */
+    readonly warnings: readonly string[];
+}
+
+/**
+ * Validate a URL for safety and resolve its hostname to an IP.
+ *
+ * Checks protocol, hostname blocklists, and private IP ranges.
+ * Performs a single DNS lookup and returns the resolved IP for
+ * pinned connections that prevent DNS rebinding.
+ *
+ * @param url - The URL string to validate.
+ * @returns The resolved IP address to pin connections to.
+ * @throws {ValidationError} If the URL is malformed or uses an unsupported protocol.
+ * @throws {SSRFError} If the hostname or resolved IP is private/internal.
+ * @throws {NetworkError} If DNS resolution fails.
+ */
+declare function validateUrl(url: string): Promise<string>;
+/**
+ * Fetch a URL and return raw text without Content-Type validation.
+ * Uses the same SSRF protections and DNS pinning as fetchWithHttp.
+ * Returns null on any failure (network error, non-2xx status, etc.).
+ */
+declare function fetchRawText(url: string, timeout: number): Promise<{
+    body: string;
+    contentType: string;
+} | null>;
+/**
+ * Fetch a page using either HTTP or headless browser.
+ *
+ * Validates the URL for SSRF and resolves DNS once.
+ * The resolved IP is pinned to the connection to prevent DNS rebinding.
+ *
+ * @param url - The URL to fetch.
+ * @param options - Fetch options.
+ * @param options.browser - If `true`, use Playwright headless browser for SPAs.
+ * @param options.timeout - Request timeout in milliseconds (clamped to 1–300 s).
+ * @returns The raw HTML and final URL after redirects.
+ * @throws {ValidationError} If the URL is invalid.
+ * @throws {SSRFError} If the URL targets a private/internal host.
+ * @throws {NetworkError} On timeout, DNS failure, or HTTP error.
+ * @throws {ContentError} If the response is too large or not HTML.
+ */
+declare function fetchPage(url: string, options: Readonly<{
+    browser: boolean;
+    timeout: number;
+}>): Promise<FetchResult>;
+
+/**
+ * Extract the main article content from raw HTML using
+ * Mozilla Readability + linkedom DOM parsing.
+ *
+ * @param html - The raw HTML string to process.
+ * @param url - The page URL, used for resolving relative links.
+ * @returns The extracted content and metadata, or `null` if extraction fails.
+ */
+declare function extractContent(html: string, url: string): ExtractResult | null;
+
+interface MdxExtractResult {
+    /** Clean markdown content (MDX components stripped). */
+    readonly markdown: string;
+    /** Metadata parsed from frontmatter. */
+    readonly metadata: PageMetadata;
+}
+/**
+ * Attempt to extract raw MDX content from Next.js RSC (React Server Components)
+ * payloads embedded in the HTML. Many Next.js documentation sites ship the full
+ * markdown source inside `self.__next_f.push([1,"..."])` script tags. This
+ * content includes everything — even collapsed accordion and tab content that
+ * never renders to the DOM.
+ *
+ * @param html - The raw HTML string of the page.
+ * @param url - The page URL, used for resolving relative links.
+ * @returns Extracted markdown and metadata, or `null` if no RSC MDX found.
+ */
+declare function extractMdx(html: string, url: string): MdxExtractResult | null;
+/**
+ * Process raw MDX/markdown text: parse frontmatter, strip JSX components,
+ * resolve relative URLs, and clean up whitespace.
+ *
+ * Used for processing .md endpoint responses and raw MDX from RSC chunks.
+ */
+declare function processRawMdx(mdx: string, url: string): MdxExtractResult;
+
+interface ConverterOptions {
+    /** Base URL for resolving relative URLs to absolute. */
+    baseUrl?: string;
+    /** If true, strip all images from output. */
+    stripImages?: boolean;
+}
+/**
+ * Convert an HTML string to Markdown.
+ *
+ * Uses Turndown with GFM extensions. Resolves relative URLs to absolute
+ * when `baseUrl` is provided. Filters dangerous protocols and optionally
+ * strips images.
+ *
+ * @param html - The HTML string to convert.
+ * @param options - Conversion options.
+ * @param options.baseUrl - Base URL for resolving relative links.
+ * @param options.stripImages - If `true`, remove all images from output.
+ * @returns A Markdown string with trailing newline.
+ * @throws {ContentError} If the HTML is too deeply nested to process.
+ */
+declare function htmlToMarkdown(html: string, options?: ConverterOptions): string;
+
+/**
+ * Custom error classes for web-to-markdown.
+ *
+ * Typed errors enable consumers to handle specific failure modes
+ * programmatically (e.g. retry on timeout, show SSRF to user, etc.)
+ * instead of parsing error message strings.
+ *
+ * All errors support the standard `cause` property for error chaining
+ * (ES2022 Error Cause), preserving the original error context.
+ */
+/** Options accepted by all web-to-markdown error constructors. */
+interface ErrorOptions {
+    cause?: unknown;
+}
+/**
+ * Base class for all web-to-markdown errors.
+ * Consumers can catch this to handle any library error.
+ */
+declare class MarkitdownError extends Error {
+    constructor(message: string, options?: ErrorOptions);
+}
+/**
+ * Thrown when a URL is invalid or uses an unsupported protocol.
+ */
+declare class ValidationError extends MarkitdownError {
+    constructor(message: string, options?: ErrorOptions);
+}
+/**
+ * Thrown when a request is blocked due to SSRF protection
+ * (private IPs, internal hostnames, DNS rebinding, etc.)
+ */
+declare class SSRFError extends MarkitdownError {
+    constructor(message: string, options?: ErrorOptions);
+}
+/**
+ * Thrown when a network request fails (timeout, DNS failure, HTTP error, etc.)
+ */
+declare class NetworkError extends MarkitdownError {
+    readonly statusCode?: number;
+    constructor(message: string, statusCode?: number, options?: ErrorOptions);
+}
+/**
+ * Thrown when content cannot be processed (too large, too deeply nested, etc.)
+ */
+declare class ContentError extends MarkitdownError {
+    constructor(message: string, options?: ErrorOptions);
+}
+
+/**
+ * Convert a URL to Markdown.
+ *
+ * Primary API for both CLI and programmatic use.
+ * Fetches the page, extracts main content via Readability,
+ * converts to Markdown, and optionally prepends YAML frontmatter.
+ *
+ * @param url - The HTTP/HTTPS URL to convert.
+ * @param options - Override default conversion options.
+ * @returns The Markdown string, page metadata, and any warnings.
+ * @throws {ValidationError} If the URL or options are invalid.
+ * @throws {SSRFError} If the URL targets a private/internal host.
+ * @throws {NetworkError} On timeout, DNS failure, or HTTP error.
+ * @throws {ContentError} If the response is too large or malformed.
+ *
+ * @example
+ * ```ts
+ * import { convert, NetworkError } from "web-to-markdown";
+ *
+ * try {
+ *   const { markdown } = await convert("https://example.com");
+ * } catch (err) {
+ *   if (err instanceof NetworkError) console.error("Network issue:", err.message);
+ * }
+ * ```
+ */
+declare function convert(url: string, options?: Partial<ConvertOptions>): Promise<ConvertResult>;
+
+export { ContentError, type ConvertOptions, type ConvertResult, MarkitdownError, NetworkError, type PageMetadata, SSRFError, ValidationError, convert, extractContent, extractMdx, fetchPage, fetchRawText, htmlToMarkdown, processRawMdx, validateUrl };