scrapex 0.5.3 → 1.0.0-alpha.1

package/dist/parsers/index.d.cts

@@ -0,0 +1,133 @@
+//#region src/parsers/types.d.ts
+/**
+ * Generic source parser interface.
+ * Parsers transform raw content into structured data with metadata.
+ *
+ * @template TData - The main data type (e.g., array of links)
+ * @template TMeta - Optional metadata type
+ */
+interface SourceParser<TData, TMeta = unknown> {
+  readonly name: string;
+  /**
+   * Check if this parser can handle the given content
+   */
+  canParse(content: string, url?: string): boolean;
+  /**
+   * Parse the content and extract structured data
+   */
+  parse(content: string, url?: string): ParserResult<TData, TMeta>;
+}
+/**
+ * Result from a parser
+ */
+interface ParserResult<TData, TMeta = unknown> {
+  data: TData;
+  meta?: TMeta;
+}
+/**
+ * Markdown link extracted from content
+ */
+interface MarkdownLink {
+  url: string;
+  text: string;
+  title?: string;
+  context?: string;
+}
+/**
+ * Markdown section (heading + content)
+ */
+interface MarkdownSection {
+  level: number;
+  title: string;
+  content: string;
+  links: MarkdownLink[];
+}
+/**
+ * Parsed markdown structure
+ */
+interface ParsedMarkdown {
+  title?: string;
+  description?: string;
+  sections: MarkdownSection[];
+  links: MarkdownLink[];
+  codeBlocks: CodeBlock[];
+  frontmatter?: Record<string, unknown>;
+}
+/**
+ * Code block from markdown
+ */
+interface CodeBlock {
+  language?: string;
+  code: string;
+  meta?: string;
+}
+/**
+ * GitHub repository metadata
+ */
+interface GitHubMeta {
+  repoOwner?: string;
+  repoName?: string;
+  stars?: number;
+  lastUpdated?: string;
+}
+//#endregion
+//#region src/parsers/github.d.ts
+/**
+ * GitHub-specific utilities for parsing repositories.
+ */
+/**
+ * Check if a URL is a GitHub repository
+ */
+declare function isGitHubRepo(url: string): boolean;
+/**
+ * Extract GitHub repo info from URL
+ */
+declare function parseGitHubUrl(url: string): {
+  owner: string;
+  repo: string;
+} | null;
+/**
+ * Convert a GitHub repo URL to raw content URL
+ */
+declare function toRawUrl(url: string, branch?: string, file?: string): string;
+/**
+ * Fetch GitHub API metadata for a repository
+ * Note: This is a placeholder - actual implementation would need GitHub API access
+ */
+declare function fetchRepoMeta(owner: string, repo: string, _token?: string): Promise<GitHubMeta>;
+/**
+ * Group links by their category/section
+ */
+declare function groupByCategory(links: MarkdownLink[]): Map<string, MarkdownLink[]>;
+//#endregion
+//#region src/parsers/markdown.d.ts
+/**
+ * Generic Markdown parser.
+ * Extracts structure, links, and code blocks from markdown content.
+ *
+ * @example
+ * ```ts
+ * const parser = new MarkdownParser();
+ * const result = parser.parse(markdownContent);
+ * console.log(result.data.sections);
+ * console.log(result.data.links);
+ * ```
+ */
+declare class MarkdownParser implements SourceParser<ParsedMarkdown> {
+  readonly name = "markdown";
+  canParse(content: string): boolean;
+  parse(content: string): ParserResult<ParsedMarkdown>;
+  private parseFrontmatter;
+  private extractDescription;
+}
+/**
+ * Extract links from a list-based markdown structure (like awesome lists)
+ */
+declare function extractListLinks(markdown: string): MarkdownLink[];
+/**
+ * Parse markdown into sections by heading level
+ */
+declare function parseByHeadings(markdown: string, minLevel?: number): MarkdownSection[];
+//#endregion
+export { type CodeBlock, type GitHubMeta, type MarkdownLink, MarkdownParser, type MarkdownSection, type ParsedMarkdown, type ParserResult, type SourceParser, extractListLinks, fetchRepoMeta, groupByCategory, isGitHubRepo, parseByHeadings, parseGitHubUrl, toRawUrl };
+//# sourceMappingURL=index.d.cts.map

package/dist/parsers/index.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.cts","names":[],"sources":["../../src/parsers/types.ts","../../src/parsers/github.ts","../../src/parsers/markdown.ts"],"sourcesContent":[],"mappings":";;AAOA;;;;;AAiBA;AAQiB,UAzBA,YAyBY,CAAA,KAAA,EAAA,QAAA,OAAA,CAAA,CAAA;EAUZ,SAAA,IAAA,EAAA,MAAe;EAUf;;;EAKH,QAAA,CAAA,OAAA,EAAA,MAAA,EAAA,GAAA,CAAA,EAAA,MAAA,CAAA,EAAA,OAAA;EACE;;AAMhB;EASiB,KAAA,CAAA,OAAU,EAAA,MAAA,EAAA,GAAA,CAAA,EAAA,MAAA,CAAA,EAvDa,YAuDb,CAvD0B,KAuD1B,EAvDiC,KAuDjC,CAAA;;;;AChE3B;AAOgB,UDQC,YCRa,CAAA,KAAA,EAAA,QAAA,OAAA,CAAA,CAAA;EAYd,IAAA,EDHR,KCGgB;EAUF,IAAA,CAAA,EDZb,KCYa;AAgBtB;;;;AAA2D,UDtB1C,YAAA,CCsB0C;;;;EC7B9C,OAAA,CAAA,EAAA,MAAA;;;;;AAAsC,UFiBlC,eAAA,CEjBkC;EAqJnC,KAAA,EAAA,MAAA;EA+BA,KAAA,EAAA,MAAA;;SF/JP;;;;;UAMQ,cAAA;;;YAGL;SACH;cACK;gBACE;;;;;UAMC,SAAA;;;;;;;;UASA,UAAA;;;;;;;;AAlEjB;;;;;AAiBA;AAQiB,iBCvBD,YAAA,CDuBa,GAAA,EAAA,MAAA,CAAA,EAAA,OAAA;AAU7B;AAUA;;AAIS,iBCxCO,cAAA,CDwCP,GAAA,EAAA,MAAA,CAAA,EAAA;EACK,KAAA,EAAA,MAAA;EACE,IAAA,EAAA,MAAA;CAAM,GAAA,IAAA;AAMtB;AASA;;iBC7CgB,QAAA;;AAnBhB;AAOA;AAYA;AAUsB,iBAAA,aAAA,CAIX,KAAA,EAAR,MAAA,EAAO,IAAA,EAAA,MAAA,EAAA,MAAA,CAAA,EAAA,MAAA,CAAA,EAAP,OAAO,CAAC,UAAD,CAAA;AAYV;;;AAAwD,iBAAxC,eAAA,CAAwC,KAAA,EAAjB,YAAiB,EAAA,CAAA,EAAA,GAAA,CAAA,MAAA,EAAY,YAAZ,EAAA,CAAA;;;AD/CxD;;;;;AAiBA;AAQA;AAUA;AAUA;;;;AAMgB,cEjCH,cAAA,YAA0B,YFiCvB,CEjCoC,cFiCpC,CAAA,CAAA;EAAM,SAAA,IAAA,GAAA,UAAA;EAML,QAAA,CAAA,OAAS,EAAA,MAAA,CAAA,EAAA,OAAA;EAST,KAAA,CAAA,OAAU,EAAA,MAAA,CAAA,EElCD,YFkCC,CElCY,cFkCZ,CAAA;;;;AChE3B;AAOA;AAYA;AAUsB,iBCwIN,gBAAA,CDpIL,QAAR,EAAA,MAAO,CAAA,ECoI0C,YDpI1C,EAAA;AAYV;;;AAAwD,iBCuJxC,eAAA,CDvJwC,QAAA,EAAA,MAAA,EAAA,QAAA,CAAA,EAAA,MAAA,CAAA,ECuJS,eDvJT,EAAA"}

package/dist/parsers/index.d.mts

@@ -0,0 +1,133 @@
+//#region src/parsers/types.d.ts
+/**
+ * Generic source parser interface.
+ * Parsers transform raw content into structured data with metadata.
+ *
+ * @template TData - The main data type (e.g., array of links)
+ * @template TMeta - Optional metadata type
+ */
+interface SourceParser<TData, TMeta = unknown> {
+  readonly name: string;
+  /**
+   * Check if this parser can handle the given content
+   */
+  canParse(content: string, url?: string): boolean;
+  /**
+   * Parse the content and extract structured data
+   */
+  parse(content: string, url?: string): ParserResult<TData, TMeta>;
+}
+/**
+ * Result from a parser
+ */
+interface ParserResult<TData, TMeta = unknown> {
+  data: TData;
+  meta?: TMeta;
+}
+/**
+ * Markdown link extracted from content
+ */
+interface MarkdownLink {
+  url: string;
+  text: string;
+  title?: string;
+  context?: string;
+}
+/**
+ * Markdown section (heading + content)
+ */
+interface MarkdownSection {
+  level: number;
+  title: string;
+  content: string;
+  links: MarkdownLink[];
+}
+/**
+ * Parsed markdown structure
+ */
+interface ParsedMarkdown {
+  title?: string;
+  description?: string;
+  sections: MarkdownSection[];
+  links: MarkdownLink[];
+  codeBlocks: CodeBlock[];
+  frontmatter?: Record<string, unknown>;
+}
+/**
+ * Code block from markdown
+ */
+interface CodeBlock {
+  language?: string;
+  code: string;
+  meta?: string;
+}
+/**
+ * GitHub repository metadata
+ */
+interface GitHubMeta {
+  repoOwner?: string;
+  repoName?: string;
+  stars?: number;
+  lastUpdated?: string;
+}
+//#endregion
+//#region src/parsers/github.d.ts
+/**
+ * GitHub-specific utilities for parsing repositories.
+ */
+/**
+ * Check if a URL is a GitHub repository
+ */
+declare function isGitHubRepo(url: string): boolean;
+/**
+ * Extract GitHub repo info from URL
+ */
+declare function parseGitHubUrl(url: string): {
+  owner: string;
+  repo: string;
+} | null;
+/**
+ * Convert a GitHub repo URL to raw content URL
+ */
+declare function toRawUrl(url: string, branch?: string, file?: string): string;
+/**
+ * Fetch GitHub API metadata for a repository
+ * Note: This is a placeholder - actual implementation would need GitHub API access
+ */
+declare function fetchRepoMeta(owner: string, repo: string, _token?: string): Promise<GitHubMeta>;
+/**
+ * Group links by their category/section
+ */
+declare function groupByCategory(links: MarkdownLink[]): Map<string, MarkdownLink[]>;
+//#endregion
+//#region src/parsers/markdown.d.ts
+/**
+ * Generic Markdown parser.
+ * Extracts structure, links, and code blocks from markdown content.
+ *
+ * @example
+ * ```ts
+ * const parser = new MarkdownParser();
+ * const result = parser.parse(markdownContent);
+ * console.log(result.data.sections);
+ * console.log(result.data.links);
+ * ```
+ */
+declare class MarkdownParser implements SourceParser<ParsedMarkdown> {
+  readonly name = "markdown";
+  canParse(content: string): boolean;
+  parse(content: string): ParserResult<ParsedMarkdown>;
+  private parseFrontmatter;
+  private extractDescription;
+}
+/**
+ * Extract links from a list-based markdown structure (like awesome lists)
+ */
+declare function extractListLinks(markdown: string): MarkdownLink[];
+/**
+ * Parse markdown into sections by heading level
+ */
+declare function parseByHeadings(markdown: string, minLevel?: number): MarkdownSection[];
+//#endregion
+export { type CodeBlock, type GitHubMeta, type MarkdownLink, MarkdownParser, type MarkdownSection, type ParsedMarkdown, type ParserResult, type SourceParser, extractListLinks, fetchRepoMeta, groupByCategory, isGitHubRepo, parseByHeadings, parseGitHubUrl, toRawUrl };
+//# sourceMappingURL=index.d.mts.map

package/dist/parsers/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.mts","names":[],"sources":["../../src/parsers/types.ts","../../src/parsers/github.ts","../../src/parsers/markdown.ts"],"sourcesContent":[],"mappings":";;AAOA;;;;;AAiBA;AAQiB,UAzBA,YAyBY,CAAA,KAAA,EAAA,QAAA,OAAA,CAAA,CAAA;EAUZ,SAAA,IAAA,EAAA,MAAe;EAUf;;;EAKH,QAAA,CAAA,OAAA,EAAA,MAAA,EAAA,GAAA,CAAA,EAAA,MAAA,CAAA,EAAA,OAAA;EACE;;AAMhB;EASiB,KAAA,CAAA,OAAU,EAAA,MAAA,EAAA,GAAA,CAAA,EAAA,MAAA,CAAA,EAvDa,YAuDb,CAvD0B,KAuD1B,EAvDiC,KAuDjC,CAAA;;;;AChE3B;AAOgB,UDQC,YCRa,CAAA,KAAA,EAAA,QAAA,OAAA,CAAA,CAAA;EAYd,IAAA,EDHR,KCGgB;EAUF,IAAA,CAAA,EDZb,KCYa;AAgBtB;;;;AAA2D,UDtB1C,YAAA,CCsB0C;;;;EC7B9C,OAAA,CAAA,EAAA,MAAA;;;;;AAAsC,UFiBlC,eAAA,CEjBkC;EAqJnC,KAAA,EAAA,MAAA;EA+BA,KAAA,EAAA,MAAA;;SF/JP;;;;;UAMQ,cAAA;;;YAGL;SACH;cACK;gBACE;;;;;UAMC,SAAA;;;;;;;;UASA,UAAA;;;;;;;;AAlEjB;;;;;AAiBA;AAQiB,iBCvBD,YAAA,CDuBa,GAAA,EAAA,MAAA,CAAA,EAAA,OAAA;AAU7B;AAUA;;AAIS,iBCxCO,cAAA,CDwCP,GAAA,EAAA,MAAA,CAAA,EAAA;EACK,KAAA,EAAA,MAAA;EACE,IAAA,EAAA,MAAA;CAAM,GAAA,IAAA;AAMtB;AASA;;iBC7CgB,QAAA;;AAnBhB;AAOA;AAYA;AAUsB,iBAAA,aAAA,CAIX,KAAA,EAAR,MAAA,EAAO,IAAA,EAAA,MAAA,EAAA,MAAA,CAAA,EAAA,MAAA,CAAA,EAAP,OAAO,CAAC,UAAD,CAAA;AAYV;;;AAAwD,iBAAxC,eAAA,CAAwC,KAAA,EAAjB,YAAiB,EAAA,CAAA,EAAA,GAAA,CAAA,MAAA,EAAY,YAAZ,EAAA,CAAA;;;AD/CxD;;;;;AAiBA;AAQA;AAUA;AAUA;;;;AAMgB,cEjCH,cAAA,YAA0B,YFiCvB,CEjCoC,cFiCpC,CAAA,CAAA;EAAM,SAAA,IAAA,GAAA,UAAA;EAML,QAAA,CAAA,OAAS,EAAA,MAAA,CAAA,EAAA,OAAA;EAST,KAAA,CAAA,OAAU,EAAA,MAAA,CAAA,EElCD,YFkCC,CElCY,cFkCZ,CAAA;;;;AChE3B;AAOA;AAYA;AAUsB,iBCwIN,gBAAA,CDpIL,QAAR,EAAA,MAAO,CAAA,ECoI0C,YDpI1C,EAAA;AAYV;;;AAAwD,iBCuJxC,eAAA,CDvJwC,QAAA,EAAA,MAAA,EAAA,QAAA,CAAA,EAAA,MAAA,CAAA,ECuJS,eDvJT,EAAA"}

package/dist/parsers/index.mjs

@@ -0,0 +1,192 @@
+import { fromMarkdown } from "mdast-util-from-markdown";
+import { toString } from "mdast-util-to-string";
+import { visit } from "unist-util-visit";
+
+//#region src/parsers/github.ts
+/**
+* GitHub-specific utilities for parsing repositories.
+*/
+/**
+* Check if a URL is a GitHub repository
+*/
+function isGitHubRepo(url) {
+	return /^https?:\/\/(www\.)?github\.com\/[^/]+\/[^/]+\/?$/.test(url);
+}
+/**
+* Extract GitHub repo info from URL
+*/
+function parseGitHubUrl(url) {
+	const match = url.match(/github\.com\/([^/]+)\/([^/]+)/);
+	if (!match || !match[1] || !match[2]) return null;
+	return {
+		owner: match[1],
+		repo: match[2].replace(/\.git$/, "")
+	};
+}
+/**
+* Convert a GitHub repo URL to raw content URL
+*/
+function toRawUrl(url, branch = "main", file = "README.md") {
+	const info = parseGitHubUrl(url);
+	if (!info) return url;
+	return `https://raw.githubusercontent.com/${info.owner}/${info.repo}/${branch}/${file}`;
+}
+/**
+* Fetch GitHub API metadata for a repository
+* Note: This is a placeholder - actual implementation would need GitHub API access
+*/
+async function fetchRepoMeta(owner, repo, _token) {
+	return {
+		repoOwner: owner,
+		repoName: repo
+	};
+}
+/**
+* Group links by their category/section
+*/
+function groupByCategory(links) {
+	const groups = /* @__PURE__ */ new Map();
+	for (const link of links) {
+		const category = link.context || "Uncategorized";
+		const existing = groups.get(category) || [];
+		existing.push(link);
+		groups.set(category, existing);
+	}
+	return groups;
+}
+
+//#endregion
+//#region src/parsers/markdown.ts
+/**
+* Generic Markdown parser.
+* Extracts structure, links, and code blocks from markdown content.
+*
+* @example
+* ```ts
+* const parser = new MarkdownParser();
+* const result = parser.parse(markdownContent);
+* console.log(result.data.sections);
+* console.log(result.data.links);
+* ```
+*/
+var MarkdownParser = class {
+	name = "markdown";
+	canParse(content) {
+		return content.includes("# ") || content.includes("## ") || content.includes("- [") || content.includes("* [") || content.includes("```");
+	}
+	parse(content) {
+		const tree = fromMarkdown(content);
+		const sections = [];
+		const allLinks = [];
+		const codeBlocks = [];
+		let frontmatter;
+		if (content.startsWith("---")) {
+			const endIndex = content.indexOf("---", 3);
+			if (endIndex !== -1) {
+				const frontmatterContent = content.slice(3, endIndex).trim();
+				frontmatter = this.parseFrontmatter(frontmatterContent);
+			}
+		}
+		let currentSection = null;
+		visit(tree, (node) => {
+			if (node.type === "heading") {
+				const heading = node;
+				const title = toString(heading);
+				if (currentSection) sections.push(currentSection);
+				currentSection = {
+					level: heading.depth,
+					title,
+					content: "",
+					links: []
+				};
+			}
+			if (node.type === "link") {
+				const link = node;
+				const text = toString(link);
+				const linkData = {
+					url: link.url,
+					text,
+					title: link.title ?? void 0,
+					context: currentSection?.title
+				};
+				allLinks.push(linkData);
+				if (currentSection) currentSection.links.push(linkData);
+			}
+			if (node.type === "code") {
+				const code = node;
+				codeBlocks.push({
+					language: code.lang ?? void 0,
+					code: code.value,
+					meta: code.meta ?? void 0
+				});
+			}
+			if (currentSection && node.type === "paragraph") {
+				const text = toString(node);
+				currentSection.content += (currentSection.content ? "\n\n" : "") + text;
+			}
+		});
+		if (currentSection) sections.push(currentSection);
+		return { data: {
+			title: frontmatter?.title ?? sections.find((s) => s.level === 1)?.title,
+			description: frontmatter?.description ?? this.extractDescription(tree),
+			sections,
+			links: allLinks,
+			codeBlocks,
+			frontmatter
+		} };
+	}
+	parseFrontmatter(content) {
+		const result = {};
+		const lines = content.split("\n");
+		for (const line of lines) {
+			const colonIndex = line.indexOf(":");
+			if (colonIndex > 0) {
+				const key = line.slice(0, colonIndex).trim();
+				let value = line.slice(colonIndex + 1).trim();
+				if (value === "true") value = true;
+				else if (value === "false") value = false;
+				else if (/^-?\d+(\.\d+)?$/.test(value)) value = Number(value);
+				else if (value.startsWith("\"") && value.endsWith("\"")) value = value.slice(1, -1);
+				else if (value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
+				result[key] = value;
+			}
+		}
+		return result;
+	}
+	extractDescription(tree) {
+		for (const node of tree.children) {
+			if (node.type === "heading") break;
+			if (node.type === "paragraph") return toString(node);
+		}
+	}
+};
+/**
+* Extract links from a list-based markdown structure (like awesome lists)
+*/
+function extractListLinks(markdown) {
+	const tree = fromMarkdown(markdown);
+	const links = [];
+	let currentHeading = "";
+	visit(tree, (node) => {
+		if (node.type === "heading") currentHeading = toString(node);
+		if (node.type === "listItem") visit(node, "link", (linkNode) => {
+			links.push({
+				url: linkNode.url,
+				text: toString(linkNode),
+				title: linkNode.title ?? void 0,
+				context: currentHeading || void 0
+			});
+		});
+	});
+	return links;
+}
+/**
+* Parse markdown into sections by heading level
+*/
+function parseByHeadings(markdown, minLevel = 2) {
+	return new MarkdownParser().parse(markdown).data.sections.filter((s) => s.level >= minLevel);
+}
+
+//#endregion
+export { MarkdownParser, extractListLinks, fetchRepoMeta, groupByCategory, isGitHubRepo, parseByHeadings, parseGitHubUrl, toRawUrl };
+//# sourceMappingURL=index.mjs.map

package/dist/parsers/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","names":["sections: MarkdownSection[]","allLinks: MarkdownLink[]","codeBlocks: CodeBlock[]","frontmatter: Record<string, unknown> | undefined","currentSection: MarkdownSection | null","mdastToString","linkData: MarkdownLink","result: Record<string, unknown>","value: string | boolean | number","links: MarkdownLink[]"],"sources":["../../src/parsers/github.ts","../../src/parsers/markdown.ts"],"sourcesContent":["import type { GitHubMeta, MarkdownLink } from './types.js';\n\n/**\n * GitHub-specific utilities for parsing repositories.\n */\n\n/**\n * Check if a URL is a GitHub repository\n */\nexport function isGitHubRepo(url: string): boolean {\n  return /^https?:\\/\\/(www\\.)?github\\.com\\/[^/]+\\/[^/]+\\/?$/.test(url);\n}\n\n/**\n * Extract GitHub repo info from URL\n */\nexport function parseGitHubUrl(url: string): { owner: string; repo: string } | null {\n  const match = url.match(/github\\.com\\/([^/]+)\\/([^/]+)/);\n  if (!match || !match[1] || !match[2]) return null;\n  return {\n    owner: match[1],\n    repo: match[2].replace(/\\.git$/, ''),\n  };\n}\n\n/**\n * Convert a GitHub repo URL to raw content URL\n */\nexport function toRawUrl(url: string, branch = 'main', file = 'README.md'): string {\n  const info = parseGitHubUrl(url);\n  if (!info) return url;\n  return `https://raw.githubusercontent.com/${info.owner}/${info.repo}/${branch}/${file}`;\n}\n\n/**\n * Fetch GitHub API metadata for a repository\n * Note: This is a placeholder - actual implementation would need GitHub API access\n */\nexport async function fetchRepoMeta(\n  owner: string,\n  repo: string,\n  _token?: string\n): Promise<GitHubMeta> {\n  // This would make actual API calls in a full implementation\n  // For now, return basic info\n  return {\n    repoOwner: owner,\n    repoName: repo,\n  };\n}\n\n/**\n * Group links by their category/section\n */\nexport function groupByCategory(links: MarkdownLink[]): Map<string, MarkdownLink[]> {\n  const groups = new Map<string, MarkdownLink[]>();\n\n  for (const link of links) {\n    const category = link.context || 'Uncategorized';\n    const existing = groups.get(category) || [];\n    existing.push(link);\n    groups.set(category, existing);\n  }\n\n  return groups;\n}\n","import type { Code, Heading, Link, ListItem, Root } from 'mdast';\nimport { fromMarkdown } from 'mdast-util-from-markdown';\nimport { toString as mdastToString } from 'mdast-util-to-string';\nimport { visit } from 'unist-util-visit';\nimport type {\n  CodeBlock,\n  MarkdownLink,\n  MarkdownSection,\n  ParsedMarkdown,\n  ParserResult,\n  SourceParser,\n} from './types.js';\n\n/**\n * Generic Markdown parser.\n * Extracts structure, links, and code blocks from markdown content.\n *\n * @example\n * ```ts\n * const parser = new MarkdownParser();\n * const result = parser.parse(markdownContent);\n * console.log(result.data.sections);\n * console.log(result.data.links);\n * ```\n */\nexport class MarkdownParser implements SourceParser<ParsedMarkdown> {\n  readonly name = 'markdown';\n\n  canParse(content: string): boolean {\n    // Check for common markdown patterns\n    return (\n      content.includes('# ') ||\n      content.includes('## ') ||\n      content.includes('- [') ||\n      content.includes('* [') ||\n      content.includes('```')\n    );\n  }\n\n  parse(content: string): ParserResult<ParsedMarkdown> {\n    const tree = fromMarkdown(content);\n    const sections: MarkdownSection[] = [];\n    const allLinks: MarkdownLink[] = [];\n    const codeBlocks: CodeBlock[] = [];\n    let frontmatter: Record<string, unknown> | undefined;\n\n    // Extract frontmatter if present\n    if (content.startsWith('---')) {\n      const endIndex = content.indexOf('---', 3);\n      if (endIndex !== -1) {\n        const frontmatterContent = content.slice(3, endIndex).trim();\n        frontmatter = this.parseFrontmatter(frontmatterContent);\n      }\n    }\n\n    // Track current section\n    let currentSection: MarkdownSection | null = null;\n\n    // Process the AST\n    visit(tree, (node) => {\n      // Handle headings\n      if (node.type === 'heading') {\n        const heading = node as Heading;\n        const title = mdastToString(heading);\n\n        // Finalize previous section\n        if (currentSection) {\n          sections.push(currentSection);\n        }\n\n        currentSection = {\n          level: heading.depth,\n          title,\n          content: '',\n          links: [],\n        };\n      }\n\n      // Handle links\n      if (node.type === 'link') {\n        const link = node as Link;\n        const text = mdastToString(link);\n        const linkData: MarkdownLink = {\n          url: link.url,\n          text,\n          title: link.title ?? undefined,\n          context: currentSection?.title,\n        };\n\n        allLinks.push(linkData);\n        if (currentSection) {\n          currentSection.links.push(linkData);\n        }\n      }\n\n      // Handle code blocks\n      if (node.type === 'code') {\n        const code = node as Code;\n        codeBlocks.push({\n          language: code.lang ?? undefined,\n          code: code.value,\n          meta: code.meta ?? undefined,\n        });\n      }\n\n      // Accumulate content for current section\n      if (currentSection && node.type === 'paragraph') {\n        const text = mdastToString(node);\n        currentSection.content += (currentSection.content ? '\\n\\n' : '') + text;\n      }\n    });\n\n    // Finalize last section\n    if (currentSection) {\n      sections.push(currentSection);\n    }\n\n    // Extract title from first h1 or frontmatter\n    const title = (frontmatter?.title as string) ?? sections.find((s) => s.level === 1)?.title;\n\n    // Extract description from frontmatter or first paragraph before any heading\n    const description = (frontmatter?.description as string) ?? this.extractDescription(tree);\n\n    return {\n      data: {\n        title,\n        description,\n        sections,\n        links: allLinks,\n        codeBlocks,\n        frontmatter,\n      },\n    };\n  }\n\n  private parseFrontmatter(content: string): Record<string, unknown> {\n    const result: Record<string, unknown> = {};\n    const lines = content.split('\\n');\n\n    for (const line of lines) {\n      const colonIndex = line.indexOf(':');\n      if (colonIndex > 0) {\n        const key = line.slice(0, colonIndex).trim();\n        let value: string | boolean | number = line.slice(colonIndex + 1).trim();\n\n        // Parse simple types\n        if (value === 'true') value = true;\n        else if (value === 'false') value = false;\n        else if (/^-?\\d+(\\.\\d+)?$/.test(value)) value = Number(value);\n        else if (value.startsWith('\"') && value.endsWith('\"')) value = value.slice(1, -1);\n        else if (value.startsWith(\"'\") && value.endsWith(\"'\")) value = value.slice(1, -1);\n\n        result[key] = value;\n      }\n    }\n\n    return result;\n  }\n\n  private extractDescription(tree: Root): string | undefined {\n    // Find first paragraph before any heading\n    for (const node of tree.children) {\n      if (node.type === 'heading') break;\n      if (node.type === 'paragraph') {\n        return mdastToString(node);\n      }\n    }\n    return undefined;\n  }\n}\n\n/**\n * Extract links from a list-based markdown structure (like awesome lists)\n */\nexport function extractListLinks(markdown: string): MarkdownLink[] {\n  const tree = fromMarkdown(markdown);\n  const links: MarkdownLink[] = [];\n  let currentHeading = '';\n\n  visit(tree, (node) => {\n    if (node.type === 'heading') {\n      currentHeading = mdastToString(node as Heading);\n    }\n\n    if (node.type === 'listItem') {\n      const listItem = node as ListItem;\n\n      // Find links in this list item\n      visit(listItem, 'link', (linkNode: Link) => {\n        links.push({\n          url: linkNode.url,\n          text: mdastToString(linkNode),\n          title: linkNode.title ?? undefined,\n          context: currentHeading || undefined,\n        });\n      });\n    }\n  });\n\n  return links;\n}\n\n/**\n * Parse markdown into sections by heading level\n */\nexport function parseByHeadings(markdown: string, minLevel = 2): MarkdownSection[] {\n  const parser = new MarkdownParser();\n  const result = parser.parse(markdown);\n  return result.data.sections.filter((s) => s.level >= minLevel);\n}\n"],"mappings":";;;;;;;;;;;AASA,SAAgB,aAAa,KAAsB;AACjD,QAAO,oDAAoD,KAAK,IAAI;;;;;AAMtE,SAAgB,eAAe,KAAqD;CAClF,MAAM,QAAQ,IAAI,MAAM,gCAAgC;AACxD,KAAI,CAAC,SAAS,CAAC,MAAM,MAAM,CAAC,MAAM,GAAI,QAAO;AAC7C,QAAO;EACL,OAAO,MAAM;EACb,MAAM,MAAM,GAAG,QAAQ,UAAU,GAAG;EACrC;;;;;AAMH,SAAgB,SAAS,KAAa,SAAS,QAAQ,OAAO,aAAqB;CACjF,MAAM,OAAO,eAAe,IAAI;AAChC,KAAI,CAAC,KAAM,QAAO;AAClB,QAAO,qCAAqC,KAAK,MAAM,GAAG,KAAK,KAAK,GAAG,OAAO,GAAG;;;;;;AAOnF,eAAsB,cACpB,OACA,MACA,QACqB;AAGrB,QAAO;EACL,WAAW;EACX,UAAU;EACX;;;;;AAMH,SAAgB,gBAAgB,OAAoD;CAClF,MAAM,yBAAS,IAAI,KAA6B;AAEhD,MAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,WAAW,KAAK,WAAW;EACjC,MAAM,WAAW,OAAO,IAAI,SAAS,IAAI,EAAE;AAC3C,WAAS,KAAK,KAAK;AACnB,SAAO,IAAI,UAAU,SAAS;;AAGhC,QAAO;;;;;;;;;;;;;;;;;ACvCT,IAAa,iBAAb,MAAoE;CAClE,AAAS,OAAO;CAEhB,SAAS,SAA0B;AAEjC,SACE,QAAQ,SAAS,KAAK,IACtB,QAAQ,SAAS,MAAM,IACvB,QAAQ,SAAS,MAAM,IACvB,QAAQ,SAAS,MAAM,IACvB,QAAQ,SAAS,MAAM;;CAI3B,MAAM,SAA+C;EACnD,MAAM,OAAO,aAAa,QAAQ;EAClC,MAAMA,WAA8B,EAAE;EACtC,MAAMC,WAA2B,EAAE;EACnC,MAAMC,aAA0B,EAAE;EAClC,IAAIC;AAGJ,MAAI,QAAQ,WAAW,MAAM,EAAE;GAC7B,MAAM,WAAW,QAAQ,QAAQ,OAAO,EAAE;AAC1C,OAAI,aAAa,IAAI;IACnB,MAAM,qBAAqB,QAAQ,MAAM,GAAG,SAAS,CAAC,MAAM;AAC5D,kBAAc,KAAK,iBAAiB,mBAAmB;;;EAK3D,IAAIC,iBAAyC;AAG7C,QAAM,OAAO,SAAS;AAEpB,OAAI,KAAK,SAAS,WAAW;IAC3B,MAAM,UAAU;IAChB,MAAM,QAAQC,SAAc,QAAQ;AAGpC,QAAI,eACF,UAAS,KAAK,eAAe;AAG/B,qBAAiB;KACf,OAAO,QAAQ;KACf;KACA,SAAS;KACT,OAAO,EAAE;KACV;;AAIH,OAAI,KAAK,SAAS,QAAQ;IACxB,MAAM,OAAO;IACb,MAAM,OAAOA,SAAc,KAAK;IAChC,MAAMC,WAAyB;KAC7B,KAAK,KAAK;KACV;KACA,OAAO,KAAK,SAAS;KACrB,SAAS,gBAAgB;KAC1B;AAED,aAAS,KAAK,SAAS;AACvB,QAAI,eACF,gBAAe,MAAM,KAAK,SAAS;;AAKvC,OAAI,KAAK,SAAS,QAAQ;IACxB,MAAM,OAAO;AACb,eAAW,KAAK;KACd,UAAU,KAAK,QAAQ;KACvB,MAAM,KAAK;KACX,MAAM,KAAK,QAAQ;KACpB,CAAC;;AAIJ,OAAI,kBAAkB,KAAK,SAAS,aAAa;IAC/C,MAAM,OAAOD,SAAc,KAAK;AAChC,mBAAe,YAAY,eAAe,UAAU,SAAS,MAAM;;IAErE;AAGF,MAAI,eACF,UAAS,KAAK,eAAe;AAS/B,SAAO,EACL,MAAM;GACJ,OAPW,aAAa,SAAoB,SAAS,MAAM,MAAM,EAAE,UAAU,EAAE,EAAE;GAQjF,aALiB,aAAa,eAA0B,KAAK,mBAAmB,KAAK;GAMrF;GACA,OAAO;GACP;GACA;GACD,EACF;;CAGH,AAAQ,iBAAiB,SAA0C;EACjE,MAAME,SAAkC,EAAE;EAC1C,MAAM,QAAQ,QAAQ,MAAM,KAAK;AAEjC,OAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,aAAa,KAAK,QAAQ,IAAI;AACpC,OAAI,aAAa,GAAG;IAClB,MAAM,MAAM,KAAK,MAAM,GAAG,WAAW,CAAC,MAAM;IAC5C,IAAIC,QAAmC,KAAK,MAAM,aAAa,EAAE,CAAC,MAAM;AAGxE,QAAI,UAAU,OAAQ,SAAQ;aACrB,UAAU,QAAS,SAAQ;aAC3B,kBAAkB,KAAK,MAAM,CAAE,SAAQ,OAAO,MAAM;aACpD,MAAM,WAAW,KAAI,IAAI,MAAM,SAAS,KAAI,CAAE,SAAQ,MAAM,MAAM,GAAG,GAAG;aACxE,MAAM,WAAW,IAAI,IAAI,MAAM,SAAS,IAAI,CAAE,SAAQ,MAAM,MAAM,GAAG,GAAG;AAEjF,WAAO,OAAO;;;AAIlB,SAAO;;CAGT,AAAQ,mBAAmB,MAAgC;AAEzD,OAAK,MAAM,QAAQ,KAAK,UAAU;AAChC,OAAI,KAAK,SAAS,UAAW;AAC7B,OAAI,KAAK,SAAS,YAChB,QAAOH,SAAc,KAAK;;;;;;;AAUlC,SAAgB,iBAAiB,UAAkC;CACjE,MAAM,OAAO,aAAa,SAAS;CACnC,MAAMI,QAAwB,EAAE;CAChC,IAAI,iBAAiB;AAErB,OAAM,OAAO,SAAS;AACpB,MAAI,KAAK,SAAS,UAChB,kBAAiBJ,SAAc,KAAgB;AAGjD,MAAI,KAAK,SAAS,WAIhB,OAHiB,MAGD,SAAS,aAAmB;AAC1C,SAAM,KAAK;IACT,KAAK,SAAS;IACd,MAAMA,SAAc,SAAS;IAC7B,OAAO,SAAS,SAAS;IACzB,SAAS,kBAAkB;IAC5B,CAAC;IACF;GAEJ;AAEF,QAAO;;;;;AAMT,SAAgB,gBAAgB,UAAkB,WAAW,GAAsB;AAGjF,QAFe,IAAI,gBAAgB,CACb,MAAM,SAAS,CACvB,KAAK,SAAS,QAAQ,MAAM,EAAE,SAAS,SAAS"}

package/dist/types-CNQZVW36.d.mts

@@ -0,0 +1,150 @@
+import { CheerioAPI } from "cheerio";
+
+//#region src/core/types.d.ts
+
+/**
+ * Content type classification for scraped URLs
+ */
+type ContentType = 'article' | 'repo' | 'docs' | 'package' | 'video' | 'tool' | 'product' | 'unknown';
+/**
+ * Extracted link from content
+ */
+interface ExtractedLink {
+  url: string;
+  text: string;
+  isExternal: boolean;
+}
+/**
+ * Extracted entities from LLM enhancement
+ */
+interface ExtractedEntities {
+  people: string[];
+  organizations: string[];
+  technologies: string[];
+  locations: string[];
+  concepts: string[];
+}
+/**
+ * Main result of metadata scraping - optimized for LLM consumption
+ */
+interface ScrapedData {
+  url: string;
+  canonicalUrl: string;
+  domain: string;
+  title: string;
+  description: string;
+  image?: string;
+  favicon?: string;
+  content: string;
+  textContent: string;
+  excerpt: string;
+  wordCount: number;
+  author?: string;
+  publishedAt?: string;
+  modifiedAt?: string;
+  siteName?: string;
+  language?: string;
+  contentType: ContentType;
+  keywords: string[];
+  jsonLd?: Record<string, unknown>[];
+  links?: ExtractedLink[];
+  summary?: string;
+  suggestedTags?: string[];
+  entities?: ExtractedEntities;
+  extracted?: Record<string, unknown>;
+  custom?: Record<string, unknown>;
+  scrapedAt: string;
+  scrapeTimeMs: number;
+  error?: string;
+}
+/**
+ * LLM enhancement types
+ */
+type EnhancementType = 'summarize' | 'tags' | 'entities' | 'classify';
+/**
+ * Schema for structured LLM extraction
+ */
+type ExtractionSchemaType = 'string' | 'number' | 'boolean' | 'string[]' | 'number[]' | `${string}?`;
+type ExtractionSchema = Record<string, ExtractionSchemaType>;
+/**
+ * Forward declaration for LLM provider (defined in llm/types.ts)
+ */
+interface LLMProvider {
+  readonly name: string;
+  complete(prompt: string, options?: CompletionOptions): Promise<string>;
+  completeJSON<T>(prompt: string, schema: unknown): Promise<T>;
+}
+interface CompletionOptions {
+  maxTokens?: number;
+  temperature?: number;
+  systemPrompt?: string;
+}
+/**
+ * Forward declaration for Fetcher (defined in fetchers/types.ts)
+ */
+interface Fetcher {
+  readonly name: string;
+  fetch(url: string, options: FetchOptions): Promise<FetchResult>;
+}
+interface FetchOptions {
+  timeout?: number;
+  userAgent?: string;
+  headers?: Record<string, string>;
+}
+interface FetchResult {
+  html: string;
+  finalUrl: string;
+  statusCode: number;
+  contentType: string;
+  headers?: Record<string, string>;
+}
+/**
+ * Forward declaration for Extractor (defined in extractors/types.ts)
+ */
+interface Extractor {
+  readonly name: string;
+  readonly priority?: number;
+  extract(context: ExtractionContext): Promise<Partial<ScrapedData>>;
+}
+/**
+ * Shared context passed to all extractors
+ */
+interface ExtractionContext {
+  url: string;
+  finalUrl: string;
+  html: string;
+  $: CheerioAPI;
+  getDocument(): Document;
+  results: Partial<ScrapedData>;
+  options: ScrapeOptions;
+}
+/**
+ * Options for scraping
+ */
+interface ScrapeOptions {
+  /** Timeout in milliseconds (default: 10000) */
+  timeout?: number;
+  /** User agent string */
+  userAgent?: string;
+  /** Whether to extract full content (default: true) */
+  extractContent?: boolean;
+  /** Maximum content length in characters (default: 50000) */
+  maxContentLength?: number;
+  /** Custom fetcher (for Puppeteer/Playwright) */
+  fetcher?: Fetcher;
+  /** Custom extractors to run */
+  extractors?: Extractor[];
+  /** If true, only run custom extractors (replace defaults) */
+  replaceDefaultExtractors?: boolean;
+  /** Check robots.txt before scraping (default: false) */
+  respectRobots?: boolean;
+  /** LLM provider for enhancements */
+  llm?: LLMProvider;
+  /** LLM enhancement types to run */
+  enhance?: EnhancementType[];
+  /** Schema for structured LLM extraction */
+  extract?: ExtractionSchema;
+}
+//#endregion
+export { ExtractedLink as a, ExtractionSchemaType as c, FetchResult as d, Fetcher as f, ScrapedData as h, ExtractedEntities as i, Extractor as l, ScrapeOptions as m, ContentType as n, ExtractionContext as o, LLMProvider as p, EnhancementType as r, ExtractionSchema as s, CompletionOptions as t, FetchOptions as u };
+//# sourceMappingURL=types-CNQZVW36.d.mts.map

package/dist/types-CNQZVW36.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types-CNQZVW36.d.mts","names":[],"sources":["../src/core/types.ts"],"sourcesContent":[],"mappings":";;;;;;AAKA;AAaiB,KAbL,WAAA,GAakB,SAAA,GAAA,MAAA,GAAA,MAAA,GAAA,SAAA,GAAA,OAAA,GAAA,MAAA,GAAA,SAAA,GAAA,SAAA;AAS9B;AAWA;;AA8BW,UAlDM,aAAA,CAkDN;EAGD,GAAA,EAAA,MAAA;EAKG,IAAA,EAAA,MAAA;EACC,UAAA,EAAA,OAAA;;;AAcd;AAKA;AAQY,UA7EK,iBAAA,CA6E6B;EAK7B,MAAA,EAAA,MAAW,EAAA;EAES,aAAA,EAAA,MAAA,EAAA;EAAoB,YAAA,EAAA,MAAA,EAAA;EACG,SAAA,EAAA,MAAA,EAAA;EAAR,QAAA,EAAA,MAAA,EAAA;;AAGpD;AASA;;AAEqD,UAxFpC,WAAA,CAwFoC;EAAR,GAAA,EAAA,MAAA;EAAO,YAAA,EAAA,MAAA;EAGnC,MAAA,EAAA,MAAA;EAMA,KAAA,EAAA,MAAA;EAWA,WAAA,EAAS,MAAA;EAGP,KAAA,CAAA,EAAA,MAAA;EAAoC,OAAA,CAAA,EAAA,MAAA;EAAR,OAAA,EAAA,MAAA;EAAR,WAAA,EAAA,MAAA;EAAO,OAAA,EAAA,MAAA;EAM7B,SAAA,EAAA,MAAA;EAOZ,MAAA,CAAA,EAAA,MAAA;EAGY,WAAA,CAAA,EAAA,MAAA;EAGE,UAAA,CAAA,EAAA,MAAA;EAAR,QAAA,CAAA,EAAA,MAAA;EAGA,QAAA,CAAA,EAAA,MAAA;EAAa,WAAA,EA3GT,WA2GS;EAMP,QAAA,EAAA,MAAa,EAAA;EAclB,MAAA,CAAA,EA3HD,MA2HC,CAAA,MAAA,EAAA,OAAA,CAAA,EAAA;EAGG,KAAA,CAAA,EA3HL,aA2HK,EAAA;EASP,OAAA,CAAA,EAAA,MAAA;EAGI,aAAA,CAAA,EAAA,MAAA,EAAA;EAGA,QAAA,CAAA,EArIC,iBAqID;EAAgB,SAAA,CAAA,EApId,MAoIc,CAAA,MAAA,EAAA,OAAA,CAAA;WAjIjB;;;;;;;;KAWC,eAAA;;;;KAKA,oBAAA;KAQA,gBAAA,GAAmB,eAAe;;;;UAK7B,WAAA;;qCAEoB,oBAAoB;oDACL,QAAQ;;UAG3C,iBAAA;;;;;;;;UASA,OAAA;;8BAEa,eAAe,QAAQ;;UAGpC,YAAA;;;YAGL;;UAGK,WAAA;;;;;YAKL;;;;;UAMK,SAAA;;;mBAGE,oBAAoB,QAAQ,QAAQ;;;;;UAMtC,iBAAA;;;;KAOZ;iBAGY;WAGN,QAAQ;WAGR;;;;;UAMM,aAAA;;;;;;;;;;YAcL;;eAGG;;;;;;QASP;;YAGI;;YAGA"}