scrapex 1.0.0-alpha.1 → 1.0.0-beta.1

package/dist/parsers/index.cjs

@@ -1,200 +1,11 @@
-const require_index = require('../index.cjs');
-let mdast_util_from_markdown = require("mdast-util-from-markdown");
-let mdast_util_to_string = require("mdast-util-to-string");
-let unist_util_visit = require("unist-util-visit");
+const require_parsers = require('../parsers-Bneuws8x.cjs');
 
-//#region src/parsers/github.ts
-/**
-* GitHub-specific utilities for parsing repositories.
-*/
-/**
-* Check if a URL is a GitHub repository
-*/
-function isGitHubRepo(url) {
-	return /^https?:\/\/(www\.)?github\.com\/[^/]+\/[^/]+\/?$/.test(url);
-}
-/**
-* Extract GitHub repo info from URL
-*/
-function parseGitHubUrl(url) {
-	const match = url.match(/github\.com\/([^/]+)\/([^/]+)/);
-	if (!match || !match[1] || !match[2]) return null;
-	return {
-		owner: match[1],
-		repo: match[2].replace(/\.git$/, "")
-	};
-}
-/**
-* Convert a GitHub repo URL to raw content URL
-*/
-function toRawUrl(url, branch = "main", file = "README.md") {
-	const info = parseGitHubUrl(url);
-	if (!info) return url;
-	return `https://raw.githubusercontent.com/${info.owner}/${info.repo}/${branch}/${file}`;
-}
-/**
-* Fetch GitHub API metadata for a repository
-* Note: This is a placeholder - actual implementation would need GitHub API access
-*/
-async function fetchRepoMeta(owner, repo, _token) {
-	return {
-		repoOwner: owner,
-		repoName: repo
-	};
-}
-/**
-* Group links by their category/section
-*/
-function groupByCategory(links) {
-	const groups = /* @__PURE__ */ new Map();
-	for (const link of links) {
-		const category = link.context || "Uncategorized";
-		const existing = groups.get(category) || [];
-		existing.push(link);
-		groups.set(category, existing);
-	}
-	return groups;
-}
-
-//#endregion
-//#region src/parsers/markdown.ts
-/**
-* Generic Markdown parser.
-* Extracts structure, links, and code blocks from markdown content.
-*
-* @example
-* ```ts
-* const parser = new MarkdownParser();
-* const result = parser.parse(markdownContent);
-* console.log(result.data.sections);
-* console.log(result.data.links);
-* ```
-*/
-var MarkdownParser = class {
-	name = "markdown";
-	canParse(content) {
-		return content.includes("# ") || content.includes("## ") || content.includes("- [") || content.includes("* [") || content.includes("```");
-	}
-	parse(content) {
-		const tree = (0, mdast_util_from_markdown.fromMarkdown)(content);
-		const sections = [];
-		const allLinks = [];
-		const codeBlocks = [];
-		let frontmatter;
-		if (content.startsWith("---")) {
-			const endIndex = content.indexOf("---", 3);
-			if (endIndex !== -1) {
-				const frontmatterContent = content.slice(3, endIndex).trim();
-				frontmatter = this.parseFrontmatter(frontmatterContent);
-			}
-		}
-		let currentSection = null;
-		(0, unist_util_visit.visit)(tree, (node) => {
-			if (node.type === "heading") {
-				const heading = node;
-				const title = (0, mdast_util_to_string.toString)(heading);
-				if (currentSection) sections.push(currentSection);
-				currentSection = {
-					level: heading.depth,
-					title,
-					content: "",
-					links: []
-				};
-			}
-			if (node.type === "link") {
-				const link = node;
-				const text = (0, mdast_util_to_string.toString)(link);
-				const linkData = {
-					url: link.url,
-					text,
-					title: link.title ?? void 0,
-					context: currentSection?.title
-				};
-				allLinks.push(linkData);
-				if (currentSection) currentSection.links.push(linkData);
-			}
-			if (node.type === "code") {
-				const code = node;
-				codeBlocks.push({
-					language: code.lang ?? void 0,
-					code: code.value,
-					meta: code.meta ?? void 0
-				});
-			}
-			if (currentSection && node.type === "paragraph") {
-				const text = (0, mdast_util_to_string.toString)(node);
-				currentSection.content += (currentSection.content ? "\n\n" : "") + text;
-			}
-		});
-		if (currentSection) sections.push(currentSection);
-		return { data: {
-			title: frontmatter?.title ?? sections.find((s) => s.level === 1)?.title,
-			description: frontmatter?.description ?? this.extractDescription(tree),
-			sections,
-			links: allLinks,
-			codeBlocks,
-			frontmatter
-		} };
-	}
-	parseFrontmatter(content) {
-		const result = {};
-		const lines = content.split("\n");
-		for (const line of lines) {
-			const colonIndex = line.indexOf(":");
-			if (colonIndex > 0) {
-				const key = line.slice(0, colonIndex).trim();
-				let value = line.slice(colonIndex + 1).trim();
-				if (value === "true") value = true;
-				else if (value === "false") value = false;
-				else if (/^-?\d+(\.\d+)?$/.test(value)) value = Number(value);
-				else if (value.startsWith("\"") && value.endsWith("\"")) value = value.slice(1, -1);
-				else if (value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
-				result[key] = value;
-			}
-		}
-		return result;
-	}
-	extractDescription(tree) {
-		for (const node of tree.children) {
-			if (node.type === "heading") break;
-			if (node.type === "paragraph") return (0, mdast_util_to_string.toString)(node);
-		}
-	}
-};
-/**
-* Extract links from a list-based markdown structure (like awesome lists)
-*/
-function extractListLinks(markdown) {
-	const tree = (0, mdast_util_from_markdown.fromMarkdown)(markdown);
-	const links = [];
-	let currentHeading = "";
-	(0, unist_util_visit.visit)(tree, (node) => {
-		if (node.type === "heading") currentHeading = (0, mdast_util_to_string.toString)(node);
-		if (node.type === "listItem") (0, unist_util_visit.visit)(node, "link", (linkNode) => {
-			links.push({
-				url: linkNode.url,
-				text: (0, mdast_util_to_string.toString)(linkNode),
-				title: linkNode.title ?? void 0,
-				context: currentHeading || void 0
-			});
-		});
-	});
-	return links;
-}
-/**
-* Parse markdown into sections by heading level
-*/
-function parseByHeadings(markdown, minLevel = 2) {
-	return new MarkdownParser().parse(markdown).data.sections.filter((s) => s.level >= minLevel);
-}
-
-//#endregion
-exports.MarkdownParser = MarkdownParser;
-exports.extractListLinks = extractListLinks;
-exports.fetchRepoMeta = fetchRepoMeta;
-exports.groupByCategory = groupByCategory;
-exports.isGitHubRepo = isGitHubRepo;
-exports.parseByHeadings = parseByHeadings;
-exports.parseGitHubUrl = parseGitHubUrl;
-exports.toRawUrl = toRawUrl;
-//# sourceMappingURL=index.cjs.map
+exports.MarkdownParser = require_parsers.MarkdownParser;
+exports.RSSParser = require_parsers.RSSParser;
+exports.extractListLinks = require_parsers.extractListLinks;
+exports.fetchRepoMeta = require_parsers.fetchRepoMeta;
+exports.groupByCategory = require_parsers.groupByCategory;
+exports.isGitHubRepo = require_parsers.isGitHubRepo;
+exports.parseByHeadings = require_parsers.parseByHeadings;
+exports.parseGitHubUrl = require_parsers.parseGitHubUrl;
+exports.toRawUrl = require_parsers.toRawUrl;

package/dist/parsers/index.d.cts

@@ -1,133 +1,2 @@
-//#region src/parsers/types.d.ts
-/**
- * Generic source parser interface.
- * Parsers transform raw content into structured data with metadata.
- *
- * @template TData - The main data type (e.g., array of links)
- * @template TMeta - Optional metadata type
- */
-interface SourceParser<TData, TMeta = unknown> {
-  readonly name: string;
-  /**
-   * Check if this parser can handle the given content
-   */
-  canParse(content: string, url?: string): boolean;
-  /**
-   * Parse the content and extract structured data
-   */
-  parse(content: string, url?: string): ParserResult<TData, TMeta>;
-}
-/**
- * Result from a parser
- */
-interface ParserResult<TData, TMeta = unknown> {
-  data: TData;
-  meta?: TMeta;
-}
-/**
- * Markdown link extracted from content
- */
-interface MarkdownLink {
-  url: string;
-  text: string;
-  title?: string;
-  context?: string;
-}
-/**
- * Markdown section (heading + content)
- */
-interface MarkdownSection {
-  level: number;
-  title: string;
-  content: string;
-  links: MarkdownLink[];
-}
-/**
- * Parsed markdown structure
- */
-interface ParsedMarkdown {
-  title?: string;
-  description?: string;
-  sections: MarkdownSection[];
-  links: MarkdownLink[];
-  codeBlocks: CodeBlock[];
-  frontmatter?: Record<string, unknown>;
-}
-/**
- * Code block from markdown
- */
-interface CodeBlock {
-  language?: string;
-  code: string;
-  meta?: string;
-}
-/**
- * GitHub repository metadata
- */
-interface GitHubMeta {
-  repoOwner?: string;
-  repoName?: string;
-  stars?: number;
-  lastUpdated?: string;
-}
-//#endregion
-//#region src/parsers/github.d.ts
-/**
- * GitHub-specific utilities for parsing repositories.
- */
-/**
- * Check if a URL is a GitHub repository
- */
-declare function isGitHubRepo(url: string): boolean;
-/**
- * Extract GitHub repo info from URL
- */
-declare function parseGitHubUrl(url: string): {
-  owner: string;
-  repo: string;
-} | null;
-/**
- * Convert a GitHub repo URL to raw content URL
- */
-declare function toRawUrl(url: string, branch?: string, file?: string): string;
-/**
- * Fetch GitHub API metadata for a repository
- * Note: This is a placeholder - actual implementation would need GitHub API access
- */
-declare function fetchRepoMeta(owner: string, repo: string, _token?: string): Promise<GitHubMeta>;
-/**
- * Group links by their category/section
- */
-declare function groupByCategory(links: MarkdownLink[]): Map<string, MarkdownLink[]>;
-//#endregion
-//#region src/parsers/markdown.d.ts
-/**
- * Generic Markdown parser.
- * Extracts structure, links, and code blocks from markdown content.
- *
- * @example
- * ```ts
- * const parser = new MarkdownParser();
- * const result = parser.parse(markdownContent);
- * console.log(result.data.sections);
- * console.log(result.data.links);
- * ```
- */
-declare class MarkdownParser implements SourceParser<ParsedMarkdown> {
-  readonly name = "markdown";
-  canParse(content: string): boolean;
-  parse(content: string): ParserResult<ParsedMarkdown>;
-  private parseFrontmatter;
-  private extractDescription;
-}
-/**
- * Extract links from a list-based markdown structure (like awesome lists)
- */
-declare function extractListLinks(markdown: string): MarkdownLink[];
-/**
- * Parse markdown into sections by heading level
- */
-declare function parseByHeadings(markdown: string, minLevel?: number): MarkdownSection[];
-//#endregion
-export { type CodeBlock, type GitHubMeta, type MarkdownLink, MarkdownParser, type MarkdownSection, type ParsedMarkdown, type ParserResult, type SourceParser, extractListLinks, fetchRepoMeta, groupByCategory, isGitHubRepo, parseByHeadings, parseGitHubUrl, toRawUrl };
-//# sourceMappingURL=index.d.cts.map
+import { _ as MarkdownSection, a as parseByHeadings, b as ParserResult, c as isGitHubRepo, d as CodeBlock, f as FeedEnclosure, g as MarkdownLink, h as GitHubMeta, i as extractListLinks, l as parseGitHubUrl, m as FeedMeta, n as RSSParserOptions, o as fetchRepoMeta, p as FeedItem, r as MarkdownParser, s as groupByCategory, t as RSSParser, u as toRawUrl, v as ParsedFeed, x as SourceParser, y as ParsedMarkdown } from "../index-CDgcRnig.cjs";
+export { CodeBlock, FeedEnclosure, FeedItem, FeedMeta, GitHubMeta, MarkdownLink, MarkdownParser, MarkdownSection, ParsedFeed, ParsedMarkdown, ParserResult, RSSParser, RSSParserOptions, SourceParser, extractListLinks, fetchRepoMeta, groupByCategory, isGitHubRepo, parseByHeadings, parseGitHubUrl, toRawUrl };

package/dist/parsers/index.d.mts

@@ -1,133 +1,2 @@
-//#region src/parsers/types.d.ts
-/**
- * Generic source parser interface.
- * Parsers transform raw content into structured data with metadata.
- *
- * @template TData - The main data type (e.g., array of links)
- * @template TMeta - Optional metadata type
- */
-interface SourceParser<TData, TMeta = unknown> {
-  readonly name: string;
-  /**
-   * Check if this parser can handle the given content
-   */
-  canParse(content: string, url?: string): boolean;
-  /**
-   * Parse the content and extract structured data
-   */
-  parse(content: string, url?: string): ParserResult<TData, TMeta>;
-}
-/**
- * Result from a parser
- */
-interface ParserResult<TData, TMeta = unknown> {
-  data: TData;
-  meta?: TMeta;
-}
-/**
- * Markdown link extracted from content
- */
-interface MarkdownLink {
-  url: string;
-  text: string;
-  title?: string;
-  context?: string;
-}
-/**
- * Markdown section (heading + content)
- */
-interface MarkdownSection {
-  level: number;
-  title: string;
-  content: string;
-  links: MarkdownLink[];
-}
-/**
- * Parsed markdown structure
- */
-interface ParsedMarkdown {
-  title?: string;
-  description?: string;
-  sections: MarkdownSection[];
-  links: MarkdownLink[];
-  codeBlocks: CodeBlock[];
-  frontmatter?: Record<string, unknown>;
-}
-/**
- * Code block from markdown
- */
-interface CodeBlock {
-  language?: string;
-  code: string;
-  meta?: string;
-}
-/**
- * GitHub repository metadata
- */
-interface GitHubMeta {
-  repoOwner?: string;
-  repoName?: string;
-  stars?: number;
-  lastUpdated?: string;
-}
-//#endregion
-//#region src/parsers/github.d.ts
-/**
- * GitHub-specific utilities for parsing repositories.
- */
-/**
- * Check if a URL is a GitHub repository
- */
-declare function isGitHubRepo(url: string): boolean;
-/**
- * Extract GitHub repo info from URL
- */
-declare function parseGitHubUrl(url: string): {
-  owner: string;
-  repo: string;
-} | null;
-/**
- * Convert a GitHub repo URL to raw content URL
- */
-declare function toRawUrl(url: string, branch?: string, file?: string): string;
-/**
- * Fetch GitHub API metadata for a repository
- * Note: This is a placeholder - actual implementation would need GitHub API access
- */
-declare function fetchRepoMeta(owner: string, repo: string, _token?: string): Promise<GitHubMeta>;
-/**
- * Group links by their category/section
- */
-declare function groupByCategory(links: MarkdownLink[]): Map<string, MarkdownLink[]>;
-//#endregion
-//#region src/parsers/markdown.d.ts
-/**
- * Generic Markdown parser.
- * Extracts structure, links, and code blocks from markdown content.
- *
- * @example
- * ```ts
- * const parser = new MarkdownParser();
- * const result = parser.parse(markdownContent);
- * console.log(result.data.sections);
- * console.log(result.data.links);
- * ```
- */
-declare class MarkdownParser implements SourceParser<ParsedMarkdown> {
-  readonly name = "markdown";
-  canParse(content: string): boolean;
-  parse(content: string): ParserResult<ParsedMarkdown>;
-  private parseFrontmatter;
-  private extractDescription;
-}
-/**
- * Extract links from a list-based markdown structure (like awesome lists)
- */
-declare function extractListLinks(markdown: string): MarkdownLink[];
-/**
- * Parse markdown into sections by heading level
- */
-declare function parseByHeadings(markdown: string, minLevel?: number): MarkdownSection[];
-//#endregion
-export { type CodeBlock, type GitHubMeta, type MarkdownLink, MarkdownParser, type MarkdownSection, type ParsedMarkdown, type ParserResult, type SourceParser, extractListLinks, fetchRepoMeta, groupByCategory, isGitHubRepo, parseByHeadings, parseGitHubUrl, toRawUrl };
-//# sourceMappingURL=index.d.mts.map
+import { _ as MarkdownSection, a as parseByHeadings, b as ParserResult, c as isGitHubRepo, d as CodeBlock, f as FeedEnclosure, g as MarkdownLink, h as GitHubMeta, i as extractListLinks, l as parseGitHubUrl, m as FeedMeta, n as RSSParserOptions, o as fetchRepoMeta, p as FeedItem, r as MarkdownParser, s as groupByCategory, t as RSSParser, u as toRawUrl, v as ParsedFeed, x as SourceParser, y as ParsedMarkdown } from "../index-piS5wtki.mjs";
+export { CodeBlock, FeedEnclosure, FeedItem, FeedMeta, GitHubMeta, MarkdownLink, MarkdownParser, MarkdownSection, ParsedFeed, ParsedMarkdown, ParserResult, RSSParser, RSSParserOptions, SourceParser, extractListLinks, fetchRepoMeta, groupByCategory, isGitHubRepo, parseByHeadings, parseGitHubUrl, toRawUrl };

package/dist/parsers/index.mjs

@@ -1,192 +1,3 @@
-import { fromMarkdown } from "mdast-util-from-markdown";
-import { toString } from "mdast-util-to-string";
-import { visit } from "unist-util-visit";
+import { a as fetchRepoMeta, c as parseGitHubUrl, i as parseByHeadings, l as toRawUrl, n as MarkdownParser, o as groupByCategory, r as extractListLinks, s as isGitHubRepo, t as RSSParser } from "../parsers-CwkYnyWY.mjs";
 
-//#region src/parsers/github.ts
-/**
-* GitHub-specific utilities for parsing repositories.
-*/
-/**
-* Check if a URL is a GitHub repository
-*/
-function isGitHubRepo(url) {
-	return /^https?:\/\/(www\.)?github\.com\/[^/]+\/[^/]+\/?$/.test(url);
-}
-/**
-* Extract GitHub repo info from URL
-*/
-function parseGitHubUrl(url) {
-	const match = url.match(/github\.com\/([^/]+)\/([^/]+)/);
-	if (!match || !match[1] || !match[2]) return null;
-	return {
-		owner: match[1],
-		repo: match[2].replace(/\.git$/, "")
-	};
-}
-/**
-* Convert a GitHub repo URL to raw content URL
-*/
-function toRawUrl(url, branch = "main", file = "README.md") {
-	const info = parseGitHubUrl(url);
-	if (!info) return url;
-	return `https://raw.githubusercontent.com/${info.owner}/${info.repo}/${branch}/${file}`;
-}
-/**
-* Fetch GitHub API metadata for a repository
-* Note: This is a placeholder - actual implementation would need GitHub API access
-*/
-async function fetchRepoMeta(owner, repo, _token) {
-	return {
-		repoOwner: owner,
-		repoName: repo
-	};
-}
-/**
-* Group links by their category/section
-*/
-function groupByCategory(links) {
-	const groups = /* @__PURE__ */ new Map();
-	for (const link of links) {
-		const category = link.context || "Uncategorized";
-		const existing = groups.get(category) || [];
-		existing.push(link);
-		groups.set(category, existing);
-	}
-	return groups;
-}
-
-//#endregion
-//#region src/parsers/markdown.ts
-/**
-* Generic Markdown parser.
-* Extracts structure, links, and code blocks from markdown content.
-*
-* @example
-* ```ts
-* const parser = new MarkdownParser();
-* const result = parser.parse(markdownContent);
-* console.log(result.data.sections);
-* console.log(result.data.links);
-* ```
-*/
-var MarkdownParser = class {
-	name = "markdown";
-	canParse(content) {
-		return content.includes("# ") || content.includes("## ") || content.includes("- [") || content.includes("* [") || content.includes("```");
-	}
-	parse(content) {
-		const tree = fromMarkdown(content);
-		const sections = [];
-		const allLinks = [];
-		const codeBlocks = [];
-		let frontmatter;
-		if (content.startsWith("---")) {
-			const endIndex = content.indexOf("---", 3);
-			if (endIndex !== -1) {
-				const frontmatterContent = content.slice(3, endIndex).trim();
-				frontmatter = this.parseFrontmatter(frontmatterContent);
-			}
-		}
-		let currentSection = null;
-		visit(tree, (node) => {
-			if (node.type === "heading") {
-				const heading = node;
-				const title = toString(heading);
-				if (currentSection) sections.push(currentSection);
-				currentSection = {
-					level: heading.depth,
-					title,
-					content: "",
-					links: []
-				};
-			}
-			if (node.type === "link") {
-				const link = node;
-				const text = toString(link);
-				const linkData = {
-					url: link.url,
-					text,
-					title: link.title ?? void 0,
-					context: currentSection?.title
-				};
-				allLinks.push(linkData);
-				if (currentSection) currentSection.links.push(linkData);
-			}
-			if (node.type === "code") {
-				const code = node;
-				codeBlocks.push({
-					language: code.lang ?? void 0,
-					code: code.value,
-					meta: code.meta ?? void 0
-				});
-			}
-			if (currentSection && node.type === "paragraph") {
-				const text = toString(node);
-				currentSection.content += (currentSection.content ? "\n\n" : "") + text;
-			}
-		});
-		if (currentSection) sections.push(currentSection);
-		return { data: {
-			title: frontmatter?.title ?? sections.find((s) => s.level === 1)?.title,
-			description: frontmatter?.description ?? this.extractDescription(tree),
-			sections,
-			links: allLinks,
-			codeBlocks,
-			frontmatter
-		} };
-	}
-	parseFrontmatter(content) {
-		const result = {};
-		const lines = content.split("\n");
-		for (const line of lines) {
-			const colonIndex = line.indexOf(":");
-			if (colonIndex > 0) {
-				const key = line.slice(0, colonIndex).trim();
-				let value = line.slice(colonIndex + 1).trim();
-				if (value === "true") value = true;
-				else if (value === "false") value = false;
-				else if (/^-?\d+(\.\d+)?$/.test(value)) value = Number(value);
-				else if (value.startsWith("\"") && value.endsWith("\"")) value = value.slice(1, -1);
-				else if (value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
-				result[key] = value;
-			}
-		}
-		return result;
-	}
-	extractDescription(tree) {
-		for (const node of tree.children) {
-			if (node.type === "heading") break;
-			if (node.type === "paragraph") return toString(node);
-		}
-	}
-};
-/**
-* Extract links from a list-based markdown structure (like awesome lists)
-*/
-function extractListLinks(markdown) {
-	const tree = fromMarkdown(markdown);
-	const links = [];
-	let currentHeading = "";
-	visit(tree, (node) => {
-		if (node.type === "heading") currentHeading = toString(node);
-		if (node.type === "listItem") visit(node, "link", (linkNode) => {
-			links.push({
-				url: linkNode.url,
-				text: toString(linkNode),
-				title: linkNode.title ?? void 0,
-				context: currentHeading || void 0
-			});
-		});
-	});
-	return links;
-}
-/**
-* Parse markdown into sections by heading level
-*/
-function parseByHeadings(markdown, minLevel = 2) {
-	return new MarkdownParser().parse(markdown).data.sections.filter((s) => s.level >= minLevel);
-}
-
-//#endregion
-export { MarkdownParser, extractListLinks, fetchRepoMeta, groupByCategory, isGitHubRepo, parseByHeadings, parseGitHubUrl, toRawUrl };
-//# sourceMappingURL=index.mjs.map
+export { MarkdownParser, RSSParser, extractListLinks, fetchRepoMeta, groupByCategory, isGitHubRepo, parseByHeadings, parseGitHubUrl, toRawUrl };