html_docs_crawler 0.1.0__py3-none-any.whl

doc_crawler/cli.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""
+Command-line interface for the universal documentation crawler.
+This provides a more convenient way to run the doc_crawler spider.
+"""
+
+import argparse
+import os
+import sys
+from urllib.parse import urlparse
+from scrapy.cmdline import execute
+
+try:
+    from importlib.metadata import version, PackageNotFoundError
+except ImportError:
+    # Python < 3.8 compatibility
+    import pkg_resources
+
+    def version(package_name):
+        try:
+            return pkg_resources.get_distribution(package_name).version
+        except pkg_resources.DistributionNotFound:
+            return None
+
+    PackageNotFoundError = pkg_resources.DistributionNotFound
+
+
+def get_version():
+    try:
+        return version("html_docs_crawler")
+    except PackageNotFoundError:
+        return "unknown"
+
+
+def get_domain_from_url(url):
+    """Extract domain from a URL string."""
+    parsed = urlparse(url)
+    domain = parsed.netloc
+    if not domain:
+        return "unknown_domain"
+    # Remove port if present
+    if ":" in domain:
+        domain = domain.split(":")[0]
+    return domain
+
+
+def get_default_output_dir(start_urls):
+    """Generate default output directory based on the first start URL."""
+    if not start_urls:
+        return "markdown_output"
+
+    # Use the first URL to determine domain
+    first_url = start_urls.split(",")[0].strip()
+    domain = get_domain_from_url(first_url)
+
+    # Build path: ~/.config/doc_crawler/_docs/{domain}
+    home = os.path.expanduser("~")
+    return os.path.join(home, ".config", "doc_crawler", "_docs", domain)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Universal documentation crawler: Convert HTML pages to Markdown with internal link correction.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  doc_crawler --start-urls https://akshare.akfamily.xyz --allowed-domains akshare.akfamily.xyz
+  doc_crawler --start-urls https://opencode.ai/docs/zh-cn/ --allow-paths /docs/zh-cn/
+  doc_crawler --start-urls https://build123d.readthedocs.io/en/stable/ --single-page
+        """,
+    )
+
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {get_version()}",
+        help="Show version information and exit",
+    )
+
+    # Required arguments
+    parser.add_argument(
+        "--start-urls", required=True, help="Starting URLs (comma-separated)"
+    )
+
+    # Optional arguments matching the spider's parameters
+    parser.add_argument(
+        "--allowed-domains", default="", help="Allowed domains (comma-separated)"
+    )
+
+    parser.add_argument(
+        "--deny-patterns", default="", help="Regex patterns to deny (comma-separated)"
+    )
+
+    parser.add_argument(
+        "--allow-paths", default="", help="Path prefixes to allow (comma-separated)"
+    )
+
+    parser.add_argument(
+        "--body-selector",
+        default="main, article, .content, .document, .body, body",
+        help="CSS selector for HTML body content",
+    )
+
+    parser.add_argument(
+        "--output-dir",
+        default="",
+        help="Output directory for Markdown files (default: ~/.config/doc_crawler/_docs/{domain})",
+    )
+
+    parser.add_argument(
+        "--converter-engine",
+        default="markitdown",
+        choices=["markitdown", "html2text"],
+        help="Converter engine: 'markitdown' (default) or 'html2text'",
+    )
+
+    parser.add_argument(
+        "--single-page",
+        default="false",
+        choices=["true", "false"],
+        help="Single page mode (don't follow links)",
+    )
+
+    parser.add_argument(
+        "--loglevel",
+        default="INFO",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+        help="Log level",
+    )
+
+    args = parser.parse_args()
+
+    # Determine output directory
+    if not args.output_dir:
+        args.output_dir = get_default_output_dir(args.start_urls)
+    # Expand user directory (~)
+    args.output_dir = os.path.expanduser(args.output_dir)
+    # Convert to absolute path before chdir
+    args.output_dir = os.path.abspath(args.output_dir)
+    # Ensure the directory exists
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    # Change to the directory containing scrapy.cfg
+    # This is necessary for scrapy to find the project settings
+    # The cli.py file is in doc_crawler/, so go up one level to find scrapy.cfg
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    project_root = os.path.dirname(script_dir)  # doc_crawler/ -> project root
+
+    # Check if scrapy.cfg exists in the project root
+    scrapy_cfg_path = os.path.join(project_root, "scrapy.cfg")
+    if os.path.exists(scrapy_cfg_path):
+        os.chdir(project_root)
+    else:
+        # If not found, try the parent directory
+        parent_dir = os.path.dirname(project_root)
+        scrapy_cfg_path = os.path.join(parent_dir, "scrapy.cfg")
+        if os.path.exists(scrapy_cfg_path):
+            os.chdir(parent_dir)
+        else:
+            print(
+                f"警告: 未找到 scrapy.cfg 文件，当前目录: {os.getcwd()}",
+                file=sys.stderr,
+            )
+
+    # Build scrapy command arguments
+    scrapy_args = [
+        "scrapy",
+        "crawl",
+        "doc_crawler",
+        "-a",
+        f"start_urls={args.start_urls}",
+        "-a",
+        f"allowed_domains={args.allowed_domains}",
+        "-a",
+        f"deny_patterns={args.deny_patterns}",
+        "-a",
+        f"allow_paths={args.allow_paths}",
+        "-a",
+        f"body_selector={args.body_selector}",
+        "-a",
+        f"output_dir={args.output_dir}",
+        "-a",
+        f"converter_engine={args.converter_engine}",
+        "-a",
+        f"single_page={args.single_page}",
+        "--loglevel",
+        args.loglevel,
+    ]
+
+    # Execute scrapy command
+    sys.exit(execute(scrapy_args))
+
+
+if __name__ == "__main__":
+    main()

doc_crawler/items.py

@@ -0,0 +1,9 @@
+# items.py
+import scrapy
+
+
+class DocCrawlerItem(scrapy.Item):
+    url = scrapy.Field()
+    markdown_content = scrapy.Field()
+    file_path = scrapy.Field()
+    media_urls = scrapy.Field()  # 存储页面中的图片、文件等媒体链接

doc_crawler/middlewares.py

@@ -0,0 +1,39 @@
+# middlewares.py
+import logging
+from fake_useragent import UserAgent
+
+class RandomUserAgentMiddleware:
+    """为每个请求随机设置 User-Agent 的下载中间件"""
+
+    def __init__(self):
+        # 初始化 UserAgent 对象
+        self.ua = UserAgent()
+        # 备用的 User-Agent 列表（当 fake-useragent 失效时使用）
+        self.fallback_ua_list = [
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15',
+            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
+            'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1',
+            'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/121.0',
+        ]
+        self.logger = logging.getLogger(__name__)
+
+    def process_request(self, request, spider):
+        """在请求发送前设置 User-Agent"""
+        try:
+            # 尝试获取随机 UA
+            ua_string = self.ua.random
+        except Exception as e:
+            # 如果获取失败（如网络问题），从备用列表中随机选择一个
+            self.logger.warning(f"获取随机 UA 失败，使用备用列表: {e}")
+            import random
+            ua_string = random.choice(self.fallback_ua_list)
+
+        # 设置请求头
+        request.headers['User-Agent'] = ua_string
+        self.logger.debug(f'使用 User-Agent: {ua_string[:50]}...')  # 可选：记录前50字符便于调试
+        return None  # 必须返回 None，表示继续处理请求
+
+    def process_response(self, request, response, spider):
+        """处理响应（此处无需特殊操作）"""
+        return response

doc_crawler/pipelines.py

@@ -0,0 +1,160 @@
+# pipelines.py
+import os
+import random
+import asyncio
+import urllib.request
+import urllib.error
+from urllib.parse import urlparse
+from itemadapter import ItemAdapter
+
+
+class SaveMarkdownPipeline:
+    def open_spider(self, spider):
+        self.output_dir = getattr(spider, "output_dir", "markdown_output")
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
+        spider.logger.info(f"Markdown 文件将保存到: {os.path.abspath(self.output_dir)}")
+
+    def process_item(self, item, spider):
+        adapter = ItemAdapter(item)
+        file_relative_path = adapter.get("file_path")
+        markdown_content = adapter.get("markdown_content", "")
+
+        if not file_relative_path or not markdown_content:
+            spider.logger.warning(
+                f"跳过 item，缺少文件路径或内容: {adapter.get('url')}"
+            )
+            return item
+
+        full_path = os.path.join(self.output_dir, file_relative_path)
+        dir_name = os.path.dirname(full_path)
+        if dir_name and not os.path.exists(dir_name):
+            os.makedirs(dir_name, exist_ok=True)
+
+        try:
+            with open(full_path, "w", encoding="utf-8") as f:
+                f.write(markdown_content)
+            spider.logger.info(f"成功保存: {full_path}")
+        except Exception as e:
+            spider.logger.error(f"保存文件失败 {full_path}: {e}")
+
+        return item
+
+
+class MediaDownloadPipeline:
+    """下载页面中的媒体文件（图片、STL等）"""
+
+    def __init__(self):
+        try:
+            from fake_useragent import UserAgent
+
+            self.ua = UserAgent()
+            self.use_fake_ua = True
+        except ImportError:
+            self.use_fake_ua = False
+
+        self.fallback_ua_list = [
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15",
+            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+            "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
+            "Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/121.0",
+        ]
+
+        self.proxy = None
+
+    def _setup_proxy(self, spider):
+        proxy = getattr(spider, "proxy", None)
+
+        if not proxy:
+            proxy = (
+                os.environ.get("HTTP_PROXY")
+                or os.environ.get("HTTPS_PROXY")
+                or os.environ.get("http_proxy")
+                or os.environ.get("https_proxy")
+            )
+
+        self.proxy = proxy
+        if self.proxy:
+            spider.logger.info(f"使用代理: {self.proxy}")
+
+    def open_spider(self, spider):
+        self.output_dir = getattr(spider, "output_dir", "markdown_output")
+        spider.logger.info(f"媒体文件将保存到: {os.path.abspath(self.output_dir)}")
+        self._setup_proxy(spider)
+
+    def _get_random_user_agent(self):
+        if self.use_fake_ua:
+            try:
+                return self.ua.random
+            except Exception:
+                pass
+        return random.choice(self.fallback_ua_list)
+
+    async def process_item(self, item, spider):
+        adapter = ItemAdapter(item)
+        media_urls = adapter.get("media_urls", [])
+
+        if not media_urls:
+            return item
+
+        tasks = [self._download_media(url, spider) for url in media_urls]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        for url, result in zip(media_urls, results):
+            if isinstance(result, Exception):
+                spider.logger.error(f"下载媒体文件失败 {url}: {result}")
+
+        return item
+
+    async def _download_media(self, url, spider):
+        parsed = urlparse(url)
+        path = parsed.path.lstrip("/")
+
+        if not path:
+            spider.logger.warning(f"跳过无效路径的URL: {url}")
+            return
+
+        local_path = os.path.join(self.output_dir, path)
+        local_dir = os.path.dirname(local_path)
+
+        if local_dir and not os.path.exists(local_dir):
+            os.makedirs(local_dir, exist_ok=True)
+
+        if os.path.exists(local_path):
+            spider.logger.debug(f"文件已存在，跳过: {local_path}")
+            return
+
+        def _sync_download():
+            spider.logger.info(f"下载媒体文件: {url} -> {local_path}")
+            user_agent = self._get_random_user_agent()
+            headers = {
+                "User-Agent": user_agent,
+                "Accept": "image/webp,image/apng,image/*,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.9",
+            }
+            req = urllib.request.Request(url, headers=headers)
+
+            if self.proxy:
+                proxy_handler = urllib.request.ProxyHandler(
+                    {"http": self.proxy, "https": self.proxy}
+                )
+                opener = urllib.request.build_opener(proxy_handler)
+                response = opener.open(req, timeout=30)
+            else:
+                response = urllib.request.urlopen(req, timeout=30)
+
+            with response:
+                content = response.read()
+                with open(local_path, "wb") as f:
+                    f.write(content)
+            spider.logger.info(f"成功下载: {local_path}")
+
+        try:
+            await asyncio.to_thread(_sync_download)
+        except urllib.error.HTTPError as e:
+            spider.logger.warning(f"HTTP错误 {e.code} 下载 {url}: {e.reason}")
+        except urllib.error.URLError as e:
+            spider.logger.warning(f"URL错误下载 {url}: {e.reason}")
+        except Exception as e:
+            spider.logger.error(f"下载失败 {url}: {e}")
+            raise

doc_crawler/settings.py

@@ -0,0 +1,58 @@
+# Scrapy settings for akshare_docs project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = "doc_crawler"
+SPIDER_MODULES = ["doc_crawler.spiders"]
+NEWSPIDER_MODULE = "doc_crawler.spiders"
+
+
+ADDONS = {}
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+# USER_AGENT = "akshare_docs (+http://www.yourdomain.com)"
+
+
+# 启用管道
+ITEM_PIPELINES = {
+    "doc_crawler.pipelines.MediaDownloadPipeline": 200,  # 先下载媒体文件
+    "doc_crawler.pipelines.SaveMarkdownPipeline": 300,  # 再保存Markdown文件
+}
+
+# 遵守 robots.txt 规则（默认 True，建议保持）
+ROBOTSTXT_OBEY = True
+
+# 设置下载延迟（秒），避免对服务器造成压力
+DOWNLOAD_DELAY = 0.5  # 根据网站承受能力调整，可以设为 1 或更大
+
+# 并发请求数（适当降低以体现温和抓取）
+CONCURRENT_REQUESTS = 8
+CONCURRENT_REQUESTS_PER_DOMAIN = 4
+
+# 启用 Cookies 中间件（如果需要维持会话）
+COOKIES_ENABLED = True
+
+# 设置默认的请求头，模拟浏览器
+DEFAULT_REQUEST_HEADERS = {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+}
+
+
+DOWNLOADER_MIDDLEWARES = {
+    "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
+    "doc_crawler.middlewares.RandomUserAgentMiddleware": 400,
+}
+
+
+# 启用 AutoThrottle 扩展（自动限速，推荐开启）
+AUTOTHROTTLE_ENABLED = True
+AUTOTHROTTLE_START_DELAY = 1.0
+AUTOTHROTTLE_MAX_DELAY = 5.0

doc_crawler/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

doc_crawler/spiders/doc_spider.py

@@ -0,0 +1,494 @@
+# spiders/doc_spider.py
+import scrapy
+from scrapy.http import HtmlResponse
+from scrapy.linkextractors import LinkExtractor
+from scrapy.spiders import CrawlSpider, Rule
+from urllib.parse import urlparse, urljoin, urldefrag
+import os
+import re
+import base64
+import hashlib
+
+from doc_crawler.items import DocCrawlerItem  # 注意：item 类名可能需要调整
+
+
+class UniversalDocSpider(CrawlSpider):
+    """
+    通用文档爬虫：将指定网站的所有 HTML 页面转换为 Markdown，
+    并自动修正内部链接为本地 .md 文件的相对路径。
+    """
+
+    name = "doc_crawler"
+
+    # 这些属性将通过命令行参数动态设置
+    def __init__(self, *args, **kwargs):
+        # 创建父类 kwargs 的副本，移除蜘蛛使用的参数
+        spider_kwargs = kwargs.copy()
+
+        # 处理 start_urls：分割并过滤空字符串
+        start_urls_raw = spider_kwargs.pop("start_urls", "")
+        self.start_urls = [
+            url.strip() for url in start_urls_raw.split(",") if url.strip()
+        ]
+        # 添加 single_page 参数处理
+        single_page = spider_kwargs.pop("single_page", "false")
+        self.single_page = single_page.lower() in ("true", "1", "yes")
+        # 单页面模式下，如果提供了多个 URL，仅使用第一个
+        if self.single_page and len(self.start_urls) > 1:
+            print(f"单页面模式下，仅处理第一个 URL: {self.start_urls[0]}")
+            self.start_urls = self.start_urls[:1]
+        # 处理 allowed_domains：分割并过滤空字符串
+        allowed_domains_raw = spider_kwargs.pop("allowed_domains", "")
+        self.allowed_domains = [
+            domain.strip()
+            for domain in allowed_domains_raw.split(",")
+            if domain.strip()
+        ]
+        # 处理 deny_patterns
+        deny_patterns_raw = spider_kwargs.pop("deny_patterns", "")
+        self.deny_patterns = deny_patterns_raw.split(",") if deny_patterns_raw else []
+
+        self.body_selector = spider_kwargs.pop(
+            "body_selector", "main, article, .content, .document, .body, body"
+        )
+        print(f"body_selector 设置为: {self.body_selector}")
+        self.output_dir = spider_kwargs.pop("output_dir", "markdown_output")
+        # 弹出 allow_paths 和 converter_engine 以避免传递给父类
+        allow_paths_raw = spider_kwargs.pop("allow_paths", "")
+        converter_engine = spider_kwargs.pop("converter_engine", "markitdown")
+
+        # ---------- 新增：路径白名单 ----------
+        # allow_paths: 逗号分隔的路径前缀，如 "/docs/zh-cn/, /help/"
+        # 只有路径以这些前缀开头的链接才会被提取和跟踪
+
+        self.allow_paths = (
+            [p.strip() for p in allow_paths_raw.split(",") if p.strip()]
+            if allow_paths_raw
+            else []
+        )
+
+        # 过滤 start_urls，只保留路径被允许的 URL
+        if self.allow_paths:
+            filtered_start_urls = []
+            for url in self.start_urls:
+                if self._url_path_allowed(url):
+                    filtered_start_urls.append(url)
+                else:
+                    print(f"警告：跳过起始 URL（路径不匹配）: {url}")
+            self.start_urls = filtered_start_urls
+
+        # ---------- 转换引擎配置 ----------
+        self.converter_engine = converter_engine.lower()
+        self._init_converter()  # 根据引擎初始化转换器
+
+        # 如果 allowed_domains 未提供，则从 start_urls 中解析域名作为默认
+        if not self.allowed_domains and self.start_urls:
+            domains = set()
+            for url in self.start_urls:
+                parsed = urlparse(url)
+                domain = parsed.netloc
+                if domain:
+                    domains.add(domain)
+            self.allowed_domains = list(domains)
+            print(f"从 start_urls 解析的 allowed_domains: {self.allowed_domains}")
+
+        # ---------- 构建 allow 和 deny 正则 ----------
+        # allow_re：如果指定了 allow_paths，则生成只匹配这些路径前缀的正则
+        if self.allow_paths:
+            # 对每个路径进行转义，确保正则安全，并添加 ^ 前缀
+            escaped_paths = [re.escape(p) for p in self.allow_paths]
+            # 组合成正则：^(path1|path2|...)
+            self.allow_re = "^(" + "|".join(escaped_paths) + ")"
+        else:
+            self.allow_re = None
+
+        # 构建 deny 正则表达式（支持多个模式，用 | 连接）
+        deny_re = "|".join(self.deny_patterns) if self.deny_patterns else None
+
+        # 根据 single_page 参数决定是否启用链接跟踪
+        if self.single_page:
+            self.rules = ()
+        else:
+            self.rules = (
+                Rule(
+                    LinkExtractor(
+                        allow_domains=self.allowed_domains,
+                        # allow=self.allow_re if self.allow_re else (),
+                        deny=deny_re if deny_re else (),
+                    ),
+                    callback="parse_item",
+                    follow=True,
+                    process_links="filter_links_by_path",  # 新增：在处理链接时调用自定义过滤函数
+                ),
+            )
+        super().__init__(*args, **spider_kwargs)
+        # 编译规则以确保 CrawlSpider 正确使用它们
+        if hasattr(self, "_compile_rules"):
+            self._compile_rules()
+        print(">>> 初始化完成，self.start_urls =", self.start_urls)
+        if self.single_page:
+            print(">>> 单页面模式已启用，将不跟随任何链接")
+
+    # 新增一个方法，用于在生成请求前过滤链接
+    def filter_links_by_path(self, links):
+        """
+        根据解析后的绝对路径过滤链接，只保留路径以 allow_paths 中任一前缀开头的链接。
+        """
+        if not self.allow_paths:
+            return links  # 如果没有设置路径白名单，返回所有链接
+
+        filtered_links = []
+        for link in links:
+            if self._url_path_allowed(link.url):
+                filtered_links.append(link)
+            else:
+                self.logger.debug(f"过滤掉链接（路径不匹配）: {link.url}")
+
+        return filtered_links
+
+    def _url_path_allowed(self, url):
+        """检查 URL 的路径是否以 allow_paths 中的任一前缀开头"""
+        if not self.allow_paths:
+            return True
+        parsed = urlparse(url)
+        path = parsed.path
+        return any(path.startswith(prefix) for prefix in self.allow_paths)
+
+    def _init_converter(self):
+        """根据 converter_engine 初始化转换器及转换函数"""
+        if self.converter_engine == "markitdown":
+            try:
+                from markitdown import MarkItDown, StreamInfo
+                import io
+
+                self.converter = MarkItDown(enable_plugins=True)
+
+                # markitdown 的 convert 方法可以直接处理 HTML 字符串
+                # 定义转换函数：明确指定流信息为 HTML
+                def convert_with_streaminfo(html):
+                    # 将字符串编码为字节流
+                    byte_stream = io.BytesIO(html.encode("utf-8"))
+                    # 创建 StreamInfo 对象，设置 mime_type 为 'text/html'
+                    # 注意：StreamInfo 的构造函数可能因版本而异，常见用法是直接传入字典或使用 kwargs
+                    # 这里采用一种兼容的方式，将信息作为关键字参数传递
+                    # 查阅最新文档，推荐使用：stream_info = StreamInfo(mime_type='text/html')
+                    # 如果上述方式不行，可以尝试：stream_info = {'mime_type': 'text/html'}
+                    try:
+                        # 尝试使用 StreamInfo 类（如果库支持）
+                        stream_info = StreamInfo(mimetype="text/html")
+                    except ImportError:
+                        # 降级方案：直接使用字典（某些旧版本支持）
+                        stream_info = {"mimetype": "text/html"}
+
+                    result = self.converter.convert_stream(
+                        byte_stream, stream_info=stream_info
+                    )
+                    return result.text_content
+
+                self.convert_func = convert_with_streaminfo
+            except ImportError:
+                raise ImportError("markitdown 未安装，请运行: pip install markitdown")
+        elif self.converter_engine == "html2text":
+            try:
+                import html2text
+
+                self.converter = html2text.HTML2Text()
+                # 设置常用选项（可在此处根据用户需求扩展）
+                self.converter.ignore_links = False
+                self.converter.body_width = 0
+                self.converter.protect_links = True
+                self.converter.mark_code = True
+                self.converter.ignore_images = False
+                # 防止将图片src作为alt文本
+                self.converter.images_to_alt = False
+                # 保留图片的width/height属性
+                self.converter.images_with_size = True
+                self.convert_func = self.converter.handle
+            except ImportError:
+                raise ImportError("html2text 未安装，请运行: pip install html2text")
+        else:
+            raise ValueError(
+                f"不支持的转换引擎: {self.converter_engine}，可选: markitdown, html2text"
+            )
+
+    def start_requests(self):
+        if self.single_page:
+            # 单页面模式下，为每个起始URL创建请求，直接调用parse_item
+            for url in self.start_urls:
+                yield scrapy.Request(url, callback=self.parse_item)
+        else:
+            # 普通模式下，使用父类的默认行为
+            yield from super().start_requests()
+
+    def parse_item(self, response):
+        # 检查 URL 路径是否被允许
+        if not self._url_path_allowed(response.url):
+            self.logger.debug(f"跳过页面（路径不匹配）: {response.url}")
+            return
+        self.logger.info(f"Parsing item from {response.url}")
+        item = DocCrawlerItem()
+        item["url"] = response.url
+
+        # 检查响应类型：如果不是 HTML 响应，则将其视为媒体文件
+        if not isinstance(response, HtmlResponse):
+            # 将当前 URL 作为媒体文件下载
+            media_urls = [response.url]
+            item["media_urls"] = media_urls
+            # 生成文件路径
+            item["file_path"] = self._url_to_file_path(response.url)
+            # 设置空的内容
+            item["markdown_content"] = ""
+            yield item
+            return
+
+        # 提取媒体文件链接（图片、STL等）
+        media_urls = self._extract_media_urls(response)
+        item["media_urls"] = media_urls
+
+        # 提取主体内容（使用用户提供的 CSS 选择器）
+        self.logger.debug(f"使用 body_selector: {self.body_selector}")
+        main_content = response.css(self.body_selector).get()
+        self.logger.debug(f"main_content 找到: {bool(main_content)}")
+        raw_html = main_content if main_content else response.text
+        if not main_content:
+            self.logger.warning(
+                f"body_selector '{self.body_selector}' 未匹配到内容，使用完整页面"
+            )
+        self.logger.debug(f"raw_html:\n{raw_html}")
+        # 清洗 HTML，只保留 <body> 部分
+        if not main_content:
+            response.selector.remove_namespaces()
+            cleaned_html = response.selector.xpath("//body").get() or raw_html
+        else:
+            cleaned_html = raw_html
+
+        # 生成当前文件的本地保存路径
+        current_file_path = self._url_to_file_path(response.url)
+        item["file_path"] = current_file_path
+
+        # 处理 HTML 中的 base64 图片：保存为本地文件并替换 src
+        cleaned_html = self._save_base64_images(cleaned_html, current_file_path)
+
+        # 转换为 Markdown
+        try:
+            markdown_text = self.convert_func(cleaned_html)
+
+        except Exception as e:
+            self.logger.error(f"转换失败 {response.url}: {e}", exc_info=True)
+            markdown_text = ""
+
+        # 转换内部链接
+        if markdown_text:
+            markdown_text = self._convert_internal_links(
+                markdown_text, current_file_path, response.url
+            )
+
+        item["markdown_content"] = markdown_text
+
+        yield item
+
+    # ---------- 辅助方法（与之前相同）----------
+    def _extract_media_urls(self, response):
+        """从响应中提取图片和文件链接"""
+        media_extensions = {
+            # 图片格式
+            ".png",
+            ".jpg",
+            ".jpeg",
+            ".gif",
+            ".svg",
+            ".webp",
+            ".bmp",
+            ".ico",
+            ".tiff",
+            # 文档/文件格式
+            ".stl",
+            ".pdf",
+            ".zip",
+            ".gz",
+            ".tar",
+            ".doc",
+            ".docx",
+            ".ppt",
+            ".pptx",
+            ".xls",
+            ".xlsx",
+            ".txt",
+            ".csv",
+            ".json",
+            ".xml",
+            ".ipynb",
+        }
+
+        media_urls = set()
+
+        # 提取图片标签
+        for img in response.css("img"):
+            src = img.attrib.get("src")
+            if src:
+                # 跳过 data: URL (base64 嵌入图片)
+                if src.startswith("data:"):
+                    continue
+                absolute_url = urljoin(response.url, src)
+                # 只下载内部链接的媒体文件
+                if self._is_internal_link(absolute_url, response.url):
+                    media_urls.add(absolute_url)
+                else:
+                    self.logger.debug(f"跳过外部媒体链接: {absolute_url}")
+
+        # 提取链接标签中的文件
+        for link in response.css("a"):
+            href = link.attrib.get("href")
+            if href:
+                # 检查是否是媒体文件
+                if any(href.lower().endswith(ext) for ext in media_extensions):
+                    absolute_url = urljoin(response.url, href)
+                    # 只下载内部链接的媒体文件
+                    if self._is_internal_link(absolute_url, response.url):
+                        media_urls.add(absolute_url)
+                    else:
+                        self.logger.debug(f"跳过外部媒体链接: {absolute_url}")
+
+        return list(media_urls)
+
+    def _save_base64_images(self, html_content, current_file_path):
+        """将 HTML 中的 base64 内嵌图片保存为本地文件，并替换 src 为相对路径"""
+        pattern = r'(<img[^>]*?src=)["\'](data:[^"\']+)["\']([^>]*?>)'
+
+        def replace_data_url(match):
+            prefix = match.group(1)
+            data_url = match.group(2)
+            suffix = match.group(3)
+
+            header, _, data = data_url.partition(",")
+            if not data:
+                return match.group(0)
+
+            if ";base64" not in header:
+                return match.group(0)
+
+            # 确定扩展名
+            ext = ".png"
+            if "image/jpeg" in header or "image/jpg" in header:
+                ext = ".jpg"
+            elif "image/png" in header:
+                ext = ".png"
+            elif "image/gif" in header:
+                ext = ".gif"
+            elif "image/svg+xml" in header:
+                ext = ".svg"
+            elif "image/webp" in header:
+                ext = ".webp"
+
+            try:
+                image_data = base64.b64decode(data)
+            except Exception:
+                return match.group(0)
+
+            file_hash = hashlib.md5(image_data).hexdigest()[:8]
+            base_name = os.path.splitext(os.path.basename(current_file_path))[0]
+            image_filename = f"{base_name}_{file_hash}{ext}"
+
+            image_dir = os.path.join(
+                self.output_dir, os.path.dirname(current_file_path)
+            )
+            image_full_path = os.path.join(image_dir, image_filename)
+            os.makedirs(image_dir, exist_ok=True)
+            with open(image_full_path, "wb") as f:
+                f.write(image_data)
+
+            self.logger.info(f"保存 base64 图片: {image_filename}")
+            return f'{prefix}"{image_filename}"{suffix}'
+
+        return re.sub(pattern, replace_data_url, html_content, flags=re.IGNORECASE)
+
+    def _url_to_file_path(self, url):
+        parsed = urlparse(url)
+        path = parsed.path
+        if path.endswith("/"):
+            path = path + "index.md"
+        elif path.endswith(".html"):
+            path = path[:-5] + ".md"
+        else:
+            if not os.path.splitext(path)[1]:
+                path = (
+                    os.path.join(path, "index.md")
+                    if path.endswith("/")
+                    else path + ".md"
+                )
+        return path.lstrip("/")
+
+    def _is_internal_link(self, url, base_url):
+        absolute = urljoin(base_url, url)
+        parsed = urlparse(absolute)
+        return any(parsed.netloc.endswith(domain) for domain in self.allowed_domains)
+
+    def _convert_url(self, url, base_url, current_file_path):
+        if not self._is_internal_link(url, base_url):
+            return url
+        target_abs = urljoin(base_url, url)
+        target_path = self._url_to_file_path(target_abs)
+        rel_path = os.path.relpath(
+            target_path, os.path.dirname(current_file_path)
+        ).replace("\\", "/")
+        if rel_path == ".":
+            rel_path = os.path.basename(target_path)
+        return rel_path
+
+    def _convert_internal_links(self, markdown_text, current_file_path, base_url):
+        # 匹配图片和链接：![alt](url) 或 [text](url)
+        pattern = r"(!?)\[([^\]]*)\]\(([^)]+)\)"
+
+        def replace_link(match):
+            is_image = match.group(1) == "!"
+            text = match.group(2)
+            inner = match.group(3).strip()
+
+            # 清理图片的alt文本：如果看起来像URL（包含路径分隔符或图片扩展名），则清空
+            if is_image:
+                # 常见的图片扩展名
+                image_extensions = {
+                    ".png",
+                    ".jpg",
+                    ".jpeg",
+                    ".gif",
+                    ".svg",
+                    ".webp",
+                    ".bmp",
+                    ".ico",
+                    ".tiff",
+                }
+                # 如果alt文本看起来像URL（包含'/'或'\'或以图片扩展名结尾），清空它
+                # 这样可以避免清空正常的alt文本如"Logo"或"Diagram"
+                text_lower = text.lower()
+                if (
+                    "/" in text
+                    or "\\" in text
+                    or any(text_lower.endswith(ext) for ext in image_extensions)
+                ):
+                    text = ""
+                    self.logger.debug(
+                        f"清空图片alt文本（看起来像URL）: {match.group(2)}"
+                    )
+
+            if inner.startswith("<"):
+                angle_match = re.match(r"<([^>]+)>", inner)
+                if not angle_match:
+                    return match.group(0)
+                old_url = angle_match.group(1)
+                new_url = self._convert_url(old_url, base_url, current_file_path)
+                new_inner = inner.replace(f"<{old_url}>", f"<{new_url}>", 1)
+            else:
+                parts = inner.split(None, 1)
+                old_url = parts[0]
+                new_url = self._convert_url(old_url, base_url, current_file_path)
+                if len(parts) == 1:
+                    new_inner = new_url
+                else:
+                    new_inner = new_url + " " + parts[1]
+
+            # 重新构建链接，保留图片标记
+            prefix = "!" if is_image else ""
+            return f"{prefix}[{text}]({new_inner})"
+
+        return re.sub(pattern, replace_link, markdown_text)

html_docs_crawler-0.1.0.dist-info/METADATA

@@ -0,0 +1,98 @@
+Metadata-Version: 2.4
+Name: html_docs_crawler
+Version: 0.1.0
+Summary: Universal documentation crawler that converts HTML pages to Markdown with internal link correction
+Project-URL: Homepage, https://github.com/zwidny/doc_crawler
+Project-URL: Source, https://github.com/zwidny/doc_crawler
+Project-URL: BugTracker, https://github.com/zwidny/doc_crawler/issues
+Keywords: scrapy,crawler,markdown,documentation,web-scraping
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: fake-useragent>=2.2.0
+Requires-Dist: html2text>=2025.4.15
+Requires-Dist: markitdown[docx,pdf,pptx]>=0.1.5
+Requires-Dist: scrapy>=2.14.1
+
+# scrapy-mth
+
+A Scrapy-based universal documentation crawler that converts HTML documentation sites to Markdown format, with automatic internal link rewriting to local `.md` relative paths. Supports multiple converter engines (markitdown / html2text), path whitelist filtering, and automatic media file download.
+
+## Parameters
+
+- `start_urls`: Starting URLs (comma-separated)
+- `allowed_domains`: Allowed domains (comma-separated)
+- `deny_patterns`: Regex deny patterns (comma-separated)
+- `allow_paths`: Allowed path prefixes (comma-separated); only URLs starting with these prefixes will be processed
+- `body_selector`: CSS selector for main HTML content (default: `"main, article, .content, .document, .body, body"`)
+- `output_dir`: Output directory (default: `"~/.config/doc_crawler/_docs/{domain_name}"`, where `{domain_name}` is extracted from `start_urls`)
+- `converter_engine`: Converter engine (default: `"markitdown"`, optional: `"html2text"`)
+- `single_page`: Single-page mode (default: `"false"`, set to `"true"` to crawl a single page without following links)
+
+## Install via UV
+
+```bash
+uv tool install git+https://github.com/zwidny/doc_crawler.git
+```
+
+After installation, you can use the `doc_crawler` command from any directory.
+
+## Usage Examples
+
+```bash
+# Crawl AKShare documentation
+doc_crawler --start-urls "https://akshare.akfamily.xyz" \
+  --allowed-domains "akshare.akfamily.xyz" \
+  --deny-patterns "/_sources/" \
+  --body-selector "main, article, .content, .document, .body" \
+  --output-dir "_docs/akshare_markdown"
+```
+
+### Single-page mode
+
+```bash
+doc_crawler --start-urls "https://build123d.readthedocs.io/en/stable/examples_1.html" \
+  --single-page true \
+  --body-selector ".wy-nav-content" \
+  --output-dir "single_page_output"
+```
+
+### Path whitelist filtering
+
+```bash
+doc_crawler --start-urls "https://opencode.ai/docs/zh-cn/" \
+  --allow-paths "/docs/zh-cn/" \
+  --body-selector "main, article, .content" \
+  --output-dir "_docs/opencode_docs_zh_cn"
+```
+
+### Crawl with html2text engine
+
+```bash
+doc_crawler --start-urls "https://akshare.akfamily.xyz/" \
+  --allowed-domains "akshare.akfamily.xyz" \
+  --deny-patterns "/_sources/" \
+  --body-selector "main, article, .content, .document, .body" \
+  --converter-engine "html2text" \
+  --output-dir "_docs/akshare_markdown_html2text"
+```
+
+### More examples
+
+```bash
+# Crawl build123d docs
+doc_crawler --start-urls "https://build123d.readthedocs.io/en/stable/" \
+  --deny-patterns "/_sources/,/latest/" \
+  --body-selector ".wy-nav-content" \
+  --output-dir "_docs/build123d"
+
+# Crawl Docusaurus docs
+doc_crawler --start-urls "https://docusaurus.io/docs" \
+  --allow-paths "/docs" \
+  --body-selector ".col.docItemCol_n6xZ" \
+  --output-dir "_docs/docusaurus"
+
+# Crawl uv documentation
+doc_crawler --start-urls "https://docs.astral.sh/uv/" \
+  --body-selector ".md-content" \
+  --output-dir "_docs/uv"
+```

html_docs_crawler-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+scrapy.cfg,sha256=TKK9gKdRaYNWn06Irgd74zC6AQ9DsI8Ksr6JaExpkEw,265
+doc_crawler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+doc_crawler/cli.py,sha256=Qejxu7V83yia-kfJZb5yEbAOLLoRLEV73WXztJK-k-A,5857
+doc_crawler/items.py,sha256=QUfYvMG6dMiHcTXZlXDqYcxhSMR5EgvDMGAnw2fywwY,240
+doc_crawler/middlewares.py,sha256=1UsJy_j5DMNvK_Ys30iczyyiJ9sI7TZR1TOX_6jOqPE,1927
+doc_crawler/pipelines.py,sha256=CzMXo1i0UyiKpXdv8MqYJRRxFYIwz-T-ptba90fi6eo,5891
+doc_crawler/settings.py,sha256=ce4obL5ALRWMHaRNA_I7VBU_Yryhzg2-piXdFZuNV8A,1953
+doc_crawler/spiders/__init__.py,sha256=ULwecZkx3_NTphkz7y_qiazBeUoHFnCCWnKSjoDCZj0,161
+doc_crawler/spiders/doc_spider.py,sha256=sUNXg6boEBwowLIbTgw-6wUksEix1qSilV50LqTcfj4,20121
+html_docs_crawler-0.1.0.dist-info/METADATA,sha256=BNjRApqgU2GPObKC_CTPXyvnHef4It77o_B50zNtDNM,3530
+html_docs_crawler-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+html_docs_crawler-0.1.0.dist-info/entry_points.txt,sha256=rM3npJjcF02jm5w_D5ZajDck2KbhwmCLzU6Ic0JyiHQ,53
+html_docs_crawler-0.1.0.dist-info/top_level.txt,sha256=lMBLRQiCumDc-fNb8HcBPLFO_aC_fiMyLr5zGdCVAV0,12
+html_docs_crawler-0.1.0.dist-info/RECORD,,

html_docs_crawler-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

html_docs_crawler-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+doc_crawler = doc_crawler.cli:main

html_docs_crawler-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+doc_crawler

scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = doc_crawler.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = doc_crawler