scrapekit-local 0.1.0__tar.gz

scrapekit_local-0.1.0/.gitignore

@@ -0,0 +1,32 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+dist/
+build/
+*.egg
+
+# 虚拟环境
+.venv/
+venv/
+env/
+
+# 开发工具
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+
+# ScrapeKit 本地数据
+~/.scrapekit/
+
+# 系统
+.DS_Store
+Thumbs.db

scrapekit_local-0.1.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: scrapekit-local
+Version: 0.1.0
+Summary: Local-first CLI web scraper with automatic strategy selection
+Requires-Python: >=3.11
+Requires-Dist: beautifulsoup4>=4.14
+Requires-Dist: httpx>=0.28
+Requires-Dist: playwright>=1.44
+Requires-Dist: typer>=0.12

scrapekit_local-0.1.0/pyproject.toml

@@ -0,0 +1,24 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "scrapekit-local"
+version = "0.1.0"
+description = "Local-first CLI web scraper with automatic strategy selection"
+requires-python = ">=3.11"
+dependencies = [
+    "httpx>=0.28",
+    "beautifulsoup4>=4.14",
+    "typer>=0.12",
+    "playwright>=1.44",
+]
+
+[project.scripts]
+scrape = "scrapekit.cli:app"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/scrapekit"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/"]

scrapekit_local-0.1.0/src/scrapekit/__init__.py

@@ -0,0 +1,3 @@
+"""ScrapeKit — local-first web scraper with automatic strategy selection."""
+
+__version__ = "0.1.0"

scrapekit_local-0.1.0/src/scrapekit/cache/__init__.py

@@ -0,0 +1,3 @@
+from scrapekit.cache import profile
+
+__all__ = ["profile"]

scrapekit_local-0.1.0/src/scrapekit/cache/profile.py

@@ -0,0 +1,172 @@
+"""
+scrapekit/cache/profile.py
+===========================
+Profile Cache — 按 domain 缓存最优抓取策略。
+
+作用：
+  第一次访问某站点时，ScrapeKit 会探索最优策略（Level 1 / 2 / 3）。
+  成功后，将"配方"写入本地 JSON 文件。
+  后续请求直接跳到已知的最优层级，跳过探索开销。
+
+存储位置：~/.scrapekit/profiles/<domain>.json
+
+配方格式：
+  {
+    "domain": "sportsbet.com.au",
+    "preferred_level": 3,
+    "state_var": "window.__PRELOADED_STATE__",   // Level 3 专用
+    "discovered_at": "2026-04-16T00:00:00Z",
+    "last_success": "2026-04-16T00:08:00Z",
+    "success_count": 12,
+    "failure_count": 0
+  }
+
+v0.1 范围：
+  - domain 级别匹配（保守，安全）
+  - 30 天过期
+  - 提取失败时立即失效，触发重新探索
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timezone, timedelta
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urlparse
+
+# 缓存目录
+CACHE_DIR = Path.home() / ".scrapekit" / "profiles"
+
+# 配方有效期（天）
+STALE_DAYS = 30
+
+
+# ---------------------------------------------------------------------------
+# 工具函数
+# ---------------------------------------------------------------------------
+
+def _domain(url: str) -> str:
+    """从 URL 提取 domain（不含 www.）。"""
+    parsed = urlparse(url)
+    host = parsed.netloc or parsed.path
+    # 去掉端口
+    host = host.split(":")[0]
+    # 去掉 www.
+    if host.startswith("www."):
+        host = host[4:]
+    return host.lower()
+
+
+def _profile_path(domain: str) -> Path:
+    return CACHE_DIR / f"{domain}.json"
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+# ---------------------------------------------------------------------------
+# 公开 API
+# ---------------------------------------------------------------------------
+
+def load(url: str) -> Optional[dict]:
+    """
+    读取 domain 的缓存配方。
+
+    Returns:
+        配方 dict，或 None（未命中 / 已过期）
+    """
+    domain = _domain(url)
+    path = _profile_path(domain)
+
+    if not path.exists():
+        return None
+
+    try:
+        with open(path, encoding="utf-8") as f:
+            profile = json.load(f)
+    except (json.JSONDecodeError, OSError):
+        return None
+
+    # 检查是否过期
+    discovered = profile.get("discovered_at", "")
+    if discovered:
+        try:
+            dt = datetime.fromisoformat(discovered)
+            if datetime.now(timezone.utc) - dt > timedelta(days=STALE_DAYS):
+                path.unlink(missing_ok=True)
+                return None
+        except ValueError:
+            pass
+
+    return profile
+
+
+def save(url: str, result_level: int, state_var: Optional[str] = None) -> None:
+    """
+    保存或更新 domain 的缓存配方。
+
+    Args:
+        url:          成功抓取的 URL
+        result_level: 实际使用的层级（1 / 2 / 3）
+        state_var:    Level 3 时用到的 JS 变量名
+    """
+    domain = _domain(url)
+    path = _profile_path(domain)
+    CACHE_DIR.mkdir(parents=True, exist_ok=True)
+
+    # 读取已有记录（更新计数器）
+    existing: dict = {}
+    if path.exists():
+        try:
+            with open(path, encoding="utf-8") as f:
+                existing = json.load(f)
+        except (json.JSONDecodeError, OSError):
+            existing = {}
+
+    now = _now_iso()
+    profile = {
+        "domain": domain,
+        "preferred_level": result_level,
+        "state_var": state_var,
+        "discovered_at": existing.get("discovered_at", now),
+        "last_success": now,
+        "success_count": existing.get("success_count", 0) + 1,
+        "failure_count": existing.get("failure_count", 0),
+    }
+
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(profile, f, ensure_ascii=False, indent=2)
+
+
+def invalidate(url: str) -> None:
+    """提取失败时调用，立即删除缓存配方，触发下次重新探索。"""
+    domain = _domain(url)
+    path = _profile_path(domain)
+    if path.exists():
+        # 增加失败计数，但不删除（保留历史，只标记失效）
+        try:
+            with open(path, encoding="utf-8") as f:
+                profile = json.load(f)
+            profile["failure_count"] = profile.get("failure_count", 0) + 1
+            profile["discovered_at"] = ""   # 清空，下次 load() 会返回 None
+            with open(path, "w", encoding="utf-8") as f:
+                json.dump(profile, f, ensure_ascii=False, indent=2)
+        except (json.JSONDecodeError, OSError):
+            path.unlink(missing_ok=True)
+
+
+def list_profiles() -> list[dict]:
+    """列出所有已缓存的配方（用于 scrape profile list 命令，Phase 2）。"""
+    if not CACHE_DIR.exists():
+        return []
+    profiles = []
+    for p in sorted(CACHE_DIR.glob("*.json")):
+        try:
+            with open(p, encoding="utf-8") as f:
+                profiles.append(json.load(f))
+        except (json.JSONDecodeError, OSError):
+            continue
+    return profiles

scrapekit_local-0.1.0/src/scrapekit/cli.py

@@ -0,0 +1,83 @@
+"""
+scrapekit/cli.py
+=================
+CLI 入口（typer）
+
+用法：
+    scrape <url>
+    scrape <url> --output md
+    scrape <url> --force-level 2
+    scrape <url> --verbose
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+from enum import Enum
+from typing import Optional
+
+import typer
+
+from scrapekit.dispatcher import dispatch
+from scrapekit.models import ExtractionError
+
+app = typer.Typer(
+    name="scrape",
+    help="Local-first web scraper with automatic strategy selection.",
+    add_completion=False,
+)
+
+
+class OutputFormat(str, Enum):
+    json = "json"
+    md   = "md"
+
+
+@app.command()
+def scrape(
+    url: str = typer.Argument(..., help="Target URL to scrape"),
+    output: OutputFormat = typer.Option(
+        OutputFormat.json,
+        "--output", "-o",
+        help="Output format: json (default) or md (markdown)",
+    ),
+    force_level: Optional[int] = typer.Option(
+        None,
+        "--force-level", "-l",
+        help="Force a specific extraction level (1=HTTP, 2=CDP)",
+        min=1,
+        max=2,
+    ),
+    cdp_endpoint: str = typer.Option(
+        "http://localhost:9222",
+        "--cdp",
+        help="Chrome DevTools endpoint",
+    ),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose", "-v",
+        help="Print escalation steps to stderr",
+    ),
+) -> None:
+    """Scrape a URL and output structured data."""
+
+    try:
+        result = asyncio.run(
+            dispatch(
+                url,
+                force_level=force_level,
+                cdp_endpoint=cdp_endpoint,
+                verbose=verbose,
+            )
+        )
+    except ExtractionError as e:
+        typer.echo(f"Error: {e.reason}", err=True)
+        raise typer.Exit(code=1)
+    except KeyboardInterrupt:
+        raise typer.Exit(code=130)
+
+    if output == OutputFormat.json:
+        typer.echo(result.to_json())
+    else:
+        typer.echo(result.to_markdown())

scrapekit_local-0.1.0/src/scrapekit/dispatcher.py

@@ -0,0 +1,158 @@
+"""
+scrapekit/dispatcher.py
+========================
+核心调度器：按策略阶梯自动升级。
+
+Level 1 → Level 2 → Level 3（v0.1 只支持到 Level 2）
+
+升级条件：
+  - Level 1 抛出 NeedsBrowserError → 升级到 Level 2
+  - Level 1 抛出 ExtractionError  → 升级到 Level 2（网络失败也值得用浏览器重试）
+  - --force-level N              → 跳过前面的层级
+
+未来扩展：
+  - Level 2 → Level 3：检测到 __PRELOADED_STATE__ 等 JS state 变量后升级
+  - Profile Cache：命中缓存时直接跳到已知层级
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Optional
+
+from scrapekit.models import ExtractResult, NeedsBrowserError, ExtractionError
+from scrapekit.extractor import http_extract, cdp_extract, is_chrome_available
+from scrapekit.cache import profile as profile_cache
+
+
+# ---------------------------------------------------------------------------
+# 主入口
+# ---------------------------------------------------------------------------
+
+async def dispatch(
+    url: str,
+    *,
+    force_level: Optional[int] = None,
+    cdp_endpoint: str = "http://localhost:9222",
+    http_timeout: float = 15.0,
+    page_timeout: float = 30.0,
+    verbose: bool = False,
+    use_cache: bool = True,
+) -> ExtractResult:
+    """
+    自动选择最优抓取策略并执行。
+
+    Args:
+        url:          目标 URL
+        force_level:  强制使用指定层级（1/2/3），跳过自动判断
+        cdp_endpoint: Chrome 调试端点
+        http_timeout: Level 1 HTTP 超时（秒）
+        page_timeout: Level 2/3 页面加载超时（秒）
+        verbose:      打印升级过程
+        use_cache:    是否使用 Profile Cache
+
+    Returns:
+        ExtractResult
+
+    Raises:
+        ExtractionError: 所有层级均失败
+    """
+    def log(msg: str) -> None:
+        if verbose:
+            print(f"[scrapekit] {msg}", flush=True)
+
+    # ── 强制指定层级 ──────────────────────────────────────────────────────
+    if force_level == 1:
+        log("forced Level 1 (HTTP)")
+        return await http_extract(url, timeout=http_timeout)
+
+    if force_level in (2, 3):
+        log(f"forced Level 2/3 (CDP)")
+        return await _try_level2(url, cdp_endpoint, page_timeout, log)
+
+    # ── Profile Cache：命中时直接跳到已知层级 ─────────────────────────────
+    if use_cache:
+        cached = profile_cache.load(url)
+        if cached:
+            preferred = cached.get("preferred_level", 1)
+            log(f"cache hit → Level {preferred} for {cached.get('domain')}")
+            if preferred == 1:
+                try:
+                    result = await http_extract(url, timeout=http_timeout)
+                    profile_cache.save(url, result.level)
+                    return result
+                except Exception:
+                    log("cache: Level 1 failed, invalidating and re-exploring")
+                    profile_cache.invalidate(url)
+            else:
+                try:
+                    result = await _try_level2(url, cdp_endpoint, page_timeout, log)
+                    profile_cache.save(url, result.level,
+                                       state_var=cached.get("state_var"))
+                    return result
+                except Exception:
+                    log("cache: Level 2 failed, invalidating and re-exploring")
+                    profile_cache.invalidate(url)
+
+    # ── 自动升级探索 ──────────────────────────────────────────────────────
+
+    # Level 1：HTTP 直取
+    log(f"Level 1 → {url}")
+    try:
+        result = await http_extract(url, timeout=http_timeout)
+        log(f"Level 1 succeeded ({result.elapsed_ms:.0f}ms)")
+        if use_cache:
+            profile_cache.save(url, result.level)
+        return result
+
+    except NeedsBrowserError as e:
+        log(f"Level 1 → needs browser ({e.reason}), escalating to Level 2/3")
+
+    except ExtractionError as e:
+        log(f"Level 1 → error ({e.reason}), escalating to Level 2/3")
+
+    # Level 2/3：CDP + 已有 Chrome（内部自动尝试 Level 3 state 提取）
+    result = await _try_level2(url, cdp_endpoint, page_timeout, log)
+    if use_cache:
+        state_var = None
+        if result.level == 3:
+            from scrapekit.extractor.state import KNOWN_STATE_VARS
+            state_var = KNOWN_STATE_VARS[0]   # v0.1 固定
+        profile_cache.save(url, result.level, state_var=state_var)
+    return result
+
+
+async def _try_level2(
+    url: str,
+    cdp_endpoint: str,
+    page_timeout: float,
+    log,
+) -> ExtractResult:
+    """尝试 Level 2，Chrome 不可用时给出清晰错误。"""
+    from scrapekit.extractor.cdp import ChromeNotRunningError
+
+    log(f"Level 2 → {url}")
+
+    # 提前探测 Chrome，给出更友好的错误
+    if not await is_chrome_available(cdp_endpoint):
+        raise ExtractionError(
+            url,
+            f"Level 1 failed and Chrome is not available at {cdp_endpoint}. "
+            "Start Chrome with: chrome --remote-debugging-port=9222",
+        )
+
+    try:
+        result = await cdp_extract(
+            url,
+            cdp_endpoint=cdp_endpoint,
+            page_timeout=page_timeout,
+        )
+        log(f"Level 2 succeeded ({result.elapsed_ms:.0f}ms)")
+        return result
+
+    except ChromeNotRunningError:
+        raise
+    except ExtractionError:
+        raise
+    except Exception as e:
+        raise ExtractionError(url, f"Level 2 unexpected error: {e}") from e

scrapekit_local-0.1.0/src/scrapekit/extractor/__init__.py

@@ -0,0 +1,4 @@
+from scrapekit.extractor.http import extract as http_extract, needs_browser
+from scrapekit.extractor.cdp import extract as cdp_extract, is_chrome_available
+
+__all__ = ["http_extract", "needs_browser", "cdp_extract", "is_chrome_available"]

scrapekit_local-0.1.0/src/scrapekit/extractor/cdp.py

@@ -0,0 +1,201 @@
+"""
+scrapekit/extractor/cdp.py
+===========================
+Level 2 — CDP + 已有 Chrome Session（Playwright connect_over_cdp）
+
+核心优势：
+  复用用户已经登录的 Chrome，无需重新认证、无需存储凭据。
+  Chrome 需提前以 --remote-debugging-port=9222 启动（见 README）。
+
+流程：
+  1. 连接到 localhost:9222
+  2. 取已有 context（继承登录态 / cookies）
+  3. 新开一个标签页，导航到目标 URL
+  4. 等待页面加载（domcontentloaded + networkidle）
+  5. 提取 HTML，复用 Level 1 的 BeautifulSoup 逻辑
+  6. 关闭标签页，不污染 Chrome 会话
+
+错误处理：
+  - Chrome 未启动 → ChromeNotRunningError（清晰提示）
+  - 页面加载超时 → ExtractionError
+  - 其他 → ExtractionError
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Optional
+
+from bs4 import BeautifulSoup
+
+from scrapekit.models import ExtractResult, ExtractionError
+from scrapekit.extractor.http import (
+    _extract_title,
+    _extract_text,
+    _extract_links,
+)
+
+# CDP 端口（可通过环境变量覆盖）
+CDP_ENDPOINT = "http://localhost:9222"
+
+# 等待策略
+WAIT_UNTIL = "domcontentloaded"   # 第一阶段：DOM 就绪
+NETWORK_IDLE_TIMEOUT = 5_000      # ms，等 networkidle 的最长时间（不强制）
+
+
+# ---------------------------------------------------------------------------
+# 专属异常
+# ---------------------------------------------------------------------------
+
+class ChromeNotRunningError(ExtractionError):
+    """
+    Chrome 未以 --remote-debugging-port=9222 运行时抛出。
+    提示用户如何启动 Chrome。
+    """
+    def __init__(self, endpoint: str = CDP_ENDPOINT):
+        super().__init__(
+            url="",
+            reason=(
+                f"Cannot connect to Chrome at {endpoint}. "
+                "Start Chrome with: "
+                "chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-debug"
+            ),
+        )
+
+
+# ---------------------------------------------------------------------------
+# 公开 API
+# ---------------------------------------------------------------------------
+
+async def extract(
+    url: str,
+    *,
+    cdp_endpoint: str = CDP_ENDPOINT,
+    page_timeout: float = 30.0,
+    wait_networkidle: bool = True,
+) -> ExtractResult:
+    """
+    Level 2 CDP 抓取。
+
+    Args:
+        url:               目标 URL
+        cdp_endpoint:      Chrome DevTools 端点，默认 http://localhost:9222
+        page_timeout:      页面导航超时（秒）
+        wait_networkidle:  是否等待 networkidle（SPA 内容渲染完毕）
+
+    Returns:
+        ExtractResult（level=2）
+
+    Raises:
+        ChromeNotRunningError: Chrome 未启动或未开启远程调试
+        ExtractionError:       页面加载或提取失败
+    """
+    try:
+        from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeout
+    except ImportError as e:
+        raise ExtractionError(url, "playwright not installed: pip install playwright") from e
+
+    t0 = time.perf_counter()
+
+    async with async_playwright() as pw:
+        # ── 1. 连接已有 Chrome ────────────────────────────────────────────
+        try:
+            browser = await pw.chromium.connect_over_cdp(
+                cdp_endpoint,
+                timeout=5_000,   # 连接超时 5s，快速失败
+            )
+        except Exception as e:
+            err_str = str(e).lower()
+            if any(k in err_str for k in ("connection refused", "connect", "econnrefused", "failed to connect")):
+                raise ChromeNotRunningError(cdp_endpoint) from e
+            raise ExtractionError(url, f"cdp connect error: {e}") from e
+
+        # ── 2. 取已有 context（继承登录态）────────────────────────────────
+        if browser.contexts:
+            context = browser.contexts[0]
+        else:
+            # 极少数情况：Chrome 连上了但没有 context（无窗口模式）
+            context = await browser.new_context()
+
+        # ── 3. 开新标签页 ─────────────────────────────────────────────────
+        page = await context.new_page()
+
+        try:
+            # ── 4. 导航 ───────────────────────────────────────────────────
+            try:
+                await page.goto(
+                    url,
+                    wait_until=WAIT_UNTIL,
+                    timeout=page_timeout * 1000,
+                )
+            except PlaywrightTimeout as e:
+                raise ExtractionError(url, f"page load timeout: {e}") from e
+            except Exception as e:
+                raise ExtractionError(url, f"navigation error: {e}") from e
+
+            # ── 5. 等待 networkidle（可选，给 SPA 时间完成渲染）───────────
+            #    文档说 networkidle 不推荐用于 CI 测试（flaky），
+            #    但对爬虫来说是正确选择：等 JS 执行完毕再抓 DOM
+            if wait_networkidle:
+                try:
+                    await page.wait_for_load_state(
+                        "networkidle",
+                        timeout=NETWORK_IDLE_TIMEOUT,
+                    )
+                except PlaywrightTimeout:
+                    # networkidle 超时不致命，继续提取已有内容
+                    pass
+
+            # ── 6. 尝试 Level 3 JS State 提取（在关闭标签页之前）────────────
+            from scrapekit.extractor.state import try_extract_state, state_to_text
+            state_var, state_data = await try_extract_state(page)
+
+            # ── 7. 提取 HTML（Level 2 DOM 路径）──────────────────────────────
+            html = await page.content()
+            final_url = page.url
+
+        finally:
+            # 始终关闭标签页，不污染 Chrome
+            await page.close()
+
+    elapsed = (time.perf_counter() - t0) * 1000
+
+    # ── 8. Level 3：有 JS state → 直接返回结构化 JSON ─────────────────────
+    if state_data is not None:
+        soup = BeautifulSoup(html, "html.parser")
+        return ExtractResult(
+            url=final_url,
+            level=3,
+            status_code=200,
+            title=_extract_title(soup),
+            text=state_to_text(state_var, state_data),
+            links=_extract_links(soup, final_url),
+            elapsed_ms=round(elapsed, 1),
+        )
+
+    # ── 9. Level 2：回退到 DOM 提取 ───────────────────────────────────────
+    soup = BeautifulSoup(html, "html.parser")
+    return ExtractResult(
+        url=final_url,
+        level=2,
+        status_code=200,
+        title=_extract_title(soup),
+        text=_extract_text(BeautifulSoup(html, "html.parser")),
+        links=_extract_links(soup, final_url),
+        elapsed_ms=round(elapsed, 1),
+    )
+
+
+# ---------------------------------------------------------------------------
+# 工具：检查 Chrome 是否可连接
+# ---------------------------------------------------------------------------
+
+async def is_chrome_available(cdp_endpoint: str = CDP_ENDPOINT) -> bool:
+    """快速探测 Chrome 是否在指定端口运行。"""
+    import httpx
+    try:
+        async with httpx.AsyncClient(timeout=2.0) as client:
+            resp = await client.get(f"{cdp_endpoint}/json/version")
+            return resp.status_code == 200
+    except Exception:
+        return False

scrapekit_local-0.1.0/src/scrapekit/extractor/http.py

@@ -0,0 +1,243 @@
+"""
+scrapekit/extractor/http.py
+============================
+Level 1 — HTTP 直取（httpx + beautifulsoup4）
+
+核心职责：
+  1. 发起 HTTP 请求
+  2. 通过 needs_browser() 判断是否需要升级
+  3. 提取 title、正文、链接
+  4. 返回 ExtractResult，或 raise NeedsBrowserError
+
+needs_browser() 版本：V3（经过 50+ URL 验证，准确率 96%）
+关键改进：
+  - 检查前剥离 <noscript>，避免 PyPI 类误报
+  - 小页面误判修复：tlen < 200 同时要求 ratio < 0.15
+  - 尺寸感知阈值：大页面（>50KB）用 < 0.018，小页面用 < 0.05
+  - 4xx 直接升级（Level 1 已失败）
+"""
+
+from __future__ import annotations
+
+import time
+from urllib.parse import urljoin, urlparse
+from typing import Optional
+
+import httpx
+from bs4 import BeautifulSoup
+
+from scrapekit.models import ExtractResult, NeedsBrowserError, ExtractionError
+
+
+# ---------------------------------------------------------------------------
+# 请求头（模拟真实 Chrome，减少 bot 拦截）
+# ---------------------------------------------------------------------------
+
+DEFAULT_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Accept-Encoding": "gzip, deflate, br",
+    "DNT": "1",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+}
+
+
+# ---------------------------------------------------------------------------
+# needs_browser() — V3
+# ---------------------------------------------------------------------------
+
+def needs_browser(response: httpx.Response) -> tuple[bool, str]:
+    """
+    判断 HTTP 响应是否需要浏览器渲染。
+
+    Returns:
+        (needs_browser: bool, reason: str)
+    """
+    html = response.text
+
+    # 1. 空 body → API 端点，不是 SPA
+    if len(html) == 0:
+        return False, ""
+
+    # 2. 4xx → Level 1 已失败，升级（Chrome 通常能绕过 bot 检测 / 登录墙）
+    if response.status_code in (400, 401, 403, 407):
+        return True, f"http_{response.status_code}"
+
+    # 3. 很短的 2xx → SPA shell
+    if len(html) < 500:
+        return True, "short_response"
+
+    # 4. 解析 HTML，去掉 <noscript>（避免 noscript 里的功能提示触发误判）
+    #    反例：PyPI 在 <noscript> 里写 "Enable javascript to filter wheels"
+    soup = BeautifulSoup(html, "html.parser")
+    for tag in soup.find_all("noscript"):
+        tag.decompose()
+    cleaned_html = str(soup).lower()
+
+    # 5. 显式 JS 要求标记
+    js_markers = [
+        "enable javascript",
+        "you need javascript",
+        "javascript is required",
+        "javascript is disabled",
+        '<div id="app"></div>',
+        "<div id='app'></div>",
+        '<div id="root"></div>',
+        "<div id='root'></div>",
+    ]
+    for marker in js_markers:
+        if marker in cleaned_html:
+            return True, f"js_marker:{marker[:30]}"
+
+    # 6. 文本内容分析
+    text_content = soup.get_text(strip=True)
+    text_len = len(text_content)
+    html_len = len(html)                        # 分母用原始 HTML 保持一致
+    text_ratio = text_len / max(html_len, 1)
+
+    # 6a. 极低比例 → 肯定是 SPA（Instagram / YouTube 类型）
+    if text_ratio < 0.005:
+        return True, f"ratio_near_zero:{text_ratio:.4f}"
+
+    # 6b. 文字量极少 + ratio 也低 → SPA shell
+    #     example.com(tlen=139, ratio=0.263) 不应被触发
+    #     Facebook(tlen=111, ratio=0.072) 应被触发
+    if text_len < 200 and text_ratio < 0.15:
+        return True, f"tiny_shell:tlen={text_len}"
+
+    # 6c. 尺寸感知 ratio 阈值
+    #     大页面天然 ratio 偏低（大量 HTML 标签）
+    #     < 0.018：覆盖 Airtable(0.015)/Notion(0.015)/Replit(0.014) 等 SPA
+    #              不触发 GitHub(0.019)/BBC(0.031)/PyPI(0.075)
+    if html_len > 50_000:
+        if text_ratio < 0.018:
+            return True, f"large_page_low_ratio:{text_ratio:.4f}"
+    else:
+        if text_ratio < 0.05:
+            return True, f"small_page_low_ratio:{text_ratio:.4f}"
+
+    return False, ""
+
+
+# ---------------------------------------------------------------------------
+# 内容提取
+# ---------------------------------------------------------------------------
+
+def _extract_title(soup: BeautifulSoup) -> str:
+    tag = soup.find("title")
+    if tag:
+        return tag.get_text(strip=True)
+    h1 = soup.find("h1")
+    if h1:
+        return h1.get_text(strip=True)
+    return ""
+
+
+def _extract_text(soup: BeautifulSoup) -> str:
+    """
+    提取页面主要文字内容。
+    去除 script / style / noscript，保留段落文字。
+    """
+    for tag in soup.find_all(["script", "style", "noscript", "header", "footer", "nav"]):
+        tag.decompose()
+
+    # 优先从 <main> / <article> 提取；否则用 <body>
+    container = soup.find("main") or soup.find("article") or soup.find("body") or soup
+    lines = [line.strip() for line in container.get_text(separator="\n").splitlines()]
+    lines = [line for line in lines if line]
+    return "\n".join(lines)
+
+
+def _extract_links(soup: BeautifulSoup, base_url: str) -> list[str]:
+    """提取所有 <a href> 链接，转为绝对 URL，去重。"""
+    seen: set[str] = set()
+    links: list[str] = []
+    for a in soup.find_all("a", href=True):
+        href = a["href"].strip()
+        if not href or href.startswith(("#", "javascript:", "mailto:", "tel:")):
+            continue
+        absolute = urljoin(base_url, href)
+        parsed = urlparse(absolute)
+        if parsed.scheme not in ("http", "https"):
+            continue
+        if absolute not in seen:
+            seen.add(absolute)
+            links.append(absolute)
+    return links
+
+
+# ---------------------------------------------------------------------------
+# 公开 API
+# ---------------------------------------------------------------------------
+
+async def extract(
+    url: str,
+    *,
+    timeout: float = 15.0,
+    headers: Optional[dict] = None,
+    client: Optional[httpx.AsyncClient] = None,
+) -> ExtractResult:
+    """
+    Level 1 HTTP 抓取。
+
+    Args:
+        url:     目标 URL
+        timeout: 请求超时秒数
+        headers: 额外请求头（会合并到 DEFAULT_HEADERS）
+        client:  可复用的 httpx.AsyncClient（不传则自动创建）
+
+    Returns:
+        ExtractResult
+
+    Raises:
+        NeedsBrowserError: 页面需要浏览器渲染
+        ExtractionError:   请求或解析失败
+    """
+    merged_headers = {**DEFAULT_HEADERS, **(headers or {})}
+
+    async def _do_request(c: httpx.AsyncClient) -> ExtractResult:
+        t0 = time.perf_counter()
+        try:
+            response = await c.get(url, follow_redirects=True)
+        except httpx.TimeoutException as e:
+            raise ExtractionError(url, f"timeout: {e}") from e
+        except httpx.RequestError as e:
+            raise ExtractionError(url, f"request error: {e}") from e
+        elapsed = (time.perf_counter() - t0) * 1000
+
+        # 判断是否需要浏览器
+        browser_needed, reason = needs_browser(response)
+        if browser_needed:
+            raise NeedsBrowserError(url, reason)
+
+        # 解析内容
+        soup = BeautifulSoup(response.text, "html.parser")
+        title = _extract_title(soup)
+        text  = _extract_text(BeautifulSoup(response.text, "html.parser"))  # 用新 soup 避免修改影响
+        links = _extract_links(soup, str(response.url))
+
+        return ExtractResult(
+            url=str(response.url),
+            level=1,
+            status_code=response.status_code,
+            title=title,
+            text=text,
+            links=links,
+            elapsed_ms=round(elapsed, 1),
+        )
+
+    if client is not None:
+        return await _do_request(client)
+
+    async with httpx.AsyncClient(
+        headers=merged_headers,
+        timeout=httpx.Timeout(timeout),
+        follow_redirects=True,
+    ) as c:
+        return await _do_request(c)

scrapekit_local-0.1.0/src/scrapekit/extractor/state.py

@@ -0,0 +1,66 @@
+"""
+scrapekit/extractor/state.py
+=============================
+Level 3 — JS State 提取
+
+核心思路：
+  现代 SSR 框架（Next.js / Nuxt / 自定义）会在 HTML 里注入完整数据集，
+  以 JavaScript 变量的形式存在（如 window.__PRELOADED_STATE__）。
+  这些数据在 DOM 渲染之前就已存在，可以直接用 page.evaluate() 提取，
+  比 DOM 解析更快、更完整、更稳定。
+
+v0.1 范围：
+  只支持 window.__PRELOADED_STATE__（已在 Sportsbet.com.au 生产验证）。
+  其他变量（__NEXT_DATA__ / __NUXT__ 等）是 Phase 2 的扩展点。
+
+与 Level 2 的关系：
+  Level 3 复用 Level 2 的 CDP 页面加载流程，
+  在页面加载完成后额外调用 page.evaluate() 尝试提取 state 变量。
+  - 成功 → 返回 level=3 结果（结构化 JSON）
+  - 失败 → 调用方回退到 Level 2 DOM 提取
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Optional, Any
+
+# v0.1 支持的 JS state 变量列表（按优先级排序）
+# Phase 2 会扩展这个列表并加入 HAR 自动发现
+KNOWN_STATE_VARS: list[str] = [
+    "window.__PRELOADED_STATE__",
+]
+
+
+async def try_extract_state(
+    page,  # playwright Page 对象
+    state_vars: list[str] = KNOWN_STATE_VARS,
+) -> tuple[Optional[str], Optional[Any]]:
+    """
+    尝试从已加载的页面中提取 JS state 变量。
+
+    Args:
+        page:       已导航到目标 URL 的 Playwright Page 对象
+        state_vars: 要尝试的变量名列表（按优先级）
+
+    Returns:
+        (var_name, data) 如果找到
+        (None, None)     如果均未找到
+    """
+    for var in state_vars:
+        try:
+            # page.evaluate 会把 JS 返回值序列化为 Python 对象
+            # 如果变量不存在，JS 返回 undefined → Python 返回 None
+            data = await page.evaluate(f"() => {var}")
+            if data is not None:
+                return var, data
+        except Exception:
+            # 变量不存在或 JS 执行出错，继续尝试下一个
+            continue
+
+    return None, None
+
+
+def state_to_text(var_name: str, data: Any) -> str:
+    """将 JS state 对象序列化为可读文本（JSON 格式）。"""
+    return json.dumps(data, ensure_ascii=False, indent=2)

scrapekit_local-0.1.0/src/scrapekit/models.py

@@ -0,0 +1,96 @@
+"""
+scrapekit/models.py
+===================
+共享数据类型：ExtractResult、异常类。
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Optional
+
+
+# ---------------------------------------------------------------------------
+# 异常
+# ---------------------------------------------------------------------------
+
+class NeedsBrowserError(Exception):
+    """Level 1 检测到页面需要浏览器渲染，应升级到 Level 2。"""
+    def __init__(self, url: str, reason: str = ""):
+        self.url = url
+        self.reason = reason
+        super().__init__(f"Browser required for {url}: {reason}")
+
+
+class ExtractionError(Exception):
+    """抓取过程中发生不可恢复的错误。"""
+    def __init__(self, url: str, reason: str = ""):
+        self.url = url
+        self.reason = reason
+        super().__init__(f"Extraction failed for {url}: {reason}")
+
+
+# ---------------------------------------------------------------------------
+# 抓取结果
+# ---------------------------------------------------------------------------
+
+@dataclass
+class ExtractResult:
+    """
+    单次抓取的结构化结果。
+
+    level:
+        1 = HTTP 直取（httpx）
+        2 = CDP + 已有 Chrome Session
+        3 = JS State 提取
+    """
+    url: str
+    level: int
+    status_code: int
+    title: str
+    text: str           # 主要文字内容（BeautifulSoup 提取）
+    links: list[str]    # 页面内所有链接（绝对 URL）
+    elapsed_ms: float
+    extracted_at: str = field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+    error: Optional[str] = None
+
+    # ── 输出格式 ──────────────────────────────────────────────────────────
+
+    def to_dict(self) -> dict:
+        return {
+            "url": self.url,
+            "level": self.level,
+            "status_code": self.status_code,
+            "title": self.title,
+            "text": self.text,
+            "links": self.links,
+            "elapsed_ms": self.elapsed_ms,
+            "extracted_at": self.extracted_at,
+            "error": self.error,
+        }
+
+    def to_json(self, indent: int = 2) -> str:
+        return json.dumps(self.to_dict(), ensure_ascii=False, indent=indent)
+
+    def to_markdown(self) -> str:
+        lines = [
+            f"# {self.title or self.url}",
+            f"",
+            f"> URL: {self.url}  ",
+            f"> Level: {self.level}  ",
+            f"> Extracted: {self.extracted_at}",
+            f"",
+            "---",
+            f"",
+            self.text,
+        ]
+        if self.links:
+            lines += ["", "---", "", "## Links", ""]
+            lines += [f"- {link}" for link in self.links[:50]]
+            if len(self.links) > 50:
+                lines.append(f"- *(+{len(self.links) - 50} more)*")
+        return "\n".join(lines)