html-reader-llm 0.0.1__py3-none-any.whl

html_reader_llm/cli.py

@@ -0,0 +1,563 @@
+"""html-reader-llm — HTML simplification and intelligent extraction for LLM.
+
+Provides three subcommands:
+- simplify: Clean and shorten HTML for LLM consumption
+- detect:   Compare HTTP vs Chrome rendering to decide if browser is needed
+- analyze:  Full pipeline — fetch, simplify, extract metadata, LLM classification + rules
+
+Output:
+- simplify: cleaned HTML to stdout, stats to stderr (--stats)
+- detect:   JSON summary to stdout, HTML bodies to files (--output-http/--output-chrome)
+- analyze:  JSON with page_type + rules to stdout, raw LLM response to stderr (--raw)
+
+Logging goes to stderr. stdout is reserved for programmatic output (piping, agent use).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+
+def cmd_simplify(args: argparse.Namespace) -> None:
+    """Run HTML simplification pipeline."""
+    from html_reader_llm.simplify import simplify_html
+
+    if args.file:
+        with open(args.file, encoding="utf-8") as f:
+            html = f.read()
+    elif not sys.stdin.isatty():
+        html = sys.stdin.read()
+    else:
+        print("Error: provide HTML via --file or stdin", file=sys.stderr)
+        sys.exit(1)
+
+    result = simplify_html(html)
+
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(result["html"])
+        print(f"Simplified HTML written to {args.output}", file=sys.stderr)
+    else:
+        print(result["html"])
+
+    if args.stats:
+        print("\n--- Stats ---", file=sys.stderr)
+        for step, s in result["stats"].items():
+            delta = s["char_before"] - s["char_after"]
+            pct = (delta / s["char_before"] * 100) if s["char_before"] else 0
+            print(
+                f"  {step}: {s['char_before']} -> {s['char_after']} ({delta} chars, {pct:.1f}%)",
+                file=sys.stderr,
+            )
+
+
+def cmd_detect(args: argparse.Namespace) -> None:
+    """Run render detection."""
+    from html_reader_llm.render_detect import detect_render
+
+    if args.list_browsers:
+        from html_reader_llm.browser import detect_browser_paths, get_browser_path
+
+        paths = detect_browser_paths()
+        auto = get_browser_path()
+        print(f"Detected browsers ({len(paths)}):", file=sys.stderr)
+        for i, p in enumerate(paths):
+            marker = " <-- auto-selected" if p == auto else ""
+            print(f"  [{i}] {p}{marker}", file=sys.stderr)
+        if not paths:
+            print("  (none found)", file=sys.stderr)
+        return
+
+    result = detect_render(
+        url=args.url,
+        chrome_path=args.chrome_path,
+        user_data_dir=args.user_data_dir,
+    )
+
+    summary = {k: v for k, v in result.items() if k not in ("http_html", "chrome_html")}
+    print(json.dumps(summary, indent=2, ensure_ascii=False))
+
+    if args.output_http:
+        with open(args.output_http, "w", encoding="utf-8") as f:
+            f.write(result["http_html"] or "")
+        print(f"HTTP HTML written to {args.output_http}", file=sys.stderr)
+
+    if args.output_chrome:
+        with open(args.output_chrome, "w", encoding="utf-8") as f:
+            f.write(result["chrome_html"] or "")
+        print(f"Chrome HTML written to {args.output_chrome}", file=sys.stderr)
+
+
+def cmd_analyze(args: argparse.Namespace) -> None:
+    """Run full analysis pipeline: fetch -> simplify -> metadata -> LLM."""
+    if args.dry_run:
+        cmd_analyze_dry_run(args)
+        return
+
+    from html_reader_llm.llm import analyze_page
+
+    result = analyze_page(
+        url=args.url,
+        base_url=args.base_url,
+        api_key=args.api_key,
+        model=args.model,
+        use_cache=not args.no_cache,
+    )
+
+    output = {k: v for k, v in result.items() if k != "raw_llm_response"}
+    print(json.dumps(output, indent=2, ensure_ascii=False))
+
+    if args.raw:
+        print("\n--- Raw LLM Response ---", file=sys.stderr)
+        print(result["raw_llm_response"], file=sys.stderr)
+
+
+def cmd_analyze_dry_run(args: argparse.Namespace) -> None:
+    """Show the prompt that would be sent to LLM, without calling LLM."""
+    from html_reader_llm import settings
+    from html_reader_llm.llm import _extract_metadata, _trim_to_budget
+    from html_reader_llm.render_detect import _fetch_http, detect_render
+    from html_reader_llm.simplify import simplify_html
+
+    _max_tokens = args.max_tokens or settings.LLM_MAX_TOKENS
+
+    status, raw_html = _fetch_http(args.url, use_cache=not args.no_cache)
+    if not raw_html:
+        print(f"Error: failed to fetch {args.url}", file=sys.stderr)
+        sys.exit(1)
+
+    render_result = detect_render(args.url, use_cache=not args.no_cache)
+    use_browser = render_result["use_browser"]
+    chrome_html = render_result["chrome_html"]
+    llm_source = chrome_html if (use_browser and chrome_html) else raw_html
+
+    metadata = _extract_metadata(raw_html)
+    simplified = simplify_html(llm_source)["html"]
+    trimmed = _trim_to_budget(simplified, _max_tokens)
+
+    meta_str = (
+        json.dumps(metadata, ensure_ascii=False, indent=2) if metadata else "None"
+    )
+    user_prompt = f"Metadata (from trafilatura):\n{meta_str}\n\nHTML:\n{trimmed}"
+
+    print(f"=== Dry Run: {args.url} ===", file=sys.stderr)
+    print(
+        f"use_browser={use_browser}, source={'chrome' if llm_source is chrome_html else 'http'}",
+        file=sys.stderr,
+    )
+    print(
+        f"raw_html={len(raw_html)} chars, simplified={len(simplified)} chars",
+        file=sys.stderr,
+    )
+    print(
+        f"prompt={len(user_prompt)} chars, est_tokens={len(user_prompt) // 3}",
+        file=sys.stderr,
+    )
+    print(f"max_tokens={_max_tokens}, budget={_max_tokens // 2}", file=sys.stderr)
+    print(file=sys.stderr)
+    print(user_prompt)
+
+
+def cmd_validate(args: argparse.Namespace) -> None:
+    """Validate CSS selectors against a URL or HTML file."""
+    import json as json_mod
+
+    from html_reader_llm.selector_validator import SelectorValidator
+
+    # Get HTML content
+    html = None
+    if args.html_file:
+        with open(args.html_file, encoding="utf-8") as f:
+            html = f.read()
+        print(f"Loaded HTML from file: {args.html_file}", file=sys.stderr)
+    elif args.url:
+        from html_reader_llm.render_detect import _fetch_chrome, _fetch_http
+
+        if args.chrome:
+            html = _fetch_chrome(args.url)
+            if not html:
+                print("Error: Chrome dump-dom failed", file=sys.stderr)
+                sys.exit(1)
+            print(f"Fetched via Chrome: {len(html)} chars", file=sys.stderr)
+        else:
+            status, html = _fetch_http(args.url, use_cache=not args.no_cache)
+            if not html:
+                print(f"Error: HTTP fetch failed (status={status})", file=sys.stderr)
+                sys.exit(1)
+            print(f"Fetched via HTTP: {len(html)} chars", file=sys.stderr)
+    else:
+        print("Error: provide URL or --html-file", file=sys.stderr)
+        sys.exit(1)
+
+    # Get rules
+    rules = None
+    if args.rules_file:
+        with open(args.rules_file, encoding="utf-8") as f:
+            rules = json_mod.load(f)
+        print(f"Loaded rules from: {args.rules_file}", file=sys.stderr)
+    else:
+        print("Error: provide --rules-file", file=sys.stderr)
+        sys.exit(1)
+
+    # Validate
+    validator = SelectorValidator(html)
+    all_results = {}
+
+    # Validate metadata
+    if "metadata" in rules:
+        results = validator.validate_rules(rules["metadata"])
+        all_results["metadata"] = results
+
+    # Validate question info
+    if "question" in rules:
+        results = validator.validate_rules(rules["question"])
+        all_results["question"] = results
+
+    # Validate answers
+    if "answers" in rules:
+        answer_rules = rules["answers"]
+        results = validator.validate_rules(
+            answer_rules["fields"], list_selector=answer_rules["list_selector"]
+        )
+        all_results["answers"] = results
+
+    # Validate page
+    if "page" in rules:
+        results = validator.validate_rules(rules["page"])
+        all_results["page"] = results
+
+    if args.json:
+        # JSON output
+        output = {
+            "valid": all(
+                r["valid"] for group in all_results.values() for r in group.values()
+            ),
+            "groups": all_results,
+        }
+        print(json_mod.dumps(output, indent=2, ensure_ascii=False))
+    else:
+        # Human-readable output
+        for group_name, results in all_results.items():
+            validator.print_validation_report(results, f"{group_name} 验证")
+
+        # Summary
+        total = sum(len(r) for r in all_results.values())
+        passed = sum(
+            1 for group in all_results.values() for r in group.values() if r["valid"]
+        )
+        print("\n=== 总结 ===")
+        print(f"总选择器: {total}, 通过: {passed}, 失败: {total - passed}")
+
+
+def cmd_fetch(args: argparse.Namespace) -> None:
+    """Fetch URL and output HTML to stdout."""
+    from html_reader_llm.render_detect import _fetch_chrome, _fetch_http, detect_render
+
+    use_chrome = args.chrome is not None  # --chrome or --chrome=<path>
+    chrome_path = args.chrome if use_chrome and args.chrome != "auto" else None
+
+    if args.json or args.detect_only:
+        # JSON mode: run full detection and output structured result
+        result = detect_render(
+            args.url,
+            chrome_path=chrome_path,
+            use_cache=not args.no_cache,
+        )
+
+        output: dict[str, object] = {
+            "url": args.url,
+            "use_browser": result["use_browser"],
+            "http_status": result["http_status"],
+            "dom_ratio": result["dom_ratio"],
+            "text_similarity": result["text_similarity"],
+            "spa_shell": result["spa_shell"],
+            "confidence": result["confidence"],
+        }
+
+        if not args.detect_only:
+            # Include HTML based on use_browser decision
+            if result["use_browser"]:
+                output["html"] = result["chrome_html"] or ""
+            else:
+                output["html"] = result["http_html"] or ""
+            output["html_chars"] = len(output["html"])
+
+            # Include both raw HTMLs if --verbose
+            if args.verbose:
+                output["http_html"] = result["http_html"] or ""
+                output["chrome_html"] = result["chrome_html"] or ""
+
+        print(json.dumps(output, indent=2, ensure_ascii=False))
+    elif use_chrome:
+        # Chrome mode (explicit --chrome)
+        html = _fetch_chrome(args.url, chrome_path=chrome_path)
+        if not html:
+            print("Error: Chrome dump-dom failed", file=sys.stderr)
+            sys.exit(1)
+        print(html)
+        print(f"Fetched via Chrome: {len(html)} chars", file=sys.stderr)
+    else:
+        # Default HTTP mode (trafilatura)
+        status, html = _fetch_http(args.url, use_cache=not args.no_cache)
+        if not html:
+            print(f"Error: HTTP fetch failed (status={status})", file=sys.stderr)
+            sys.exit(1)
+        print(html)
+        print(f"Fetched via HTTP: {len(html)} chars", file=sys.stderr)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="html-reader-llm",
+        description="HTML simplification and intelligent extraction for LLM.",
+        epilog="stdout: programmatic output (HTML/JSON). stderr: logs/diagnostics.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    sub = parser.add_subparsers(dest="command")
+
+    # --- simplify ---
+    p_simplify = sub.add_parser(
+        "simplify",
+        help="Clean and shorten HTML for LLM consumption",
+        description=(
+            "7-step pipeline: strip noise -> media placeholders -> "
+            "clean attrs -> truncate lists -> truncate text -> "
+            "unwrap bare -> normalize whitespace."
+        ),
+        epilog="""
+examples:
+  html-reader-llm simplify < page.html
+  cat page.html | html-reader-llm simplify --stats
+  html-reader-llm simplify --file page.html --output clean.html
+
+input:  HTML via stdin or --file
+output: cleaned HTML to stdout
+        per-step char stats to stderr (--stats)
+        """,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p_simplify.add_argument("--file", "-f", help="Input HTML file (default: stdin)")
+    p_simplify.add_argument("--output", "-o", help="Output file (default: stdout)")
+    p_simplify.add_argument(
+        "--stats",
+        "-s",
+        action="store_true",
+        help="Print per-step character statistics to stderr",
+    )
+
+    # --- detect ---
+    p_detect = sub.add_parser(
+        "detect",
+        help="Detect if URL needs browser rendering",
+        description="Compare HTTP vs Chrome --dump-dom: DOM ratio, text Jaccard similarity, SPA shell detection.",
+        epilog="""
+examples:
+  html-reader-llm detect https://example.com
+  html-reader-llm detect --list-browsers
+  html-reader-llm detect https://example.com --chrome-path "C:/chrome.exe"
+
+input:  URL (positional)
+output: JSON to stdout:
+  {"http_status":200, "dom_ratio":0.983, "text_similarity":0.929,
+   "spa_shell":false, "confidence":0.967, "use_browser":false}
+        """,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p_detect.add_argument("url", nargs="?", help="URL to analyze")
+    p_detect.add_argument(
+        "--chrome-path", help="Chrome/Edge executable path (default: auto-detect)"
+    )
+    p_detect.add_argument(
+        "--user-data-dir", help="Chrome user-data-dir for caching (default: none)"
+    )
+    p_detect.add_argument(
+        "--list-browsers", action="store_true", help="List detected browsers and exit"
+    )
+    p_detect.add_argument("--output-http", help="Save HTTP-fetched HTML to file")
+    p_detect.add_argument("--output-chrome", help="Save Chrome-rendered HTML to file")
+
+    # --- analyze ---
+    p_analyze = sub.add_parser(
+        "analyze",
+        help="Full analysis: fetch, simplify, metadata, LLM classification + rules",
+        description=(
+            "End-to-end: fetch -> detect render -> simplify -> "
+            "trafilatura metadata -> LLM classification (list/detail) "
+            "+ CSS selector rule generation."
+        ),
+        epilog="""
+requires: HTMLREADER_LLM_BASE_URL + HTMLREADER_LLM_API_KEY in .env or --base-url/--api-key
+
+examples:
+  html-reader-llm analyze https://example.com
+  html-reader-llm analyze https://example.com --dry-run
+  html-reader-llm analyze https://example.com --raw
+
+input:  URL (positional)
+output: JSON to stdout:
+  {"page_type":"detail",
+   "metadata":{"title":"...", "author":"...", "date":"..."},
+   "detail_rules":{
+     "title":{"css":"h1", "method":"$text"},
+     "content":{"css":".article-body", "method":"$text"}
+   }}
+        """,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p_analyze.add_argument("url", help="URL to analyze")
+    p_analyze.add_argument(
+        "--base-url", help="LLM API base URL (default: env HTMLREADER_LLM_BASE_URL)"
+    )
+    p_analyze.add_argument(
+        "--api-key", help="LLM API key (default: env HTMLREADER_LLM_API_KEY)"
+    )
+    p_analyze.add_argument(
+        "--model", help="LLM model name (default: env HTMLREADER_LLM_MODEL)"
+    )
+    p_analyze.add_argument(
+        "--max-tokens",
+        type=int,
+        help="Model context window (default: env HTMLREADER_LLM_MAX_TOKENS)",
+    )
+    p_analyze.add_argument(
+        "--no-cache", action="store_true", help="Bypass HTTP response cache"
+    )
+    p_analyze.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Show prompt to stdout without calling LLM",
+    )
+    p_analyze.add_argument(
+        "--raw", action="store_true", help="Print raw LLM response to stderr"
+    )
+
+    # --- fetch ---
+    p_fetch = sub.add_parser(
+        "fetch",
+        help="Download HTML from a URL",
+        description=(
+            "Fetch a URL and output raw HTML to stdout. "
+            "Default: HTTP via trafilatura. "
+            "Use --chrome for headless Chrome."
+        ),
+        epilog="""
+examples:
+  html-reader-llm fetch https://example.com
+  html-reader-llm fetch https://example.com --chrome=auto
+  html-reader-llm fetch https://example.com --chrome="C:/chrome.exe"
+  html-reader-llm fetch https://example.com -H "Accept-Language: zh-CN"
+  html-reader-llm fetch https://example.com --timeout 30
+  html-reader-llm fetch https://example.com --json
+  html-reader-llm fetch https://example.com --json --verbose
+  html-reader-llm fetch https://example.com --json --detect-only
+
+input:  URL (positional)
+output: raw HTML to stdout (default)
+        fetch method + char count to stderr
+        JSON with detection + optimal HTML to stdout (--json)
+        JSON with both http/chrome HTML to stdout (--json --verbose)
+        """,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p_fetch.add_argument("url", help="URL to fetch")
+    p_fetch.add_argument(
+        "-H",
+        "--header",
+        action="append",
+        dest="headers",
+        help="Custom header (repeatable), format: 'Name: Value'",
+    )
+    p_fetch.add_argument("--timeout", type=int, help="Request timeout in seconds")
+    p_fetch.add_argument(
+        "--chrome",
+        nargs="?",
+        const="auto",
+        default=None,
+        help="Use Chrome --dump-dom (default path or specify path)",
+    )
+    p_fetch.add_argument(
+        "--no-cache", action="store_true", help="Bypass HTTP response cache"
+    )
+    p_fetch.add_argument(
+        "--json",
+        action="store_true",
+        help="Output JSON with detection results and optimal HTML",
+    )
+    p_fetch.add_argument(
+        "--detect-only",
+        action="store_true",
+        help="Only detect if browser is needed, without outputting HTML (use with --json)",
+    )
+    p_fetch.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Include both http_html and chrome_html in JSON output",
+    )
+
+    # --- validate ---
+    p_validate = sub.add_parser(
+        "validate",
+        help="Validate CSS selectors against a URL or HTML file",
+        description=(
+            "Test if CSS selectors can extract target content from a page. "
+            "Supports QQ News Q&A page selectors out of the box."
+        ),
+        epilog="""
+examples:
+  html-reader-llm validate https://example.com --rules-file selectors.json
+  html-reader-llm validate --html-file page.html --rules-file selectors.json
+  html-reader-llm validate https://example.com --chrome --rules-file selectors.json
+
+input:  URL (positional) or --html-file
+output: validation report to stderr
+        JSON results to stdout (--json)
+        """,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p_validate.add_argument("url", nargs="?", help="URL to validate against")
+    p_validate.add_argument(
+        "--html-file", help="HTML file to validate against (instead of URL)"
+    )
+    p_validate.add_argument(
+        "--chrome",
+        action="store_true",
+        help="Use Chrome to fetch URL (for SPA pages)",
+    )
+    p_validate.add_argument(
+        "--rules-file",
+        required=True,
+        help="JSON file with selectors to validate",
+    )
+    p_validate.add_argument(
+        "--json",
+        action="store_true",
+        help="Output validation results as JSON",
+    )
+    p_validate.add_argument(
+        "--no-cache", action="store_true", help="Bypass HTTP response cache"
+    )
+
+    args = parser.parse_args()
+
+    if args.command == "simplify":
+        cmd_simplify(args)
+    elif args.command == "detect":
+        if not args.url and not args.list_browsers:
+            p_detect.error("url is required (or use --list-browsers)")
+        cmd_detect(args)
+    elif args.command == "analyze":
+        cmd_analyze(args)
+    elif args.command == "fetch":
+        cmd_fetch(args)
+    elif args.command == "validate":
+        cmd_validate(args)
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()