omk-crawl 2.0.0__py3-none-any.whl

omk_crawl/__init__.py

@@ -0,0 +1,16 @@
+"""omk-crawl — Smart web crawling toolbox.
+
+Auto-escalating router across 6 adapters:
+  curl_cffi → crawl4ai → scrapling → browser-use (+ autoscraper, markitdown)
+
+Usage:
+    from omk_crawl import crawl
+    result = crawl("https://example.com")
+    print(result.markdown)
+"""
+
+from omk_crawl.result import CrawlResult, CrawlStatus
+from omk_crawl.router import SmartRouter, crawl, crawl_async
+
+__all__ = ["CrawlResult", "CrawlStatus", "SmartRouter", "crawl", "crawl_async"]
+__version__ = "2.0.0"

omk_crawl/cli.py

@@ -0,0 +1,137 @@
+"""CLI — omk-crawl <url> [options]
+
+Usage:
+    omk-crawl https://example.com                    # auto-escalate
+    omk-crawl https://example.com --tool curl_cffi   # force tool
+    omk-crawl https://example.com -o out.md          # save markdown
+    omk-crawl https://example.com --json             # JSON output
+    omk-crawl https://example.com -v                 # verbose escalation
+    omk-crawl --diagnose https://example.com         # dry-run: what would we try?
+    omk-crawl --tools                                # list installed tools
+    omk-crawl report.pdf                             # file → markdown (markitdown)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from omk_crawl.detect import available_tools, missing_tools
+from omk_crawl.result import CrawlResult
+from omk_crawl.router import SmartRouter, crawl
+
+
+def _print_result(r: CrawlResult, *, as_json: bool = False, output: str | None = None) -> None:
+    if as_json:
+        data = {
+            "url": r.url,
+            "status": r.status.value,
+            "status_code": r.status_code,
+            "tool": r.tool,
+            "elapsed_ms": round(r.elapsed_ms, 1),
+            "content_length": len(r.content) if r.content else 0,
+            "markdown": r.markdown,
+            "fit_markdown": r.fit_markdown,
+            "extracted": r.extracted,
+            "error": r.error,
+            "metadata": r.metadata,
+        }
+        text = json.dumps(data, ensure_ascii=False, indent=2)
+    else:
+        text = r.content or r.error or "(empty)"
+
+    if output:
+        Path(output).write_text(text, encoding="utf-8")
+        print(f"Saved {len(text)} chars → {output}")
+    else:
+        print(text)
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(
+        prog="omk-crawl",
+        description="Smart web crawling toolbox — 6 adapters, auto-escalating router",
+        epilog=(
+            "This product uses Crawl4AI"
+            " (https://github.com/unclecode/crawl4ai) for web data extraction."
+        ),
+    )
+    parser.add_argument("url", nargs="?", help="URL or file path to crawl/convert")
+    parser.add_argument("--tool", "-t", help="Force a specific tool (skip auto-escalation)")
+    parser.add_argument("--output", "-o", help="Save output to file")
+    parser.add_argument("--json", "-j", action="store_true", help="JSON output")
+    parser.add_argument("--verbose", "-v", action="store_true", help="Verbose escalation log")
+    parser.add_argument(
+        "--no-robots", action="store_true",
+        help="Skip robots.txt check (use responsibly)",
+    )
+    parser.add_argument(
+        "--min-delay", type=float, default=0.5,
+        help="Minimum seconds between requests to same domain (default: 0.5)",
+    )
+    parser.add_argument(
+        "--diagnose", action="store_true",
+        help="Dry-run: show what tools would be tried",
+    )
+    parser.add_argument("--tools", action="store_true", help="List installed/missing tools")
+    parser.add_argument("--version", action="version", version="omk-crawl 2.0.0")
+
+    args = parser.parse_args(argv)
+
+    if args.tools:
+        avail = available_tools()
+        miss = missing_tools()
+        print("Installed tools:")
+        for t in avail:
+            print(f"  ✅ {t}")
+        if miss:
+            print("\nMissing tools:")
+            for t in miss:
+                print(f"  ❌ {t}")
+            print("\nInstall all: pip install omk-crawl[all]")
+        return
+
+    if not args.url:
+        parser.print_help()
+        sys.exit(1)
+
+    if args.diagnose:
+        router = SmartRouter(verbose=True)
+        info = router.diagnose(args.url)
+        print(json.dumps(info, indent=2))
+        return
+
+    # File path → markitdown
+    if Path(args.url).is_file():
+        from omk_crawl.tools.markitdown_tool import MarkitdownTool
+
+        tool = MarkitdownTool()
+        r = tool.fetch(args.url)
+        _print_result(r, as_json=args.json, output=args.output)
+        sys.exit(0 if r.ok else 1)
+
+    # Web crawl with auto-escalation
+    if args.verbose:
+        import logging
+
+        logging.basicConfig(
+            level=logging.INFO,
+            format="  [omk-crawl] %(message)s",
+        )
+
+    r = crawl(
+        args.url, tool=args.tool, verbose=args.verbose,
+        respect_robots=not args.no_robots, min_delay=args.min_delay,
+    )
+
+    if args.verbose:
+        print(f"\n--- {r.summary()} ---\n", file=sys.stderr)
+
+    _print_result(r, as_json=args.json, output=args.output)
+    sys.exit(0 if r.ok else 1)
+
+
+if __name__ == "__main__":
+    main()

omk_crawl/detect.py

@@ -0,0 +1,189 @@
+"""Detection heuristics — what's blocking us, what do we need?"""
+
+from __future__ import annotations
+
+import importlib.util
+from dataclasses import dataclass
+from enum import Flag, auto
+
+from omk_crawl.result import CrawlStatus
+
+
+class BlockType(Flag):
+    """Detected blocking mechanism."""
+
+    NONE = 0
+    TLS_FINGERPRINT = auto()  # JA3/HTTP2 fingerprint rejection
+    JS_REQUIRED = auto()  # empty body / noscript / JS framework
+    CLOUDFLARE = auto()  # CF challenge / Turnstile
+    WAF = auto()  # generic WAF (403 + challenge page)
+    RATE_LIMIT = auto()  # 429
+    AUTH_REQUIRED = auto()  # 401 / login redirect
+
+
+@dataclass
+class Detection:
+    """Result of analyzing a response."""
+
+    block: BlockType = BlockType.NONE
+    status_code: int | None = None
+    needs_browser: bool = False
+    needs_stealth: bool = False
+    needs_llm_agent: bool = False
+    confidence: float = 0.0
+    detail: str = ""
+
+
+# --- Tool availability ---
+
+_TOOL_MODULES: dict[str, str] = {
+    "curl_cffi": "curl_cffi",
+    "crawl4ai": "crawl4ai",
+    "scrapling": "scrapling",
+    "scrapy": "scrapy",
+    "crawlee": "crawlee",
+    "browser_use": "browser_use",
+    "autoscraper": "autoscraper",
+    "markitdown": "markitdown",
+}
+
+
+def tool_available(name: str) -> bool:
+    """Check if a tool's Python module is importable."""
+    mod = _TOOL_MODULES.get(name, name)
+    return importlib.util.find_spec(mod) is not None
+
+
+def available_tools() -> list[str]:
+    """List installed tools."""
+    return [name for name in _TOOL_MODULES if tool_available(name)]
+
+
+def missing_tools() -> list[str]:
+    """List tools that need `pip install`."""
+    return [name for name in _TOOL_MODULES if not tool_available(name)]
+
+
+# --- Response analysis ---
+
+_CF_MARKERS = ("cf-browser-verification", "cf_chl_opt", "turnstile", "challenge-platform")
+_JS_MARKERS = (
+    "<noscript>", 'id="__next"', 'id="root"', 'id="app"', "ng-app", "data-reactroot",
+)
+_WAF_MARKERS = ("access denied", "blocked", "captcha", "are you a robot", "unusual traffic")
+
+
+def detect_block(html: str | None, status_code: int | None) -> Detection:
+    """Analyze response HTML + status to detect blocking type."""
+    d = Detection(status_code=status_code)
+
+    if status_code == 429:
+        d.block = BlockType.RATE_LIMIT
+        d.confidence = 0.9
+        d.detail = "Rate limited (429)"
+        return d
+
+    if status_code == 401:
+        d.block = BlockType.AUTH_REQUIRED
+        d.confidence = 0.9
+        d.detail = "Authentication required (401)"
+        return d
+
+    if not html:
+        if status_code == 403:
+            d.block = BlockType.TLS_FINGERPRINT | BlockType.WAF
+            d.confidence = 0.7
+            d.detail = "403 with empty body — likely TLS fingerprint or WAF"
+        return d
+
+    lower = html.lower()
+
+    # Cloudflare
+    if any(m in lower for m in _CF_MARKERS):
+        d.block |= BlockType.CLOUDFLARE
+        d.needs_stealth = True
+        d.confidence = 0.9
+        d.detail = "Cloudflare challenge detected"
+
+    # JS-required
+    if any(m in lower for m in _JS_MARKERS) and len(html) < 5000:
+        d.block |= BlockType.JS_REQUIRED
+        d.needs_browser = True
+        d.confidence = max(d.confidence, 0.8)
+        d.detail = "JS framework shell detected (needs rendering)"
+
+    # Generic WAF
+    if status_code == 403 and any(m in lower for m in _WAF_MARKERS):
+        d.block |= BlockType.WAF
+        d.confidence = max(d.confidence, 0.7)
+        d.detail = "WAF challenge page"
+
+    # TLS fingerprint: 403 + very short/empty response
+    if status_code == 403 and len(html) < 500 and not d.block:
+        d.block = BlockType.TLS_FINGERPRINT
+        d.confidence = 0.6
+        d.detail = "Short 403 — possible TLS fingerprint block"
+
+    if not d.block:
+        d.block = BlockType.NONE
+        d.confidence = 1.0
+        d.detail = "No blocking detected"
+
+    return d
+
+
+def detection_to_status(det: Detection) -> CrawlStatus:
+    """Map a Detection to the appropriate CrawlStatus.
+
+    Priority: block signals > HTTP client/server errors > OK.
+    Block signals (CF, WAF, TLS, JS) take precedence over raw HTTP codes
+    because anti-bot systems commonly use 403/503 for challenge pages.
+    """
+    # 1. Blocking detection FIRST — anti-bot challenges use 403/503
+    if BlockType.TLS_FINGERPRINT in det.block:
+        return CrawlStatus.TLS_BLOCKED
+    if BlockType.JS_REQUIRED in det.block and det.needs_browser:
+        return CrawlStatus.JS_REQUIRED
+    if det.block.value != 0:
+        return CrawlStatus.BLOCKED
+
+    # 2. HTTP-level errors with NO blocking signal — genuine errors
+    sc = det.status_code
+    if sc is not None and sc >= 400:
+        if sc in (404, 405, 410):
+            return CrawlStatus.ERROR  # client error — not a block
+        if sc == 401:
+            return CrawlStatus.ERROR  # auth — no point escalating
+        if sc >= 500:
+            return CrawlStatus.ERROR  # server error — not a block
+        return CrawlStatus.ERROR  # other 4xx without markers
+
+    return CrawlStatus.OK
+
+
+# --- robots.txt ---
+
+def check_robots_txt(url: str, user_agent: str = "*") -> bool:
+    """Check if a URL is allowed by the site's robots.txt.
+
+    Returns True if crawling is allowed (or robots.txt is unavailable).
+    Returns False if the URL is explicitly disallowed.
+
+    This is a best-effort check — it fetches and caches robots.txt
+    per domain. Network errors are treated as 'allowed' (fail-open).
+    """
+    import urllib.request  # noqa: F401 — needed by RobotFileParser internally
+    from urllib.parse import urlparse
+    from urllib.robotparser import RobotFileParser
+
+    parsed = urlparse(url)
+    robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
+
+    rp = RobotFileParser()
+    try:
+        rp.set_url(robots_url)
+        rp.read()
+    except Exception:
+        return True  # fail-open: can't read robots.txt → allow
+
+    return rp.can_fetch(user_agent, url)

omk_crawl/pipeline.py

@@ -0,0 +1,80 @@
+"""Pipeline — compose fetch → extract → convert steps."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from omk_crawl.result import CrawlResult, CrawlStatus
+from omk_crawl.router import SmartRouter
+
+
+@dataclass
+class Pipeline:
+    """Composable crawl pipeline.
+
+    Usage:
+        p = Pipeline()
+        p.fetch()                          # auto-escalating fetch
+        p.extract_css("div.product", {     # CSS extraction
+            "title": "h2", "price": ".price"
+        })
+        p.to_markdown()                    # ensure markdown output
+        result = p.run("https://example.com")
+    """
+
+    steps: list[Callable[[CrawlResult], CrawlResult]] = field(default_factory=list)
+    router: SmartRouter = field(default_factory=SmartRouter)
+
+    def fetch(self, tool: str | None = None, **kwargs: Any) -> Pipeline:
+        """Step 1: fetch with auto-escalation (or specific tool)."""
+        def _fetch(r: CrawlResult) -> CrawlResult:
+            if tool:
+                self.router.tools = [tool]
+            return self.router.crawl(r.url, **kwargs)
+        self.steps.append(_fetch)
+        return self
+
+    def extract_css(self, base: str, fields: dict[str, str]) -> Pipeline:
+        """Step 2: CSS extraction on HTML (requires selectolax)."""
+        def _extract(r: CrawlResult) -> CrawlResult:
+            if not r.html:
+                r.error = "No HTML to extract from"
+                return r
+            try:
+                from selectolax.parser import HTMLParser
+
+                tree = HTMLParser(r.html)
+                rows = []
+                for node in tree.css(base):
+                    row = {}
+                    for name, sel in fields.items():
+                        child = node.css_first(sel)
+                        row[name] = child.text(strip=True) if child else None
+                    rows.append(row)
+                r.extracted = rows
+                r.metadata["extract_count"] = len(rows)
+            except ImportError:
+                # selectolax not installed — no extraction possible
+                r.metadata["extract_note"] = "pip install selectolax for CSS extraction"
+            return r
+        self.steps.append(_extract)
+        return self
+
+    def to_markdown(self) -> Pipeline:
+        """Step 3: ensure markdown output (delegates to SmartRouter._ensure_markdown)."""
+        def _convert(r: CrawlResult) -> CrawlResult:
+            SmartRouter._ensure_markdown(r)
+            return r
+        self.steps.append(_convert)
+        return self
+
+    def run(self, url: str) -> CrawlResult:
+        """Execute the pipeline."""
+        result = CrawlResult(url=url, status=CrawlStatus.OK)
+        for step in self.steps:
+            result = step(result)
+            if result.status is CrawlStatus.ERROR:
+                break
+        return result

omk_crawl/result.py

@@ -0,0 +1,73 @@
+"""Unified crawl result across all tools."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+class CrawlStatus(Enum):
+    """Outcome of a crawl attempt."""
+
+    OK = "ok"
+    BLOCKED = "blocked"  # 403 / WAF / anti-bot
+    TLS_BLOCKED = "tls_blocked"  # fingerprint-based rejection
+    JS_REQUIRED = "js_required"  # empty body, needs rendering
+    ERROR = "error"
+    TOOL_MISSING = "tool_missing"
+
+
+@dataclass
+class CrawlResult:
+    """Tool-agnostic crawl output."""
+
+    url: str
+    status: CrawlStatus
+    status_code: int | None = None
+    html: str | None = None
+    markdown: str | None = None
+    fit_markdown: str | None = None
+    extracted: list[dict[str, Any]] | None = None
+    tool: str = ""
+    elapsed_ms: float = 0.0
+    error: str | None = None
+    headers: dict[str, str] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def ok(self) -> bool:
+        return self.status is CrawlStatus.OK
+
+    @property
+    def blocked(self) -> bool:
+        return self.status in (CrawlStatus.BLOCKED, CrawlStatus.TLS_BLOCKED)
+
+    @property
+    def content(self) -> str | None:
+        """Best available text content."""
+        return self.fit_markdown or self.markdown or self.html
+
+    def summary(self) -> str:
+        parts = [f"[{self.status.value}] {self.url}"]
+        if self.tool:
+            parts.append(f"via {self.tool}")
+        if self.status_code:
+            parts.append(f"HTTP {self.status_code}")
+        parts.append(f"{self.elapsed_ms:.0f}ms")
+        if self.content:
+            parts.append(f"{len(self.content)} chars")
+        if self.error:
+            parts.append(f"err={self.error}")
+        return " | ".join(parts)
+
+
+def _timer() -> tuple[float, Any]:
+    """Return (start_time, stop_fn). Call stop_fn() to get elapsed ms."""
+    t0 = time.perf_counter()
+
+    def stop() -> float:
+        return (time.perf_counter() - t0) * 1000
+
+    return t0, stop