markgrab 0.1.0__py3-none-any.whl

markgrab/__init__.py

@@ -0,0 +1,7 @@
+"""MarkGrab — Universal web content extraction."""
+
+from markgrab.core import extract
+from markgrab.result import ExtractResult
+
+__all__ = ["extract", "ExtractResult"]
+__version__ = "0.1.0"

markgrab/__main__.py

@@ -0,0 +1,70 @@
+"""CLI entry point — python -m markgrab or `markgrab` command."""
+
+import argparse
+import asyncio
+import json
+import sys
+
+from markgrab import extract
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="markgrab",
+        description="MarkGrab — extract web content as LLM-ready markdown",
+    )
+    parser.add_argument("url", help="URL to extract content from")
+    parser.add_argument("--max-chars", type=int, default=50_000, help="Max output characters (default: 50000)")
+    parser.add_argument("--browser", action="store_true", help="Force Playwright browser rendering")
+    parser.add_argument("--timeout", type=float, default=30.0, help="Request timeout in seconds (default: 30)")
+    parser.add_argument("--proxy", help="Proxy URL (e.g., http://proxy:8080)")
+    parser.add_argument(
+        "--format", "-f",
+        choices=["markdown", "text", "json"],
+        default="markdown",
+        help="Output format (default: markdown)",
+    )
+    args = parser.parse_args()
+
+    try:
+        result = asyncio.run(extract(
+            args.url,
+            max_chars=args.max_chars,
+            use_browser=args.browser,
+            timeout=args.timeout,
+            proxy=args.proxy,
+        ))
+    except KeyboardInterrupt:
+        sys.exit(130)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    if args.format == "json":
+        output = {
+            "title": result.title,
+            "text": result.text,
+            "markdown": result.markdown,
+            "word_count": result.word_count,
+            "language": result.language,
+            "content_type": result.content_type,
+            "source_url": result.source_url,
+            "metadata": result.metadata,
+        }
+        print(json.dumps(output, ensure_ascii=False, indent=2))
+    elif args.format == "text":
+        if result.title:
+            print(f"Title: {result.title}")
+            print(f"Words: {result.word_count} | Language: {result.language} | Type: {result.content_type}")
+            print("---")
+        print(result.text)
+    else:
+        if result.title:
+            print(f"# {result.title}")
+            print(f"<!-- words: {result.word_count} | lang: {result.language} | type: {result.content_type} -->")
+            print()
+        print(result.markdown)
+
+
+if __name__ == "__main__":
+    main()

markgrab/anti_bot/stealth.py

@@ -0,0 +1,45 @@
+"""Stealth settings for Playwright to avoid bot detection."""
+
+_STEALTH_SCRIPT = """\
+// Remove webdriver flag
+Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
+
+// Realistic languages
+Object.defineProperty(navigator, 'languages', {get: () => ['en-US', 'en', 'ko']});
+
+// Mock plugins (Chrome always has these)
+Object.defineProperty(navigator, 'plugins', {
+    get: () => {
+        const plugins = [
+            {name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer'},
+            {name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai'},
+            {name: 'Native Client', filename: 'internal-nacl-plugin'},
+        ];
+        plugins.length = 3;
+        return plugins;
+    }
+});
+
+// Mock permissions
+const originalQuery = window.navigator.permissions.query;
+window.navigator.permissions.query = (parameters) =>
+    parameters.name === 'notifications'
+        ? Promise.resolve({state: Notification.permission})
+        : originalQuery(parameters);
+
+// Chrome runtime mock
+window.chrome = {runtime: {}, loadTimes: function() {}, csi: function() {}};
+
+// WebGL vendor/renderer (Intel is the most common)
+const getParameter = WebGLRenderingContext.prototype.getParameter;
+WebGLRenderingContext.prototype.getParameter = function(parameter) {
+    if (parameter === 37445) return 'Intel Inc.';           // UNMASKED_VENDOR_WEBGL
+    if (parameter === 37446) return 'Intel Iris OpenGL Engine'; // UNMASKED_RENDERER_WEBGL
+    return getParameter.call(this, parameter);
+};
+"""
+
+
+async def apply_stealth(context) -> None:
+    """Apply stealth settings to a Playwright browser context."""
+    await context.add_init_script(_STEALTH_SCRIPT)

markgrab/core.py

@@ -0,0 +1,196 @@
+"""Main orchestrator — route URL to appropriate engine and parser."""
+
+import logging
+import random
+from urllib.parse import urlparse
+
+import httpx
+
+from markgrab.engine.base import USER_AGENTS, Engine
+from markgrab.engine.browser import BrowserEngine
+from markgrab.engine.http import HttpEngine
+from markgrab.filter.truncate import truncate_result
+from markgrab.parser.html import HtmlParser
+from markgrab.parser.youtube import YouTubeParser, _extract_video_id
+from markgrab.result import ExtractResult
+
+logger = logging.getLogger(__name__)
+
+# Minimum word count — below this, content is likely SPA/JS-only
+_MIN_WORD_COUNT = 50
+
+_OEMBED_URL = "https://www.youtube.com/oembed?url={url}&format=json"
+
+try:
+    import playwright  # noqa: F401
+
+    _BROWSER_AVAILABLE = True
+except ImportError:
+    _BROWSER_AVAILABLE = False
+
+
+def _detect_type_from_url(url: str) -> str:
+    """Detect content type from URL pattern."""
+    parsed = urlparse(url)
+    path = parsed.path.lower()
+
+    if "youtube.com" in parsed.netloc or "youtu.be" in parsed.netloc:
+        return "youtube"
+    if path.endswith(".pdf"):
+        return "pdf"
+    if path.endswith(".docx"):
+        return "docx"
+
+    return "html"
+
+
+async def _fetch_with_fallback(
+    url: str,
+    *,
+    engine: Engine | None = None,
+    timeout: float = 30.0,
+    proxy: str | None = None,
+    stealth: bool = False,
+):
+    """Fetch via HTTP, fallback to browser on error."""
+    http_engine = engine or HttpEngine(proxy=proxy)
+    try:
+        return await http_engine.fetch(url, timeout=timeout)
+    except Exception as exc:
+        if _BROWSER_AVAILABLE:
+            logger.info("HTTP failed for %s (%s), falling back to browser", url, type(exc).__name__)
+            return await BrowserEngine(proxy=proxy, stealth=stealth).fetch(url, timeout=timeout)
+        raise
+
+
+async def _fetch_youtube_title(url: str, timeout: float = 30.0) -> str:
+    """Fetch YouTube video title via oEmbed API."""
+    try:
+        oembed_url = _OEMBED_URL.format(url=url)
+        async with httpx.AsyncClient(timeout=httpx.Timeout(timeout)) as client:
+            resp = await client.get(oembed_url)
+            if resp.status_code == 200:
+                return resp.json().get("title", "")
+    except Exception:
+        logger.debug("Failed to fetch YouTube oEmbed title for %s", url)
+    return ""
+
+
+async def _fetch_bytes(url: str, *, timeout: float = 30.0, proxy: str | None = None) -> tuple[bytes, str]:
+    """Fetch URL as raw bytes. Returns (data, final_url)."""
+    headers = {
+        "User-Agent": random.choice(USER_AGENTS),
+        "Accept": "*/*",
+    }
+    async with httpx.AsyncClient(
+        headers=headers,
+        follow_redirects=True,
+        timeout=httpx.Timeout(timeout),
+        proxy=proxy,
+    ) as client:
+        resp = await client.get(url)
+        resp.raise_for_status()
+        return resp.content, str(resp.url)
+
+
+async def _extract_youtube(url: str, *, timeout: float = 30.0, max_chars: int = 50_000) -> ExtractResult:
+    """Extract YouTube video transcript."""
+    video_id = _extract_video_id(url)
+    title = await _fetch_youtube_title(url, timeout=timeout)
+
+    parser = YouTubeParser()
+    result = parser.parse(video_id=video_id, url=url, title=title)
+    return truncate_result(result, max_chars=max_chars)
+
+
+async def _extract_binary(
+    url: str,
+    content_type: str,
+    *,
+    timeout: float = 30.0,
+    max_chars: int = 50_000,
+    proxy: str | None = None,
+) -> ExtractResult:
+    """Extract content from binary URLs (PDF, DOCX)."""
+    data, final_url = await _fetch_bytes(url, timeout=timeout, proxy=proxy)
+
+    if content_type == "pdf":
+        from markgrab.parser.pdf import PdfParser
+
+        result = PdfParser().parse(data, url=final_url)
+    elif content_type == "docx":
+        from markgrab.parser.docx import DocxParser
+
+        result = DocxParser().parse(data, url=final_url)
+    else:
+        raise ValueError(f"Unknown binary content type: {content_type}")
+
+    return truncate_result(result, max_chars=max_chars)
+
+
+async def extract(
+    url: str,
+    *,
+    engine: Engine | None = None,
+    max_chars: int = 50_000,
+    use_browser: bool = False,
+    stealth: bool = False,
+    timeout: float = 30.0,
+    proxy: str | None = None,
+) -> ExtractResult:
+    """Extract content from URL and return ExtractResult.
+
+    Args:
+        url: Target URL to extract content from.
+        engine: Custom engine instance (default: HttpEngine, with browser fallback).
+        max_chars: Maximum characters for text/markdown (default 50K).
+        use_browser: Force Playwright browser rendering.
+        stealth: Apply anti-bot stealth scripts when using browser (default: False).
+        timeout: Request timeout in seconds.
+        proxy: Proxy URL (e.g., "http://proxy:8080", "socks5://proxy:1080").
+    """
+    url_type = _detect_type_from_url(url)
+
+    # YouTube — dedicated parser (no engine needed)
+    if url_type == "youtube":
+        return await _extract_youtube(url, timeout=timeout, max_chars=max_chars)
+
+    # PDF / DOCX — binary fetch + dedicated parser
+    if url_type in ("pdf", "docx"):
+        return await _extract_binary(url, url_type, timeout=timeout, max_chars=max_chars, proxy=proxy)
+
+    # HTML flow — engine + parser + fallback
+    if use_browser:
+        if not _BROWSER_AVAILABLE:
+            raise ImportError("Playwright not installed. Run: pip install 'markgrab[browser]'")
+        fetch_result = await (engine or BrowserEngine(proxy=proxy, stealth=stealth)).fetch(url, timeout=timeout)
+    else:
+        if _BROWSER_AVAILABLE:
+            fetch_result = await _fetch_with_fallback(url, engine=engine, timeout=timeout, proxy=proxy, stealth=stealth)
+        else:
+            fetch_result = await (engine or HttpEngine(proxy=proxy)).fetch(url, timeout=timeout)
+
+    # Content-Type header may reveal PDF even without .pdf extension
+    if "application/pdf" in fetch_result.content_type:
+        data, final_url = await _fetch_bytes(url, timeout=timeout, proxy=proxy)
+        from markgrab.parser.pdf import PdfParser
+
+        result = PdfParser().parse(data, url=final_url)
+        return truncate_result(result, max_chars=max_chars)
+
+    # Parse HTML
+    parser = HtmlParser()
+    result = parser.parse(fetch_result.html, url=fetch_result.final_url)
+
+    # Auto-fallback: thin content likely means SPA/JS-only page
+    if not use_browser and _BROWSER_AVAILABLE and result.word_count < _MIN_WORD_COUNT:
+        logger.info("Thin content (%d words) for %s, retrying with browser", result.word_count, url)
+        try:
+            browser_result = await BrowserEngine(proxy=proxy, stealth=stealth).fetch(url, timeout=timeout)
+            browser_parsed = parser.parse(browser_result.html, url=browser_result.final_url)
+            if browser_parsed.word_count > result.word_count:
+                result = browser_parsed
+        except Exception:
+            pass  # Keep original result
+
+    return truncate_result(result, max_chars=max_chars)

markgrab/engine/__init__.py

@@ -0,0 +1,7 @@
+"""Content fetching engines."""
+
+from markgrab.engine.base import USER_AGENTS, Engine, FetchResult
+from markgrab.engine.browser import BrowserEngine
+from markgrab.engine.http import HttpEngine
+
+__all__ = ["USER_AGENTS", "Engine", "FetchResult", "HttpEngine", "BrowserEngine"]

markgrab/engine/base.py

@@ -0,0 +1,42 @@
+"""Engine base — content fetching abstraction."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+@dataclass
+class FetchResult:
+    """Raw result from fetching a URL."""
+
+    html: str
+    status_code: int
+    content_type: str
+    final_url: str
+
+
+# Shared User-Agent pool — used by HttpEngine, core._fetch_bytes, etc.
+USER_AGENTS = [
+    (
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
+        " (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+    ),
+    (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+        " (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+    ),
+    (
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
+        " (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+    ),
+]
+
+
+class Engine(ABC):
+    """Abstract base for content fetching engines."""
+
+    def __init__(self, *, proxy: str | None = None):
+        self.proxy = proxy
+
+    @abstractmethod
+    async def fetch(self, url: str, *, timeout: float = 30.0) -> FetchResult:
+        ...

markgrab/engine/browser.py

@@ -0,0 +1,72 @@
+"""Browser engine — Playwright headless for JS-rendered and bot-protected pages."""
+
+import logging
+
+from markgrab.engine.base import Engine, FetchResult
+
+logger = logging.getLogger(__name__)
+
+
+class BrowserEngine(Engine):
+    """Playwright-based browser engine for JS-heavy and bot-protected sites.
+
+    Requires: pip install markgrab[browser]
+    Playwright is imported lazily — the class can be imported without playwright installed.
+
+    Args:
+        proxy: Proxy URL.
+        stealth: Apply anti-bot stealth scripts (default: False).
+    """
+
+    def __init__(self, *, proxy: str | None = None, stealth: bool = False):
+        super().__init__(proxy=proxy)
+        self.stealth = stealth
+
+    async def fetch(self, url: str, *, timeout: float = 30.0) -> FetchResult:
+        from playwright.async_api import async_playwright
+
+        timeout_ms = int(timeout * 1000)
+
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            try:
+                context_kwargs: dict = {
+                    "viewport": {"width": 1920, "height": 1080},
+                    "locale": "en-US",
+                    "timezone_id": "America/New_York",
+                }
+                if self.proxy:
+                    context_kwargs["proxy"] = {"server": self.proxy}
+
+                context = await browser.new_context(**context_kwargs)
+                if self.stealth:
+                    from markgrab.anti_bot.stealth import apply_stealth
+
+                    await apply_stealth(context)
+
+                page = await context.new_page()
+                response = await page.goto(
+                    url,
+                    wait_until="domcontentloaded",
+                    timeout=timeout_ms,
+                )
+
+                # Best-effort wait for JS rendering (max 5s or half timeout)
+                networkidle_ms = min(5000, timeout_ms // 2)
+                try:
+                    await page.wait_for_load_state("networkidle", timeout=networkidle_ms)
+                except Exception:
+                    pass  # DOM content is enough
+
+                html = await page.content()
+                status = response.status if response else 200
+                headers = response.headers if response else {}
+
+                return FetchResult(
+                    html=html,
+                    status_code=status,
+                    content_type=headers.get("content-type", "text/html"),
+                    final_url=page.url,
+                )
+            finally:
+                await browser.close()

markgrab/engine/http.py

@@ -0,0 +1,37 @@
+"""HTTP engine — lightweight fetching with httpx."""
+
+import logging
+import random
+
+import httpx
+
+from markgrab.engine.base import USER_AGENTS, Engine, FetchResult
+
+logger = logging.getLogger(__name__)
+
+
+class HttpEngine(Engine):
+    """Lightweight HTTP engine using httpx."""
+
+    async def fetch(self, url: str, *, timeout: float = 30.0) -> FetchResult:
+        headers = {
+            "User-Agent": random.choice(USER_AGENTS),
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.9,ko;q=0.8",
+        }
+
+        async with httpx.AsyncClient(
+            headers=headers,
+            follow_redirects=True,
+            timeout=httpx.Timeout(timeout),
+            proxy=self.proxy,
+        ) as client:
+            response = await client.get(url)
+            response.raise_for_status()
+
+            return FetchResult(
+                html=response.text,
+                status_code=response.status_code,
+                content_type=response.headers.get("content-type", ""),
+                final_url=str(response.url),
+            )

markgrab/filter/__init__.py

@@ -0,0 +1,7 @@
+"""Content filters."""
+
+from markgrab.filter.density import filter_low_density
+from markgrab.filter.noise import clean_soup
+from markgrab.filter.truncate import truncate_result
+
+__all__ = ["clean_soup", "filter_low_density", "truncate_result"]

markgrab/filter/density.py

@@ -0,0 +1,79 @@
+"""Content density filter — remove sidebars and navigation from content area."""
+
+import logging
+
+from bs4 import Tag
+
+logger = logging.getLogger(__name__)
+
+# Block-level elements to analyze for link density
+_BLOCK_TAGS = frozenset({"div", "section", "ul", "ol", "table", "form", "dl"})
+
+# Class/id patterns that indicate sidebar/non-content blocks
+_SIDEBAR_PATTERNS = (
+    "sidebar",
+    "related",
+    "widget",
+    "toc",
+    "breadcrumb",
+    "social",
+    "share",
+    "comment",
+    "advert",
+    "promo",
+    "recommend",
+    "popular",
+    "trending",
+    "signup",
+    "newsletter",
+    "subscribe",
+)
+
+# Link density above this = likely navigation, not content
+_LINK_DENSITY_THRESHOLD = 0.5
+
+# Minimum text length to consider for link density analysis
+_MIN_BLOCK_TEXT = 25
+
+
+def filter_low_density(content: Tag) -> None:
+    """Remove low-density sidebar/navigation blocks from content area in-place.
+
+    Three-pass approach:
+    1. Remove <aside>/<nav> tags (semantically non-content)
+    2. Remove elements matching sidebar class/id patterns
+    3. Remove direct block children with high link density
+    """
+    # Pass 1: semantic non-content tags inside content
+    for tag in content.find_all(["aside", "nav"]):
+        logger.debug("Removing <%s> from content", tag.name)
+        tag.decompose()
+
+    # Pass 2: sidebar/widget pattern matching
+    for pattern in _SIDEBAR_PATTERNS:
+        for selector in (f"[class*='{pattern}']", f"[id*='{pattern}']"):
+            for el in content.select(selector):
+                if el.attrs is None:
+                    continue  # Already decomposed by a prior pattern
+                logger.debug("Removing sidebar pattern '%s': %s", pattern, el.get("class") or el.get("id"))
+                el.decompose()
+
+    # Pass 3: link density on direct block children
+    for child in list(content.children):
+        if not isinstance(child, Tag):
+            continue
+        if child.name not in _BLOCK_TAGS:
+            continue
+
+        text = child.get_text(strip=True)
+        if not text or len(text) < _MIN_BLOCK_TEXT:
+            continue
+
+        links_text = "".join(a.get_text(strip=True) for a in child.find_all("a"))
+        if not links_text:
+            continue
+
+        link_ratio = len(links_text) / len(text)
+        if link_ratio > _LINK_DENSITY_THRESHOLD:
+            logger.debug("Removing high link-density block (%.0f%%): <%s>", link_ratio * 100, child.name)
+            child.decompose()

markgrab/filter/noise.py

@@ -0,0 +1,42 @@
+"""Noise filter — remove ads, navigation, popups from HTML."""
+
+import re
+
+from bs4 import BeautifulSoup
+
+_NOISE_TAGS = frozenset({"script", "style", "noscript", "svg", "iframe"})
+
+_POPUP_SELECTORS = [
+    "[class*='cookie']",
+    "[class*='consent']",
+    "[class*='popup']",
+    "[class*='modal']",
+    "[id*='cookie']",
+    "[id*='consent']",
+]
+
+
+def clean_soup(soup: BeautifulSoup) -> None:
+    """Remove noise elements from soup in-place.
+
+    Removes: script/style/noscript/svg/iframe tags,
+    cookie/consent/popup/modal elements, hidden elements.
+    """
+    for tag in soup.find_all(list(_NOISE_TAGS)):
+        tag.decompose()
+
+    for selector in _POPUP_SELECTORS:
+        for el in soup.select(selector):
+            if el.attrs is None:
+                continue  # Already decomposed by a prior selector
+            el.decompose()
+
+    for el in soup.find_all(attrs={"aria-hidden": "true"}):
+        if el.attrs is None:
+            continue
+        el.decompose()
+
+    for el in soup.find_all(style=re.compile(r"display:\s*none")):
+        if el.attrs is None:
+            continue
+        el.decompose()

markgrab/filter/truncate.py

@@ -0,0 +1,33 @@
+"""Truncate filter — limit content length."""
+
+from markgrab.result import ExtractResult
+
+
+def truncate_result(result: ExtractResult, *, max_chars: int = 50_000) -> ExtractResult:
+    """Truncate text and markdown fields to max_chars.
+
+    Tries to break at the last newline before the limit.
+    Returns the original result if no truncation needed.
+    """
+    if max_chars <= 0 or (len(result.text) <= max_chars and len(result.markdown) <= max_chars):
+        return result
+
+    text = result.text
+    markdown = result.markdown
+
+    if len(text) > max_chars:
+        text = text[:max_chars].rsplit("\n", 1)[0] + "\n\n[truncated]"
+
+    if len(markdown) > max_chars:
+        markdown = markdown[:max_chars].rsplit("\n", 1)[0] + "\n\n[truncated]"
+
+    return ExtractResult(
+        title=result.title,
+        text=text,
+        markdown=markdown,
+        word_count=len(text.split()),
+        language=result.language,
+        content_type=result.content_type,
+        source_url=result.source_url,
+        metadata=result.metadata,
+    )

markgrab/parser/__init__.py

@@ -0,0 +1,9 @@
+"""Content parsers."""
+
+from markgrab.parser.base import Parser
+from markgrab.parser.docx import DocxParser
+from markgrab.parser.html import HtmlParser
+from markgrab.parser.pdf import PdfParser
+from markgrab.parser.youtube import YouTubeParser
+
+__all__ = ["Parser", "HtmlParser", "YouTubeParser", "PdfParser", "DocxParser"]

markgrab/parser/base.py

@@ -0,0 +1,13 @@
+"""Parser base — content parsing abstraction."""
+
+from abc import ABC, abstractmethod
+
+from markgrab.result import ExtractResult
+
+
+class Parser(ABC):
+    """Abstract base for content parsers."""
+
+    @abstractmethod
+    def parse(self, html: str, url: str) -> ExtractResult:
+        ...