markgrab 0.1.0__tar.gz

markgrab-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 hmj
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

markgrab-0.1.0/PKG-INFO

@@ -0,0 +1,179 @@
+Metadata-Version: 2.4
+Name: markgrab
+Version: 0.1.0
+Summary: Universal web content extraction — URL to LLM-ready markdown
+Author: hmj
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/QuartzUnit/markgrab
+Project-URL: Repository, https://github.com/QuartzUnit/markgrab
+Project-URL: Issues, https://github.com/QuartzUnit/markgrab/issues
+Keywords: web-scraping,content-extraction,markdown,llm,rag
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: httpx>=0.28
+Requires-Dist: beautifulsoup4>=4.13
+Requires-Dist: markdownify>=0.14
+Provides-Extra: browser
+Requires-Dist: playwright>=1.49; extra == "browser"
+Provides-Extra: youtube
+Requires-Dist: youtube-transcript-api>=1.0; extra == "youtube"
+Provides-Extra: pdf
+Requires-Dist: pdfplumber>=0.11; extra == "pdf"
+Provides-Extra: docx
+Requires-Dist: python-docx>=1.1; extra == "docx"
+Provides-Extra: all
+Requires-Dist: playwright>=1.49; extra == "all"
+Requires-Dist: youtube-transcript-api>=1.0; extra == "all"
+Requires-Dist: pdfplumber>=0.11; extra == "all"
+Requires-Dist: python-docx>=1.1; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
+Requires-Dist: ruff>=0.9; extra == "dev"
+Dynamic: license-file
+
+# MarkGrab
+
+> [한국어 문서](README.ko.md)
+
+Universal web content extraction — any URL to LLM-ready markdown.
+
+```python
+from markgrab import extract
+
+result = await extract("https://example.com/article")
+print(result.markdown)    # clean markdown
+print(result.title)       # "Article Title"
+print(result.word_count)  # 1234
+print(result.language)    # "en"
+```
+
+## Features
+
+- **HTML** — BeautifulSoup + content density filtering (removes nav, sidebar, ads)
+- **YouTube** — transcript extraction with timestamps
+- **PDF** — text extraction with page structure
+- **DOCX** — paragraph and heading extraction
+- **Auto-fallback** — tries lightweight httpx first, falls back to Playwright for JS-heavy pages
+- **Async-first** — built on httpx and Playwright async APIs
+
+## Install
+
+```bash
+pip install markgrab
+```
+
+Optional extras for specific content types:
+
+```bash
+pip install "markgrab[browser]"    # Playwright for JS-rendered pages
+pip install "markgrab[youtube]"    # YouTube transcript extraction
+pip install "markgrab[pdf]"       # PDF text extraction
+pip install "markgrab[docx]"      # DOCX text extraction
+pip install "markgrab[all]"       # everything
+```
+
+## Usage
+
+### Python API
+
+```python
+import asyncio
+from markgrab import extract
+
+async def main():
+    # HTML (auto-detects content type)
+    result = await extract("https://example.com/article")
+
+    # YouTube transcript
+    result = await extract("https://youtube.com/watch?v=dQw4w9WgXcQ")
+
+    # PDF
+    result = await extract("https://arxiv.org/pdf/1706.03762")
+
+    # Options
+    result = await extract(
+        "https://example.com",
+        max_chars=30_000,       # limit output length (default: 50K)
+        use_browser=True,       # force Playwright rendering
+        stealth=True,           # anti-bot stealth scripts (opt-in)
+        timeout=60.0,           # request timeout in seconds
+        proxy="http://proxy:8080",
+    )
+
+asyncio.run(main())
+```
+
+### CLI
+
+```bash
+markgrab https://example.com                     # markdown output
+markgrab https://example.com -f text             # plain text
+markgrab https://example.com -f json             # structured JSON
+markgrab https://example.com --browser           # force browser rendering
+markgrab https://example.com --max-chars 10000   # limit output
+```
+
+### ExtractResult
+
+```python
+result.title        # page title
+result.text         # plain text
+result.markdown     # LLM-ready markdown
+result.word_count   # word count
+result.language     # detected language ("en", "ko", ...)
+result.content_type # "article", "video", "pdf", "docx"
+result.source_url   # final URL (after redirects)
+result.metadata     # extra metadata (video_id, page_count, etc.)
+```
+
+## How it works
+
+```
+markgrab.extract(url)
+    1. Detect content type (URL pattern)
+    2. Fetch content (httpx first, Playwright fallback)
+    3. Parse (HTML/YouTube/PDF/DOCX)
+    4. Filter (noise removal + content density + truncation)
+    5. Return ExtractResult
+```
+
+For HTML pages, if the initial httpx fetch yields fewer than 50 words, MarkGrab automatically retries with Playwright to handle JavaScript-rendered content.
+
+## Disclaimer
+
+**This software is provided for legitimate purposes only.** By using MarkGrab, you agree to the following:
+
+- **robots.txt**: MarkGrab does **not** check or enforce `robots.txt`. Users are solely responsible for checking and respecting `robots.txt` directives and the terms of service of any website they access.
+
+- **Rate limiting**: MarkGrab does **not** include built-in rate limiting or request throttling. Users must implement their own rate limiting to avoid overloading target servers. Abusive request patterns may violate applicable laws and website terms of service.
+
+- **YouTube transcripts**: YouTube transcript extraction relies on the third-party `youtube-transcript-api` library, which uses YouTube's internal (unofficial) caption API. This may not comply with YouTube's Terms of Service. Use at your own discretion and risk.
+
+- **Stealth mode**: The optional `stealth=True` feature modifies browser fingerprinting signals to reduce bot detection. This feature is intended for legitimate use cases such as testing, research, and accessing content that is publicly available to regular browser users. Users are responsible for ensuring their use complies with applicable laws and the terms of service of target websites.
+
+- **Legal compliance**: Users are responsible for ensuring that their use of MarkGrab complies with all applicable laws, including but not limited to the Computer Fraud and Abuse Act (CFAA), the Digital Millennium Copyright Act (DMCA), GDPR, and equivalent legislation in their jurisdiction.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND. See the [LICENSE](LICENSE) file for the full MIT license text.
+
+## Acknowledgments
+
+MarkGrab builds on excellent open-source work and well-established techniques:
+
+- **[puppeteer-extra-plugin-stealth](https://github.com/nicoleahmed/puppeteer-extra-plugin-stealth)** — stealth evasion patterns (webdriver removal, plugin mocking, WebGL spoofing) that inspired the opt-in `anti_bot/stealth.py` module
+- **[Mozilla Readability](https://github.com/mozilla/readability)** — content area detection priority (`article > main > body`) and link density filtering concepts used in the density filter
+- **[Boilerpipe](https://github.com/kohlschutter/boilerpipe)** (Kohlschutter et al., 2010) — the academic origin of link density ratio algorithms for boilerplate removal
+- **[Jina Reader](https://github.com/jina-ai/reader)** — validated the market need for URL-to-markdown extraction; MarkGrab aims to be a lightweight, self-hosted alternative
+
+Built with [httpx](https://github.com/encode/httpx), [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/), [markdownify](https://github.com/matthewwithanm/python-markdownify), [Playwright](https://github.com/microsoft/playwright-python), [youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api), [pdfplumber](https://github.com/jsvine/pdfplumber), and [python-docx](https://github.com/python-openxml/python-docx).
+
+## License
+
+[MIT](LICENSE)

markgrab-0.1.0/README.md

@@ -0,0 +1,138 @@
+# MarkGrab
+
+> [한국어 문서](README.ko.md)
+
+Universal web content extraction — any URL to LLM-ready markdown.
+
+```python
+from markgrab import extract
+
+result = await extract("https://example.com/article")
+print(result.markdown)    # clean markdown
+print(result.title)       # "Article Title"
+print(result.word_count)  # 1234
+print(result.language)    # "en"
+```
+
+## Features
+
+- **HTML** — BeautifulSoup + content density filtering (removes nav, sidebar, ads)
+- **YouTube** — transcript extraction with timestamps
+- **PDF** — text extraction with page structure
+- **DOCX** — paragraph and heading extraction
+- **Auto-fallback** — tries lightweight httpx first, falls back to Playwright for JS-heavy pages
+- **Async-first** — built on httpx and Playwright async APIs
+
+## Install
+
+```bash
+pip install markgrab
+```
+
+Optional extras for specific content types:
+
+```bash
+pip install "markgrab[browser]"    # Playwright for JS-rendered pages
+pip install "markgrab[youtube]"    # YouTube transcript extraction
+pip install "markgrab[pdf]"       # PDF text extraction
+pip install "markgrab[docx]"      # DOCX text extraction
+pip install "markgrab[all]"       # everything
+```
+
+## Usage
+
+### Python API
+
+```python
+import asyncio
+from markgrab import extract
+
+async def main():
+    # HTML (auto-detects content type)
+    result = await extract("https://example.com/article")
+
+    # YouTube transcript
+    result = await extract("https://youtube.com/watch?v=dQw4w9WgXcQ")
+
+    # PDF
+    result = await extract("https://arxiv.org/pdf/1706.03762")
+
+    # Options
+    result = await extract(
+        "https://example.com",
+        max_chars=30_000,       # limit output length (default: 50K)
+        use_browser=True,       # force Playwright rendering
+        stealth=True,           # anti-bot stealth scripts (opt-in)
+        timeout=60.0,           # request timeout in seconds
+        proxy="http://proxy:8080",
+    )
+
+asyncio.run(main())
+```
+
+### CLI
+
+```bash
+markgrab https://example.com                     # markdown output
+markgrab https://example.com -f text             # plain text
+markgrab https://example.com -f json             # structured JSON
+markgrab https://example.com --browser           # force browser rendering
+markgrab https://example.com --max-chars 10000   # limit output
+```
+
+### ExtractResult
+
+```python
+result.title        # page title
+result.text         # plain text
+result.markdown     # LLM-ready markdown
+result.word_count   # word count
+result.language     # detected language ("en", "ko", ...)
+result.content_type # "article", "video", "pdf", "docx"
+result.source_url   # final URL (after redirects)
+result.metadata     # extra metadata (video_id, page_count, etc.)
+```
+
+## How it works
+
+```
+markgrab.extract(url)
+    1. Detect content type (URL pattern)
+    2. Fetch content (httpx first, Playwright fallback)
+    3. Parse (HTML/YouTube/PDF/DOCX)
+    4. Filter (noise removal + content density + truncation)
+    5. Return ExtractResult
+```
+
+For HTML pages, if the initial httpx fetch yields fewer than 50 words, MarkGrab automatically retries with Playwright to handle JavaScript-rendered content.
+
+## Disclaimer
+
+**This software is provided for legitimate purposes only.** By using MarkGrab, you agree to the following:
+
+- **robots.txt**: MarkGrab does **not** check or enforce `robots.txt`. Users are solely responsible for checking and respecting `robots.txt` directives and the terms of service of any website they access.
+
+- **Rate limiting**: MarkGrab does **not** include built-in rate limiting or request throttling. Users must implement their own rate limiting to avoid overloading target servers. Abusive request patterns may violate applicable laws and website terms of service.
+
+- **YouTube transcripts**: YouTube transcript extraction relies on the third-party `youtube-transcript-api` library, which uses YouTube's internal (unofficial) caption API. This may not comply with YouTube's Terms of Service. Use at your own discretion and risk.
+
+- **Stealth mode**: The optional `stealth=True` feature modifies browser fingerprinting signals to reduce bot detection. This feature is intended for legitimate use cases such as testing, research, and accessing content that is publicly available to regular browser users. Users are responsible for ensuring their use complies with applicable laws and the terms of service of target websites.
+
+- **Legal compliance**: Users are responsible for ensuring that their use of MarkGrab complies with all applicable laws, including but not limited to the Computer Fraud and Abuse Act (CFAA), the Digital Millennium Copyright Act (DMCA), GDPR, and equivalent legislation in their jurisdiction.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND. See the [LICENSE](LICENSE) file for the full MIT license text.
+
+## Acknowledgments
+
+MarkGrab builds on excellent open-source work and well-established techniques:
+
+- **[puppeteer-extra-plugin-stealth](https://github.com/nicoleahmed/puppeteer-extra-plugin-stealth)** — stealth evasion patterns (webdriver removal, plugin mocking, WebGL spoofing) that inspired the opt-in `anti_bot/stealth.py` module
+- **[Mozilla Readability](https://github.com/mozilla/readability)** — content area detection priority (`article > main > body`) and link density filtering concepts used in the density filter
+- **[Boilerpipe](https://github.com/kohlschutter/boilerpipe)** (Kohlschutter et al., 2010) — the academic origin of link density ratio algorithms for boilerplate removal
+- **[Jina Reader](https://github.com/jina-ai/reader)** — validated the market need for URL-to-markdown extraction; MarkGrab aims to be a lightweight, self-hosted alternative
+
+Built with [httpx](https://github.com/encode/httpx), [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/), [markdownify](https://github.com/matthewwithanm/python-markdownify), [Playwright](https://github.com/microsoft/playwright-python), [youtube-transcript-api](https://github.com/jdepoix/youtube-transcript-api), [pdfplumber](https://github.com/jsvine/pdfplumber), and [python-docx](https://github.com/python-openxml/python-docx).
+
+## License
+
+[MIT](LICENSE)

markgrab-0.1.0/markgrab/__init__.py

@@ -0,0 +1,7 @@
+"""MarkGrab — Universal web content extraction."""
+
+from markgrab.core import extract
+from markgrab.result import ExtractResult
+
+__all__ = ["extract", "ExtractResult"]
+__version__ = "0.1.0"

markgrab-0.1.0/markgrab/__main__.py

@@ -0,0 +1,70 @@
+"""CLI entry point — python -m markgrab or `markgrab` command."""
+
+import argparse
+import asyncio
+import json
+import sys
+
+from markgrab import extract
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="markgrab",
+        description="MarkGrab — extract web content as LLM-ready markdown",
+    )
+    parser.add_argument("url", help="URL to extract content from")
+    parser.add_argument("--max-chars", type=int, default=50_000, help="Max output characters (default: 50000)")
+    parser.add_argument("--browser", action="store_true", help="Force Playwright browser rendering")
+    parser.add_argument("--timeout", type=float, default=30.0, help="Request timeout in seconds (default: 30)")
+    parser.add_argument("--proxy", help="Proxy URL (e.g., http://proxy:8080)")
+    parser.add_argument(
+        "--format", "-f",
+        choices=["markdown", "text", "json"],
+        default="markdown",
+        help="Output format (default: markdown)",
+    )
+    args = parser.parse_args()
+
+    try:
+        result = asyncio.run(extract(
+            args.url,
+            max_chars=args.max_chars,
+            use_browser=args.browser,
+            timeout=args.timeout,
+            proxy=args.proxy,
+        ))
+    except KeyboardInterrupt:
+        sys.exit(130)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    if args.format == "json":
+        output = {
+            "title": result.title,
+            "text": result.text,
+            "markdown": result.markdown,
+            "word_count": result.word_count,
+            "language": result.language,
+            "content_type": result.content_type,
+            "source_url": result.source_url,
+            "metadata": result.metadata,
+        }
+        print(json.dumps(output, ensure_ascii=False, indent=2))
+    elif args.format == "text":
+        if result.title:
+            print(f"Title: {result.title}")
+            print(f"Words: {result.word_count} | Language: {result.language} | Type: {result.content_type}")
+            print("---")
+        print(result.text)
+    else:
+        if result.title:
+            print(f"# {result.title}")
+            print(f"<!-- words: {result.word_count} | lang: {result.language} | type: {result.content_type} -->")
+            print()
+        print(result.markdown)
+
+
+if __name__ == "__main__":
+    main()

markgrab-0.1.0/markgrab/anti_bot/stealth.py

@@ -0,0 +1,45 @@
+"""Stealth settings for Playwright to avoid bot detection."""
+
+_STEALTH_SCRIPT = """\
+// Remove webdriver flag
+Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
+
+// Realistic languages
+Object.defineProperty(navigator, 'languages', {get: () => ['en-US', 'en', 'ko']});
+
+// Mock plugins (Chrome always has these)
+Object.defineProperty(navigator, 'plugins', {
+    get: () => {
+        const plugins = [
+            {name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer'},
+            {name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai'},
+            {name: 'Native Client', filename: 'internal-nacl-plugin'},
+        ];
+        plugins.length = 3;
+        return plugins;
+    }
+});
+
+// Mock permissions
+const originalQuery = window.navigator.permissions.query;
+window.navigator.permissions.query = (parameters) =>
+    parameters.name === 'notifications'
+        ? Promise.resolve({state: Notification.permission})
+        : originalQuery(parameters);
+
+// Chrome runtime mock
+window.chrome = {runtime: {}, loadTimes: function() {}, csi: function() {}};
+
+// WebGL vendor/renderer (Intel is the most common)
+const getParameter = WebGLRenderingContext.prototype.getParameter;
+WebGLRenderingContext.prototype.getParameter = function(parameter) {
+    if (parameter === 37445) return 'Intel Inc.';           // UNMASKED_VENDOR_WEBGL
+    if (parameter === 37446) return 'Intel Iris OpenGL Engine'; // UNMASKED_RENDERER_WEBGL
+    return getParameter.call(this, parameter);
+};
+"""
+
+
+async def apply_stealth(context) -> None:
+    """Apply stealth settings to a Playwright browser context."""
+    await context.add_init_script(_STEALTH_SCRIPT)

markgrab-0.1.0/markgrab/core.py

@@ -0,0 +1,196 @@
+"""Main orchestrator — route URL to appropriate engine and parser."""
+
+import logging
+import random
+from urllib.parse import urlparse
+
+import httpx
+
+from markgrab.engine.base import USER_AGENTS, Engine
+from markgrab.engine.browser import BrowserEngine
+from markgrab.engine.http import HttpEngine
+from markgrab.filter.truncate import truncate_result
+from markgrab.parser.html import HtmlParser
+from markgrab.parser.youtube import YouTubeParser, _extract_video_id
+from markgrab.result import ExtractResult
+
+logger = logging.getLogger(__name__)
+
+# Minimum word count — below this, content is likely SPA/JS-only
+_MIN_WORD_COUNT = 50
+
+_OEMBED_URL = "https://www.youtube.com/oembed?url={url}&format=json"
+
+try:
+    import playwright  # noqa: F401
+
+    _BROWSER_AVAILABLE = True
+except ImportError:
+    _BROWSER_AVAILABLE = False
+
+
+def _detect_type_from_url(url: str) -> str:
+    """Detect content type from URL pattern."""
+    parsed = urlparse(url)
+    path = parsed.path.lower()
+
+    if "youtube.com" in parsed.netloc or "youtu.be" in parsed.netloc:
+        return "youtube"
+    if path.endswith(".pdf"):
+        return "pdf"
+    if path.endswith(".docx"):
+        return "docx"
+
+    return "html"
+
+
+async def _fetch_with_fallback(
+    url: str,
+    *,
+    engine: Engine | None = None,
+    timeout: float = 30.0,
+    proxy: str | None = None,
+    stealth: bool = False,
+):
+    """Fetch via HTTP, fallback to browser on error."""
+    http_engine = engine or HttpEngine(proxy=proxy)
+    try:
+        return await http_engine.fetch(url, timeout=timeout)
+    except Exception as exc:
+        if _BROWSER_AVAILABLE:
+            logger.info("HTTP failed for %s (%s), falling back to browser", url, type(exc).__name__)
+            return await BrowserEngine(proxy=proxy, stealth=stealth).fetch(url, timeout=timeout)
+        raise
+
+
+async def _fetch_youtube_title(url: str, timeout: float = 30.0) -> str:
+    """Fetch YouTube video title via oEmbed API."""
+    try:
+        oembed_url = _OEMBED_URL.format(url=url)
+        async with httpx.AsyncClient(timeout=httpx.Timeout(timeout)) as client:
+            resp = await client.get(oembed_url)
+            if resp.status_code == 200:
+                return resp.json().get("title", "")
+    except Exception:
+        logger.debug("Failed to fetch YouTube oEmbed title for %s", url)
+    return ""
+
+
+async def _fetch_bytes(url: str, *, timeout: float = 30.0, proxy: str | None = None) -> tuple[bytes, str]:
+    """Fetch URL as raw bytes. Returns (data, final_url)."""
+    headers = {
+        "User-Agent": random.choice(USER_AGENTS),
+        "Accept": "*/*",
+    }
+    async with httpx.AsyncClient(
+        headers=headers,
+        follow_redirects=True,
+        timeout=httpx.Timeout(timeout),
+        proxy=proxy,
+    ) as client:
+        resp = await client.get(url)
+        resp.raise_for_status()
+        return resp.content, str(resp.url)
+
+
+async def _extract_youtube(url: str, *, timeout: float = 30.0, max_chars: int = 50_000) -> ExtractResult:
+    """Extract YouTube video transcript."""
+    video_id = _extract_video_id(url)
+    title = await _fetch_youtube_title(url, timeout=timeout)
+
+    parser = YouTubeParser()
+    result = parser.parse(video_id=video_id, url=url, title=title)
+    return truncate_result(result, max_chars=max_chars)
+
+
+async def _extract_binary(
+    url: str,
+    content_type: str,
+    *,
+    timeout: float = 30.0,
+    max_chars: int = 50_000,
+    proxy: str | None = None,
+) -> ExtractResult:
+    """Extract content from binary URLs (PDF, DOCX)."""
+    data, final_url = await _fetch_bytes(url, timeout=timeout, proxy=proxy)
+
+    if content_type == "pdf":
+        from markgrab.parser.pdf import PdfParser
+
+        result = PdfParser().parse(data, url=final_url)
+    elif content_type == "docx":
+        from markgrab.parser.docx import DocxParser
+
+        result = DocxParser().parse(data, url=final_url)
+    else:
+        raise ValueError(f"Unknown binary content type: {content_type}")
+
+    return truncate_result(result, max_chars=max_chars)
+
+
+async def extract(
+    url: str,
+    *,
+    engine: Engine | None = None,
+    max_chars: int = 50_000,
+    use_browser: bool = False,
+    stealth: bool = False,
+    timeout: float = 30.0,
+    proxy: str | None = None,
+) -> ExtractResult:
+    """Extract content from URL and return ExtractResult.
+
+    Args:
+        url: Target URL to extract content from.
+        engine: Custom engine instance (default: HttpEngine, with browser fallback).
+        max_chars: Maximum characters for text/markdown (default 50K).
+        use_browser: Force Playwright browser rendering.
+        stealth: Apply anti-bot stealth scripts when using browser (default: False).
+        timeout: Request timeout in seconds.
+        proxy: Proxy URL (e.g., "http://proxy:8080", "socks5://proxy:1080").
+    """
+    url_type = _detect_type_from_url(url)
+
+    # YouTube — dedicated parser (no engine needed)
+    if url_type == "youtube":
+        return await _extract_youtube(url, timeout=timeout, max_chars=max_chars)
+
+    # PDF / DOCX — binary fetch + dedicated parser
+    if url_type in ("pdf", "docx"):
+        return await _extract_binary(url, url_type, timeout=timeout, max_chars=max_chars, proxy=proxy)
+
+    # HTML flow — engine + parser + fallback
+    if use_browser:
+        if not _BROWSER_AVAILABLE:
+            raise ImportError("Playwright not installed. Run: pip install 'markgrab[browser]'")
+        fetch_result = await (engine or BrowserEngine(proxy=proxy, stealth=stealth)).fetch(url, timeout=timeout)
+    else:
+        if _BROWSER_AVAILABLE:
+            fetch_result = await _fetch_with_fallback(url, engine=engine, timeout=timeout, proxy=proxy, stealth=stealth)
+        else:
+            fetch_result = await (engine or HttpEngine(proxy=proxy)).fetch(url, timeout=timeout)
+
+    # Content-Type header may reveal PDF even without .pdf extension
+    if "application/pdf" in fetch_result.content_type:
+        data, final_url = await _fetch_bytes(url, timeout=timeout, proxy=proxy)
+        from markgrab.parser.pdf import PdfParser
+
+        result = PdfParser().parse(data, url=final_url)
+        return truncate_result(result, max_chars=max_chars)
+
+    # Parse HTML
+    parser = HtmlParser()
+    result = parser.parse(fetch_result.html, url=fetch_result.final_url)
+
+    # Auto-fallback: thin content likely means SPA/JS-only page
+    if not use_browser and _BROWSER_AVAILABLE and result.word_count < _MIN_WORD_COUNT:
+        logger.info("Thin content (%d words) for %s, retrying with browser", result.word_count, url)
+        try:
+            browser_result = await BrowserEngine(proxy=proxy, stealth=stealth).fetch(url, timeout=timeout)
+            browser_parsed = parser.parse(browser_result.html, url=browser_result.final_url)
+            if browser_parsed.word_count > result.word_count:
+                result = browser_parsed
+        except Exception:
+            pass  # Keep original result
+
+    return truncate_result(result, max_chars=max_chars)

markgrab-0.1.0/markgrab/engine/__init__.py

@@ -0,0 +1,7 @@
+"""Content fetching engines."""
+
+from markgrab.engine.base import USER_AGENTS, Engine, FetchResult
+from markgrab.engine.browser import BrowserEngine
+from markgrab.engine.http import HttpEngine
+
+__all__ = ["USER_AGENTS", "Engine", "FetchResult", "HttpEngine", "BrowserEngine"]