patchtroy 0.4.0__py3-none-any.whl

patchtroy/__init__.py

@@ -0,0 +1,26 @@
+"""Patchtroy — Undetected stealth web scraper & markdown extractor for LLMs."""
+
+from patchtroy.chunker import TextChunk, chunk_markdown, count_tokens
+from patchtroy.crawler import AsyncPatchtroy, Patchtroy
+from patchtroy.models import LinkItem, PatchtroyConfig, ScrapeResult
+from patchtroy.pool import BrowserContextPool
+from patchtroy.proxy import ProxyItem, ProxyManager
+
+__version__ = "0.4.0"
+__author__ = "Marcus Zou"
+__license__ = "Apache-2.0"
+__copyright__ = "Copyright 2026 Alfazen Inc."
+
+__all__ = [
+    "AsyncPatchtroy",
+    "BrowserContextPool",
+    "LinkItem",
+    "Patchtroy",
+    "PatchtroyConfig",
+    "ProxyItem",
+    "ProxyManager",
+    "ScrapeResult",
+    "TextChunk",
+    "chunk_markdown",
+    "count_tokens",
+]

patchtroy/chunker.py

@@ -0,0 +1,178 @@
+"""Structure-aware Markdown chunking and token counting utility for LLMs."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+# Regex pattern for fast BPE-like token estimation without heavy ML dependencies
+_TOKEN_PATTERN = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\w+| ?\d+| ?[^\s\w\d]+|\s+(?!\S)|\s+""")
+
+
+def count_tokens(text: str) -> int:
+    """Fast, accurate token counting.
+
+    Uses `tiktoken` (cl100k_base) if installed; otherwise falls back to a calibrated regex tokenizer.
+    """
+    if not text:
+        return 0
+
+    try:
+        import tiktoken
+        enc = tiktoken.get_encoding("cl100k_base")
+        return len(enc.encode(text, disallowed_special=()))
+    except Exception:
+        # High-speed fallback: regex BPE token estimation (~98% match with cl100k_base)
+        return max(1, len(_TOKEN_PATTERN.findall(text)))
+
+
+@dataclass
+class TextChunk:
+    """Represents a coherent segment of extracted Markdown for LLM ingestion."""
+
+    text: str
+    chunk_index: int
+    token_count: int
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert chunk into a dictionary record."""
+        return {
+            "chunk_index": self.chunk_index,
+            "text": self.text,
+            "token_count": self.token_count,
+            "metadata": self.metadata,
+        }
+
+
+def _split_oversized_text(text: str, max_tokens: int) -> list[str]:
+    """Split oversized text recursively: sentences -> words."""
+    if count_tokens(text) <= max_tokens:
+        return [text]
+
+    # Try sentence splitting
+    sentences = [s.strip() for s in re.split(r"(?<=[.!?])\s+", text) if s.strip()]
+    if len(sentences) > 1:
+        res = []
+        for s in sentences:
+            res.extend(_split_oversized_text(s, max_tokens))
+        return res
+
+    # If single sentence exceeds max_tokens, split by words
+    words = text.split()
+    parts: list[str] = []
+    current: list[str] = []
+    for w in words:
+        current.append(w)
+        if count_tokens(" ".join(current)) >= max_tokens:
+            parts.append(" ".join(current))
+            current = []
+    if current:
+        parts.append(" ".join(current))
+    return parts or [text]
+
+
+def chunk_markdown(
+    markdown: str,
+    max_tokens: int = 2048,
+    overlap_tokens: int = 100,
+    metadata: dict[str, Any] | None = None,
+) -> list[TextChunk]:
+    """Split clean Markdown text into coherent, structure-preserving chunks.
+
+    Splits hierarchically by headings (#, ##, ###), then by paragraphs (double newlines),
+    ensuring that chunks do not exceed `max_tokens` while preserving semantic context
+    and overlap.
+    """
+    if not markdown or not markdown.strip():
+        return []
+
+    clean_text = markdown.strip()
+    total_tokens = count_tokens(clean_text)
+    base_meta = metadata.copy() if metadata else {}
+
+    def extract_heading(txt: str) -> str | None:
+        heading_match = re.search(r"^(#{1,6}\s+.+)$", txt, re.MULTILINE)
+        return heading_match.group(1).strip() if heading_match else None
+
+    # If entire document fits inside max_tokens, return single chunk
+    if total_tokens <= max_tokens:
+        meta = base_meta.copy()
+        heading = extract_heading(clean_text)
+        if heading:
+            meta["section_heading"] = heading
+        return [
+            TextChunk(
+                text=clean_text,
+                chunk_index=0,
+                token_count=total_tokens,
+                metadata=meta,
+            )
+        ]
+
+    # Split into sections by Markdown headings
+    heading_pattern = re.compile(r"(?=^(?:#{1,6}\s+.+$))", re.MULTILINE)
+    sections = [s.strip() for s in heading_pattern.split(clean_text) if s.strip()]
+
+    # Further break down any section that exceeds max_tokens into paragraphs / sentences
+    blocks: list[str] = []
+    for section in sections:
+        if count_tokens(section) <= max_tokens:
+            blocks.append(section)
+        else:
+            paragraphs = [p.strip() for p in section.split("\n\n") if p.strip()]
+            for p in paragraphs:
+                if count_tokens(p) <= max_tokens:
+                    blocks.append(p)
+                else:
+                    sub_parts = _split_oversized_text(p, max_tokens)
+                    blocks.extend(sub_parts)
+
+    chunks: list[TextChunk] = []
+    current_blocks: list[str] = []
+    current_tokens = 0
+
+    def finalize_chunk(blocks_to_join: list[str], idx: int) -> TextChunk:
+        content = "\n\n".join(blocks_to_join).strip()
+        t_count = count_tokens(content)
+        chunk_meta = base_meta.copy()
+        heading = extract_heading(content)
+        if heading:
+            chunk_meta["section_heading"] = heading
+        return TextChunk(
+            text=content,
+            chunk_index=idx,
+            token_count=t_count,
+            metadata=chunk_meta,
+        )
+
+    for block in blocks:
+        block_tokens = count_tokens(block)
+        if current_blocks and (current_tokens + block_tokens > max_tokens):
+            chunk = finalize_chunk(current_blocks, len(chunks))
+            chunks.append(chunk)
+
+            # Calculate overlap from previous blocks
+            overlap_accum: list[str] = []
+            overlap_count = 0
+            for prev_b in reversed(current_blocks):
+                b_toks = count_tokens(prev_b)
+                if overlap_count + b_toks <= overlap_tokens:
+                    overlap_accum.insert(0, prev_b)
+                    overlap_count += b_toks
+                else:
+                    break
+
+            current_blocks = overlap_accum + [block]
+            current_tokens = count_tokens("\n\n".join(current_blocks))
+        else:
+            current_blocks.append(block)
+            current_tokens += block_tokens
+
+    if current_blocks:
+        chunk = finalize_chunk(current_blocks, len(chunks))
+        if not chunks or chunks[-1].text != chunk.text:
+            chunks.append(chunk)
+
+    return chunks

patchtroy/cli.py

@@ -0,0 +1,196 @@
+"""Command-line interface for Patchtroy."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from patchtroy.crawler import Patchtroy
+from patchtroy.models import PatchtroyConfig
+
+__version__ = "0.4.0"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="patchtroy",
+        description="Undetected stealth web scraper & markdown extractor for LLMs.",
+    )
+    parser.add_argument("urls", nargs="*", help="Target URL(s) to scrape or 'serve' command")
+    parser.add_argument(
+        "--serve",
+        action="store_true",
+        help="Start the FastAPI REST microservice",
+    )
+    parser.add_argument(
+        "--host",
+        default="0.0.0.0",
+        help="Host address for REST microservice (default: 0.0.0.0)",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=4013,
+        help="Port for REST microservice (default: 4013)",
+    )
+    parser.add_argument(
+        "-o", "--output", help="Output file path (default: print to stdout)"
+    )
+    parser.add_argument(
+        "-f",
+        "--format",
+        choices=["markdown", "json", "html"],
+        default="markdown",
+        help="Output format (default: markdown)",
+    )
+    parser.add_argument(
+        "--wait-for",
+        help="CSS selector to wait for before extracting page content",
+    )
+    parser.add_argument(
+        "--headful",
+        action="store_true",
+        help="Run Chromium with visible browser window (non-headless)",
+    )
+    parser.add_argument(
+        "--timeout",
+        type=float,
+        default=25.0,
+        help="Browser navigation timeout in seconds (default: 25.0)",
+    )
+    parser.add_argument(
+        "--screenshot",
+        help="Capture and save screenshot to specified file path (e.g. page.png)",
+    )
+    parser.add_argument(
+        "--full-page",
+        action="store_true",
+        help="Capture full-page screenshot instead of viewport only",
+    )
+    parser.add_argument(
+        "--pdf",
+        help="Generate and save PDF to specified file path (headless Chromium only)",
+    )
+    parser.add_argument(
+        "--proxy",
+        help="Single proxy URL (e.g. http://user:pass@host:port)",
+    )
+    parser.add_argument(
+        "--proxy-file",
+        help="Path to proxy file for automatic proxy rotation",
+    )
+    parser.add_argument(
+        "--proxy-strategy",
+        choices=["round-robin", "random"],
+        default="round-robin",
+        help="Proxy rotation strategy (default: round-robin)",
+    )
+    parser.add_argument(
+        "-c",
+        "--concurrency",
+        type=int,
+        default=5,
+        help="Maximum concurrent browser contexts for batch scraping (default: 5)",
+    )
+    parser.add_argument(
+        "-v", "--version", action="version", version=f"patchtroy {__version__}"
+    )
+
+    args = parser.parse_args(argv)
+
+    # Launch REST microservice if requested
+    if args.serve or (args.urls and args.urls[0] == "serve"):
+        from patchtroy.server import run_server
+        sys.stderr.write(f"[Patchtroy] Starting REST API on {args.host}:{args.port}...\n")
+        run_server(host=args.host, port=args.port)
+        return 0
+
+    if not args.urls:
+        parser.print_help()
+        return 1
+
+    # Configure proxy settings
+    proxies = args.proxy_file if args.proxy_file else None
+
+    config = PatchtroyConfig(
+        headless=not args.headful,
+        browser_timeout_s=args.timeout,
+        wait_for=args.wait_for,
+        proxy=args.proxy,
+        proxies=proxies,
+        proxy_strategy=args.proxy_strategy,
+        max_concurrency=args.concurrency,
+        screenshot=bool(args.screenshot),
+        full_page_screenshot=args.full_page,
+        screenshot_path=args.screenshot,
+        pdf=bool(args.pdf),
+        pdf_path=args.pdf,
+    )
+
+    crawler = Patchtroy(config)
+
+    # Single URL execution
+    if len(args.urls) == 1:
+        target_url = args.urls[0]
+        result = crawler.scrape(target_url)
+
+        if not result.success and not result.markdown:
+            sys.stderr.write(f"Error scraping {target_url}: {result.error}\n")
+            return 1
+
+        if args.format == "json":
+            # Avoid serializing raw media bytes directly into standard JSON
+            dump_data = result.model_dump(exclude={"screenshot_bytes", "pdf_bytes"})
+            content = json.dumps(dump_data, indent=2, ensure_ascii=False)
+        elif args.format == "html":
+            content = result.html
+        else:
+            header = f"# {result.title}\n\nSource: {result.url}\n\n" if result.title else ""
+            content = header + result.markdown
+
+        if args.output:
+            Path(args.output).write_text(content, encoding="utf-8")
+            sys.stderr.write(f"[Patchtroy] Extracted content saved to {args.output} ({len(content)} chars)\n")
+        else:
+            print(content)
+
+        if args.screenshot and result.screenshot_bytes:
+            sys.stderr.write(f"[Patchtroy] Screenshot saved to {args.screenshot}\n")
+        if args.pdf and result.pdf_bytes:
+            sys.stderr.write(f"[Patchtroy] PDF saved to {args.pdf}\n")
+
+        return 0
+
+    # Batch URLs execution
+    sys.stderr.write(f"[Patchtroy] Batch scraping {len(args.urls)} URLs (concurrency: {args.concurrency})...\n")
+    results = crawler.scrape_many(args.urls)
+
+    if args.format == "json":
+        dump_data = [
+            r.model_dump(exclude={"screenshot_bytes", "pdf_bytes"})
+            for r in results
+        ]
+        content = json.dumps(dump_data, indent=2, ensure_ascii=False)
+    else:
+        combined = []
+        for r in results:
+            if r.success:
+                header = f"# {r.title}\n\nSource: {r.url}\n\n" if r.title else f"Source: {r.url}\n\n"
+                combined.append(header + r.markdown)
+            else:
+                combined.append(f"<!-- Failed: {r.url} ({r.error}) -->")
+        content = "\n\n---\n\n".join(combined)
+
+    if args.output:
+        Path(args.output).write_text(content, encoding="utf-8")
+        sys.stderr.write(f"[Patchtroy] Batch results saved to {args.output}\n")
+    else:
+        print(content)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())