ghostscraper 0.0.1__tar.gz

ghostscraper-0.0.1/PKG-INFO

@@ -0,0 +1,223 @@
+Metadata-Version: 2.4
+Name: ghostscraper
+Version: 0.0.1
+Summary: An asynchronous web scraper using Playwright with HTML to Markdown conversion
+Project-URL: Homepage, https://github.com/Redundando/ghostscraper
+Project-URL: Issues, https://github.com/Redundando/ghostscraper/issues
+Author-email: Arved Klöhn <arved.kloehn@gmail.com>
+License: MIT
+Keywords: async,converter,html,markdown,playwright,web scraping
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Requires-Python: >=3.8
+Requires-Dist: beautifulsoup4>=4.10.0
+Requires-Dist: cacherator
+Requires-Dist: logorator
+Requires-Dist: playwright>=1.30.0
+Requires-Dist: python-slugify>=8.0.0
+Description-Content-Type: text/markdown
+
+# GhostScraper
+
+GhostScraper is an asynchronous web scraping library built on top of Playwright that makes it easy to fetch and convert web content to Markdown format. It handles browser management, retries, and provides a clean interface for working with web content.
+
+## Features
+
+- Asynchronous web scraping with Playwright
+- HTML to Markdown conversion
+- Built-in retry mechanism with exponential backoff
+- Result caching using JSONCache
+- Smart content extraction
+- Support for multiple browser types (Chromium, Firefox, WebKit)
+
+## Installation
+
+```bash
+pip install ghostscraper
+```
+
+GhostScraper will automatically install and manage required browsers during the first run.
+
+## Basic Usage
+
+```python
+import asyncio
+from ghostscraper import GhostScraper
+
+async def main():
+    # Create a scraper instance
+    scraper = GhostScraper(url="https://example.com")
+    
+    # Get the HTML content
+    html = await scraper.html()
+    
+    # Get the Markdown converted content
+    markdown = await scraper.markdown()
+    
+    # Get the response code
+    status_code = await scraper.response_code()
+    
+    print(f"Status code: {status_code}")
+    print(f"Markdown content:\n{markdown}")
+
+# Run the async function
+asyncio.run(main())
+```
+
+## API Reference
+
+### GhostScraper
+
+The main class for scraping and converting web content.
+
+#### Constructor
+
+```python
+GhostScraper(
+    url: str = "",
+    clear_cache: bool = False,
+    markdown_options: Optional[Dict[str, Any]] = None,
+    **kwargs
+)
+```
+
+- `url`: The URL to scrape
+- `clear_cache`: Whether to clear the cache before scraping
+- `markdown_options`: Options for the Markdown converter
+- `**kwargs`: Additional arguments passed to the PlaywrightScraper
+
+#### Methods
+
+- `async html() -> str`: Get the HTML content of the URL
+- `async response_code() -> int`: Get the HTTP response code
+- `async markdown() -> str`: Get the content converted to Markdown
+- `async soup() -> BeautifulSoup`: Get a BeautifulSoup object for the HTML content
+
+### **kwargs Keywords
+
+The GhostScraper constructor accepts any keyword arguments and passes them directly to the underlying PlaywrightScraper. This allows you to customize the browser behavior without directly interacting with the PlaywrightScraper class.
+
+```python
+# GhostScraper accepts all these keyword arguments which are passed to PlaywrightScraper
+scraper = GhostScraper(
+    url="https://example.com",
+    browser_type="chromium",     # Browser to use: "chromium", "firefox", or "webkit"
+    headless=True,               # Run browser in headless mode
+    browser_args={},             # Arguments for browser launcher
+    context_args={},             # Arguments for browser context
+    max_retries=3,               # Maximum retry attempts
+    backoff_factor=2.0,          # Exponential backoff factor
+    network_idle_timeout=10000,  # Network idle timeout (ms)
+    load_timeout=30000,          # Page load timeout (ms)
+    wait_for_selectors=[]        # CSS selectors to wait for
+)
+```
+
+These keyword arguments configure how the page is loaded, browser behavior, and retry mechanisms.
+
+## Advanced Usage
+
+### Custom Markdown Options
+
+```python
+from ghostscraper import GhostScraper
+
+# Configure the Markdown converter
+markdown_options = {
+    "strip_tags": ["script", "style", "nav", "footer", "header", "aside"],
+    "keep_tags": ["article", "main", "div", "section", "p"],
+    "content_selectors": ["article", "main", ".content", "#content"],
+    "preserve_images": True,
+    "preserve_links": True,
+    "preserve_tables": True,
+    "include_title": True,
+    "compact_output": False
+}
+
+# Create a scraper with custom Markdown options
+scraper = GhostScraper(
+    url="https://example.com",
+    markdown_options=markdown_options
+)
+```
+
+### Custom Browser Configuration
+
+```python
+from ghostscraper import GhostScraper
+
+# Create a scraper with custom browser settings
+scraper = GhostScraper(
+    url="https://example.com",
+    # Browser configuration options (passed to PlaywrightScraper)
+    browser_type="firefox",                         # Use Firefox instead of Chromium
+    headless=False,                                 # Show the browser window
+    max_retries=5,                                  # Increase retry attempts
+    load_timeout=60000,                             # Increase load timeout to 60 seconds
+    wait_for_selectors=[".content", ".main-article"] # Wait for these selectors
+)
+
+# You can also pass browser-specific arguments
+scraper = GhostScraper(
+    url="https://example.com",
+    browser_args={
+        "proxy": {                                  # Set up a proxy
+            "server": "http://myproxy.com:8080",
+            "username": "user",
+            "password": "pass"
+        },
+        "slowMo": 50,                               # Slow down browser operations by 50ms
+    },
+    context_args={
+        "userAgent": "Custom User Agent",           # Set custom user agent
+        "viewport": {"width": 1920, "height": 1080} # Set viewport size
+    }
+)
+```
+
+### Progressive Loading Strategy
+
+GhostScraper uses a progressive loading strategy that tries different methods to load the page:
+
+1. First tries with `networkidle` - waits until network is idle
+2. If that fails, tries with `load` - waits for the load event
+3. If that fails, tries with `domcontentloaded` - waits for DOM content loaded
+
+This ensures maximum compatibility with different websites.
+
+### Browser Installation
+
+GhostScraper automatically checks if the required browser is installed and installs it if needed:
+
+```python
+# Install browsers manually if needed
+from ghostscraper import install_browser
+
+# Install a specific browser type
+install_browser("chromium")
+install_browser("firefox")
+install_browser("webkit")
+```
+
+### Using Caching
+
+By default, GhostScraper caches results in the `data/ghostscraper` directory. To clear the cache:
+
+```python
+# Clear cache for a specific URL
+scraper = GhostScraper(url="https://example.com", clear_cache=True)
+```
+
+## License
+
+MIT

ghostscraper-0.0.1/README.md

@@ -0,0 +1,194 @@
+# GhostScraper
+
+GhostScraper is an asynchronous web scraping library built on top of Playwright that makes it easy to fetch and convert web content to Markdown format. It handles browser management, retries, and provides a clean interface for working with web content.
+
+## Features
+
+- Asynchronous web scraping with Playwright
+- HTML to Markdown conversion
+- Built-in retry mechanism with exponential backoff
+- Result caching using JSONCache
+- Smart content extraction
+- Support for multiple browser types (Chromium, Firefox, WebKit)
+
+## Installation
+
+```bash
+pip install ghostscraper
+```
+
+GhostScraper will automatically install and manage required browsers during the first run.
+
+## Basic Usage
+
+```python
+import asyncio
+from ghostscraper import GhostScraper
+
+async def main():
+    # Create a scraper instance
+    scraper = GhostScraper(url="https://example.com")
+    
+    # Get the HTML content
+    html = await scraper.html()
+    
+    # Get the Markdown converted content
+    markdown = await scraper.markdown()
+    
+    # Get the response code
+    status_code = await scraper.response_code()
+    
+    print(f"Status code: {status_code}")
+    print(f"Markdown content:\n{markdown}")
+
+# Run the async function
+asyncio.run(main())
+```
+
+## API Reference
+
+### GhostScraper
+
+The main class for scraping and converting web content.
+
+#### Constructor
+
+```python
+GhostScraper(
+    url: str = "",
+    clear_cache: bool = False,
+    markdown_options: Optional[Dict[str, Any]] = None,
+    **kwargs
+)
+```
+
+- `url`: The URL to scrape
+- `clear_cache`: Whether to clear the cache before scraping
+- `markdown_options`: Options for the Markdown converter
+- `**kwargs`: Additional arguments passed to the PlaywrightScraper
+
+#### Methods
+
+- `async html() -> str`: Get the HTML content of the URL
+- `async response_code() -> int`: Get the HTTP response code
+- `async markdown() -> str`: Get the content converted to Markdown
+- `async soup() -> BeautifulSoup`: Get a BeautifulSoup object for the HTML content
+
+### **kwargs Keywords
+
+The GhostScraper constructor accepts any keyword arguments and passes them directly to the underlying PlaywrightScraper. This allows you to customize the browser behavior without directly interacting with the PlaywrightScraper class.
+
+```python
+# GhostScraper accepts all these keyword arguments which are passed to PlaywrightScraper
+scraper = GhostScraper(
+    url="https://example.com",
+    browser_type="chromium",     # Browser to use: "chromium", "firefox", or "webkit"
+    headless=True,               # Run browser in headless mode
+    browser_args={},             # Arguments for browser launcher
+    context_args={},             # Arguments for browser context
+    max_retries=3,               # Maximum retry attempts
+    backoff_factor=2.0,          # Exponential backoff factor
+    network_idle_timeout=10000,  # Network idle timeout (ms)
+    load_timeout=30000,          # Page load timeout (ms)
+    wait_for_selectors=[]        # CSS selectors to wait for
+)
+```
+
+These keyword arguments configure how the page is loaded, browser behavior, and retry mechanisms.
+
+## Advanced Usage
+
+### Custom Markdown Options
+
+```python
+from ghostscraper import GhostScraper
+
+# Configure the Markdown converter
+markdown_options = {
+    "strip_tags": ["script", "style", "nav", "footer", "header", "aside"],
+    "keep_tags": ["article", "main", "div", "section", "p"],
+    "content_selectors": ["article", "main", ".content", "#content"],
+    "preserve_images": True,
+    "preserve_links": True,
+    "preserve_tables": True,
+    "include_title": True,
+    "compact_output": False
+}
+
+# Create a scraper with custom Markdown options
+scraper = GhostScraper(
+    url="https://example.com",
+    markdown_options=markdown_options
+)
+```
+
+### Custom Browser Configuration
+
+```python
+from ghostscraper import GhostScraper
+
+# Create a scraper with custom browser settings
+scraper = GhostScraper(
+    url="https://example.com",
+    # Browser configuration options (passed to PlaywrightScraper)
+    browser_type="firefox",                         # Use Firefox instead of Chromium
+    headless=False,                                 # Show the browser window
+    max_retries=5,                                  # Increase retry attempts
+    load_timeout=60000,                             # Increase load timeout to 60 seconds
+    wait_for_selectors=[".content", ".main-article"] # Wait for these selectors
+)
+
+# You can also pass browser-specific arguments
+scraper = GhostScraper(
+    url="https://example.com",
+    browser_args={
+        "proxy": {                                  # Set up a proxy
+            "server": "http://myproxy.com:8080",
+            "username": "user",
+            "password": "pass"
+        },
+        "slowMo": 50,                               # Slow down browser operations by 50ms
+    },
+    context_args={
+        "userAgent": "Custom User Agent",           # Set custom user agent
+        "viewport": {"width": 1920, "height": 1080} # Set viewport size
+    }
+)
+```
+
+### Progressive Loading Strategy
+
+GhostScraper uses a progressive loading strategy that tries different methods to load the page:
+
+1. First tries with `networkidle` - waits until network is idle
+2. If that fails, tries with `load` - waits for the load event
+3. If that fails, tries with `domcontentloaded` - waits for DOM content loaded
+
+This ensures maximum compatibility with different websites.
+
+### Browser Installation
+
+GhostScraper automatically checks if the required browser is installed and installs it if needed:
+
+```python
+# Install browsers manually if needed
+from ghostscraper import install_browser
+
+# Install a specific browser type
+install_browser("chromium")
+install_browser("firefox")
+install_browser("webkit")
+```
+
+### Using Caching
+
+By default, GhostScraper caches results in the `data/ghostscraper` directory. To clear the cache:
+
+```python
+# Clear cache for a specific URL
+scraper = GhostScraper(url="https://example.com", clear_cache=True)
+```
+
+## License
+
+MIT

ghostscraper-0.0.1/data/ghostscraper/https-www-example-com.json

@@ -0,0 +1,12 @@
+{
+    "_json_cache_func_cache": {},
+    "_json_cache_variable_cache": {
+        "_html": "<!DOCTYPE html><html><head>\n    <title>Example Domain</title>\n\n    <meta charset=\"utf-8\">\n    <meta http-equiv=\"Content-type\" content=\"text/html; charset=utf-8\">\n    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n    <style type=\"text/css\">\n    body {\n        background-color: #f0f0f2;\n        margin: 0;\n        padding: 0;\n        font-family: -apple-system, system-ui, BlinkMacSystemFont, \"Segoe UI\", \"Open Sans\", \"Helvetica Neue\", Helvetica, Arial, sans-serif;\n        \n    }\n    div {\n        width: 600px;\n        margin: 5em auto;\n        padding: 2em;\n        background-color: #fdfdff;\n        border-radius: 0.5em;\n        box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);\n    }\n    a:link, a:visited {\n        color: #38488f;\n        text-decoration: none;\n    }\n    @media (max-width: 700px) {\n        div {\n            margin: 0 auto;\n            width: auto;\n        }\n    }\n    </style>    \n</head>\n\n<body>\n<div>\n    <h1>Example Domain</h1>\n    <p>This domain is for use in illustrative examples in documents. You may use this\n    domain in literature without prior coordination or asking for permission.</p>\n    <p><a href=\"https://www.iana.org/domains/example\">More information...</a></p>\n</div>\n\n\n</body></html>",
+        "_markdown": null,
+        "_markdown_options": {},
+        "_response_code": 200,
+        "_soup": null,
+        "kwargs": {},
+        "url": "https://www.example.com"
+    }
+}

ghostscraper-0.0.1/ghostscraper/__init__.py

@@ -0,0 +1,4 @@
+from .playwright_scraper import PlaywrightScraper
+from .markdown_converter import MarkdownConverter
+from .ghost_scraper import GhostScraper
+from .playwright_installer import check_browser_installed, install_browser

ghostscraper-0.0.1/ghostscraper/ghost_scraper.py

@@ -0,0 +1,59 @@
+from logorator import Logger
+from typing import Any, Dict, Optional
+
+from bs4 import BeautifulSoup
+from cacherator import Cached, JSONCache
+from slugify import slugify
+
+from .markdown_converter import MarkdownConverter
+from .playwright_scraper import PlaywrightScraper
+
+
+class GhostScraper(JSONCache):
+    def __init__(self, url="", clear_cache=False, ttl=999,markdown_options: Optional[Dict[str, Any]] = None, **kwargs):
+        self.url = url
+        self._html: str | None = None
+        self._soup: BeautifulSoup | None = None
+        self._markdown: str | None = None
+        self._response_code: int | None = None
+        self.kwargs = kwargs
+        self._markdown_options = markdown_options or {}
+
+        JSONCache.__init__(self, data_id=f"{slugify(self.url)}", directory="data/ghostscraper", clear_cache=clear_cache, ttl=ttl)
+
+    def __str__(self):
+        return f"{self.url}"
+
+    def __repr__(self):
+        return self.__str__()
+
+    @property
+    @Cached()
+    def _playwright_scraper(self):
+        return PlaywrightScraper(url=self.url, **self.kwargs)
+
+    @Logger(override_function_name="Fetching URL via Playwright")
+    async def _fetch_response(self):
+        return await self._playwright_scraper.fetch_and_close()
+
+    async def get_response(self):
+        if self._response_code is None or self._html is None:
+            (self._html, self._response_code) = await self._fetch_response()
+        return {"html": self._html, "response_code": self._response_code}
+
+    async def html(self):
+        return (await self.get_response())["html"]
+
+    async def response_code(self):
+        return (await self.get_response())["response_code"]
+
+    async def markdown(self) -> str:
+        if self._markdown is None:
+            converter = MarkdownConverter(**self._markdown_options)
+            self._markdown = converter.convert(await self.html())
+        return self._markdown
+
+    async def soup(self) -> BeautifulSoup:
+        if self._soup is None:
+            self._soup = BeautifulSoup(await self.html(), "html.parser")
+        return self._soup

ghostscraper-0.0.1/ghostscraper/markdown_converter.py

@@ -0,0 +1,277 @@
+from typing import Optional, Dict, Any, List, Union, Tuple, Set
+from bs4 import BeautifulSoup, Tag, NavigableString
+import re
+
+
+class MarkdownConverter:
+    def __init__(
+            self,
+            strip_tags: Optional[List[str]] = None,
+            keep_tags: Optional[List[str]] = None,
+            content_selectors: Optional[List[str]] = None,
+            preserve_images: bool = True,
+            preserve_links: bool = True,
+            preserve_tables: bool = True,
+            include_title: bool = True,
+            compact_output: bool = False
+    ):
+        self.strip_tags = strip_tags or ["script", "style", "nav", "footer", "header", "aside", "iframe", "noscript"]
+        self.keep_tags = keep_tags or ["article", "main", "div", "section", "p", "h1", "h2", "h3", "h4", "h5", "h6"]
+        self.content_selectors = content_selectors or [
+                "article", "main", ".content", "#content", ".post-content",
+                ".article-content", ".entry-content", "[role='main']"
+        ]
+        self.preserve_images = preserve_images
+        self.preserve_links = preserve_links
+        self.preserve_tables = preserve_tables
+        self.include_title = include_title
+        self.compact_output = compact_output
+
+    def _extract_title(self, soup: BeautifulSoup) -> str:
+        title_tag = soup.title
+        if title_tag:
+            return title_tag.string.strip()
+        h1_tag = soup.find("h1")
+        if h1_tag:
+            return h1_tag.get_text().strip()
+        return ""
+
+    def _clean_text(self, text: str) -> str:
+        text = re.sub(r'\s+', ' ', text).strip()
+        text = re.sub(r'\n\s*\n', '\n\n', text)
+        return text
+
+    def _handle_heading(self, tag: Tag, level: int) -> str:
+        text = tag.get_text().strip()
+        return f"{'#' * level} {text}\n\n"
+
+    def _handle_paragraph(self, tag: Tag) -> str:
+        text = tag.get_text().strip()
+        if not text:
+            return ""
+        return f"{text}\n\n"
+
+    def _handle_list(self, tag: Tag, ordered: bool = False) -> str:
+        result = []
+        for i, item in enumerate(tag.find_all("li", recursive=False)):
+            prefix = f"{i + 1}. " if ordered else "* "
+            text = item.get_text().strip()
+            result.append(f"{prefix}{text}")
+        return "\n".join(result) + "\n\n"
+
+    def _handle_link(self, tag: Tag) -> str:
+        if not self.preserve_links:
+            return tag.get_text().strip()
+
+        text = tag.get_text().strip()
+        href = tag.get("href", "")
+        title = tag.get("title", "")
+
+        if not href or not text:
+            return text
+
+        if title:
+            return f"[{text}]({href} \"{title}\")"
+        return f"[{text}]({href})"
+
+    def _handle_image(self, tag: Tag) -> str:
+        if not self.preserve_images:
+            return ""
+
+        alt = tag.get("alt", "")
+        src = tag.get("src", "")
+        title = tag.get("title", "")
+
+        if not src:
+            return ""
+
+        if src.startswith("/"):
+            parent_link = tag.find_parent("a")
+            if parent_link and parent_link.get("href"):
+                href = parent_link.get("href", "")
+                if href.startswith("http"):
+                    base = href.split("//")[0] + "//" + href.split("//")[1].split("/")[0]
+                    src = base + src
+
+        if title:
+            return f"![{alt}]({src} \"{title}\")"
+        return f"![{alt}]({src})"
+
+    def _handle_table(self, tag: Tag) -> str:
+        if not self.preserve_tables:
+            return tag.get_text().strip() + "\n\n"
+
+        result = []
+
+        headers = []
+        header_row = tag.find("thead")
+        if header_row:
+            for th in header_row.find_all("th"):
+                headers.append(th.get_text().strip())
+
+        if not headers and tag.find("tr"):
+            first_row = tag.find("tr")
+            for cell in first_row.find_all(["th", "td"]):
+                headers.append(cell.get_text().strip())
+
+        if not headers:
+            first_row = tag.find("tr")
+            if first_row:
+                cell_count = len(first_row.find_all(["td", "th"]))
+                headers = [f"Column {i + 1}" for i in range(cell_count)]
+            else:
+                return tag.get_text().strip() + "\n\n"
+
+        result.append("| " + " | ".join(headers) + " |")
+        result.append("| " + " | ".join(["---"] * len(headers)) + " |")
+
+        body = tag.find("tbody") or tag
+        for row in body.find_all("tr"):
+            if not header_row and row == tag.find("tr"):
+                continue
+
+            cells = []
+            row_cells = row.find_all(["td", "th"])
+
+            for cell in row_cells:
+                content = cell.get_text().strip()
+                colspan = int(cell.get("colspan", 1))
+                if colspan > 1:
+                    cells.extend([content] + [""] * (colspan - 1))
+                else:
+                    cells.append(content)
+
+            while len(cells) < len(headers):
+                cells.append("")
+
+            cells = cells[:len(headers)]
+
+            if cells:
+                result.append("| " + " | ".join(cells) + " |")
+
+        return "\n".join(result) + "\n\n"
+
+    def _handle_blockquote(self, tag: Tag) -> str:
+        lines = tag.get_text().strip().split("\n")
+        result = []
+        for line in lines:
+            result.append(f"> {line}")
+        return "\n".join(result) + "\n\n"
+
+    def _handle_code(self, tag: Tag) -> str:
+        language = tag.get("class", [""])[0].replace("language-", "") if tag.get("class") else ""
+        code = tag.get_text()
+        if language:
+            return f"```{language}\n{code}\n```\n\n"
+        return f"```\n{code}\n```\n\n"
+
+    def _handle_inline_code(self, tag: Tag) -> str:
+        return f"`{tag.get_text()}`"
+
+    def _handle_strong(self, tag: Tag) -> str:
+        return f"**{tag.get_text()}**"
+
+    def _handle_em(self, tag: Tag) -> str:
+        return f"*{tag.get_text()}*"
+
+    def _handle_hr(self, tag: Tag) -> str:
+        return "---\n\n"
+
+    def _process_node(self, node: Union[Tag, NavigableString]) -> str:
+        if isinstance(node, NavigableString):
+            return str(node)
+
+        tag_name = node.name
+
+        if tag_name in self.strip_tags:
+            return ""
+
+        handlers = {
+                "h1"        : lambda t: self._handle_heading(t, 1),
+                "h2"        : lambda t: self._handle_heading(t, 2),
+                "h3"        : lambda t: self._handle_heading(t, 3),
+                "h4"        : lambda t: self._handle_heading(t, 4),
+                "h5"        : lambda t: self._handle_heading(t, 5),
+                "h6"        : lambda t: self._handle_heading(t, 6),
+                "p"         : self._handle_paragraph,
+                "ul"        : lambda t: self._handle_list(t, ordered=False),
+                "ol"        : lambda t: self._handle_list(t, ordered=True),
+                "a"         : self._handle_link,
+                "img"       : self._handle_image,
+                "table"     : self._handle_table,
+                "blockquote": self._handle_blockquote,
+                "pre"       : self._handle_code,
+                "code"      : self._handle_inline_code,
+                "strong"    : self._handle_strong,
+                "b"         : self._handle_strong,
+                "em"        : self._handle_em,
+                "i"         : self._handle_em,
+                "hr"        : self._handle_hr,
+        }
+
+        if tag_name in handlers:
+            return handlers[tag_name](node)
+
+        result = ""
+        for child in node.children:
+            result += self._process_node(child)
+
+        return result
+
+    def _find_content_container(self, soup: BeautifulSoup) -> Optional[Tag]:
+        for selector in self.content_selectors:
+            if selector.startswith("."):
+                containers = soup.find_all(class_=selector[1:])
+            elif selector.startswith("#"):
+                container = soup.find(id=selector[1:])
+                containers = [container] if container else []
+            elif "[" in selector and "]" in selector:
+                attr_name = selector.split("[")[1].split("=")[0]
+                attr_value = selector.split("=")[1].split("]")[0].strip("'\"")
+                containers = soup.find_all(attrs={attr_name: attr_value})
+            else:
+                containers = soup.find_all(selector)
+
+            if containers:
+                if len(containers) == 1:
+                    return containers[0]
+
+                containers_with_length = [(c, len(c.get_text())) for c in containers]
+                containers_with_length.sort(key=lambda x: x[1], reverse=True)
+                return containers_with_length[0][0]
+
+        for tag_name in self.keep_tags:
+            tags = soup.find_all(tag_name)
+
+            if tags:
+                tags_with_length = [(tag, len(tag.get_text())) for tag in tags]
+                tags_with_length.sort(key=lambda x: x[1], reverse=True)
+                return tags_with_length[0][0]
+
+        return soup.body
+
+    def convert(self, html: str) -> str:
+        if not html:
+            return ""
+
+        soup = BeautifulSoup(html, "html.parser")
+
+        for tag_name in self.strip_tags:
+            for tag in soup.find_all(tag_name):
+                tag.decompose()
+
+        content = self._find_content_container(soup)
+        if not content:
+            content = soup
+
+        title = self._extract_title(soup) if self.include_title else ""
+        result = f"# {title}\n\n" if title else ""
+
+        markdown = result + self._process_node(content)
+
+        markdown = re.sub(r'\n{3,}', '\n\n', markdown)
+
+        if self.compact_output:
+            markdown = re.sub(r'\n\n+', '\n\n', markdown)
+
+        return markdown.strip()

ghostscraper-0.0.1/ghostscraper/playwright_installer.py

@@ -0,0 +1,51 @@
+import subprocess
+import sys
+import os
+from playwright.async_api import async_playwright, Browser, BrowserContext
+from logorator import Logger
+
+async def check_browser_installed(browser_name: str) -> bool:
+    async with async_playwright() as p:
+        browsers = {"chromium": p.chromium, "firefox": p.firefox, "webkit": p.webkit, }
+        if browser_name not in browsers:
+            Logger.note(f"❌ Invalid browser name: {browser_name}")
+            return False
+
+        try:
+            browser = await browsers[browser_name].launch()
+            await browser.close()
+            Logger.note(f"✅ {browser_name} is installed and working!")
+            return True
+        except Exception as e:
+            Logger.note(f"❌ {browser_name} is NOT installed or failed to launch: {e}")
+            return False
+
+@Logger()
+def install_browser(browser_type: str) -> bool:
+    try:
+        Logger.note(f"\n[Ghostscraper] Installing {browser_type} browser (first-time setup)")
+        Logger.note("[Ghostscraper] This may take a few minutes...")
+
+        subprocess.check_call([
+                sys.executable, "-m", "playwright", "install", browser_type
+        ])
+
+        Logger.note(f"[Ghostscraper] Successfully installed {browser_type} browser.")
+        return True
+
+    except subprocess.CalledProcessError as e:
+        Logger.note(f"\n[Ghostscraper] Failed to install {browser_type} browser. Error code: {e.returncode}")
+
+        if os.name == 'posix' and os.geteuid() != 0:
+            Logger.note("[Ghostscraper] You may need to run with sudo privileges.")
+            Logger.note(f"[Ghostscraper] Try: sudo playwright install {browser_type}")
+        else:
+            Logger.note("[Ghostscraper] You may need administrator privileges.")
+            Logger.note(f"[Ghostscraper] Try running: playwright install {browser_type}")
+
+        return False
+
+    except Exception as e:
+        Logger.note(f"\n[Ghostscraper] An unexpected error occurred: {str(e)}")
+        Logger.note(f"[Ghostscraper] Please run 'playwright install {browser_type}' manually.")
+        return False

ghostscraper-0.0.1/ghostscraper/playwright_scraper.py

@@ -0,0 +1,207 @@
+import asyncio
+from typing import Any, Dict, List, Literal, Optional, Tuple
+
+from logorator import Logger
+from playwright.async_api import async_playwright, Browser, BrowserContext, Page, Playwright, TimeoutError as PlaywrightTimeoutError
+
+from .playwright_installer import check_browser_installed, install_browser
+
+
+class PlaywrightScraper:
+    BROWSERS_CHECKED = {}
+
+    def __init__(self, url: str = "", browser_type: Literal["chromium", "firefox", "webkit"] = "chromium", headless: bool = True, browser_args: Optional[Dict[str, Any]] = None,
+            context_args: Optional[Dict[str, Any]] = None, max_retries: int = 3, backoff_factor: float = 2.0, network_idle_timeout: int = 10000,  # 10 seconds
+            load_timeout: int = 30000,  # 30 seconds
+            wait_for_selectors: Optional[List[str]] = None  # CSS selectors to wait for
+    ):
+        self.url = url
+        self.browser_type: str = browser_type
+        self.headless: bool = headless
+        self.browser_args: Dict[str, Any] = browser_args or {}
+        self.context_args: Dict[str, Any] = context_args or {}
+        self.max_retries: int = max_retries
+        self.backoff_factor: float = backoff_factor
+        self.network_idle_timeout: int = network_idle_timeout
+        self.load_timeout: int = load_timeout
+        self.wait_for_selectors: List[str] = wait_for_selectors or []
+        self._playwright: Optional[Playwright] = None
+        self._browser: Optional[Browser] = None
+        self._context: Optional[BrowserContext] = None
+        self.last_status_code: int = 200
+
+    def __str__(self):
+        return self.url
+
+    def __repr__(self):
+        return self.__str__()
+
+    async def check_and_install_browser(self):
+        if PlaywrightScraper.BROWSERS_CHECKED.get(self.browser_type) is not None:
+            return PlaywrightScraper.BROWSERS_CHECKED.get(self.browser_type)
+        if await check_browser_installed(self.browser_type):
+            PlaywrightScraper.BROWSERS_CHECKED[self.browser_type] = True
+            return True
+        else:
+            install_browser(self.browser_type)
+            PlaywrightScraper.BROWSERS_CHECKED[self.browser_type] = asyncio.run(check_browser_installed(self.browser_type))
+            return PlaywrightScraper.BROWSERS_CHECKED[self.browser_type]
+
+    async def _ensure_browser(self) -> None:
+        await self.check_and_install_browser()
+        if self._playwright is None:
+            self._playwright = await async_playwright().start()
+
+            if self.browser_type == "chromium":
+                browser_launcher = self._playwright.chromium
+            elif self.browser_type == "firefox":
+                browser_launcher = self._playwright.firefox
+            elif self.browser_type == "webkit":
+                browser_launcher = self._playwright.webkit
+            else:
+                raise ValueError(f"Unknown browser type: {self.browser_type}")
+
+            self._browser = await browser_launcher.launch(headless=self.headless, **self.browser_args)
+
+            self._context = await self._browser.new_context(**self.context_args)
+
+    async def _try_progressive_load(self, page: Page, url: str) -> Tuple[bool, int]:
+        # Strategy 1: Try with networkidle first (strictest, but most reliable)
+        try:
+            Logger.note(f"GhostScraper: Attempting to load with 'networkidle' (timeout: {self.network_idle_timeout}ms)")
+            response = await page.goto(url, wait_until="networkidle", timeout=self.network_idle_timeout)
+            status_code = response.status if response else 200
+            return True, status_code
+        except PlaywrightTimeoutError:
+            Logger.note("GhostScraper: 'networkidle' timed out, falling back to 'load'")
+            pass
+
+        # Strategy 2: Fallback to load event (less strict)
+        try:
+            Logger.note(f"GhostScraper: Attempting to load with 'load' (timeout: {self.load_timeout}ms)")
+            response = await page.goto(url, wait_until="load", timeout=self.load_timeout)
+            status_code = response.status if response else 200
+            return True, status_code
+        except PlaywrightTimeoutError:
+            Logger.note("GhostScraper: 'load' timed out, falling back to 'domcontentloaded'")
+            pass
+
+        # Strategy 3: Fallback to domcontentloaded (least strict)
+        try:
+            Logger.note("GhostScraper: Attempting to load with 'domcontentloaded'")
+            response = await page.goto(url, wait_until="domcontentloaded", timeout=self.load_timeout)
+            status_code = response.status if response else 200
+            return True, status_code
+        except PlaywrightTimeoutError:
+            Logger.note("GhostScraper: All loading strategies failed")
+            return False, 408  # Request Timeout
+
+    async def _wait_for_selectors(self, page: Page) -> bool:
+        if not self.wait_for_selectors:
+            return True
+
+        try:
+            for selector in self.wait_for_selectors:
+                try:
+                    Logger.note(f"GhostScraper: Waiting for selector '{selector}'")
+                    await page.wait_for_selector(selector, timeout=5000)
+                    Logger.note(f"GhostScraper: Found selector '{selector}'")
+                except PlaywrightTimeoutError:
+                    Logger.note(f"GhostScraper: Selector '{selector}' not found, continuing anyway")
+            return True
+        except Exception as e:
+            Logger.note(f"GhostScraper: Error waiting for selectors: {str(e)}")
+            return False
+
+    async def fetch(self) -> Tuple[str, int]:
+        await self._ensure_browser()
+        attempts = 0
+
+        while attempts <= self.max_retries:
+            page: Page = await self._context.new_page()
+            try:
+                # Set a default navigation timeout
+                page.set_default_navigation_timeout(self.load_timeout)
+                # Try progressive loading strategies
+                load_success, status_code = await self._try_progressive_load(page, self.url)
+                self.last_status_code = status_code
+
+                if not load_success:
+                    if attempts == self.max_retries:
+                        Logger.note(f"GhostScraper: Max retries reached. All loading strategies failed.")
+                        return "", 408
+                    wait_time = self.backoff_factor ** attempts
+                    Logger.note(f"GhostScraper: All loading strategies failed. Retrying in {wait_time:.2f}s (attempt {attempts + 1}/{self.max_retries})")
+                    await asyncio.sleep(wait_time)
+                    attempts += 1
+                    continue
+
+                if status_code >= 400:
+                    if attempts == self.max_retries:
+                        Logger.note(f"GhostScraper: Max retries reached with status code {status_code}. Returning empty response.")
+                        return "", status_code
+
+                    wait_time = self.backoff_factor ** attempts
+                    Logger.note(f"GhostScraper: Status code {status_code} received. Retrying in {wait_time:.2f}s (attempt {attempts + 1}/{self.max_retries})")
+                    await asyncio.sleep(wait_time)
+                    attempts += 1
+                    continue
+
+                # Try to wait for specified selectors (if any)
+                await self._wait_for_selectors(page)
+
+                # If we reached here, we consider it a success. Grab the content and return.
+                html: str = await page.content()
+                return html, status_code
+
+            except PlaywrightTimeoutError as e:
+                if attempts == self.max_retries:
+                    Logger.note(f"GhostScraper: Max retries reached after timeout. Returning empty response with 408 status.")
+                    return "", 408
+
+                wait_time = self.backoff_factor ** attempts
+                Logger.note(f"GhostScraper: Timeout error occurred: {str(e)}. Retrying in {wait_time:.2f}s (attempt {attempts + 1}/{self.max_retries})")
+                await asyncio.sleep(wait_time)
+                attempts += 1
+
+            except Exception as e:
+                if attempts == self.max_retries:
+                    Logger.note(f"GhostScraper: Max retries reached after exception: {str(e)}. Returning empty response with 500 status.")
+                    return "", 500
+
+                wait_time = self.backoff_factor ** attempts
+                Logger.note(f"GhostScraper: Exception occurred: {str(e)}. Retrying in {wait_time:.2f}s (attempt {attempts + 1}/{self.max_retries})")
+                await asyncio.sleep(wait_time)
+                attempts += 1
+
+            finally:
+                await page.close()
+
+        # This should not be reached, but just in case
+        return "", 500
+
+    async def close(self) -> None:
+        if self._context:
+            await self._context.close()
+            self._context = None
+
+        if self._browser:
+            await self._browser.close()
+            self._browser = None
+
+        if self._playwright:
+            await self._playwright.stop()
+            self._playwright = None
+
+    async def fetch_and_close(self) -> Tuple[str, int]:
+        try:
+            return await self.fetch()
+        finally:
+            await self.close()
+
+    async def __aenter__(self):
+        await self._ensure_browser()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()

ghostscraper-0.0.1/pyproject.toml

@@ -0,0 +1,46 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "ghostscraper"
+version = "0.0.1"
+description = "An asynchronous web scraper using Playwright with HTML to Markdown conversion"
+readme = "README.md"
+authors = [
+    {name = "Arved Klöhn", email = "arved.kloehn@gmail.com"},
+]
+license = {text = "MIT"}
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Text Processing :: Markup :: HTML",
+    "Topic :: Text Processing :: Markup :: Markdown",
+]
+keywords = ["web scraping", "playwright", "markdown", "html", "converter", "async"]
+dependencies = [
+    "playwright>=1.30.0",
+    "beautifulsoup4>=4.10.0",
+    "cacherator",
+    "logorator",
+    "python-slugify>=8.0.0",
+]
+requires-python = ">=3.8"
+
+[project.urls]
+Homepage = "https://github.com/Redundando/ghostscraper"
+Issues = "https://github.com/Redundando/ghostscraper/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["ghostscraper"]
+
+[tool.hatch.build.targets.sdist]
+include = ["ghostscraper"]