pg2md 1.0.1__tar.gz

pg2md-1.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pg2md-1.0.1/PKG-INFO

@@ -0,0 +1,302 @@
+Metadata-Version: 2.4
+Name: pg2md
+Version: 1.0.1
+Summary: HTML to Markdown converter with Requests or Playwright backend
+Author-email: Your Name <your@email.com>
+License: MIT
+Project-URL: Homepage, https://github.com/yourname/pg2md
+Project-URL: Repository, https://github.com/yourname/pg2md
+Project-URL: Issues, https://github.com/yourname/pg2md/issues
+Keywords: html,markdown,converter,playwright,requests,scraper
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.28.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: html-to-markdown>=1.1.0
+Provides-Extra: playwright
+Requires-Dist: playwright>=1.40.0; extra == "playwright"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: build>=1.0.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+Dynamic: license-file
+
+# pg2md
+
+**HTML to Markdown converter** with Requests or Playwright backend.
+
+Convert any webpage to clean Markdown. Choose between fast `requests` or full browser `playwright` for JavaScript-rendered pages.
+
+## Features
+
+- **Two backends**: `Pg2MdRequests` (fast) or `Pg2MdPlaywright` (JS support)
+- **Browser reuse**: Playwright instances share a single browser
+- **Proxy support**: HTTP/HTTPS proxies with authentication
+- **Custom headers & cookies**: Full control over requests
+- **Clean output**: Optional removal of images and links
+- **Context manager**: Auto-cleanup with `with` statement
+
+## Installation
+
+```bash
+pip install pg2md
+
+# For Playwright backend:
+pip install pg2md[playwright]
+playwright install chromium
+```
+
+## Quick Start
+
+```python
+from pg2md import Pg2MdRequests, Pg2MdPlaywright
+
+# Simple usage with Requests
+pg = Pg2MdRequests()
+markdown = pg.run("https://example.com")
+print(markdown)
+
+# Playwright for JS-heavy sites
+pg = Pg2MdPlaywright()
+markdown = pg.run("https://spa-example.com")
+pg.close()
+```
+
+## Usage
+
+### Basic Conversion
+
+```python
+from pg2md import Pg2MdRequests
+
+pg = Pg2MdRequests(with_image=False, with_link=False)
+md = pg.run("https://news.ycombinator.com")
+```
+
+### With Proxy
+
+```python
+from pg2md import Pg2MdRequests, Pg2MdPlaywright
+
+# Format: http://user:password@host:port
+# Or: host:port:user:password
+proxy = "http://user:pass@proxy.example.com:8080"
+
+# Requests
+pg = Pg2MdRequests()
+md = pg.run("https://example.com", proxy=proxy)
+
+# Playwright
+pg = Pg2MdPlaywright()
+md = pg.run("https://example.com", proxy=proxy)
+pg.close()
+```
+
+### Custom Headers & User-Agent
+
+```python
+from pg2md import Pg2MdRequests
+
+pg = Pg2MdRequests()
+md = pg.run(
+    "https://api.example.com/data",
+    headers={
+        "X-API-Key": "secret123",
+        "Accept": "application/json",
+    },
+    user_agent="MyBot/1.0",
+)
+```
+
+### With Cookies
+
+```python
+from pg2md import Pg2MdRequests
+
+pg = Pg2MdRequests()
+md = pg.run(
+    "https://example.com/dashboard",
+    cookies={
+        "session": "abc123",
+        "auth_token": "xyz789",
+    },
+)
+```
+
+### Save to File
+
+```python
+from pg2md import Pg2MdRequests
+
+pg = Pg2MdRequests()
+pg.save("output.md", "https://example.com")
+
+# With options
+pg.save(
+    "article.md",
+    "https://blog.example.com/post",
+    proxy="http://user:pass@host:port",
+    user_agent="MyBot/1.0",
+)
+```
+
+### Context Manager
+
+```python
+from pg2md import Pg2MdPlaywright
+
+with Pg2MdPlaywright() as pg:
+    md1 = pg.run("https://site1.com")
+    md2 = pg.run("https://site2.com")
+    # Browser closed automatically
+```
+
+### Multiple Instances
+
+```python
+from pg2md import Pg2MdPlaywright
+
+# Both share the same browser (efficient)
+pg1 = Pg2MdPlaywright()
+pg2 = Pg2MdPlaywright()
+
+md1 = pg1.run("https://site1.com")
+md2 = pg2.run("https://site2.com")
+
+Pg2MdPlaywright.close_all()  # Close shared browser
+```
+
+## API Reference
+
+### Pg2MdRequests
+
+```python
+Pg2MdRequests(with_image=False, with_link=False)
+```
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `with_image` | bool | False | Include images in output |
+| `with_link` | bool | False | Include links in output |
+
+### Pg2MdPlaywright
+
+```python
+Pg2MdPlaywright(
+    browser=None,       # Custom Browser instance
+    headless=True,      # Headless mode
+    with_image=False,
+    with_link=False,
+)
+```
+
+### Methods
+
+#### `run(url, proxy=None, headers=None, cookies=None, user_agent=None, timeout=30)`
+
+Fetch URL and convert to Markdown.
+
+Returns: `str` (Markdown)
+
+#### `fetch(url, proxy=None, headers=None, cookies=None, user_agent=None, timeout=30)`
+
+Fetch HTML only.
+
+Returns: `str` (HTML)
+
+#### `convert(html)`
+
+Convert HTML to Markdown.
+
+Returns: `str` (Markdown)
+
+#### `save(filepath, url, **kwargs)`
+
+Fetch, convert, and save to file.
+
+#### `close()`
+
+Close browser (Playwright only).
+
+#### `close_all()` (classmethod, Playwright only)
+
+Close all shared browsers.
+
+## When to Use Which Backend?
+
+| Use Requests | Use Playwright |
+|--------------|----------------|
+| Static HTML pages | SPA / JavaScript apps |
+| Speed matters | Need rendered content |
+| Simple scraping | Bypass anti-bot (sometimes) |
+| Low memory | Modern web apps |
+
+## Examples
+
+### Scrape Multiple URLs
+
+```python
+from pg2md import Pg2MdRequests
+
+urls = [
+    "https://blog.example.com/post1",
+    "https://blog.example.com/post2",
+    "https://blog.example.com/post3",
+]
+
+pg = Pg2MdRequests(with_image=False, with_link=False)
+
+for i, url in enumerate(urls):
+    pg.save(f"post_{i+1}.md", url)
+    print(f"Saved: {url}")
+```
+
+### Batch with Proxies
+
+```python
+from pg2md import Pg2MdRequests
+
+urls = ["https://site1.com", "https://site2.com", "https://site3.com"]
+proxies = [
+    "http://user1:pass1@proxy1:8080",
+    "http://user2:pass2@proxy2:8080",
+]
+
+pg = Pg2MdRequests()
+
+for i, url in enumerate(urls):
+    proxy = proxies[i % len(proxies)]
+    md = pg.run(url, proxy=proxy)
+    print(f"[{i+1}] {len(md)} chars")
+```
+
+### Extract Article Content
+
+```python
+from pg2md import Pg2MdPlaywright
+
+with Pg2MdPlaywright() as pg:
+    md = pg.run(
+        "https://medium.com/some-article",
+        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+    )
+    
+    # Save clean text
+    with open("article.md", "w") as f:
+        f.write(md)
+```
+
+## License
+
+MIT

pg2md-1.0.1/README.md

@@ -0,0 +1,268 @@
+# pg2md
+
+**HTML to Markdown converter** with Requests or Playwright backend.
+
+Convert any webpage to clean Markdown. Choose between fast `requests` or full browser `playwright` for JavaScript-rendered pages.
+
+## Features
+
+- **Two backends**: `Pg2MdRequests` (fast) or `Pg2MdPlaywright` (JS support)
+- **Browser reuse**: Playwright instances share a single browser
+- **Proxy support**: HTTP/HTTPS proxies with authentication
+- **Custom headers & cookies**: Full control over requests
+- **Clean output**: Optional removal of images and links
+- **Context manager**: Auto-cleanup with `with` statement
+
+## Installation
+
+```bash
+pip install pg2md
+
+# For Playwright backend:
+pip install pg2md[playwright]
+playwright install chromium
+```
+
+## Quick Start
+
+```python
+from pg2md import Pg2MdRequests, Pg2MdPlaywright
+
+# Simple usage with Requests
+pg = Pg2MdRequests()
+markdown = pg.run("https://example.com")
+print(markdown)
+
+# Playwright for JS-heavy sites
+pg = Pg2MdPlaywright()
+markdown = pg.run("https://spa-example.com")
+pg.close()
+```
+
+## Usage
+
+### Basic Conversion
+
+```python
+from pg2md import Pg2MdRequests
+
+pg = Pg2MdRequests(with_image=False, with_link=False)
+md = pg.run("https://news.ycombinator.com")
+```
+
+### With Proxy
+
+```python
+from pg2md import Pg2MdRequests, Pg2MdPlaywright
+
+# Format: http://user:password@host:port
+# Or: host:port:user:password
+proxy = "http://user:pass@proxy.example.com:8080"
+
+# Requests
+pg = Pg2MdRequests()
+md = pg.run("https://example.com", proxy=proxy)
+
+# Playwright
+pg = Pg2MdPlaywright()
+md = pg.run("https://example.com", proxy=proxy)
+pg.close()
+```
+
+### Custom Headers & User-Agent
+
+```python
+from pg2md import Pg2MdRequests
+
+pg = Pg2MdRequests()
+md = pg.run(
+    "https://api.example.com/data",
+    headers={
+        "X-API-Key": "secret123",
+        "Accept": "application/json",
+    },
+    user_agent="MyBot/1.0",
+)
+```
+
+### With Cookies
+
+```python
+from pg2md import Pg2MdRequests
+
+pg = Pg2MdRequests()
+md = pg.run(
+    "https://example.com/dashboard",
+    cookies={
+        "session": "abc123",
+        "auth_token": "xyz789",
+    },
+)
+```
+
+### Save to File
+
+```python
+from pg2md import Pg2MdRequests
+
+pg = Pg2MdRequests()
+pg.save("output.md", "https://example.com")
+
+# With options
+pg.save(
+    "article.md",
+    "https://blog.example.com/post",
+    proxy="http://user:pass@host:port",
+    user_agent="MyBot/1.0",
+)
+```
+
+### Context Manager
+
+```python
+from pg2md import Pg2MdPlaywright
+
+with Pg2MdPlaywright() as pg:
+    md1 = pg.run("https://site1.com")
+    md2 = pg.run("https://site2.com")
+    # Browser closed automatically
+```
+
+### Multiple Instances
+
+```python
+from pg2md import Pg2MdPlaywright
+
+# Both share the same browser (efficient)
+pg1 = Pg2MdPlaywright()
+pg2 = Pg2MdPlaywright()
+
+md1 = pg1.run("https://site1.com")
+md2 = pg2.run("https://site2.com")
+
+Pg2MdPlaywright.close_all()  # Close shared browser
+```
+
+## API Reference
+
+### Pg2MdRequests
+
+```python
+Pg2MdRequests(with_image=False, with_link=False)
+```
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `with_image` | bool | False | Include images in output |
+| `with_link` | bool | False | Include links in output |
+
+### Pg2MdPlaywright
+
+```python
+Pg2MdPlaywright(
+    browser=None,       # Custom Browser instance
+    headless=True,      # Headless mode
+    with_image=False,
+    with_link=False,
+)
+```
+
+### Methods
+
+#### `run(url, proxy=None, headers=None, cookies=None, user_agent=None, timeout=30)`
+
+Fetch URL and convert to Markdown.
+
+Returns: `str` (Markdown)
+
+#### `fetch(url, proxy=None, headers=None, cookies=None, user_agent=None, timeout=30)`
+
+Fetch HTML only.
+
+Returns: `str` (HTML)
+
+#### `convert(html)`
+
+Convert HTML to Markdown.
+
+Returns: `str` (Markdown)
+
+#### `save(filepath, url, **kwargs)`
+
+Fetch, convert, and save to file.
+
+#### `close()`
+
+Close browser (Playwright only).
+
+#### `close_all()` (classmethod, Playwright only)
+
+Close all shared browsers.
+
+## When to Use Which Backend?
+
+| Use Requests | Use Playwright |
+|--------------|----------------|
+| Static HTML pages | SPA / JavaScript apps |
+| Speed matters | Need rendered content |
+| Simple scraping | Bypass anti-bot (sometimes) |
+| Low memory | Modern web apps |
+
+## Examples
+
+### Scrape Multiple URLs
+
+```python
+from pg2md import Pg2MdRequests
+
+urls = [
+    "https://blog.example.com/post1",
+    "https://blog.example.com/post2",
+    "https://blog.example.com/post3",
+]
+
+pg = Pg2MdRequests(with_image=False, with_link=False)
+
+for i, url in enumerate(urls):
+    pg.save(f"post_{i+1}.md", url)
+    print(f"Saved: {url}")
+```
+
+### Batch with Proxies
+
+```python
+from pg2md import Pg2MdRequests
+
+urls = ["https://site1.com", "https://site2.com", "https://site3.com"]
+proxies = [
+    "http://user1:pass1@proxy1:8080",
+    "http://user2:pass2@proxy2:8080",
+]
+
+pg = Pg2MdRequests()
+
+for i, url in enumerate(urls):
+    proxy = proxies[i % len(proxies)]
+    md = pg.run(url, proxy=proxy)
+    print(f"[{i+1}] {len(md)} chars")
+```
+
+### Extract Article Content
+
+```python
+from pg2md import Pg2MdPlaywright
+
+with Pg2MdPlaywright() as pg:
+    md = pg.run(
+        "https://medium.com/some-article",
+        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+    )
+    
+    # Save clean text
+    with open("article.md", "w") as f:
+        f.write(md)
+```
+
+## License
+
+MIT

pg2md-1.0.1/pg2md/__init__.py

@@ -0,0 +1,257 @@
+"""
+Pg2Md — HTML to Markdown converter with Requests or Playwright backend.
+
+Usage:
+    from pg2md import Pg2MdRequests, Pg2MdPlaywright
+
+    # Requests
+    pg = Pg2MdRequests(with_image=False, with_link=False)
+    md = pg.run("https://example.com", proxy="http://user:pass@host:port")
+
+    # Playwright
+    pg = Pg2MdPlaywright()
+    md = pg.run("https://example.com")
+    pg.close()
+"""
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urlparse
+
+from requests import Session
+from playwright.sync_api import sync_playwright, Browser
+
+from .html_to_md import HtmlToMarkdown
+
+
+class Pg2Md(ABC):
+    """Base class for HTML to Markdown conversion."""
+
+    def __init__(self, with_image: bool = False, with_link: bool = False):
+        self._converter = HtmlToMarkdown(
+            with_image=with_image,
+            with_link=with_link,
+        )
+
+    @abstractmethod
+    def fetch(
+        self,
+        url: str,
+        proxy: Optional[str] = None,
+        headers: Optional[dict] = None,
+        cookies: Optional[dict] = None,
+        user_agent: Optional[str] = None,
+        timeout: int = 30,
+    ) -> str:
+        """Fetch HTML from URL."""
+        ...
+
+    def convert(self, html: str) -> str:
+        """Convert HTML to Markdown."""
+        return self._converter.convert(html)
+
+    def run(
+        self,
+        url: str,
+        proxy: Optional[str] = None,
+        headers: Optional[dict] = None,
+        cookies: Optional[dict] = None,
+        user_agent: Optional[str] = None,
+        timeout: int = 30,
+    ) -> str:
+        """Fetch URL and convert to Markdown."""
+        html = self.fetch(url, proxy, headers, cookies, user_agent, timeout)
+        return self.convert(html)
+
+    def save(
+        self,
+        filepath: str,
+        url: str,
+        proxy: Optional[str] = None,
+        headers: Optional[dict] = None,
+        cookies: Optional[dict] = None,
+        user_agent: Optional[str] = None,
+        timeout: int = 30,
+    ) -> None:
+        """Fetch, convert and save to file."""
+        md = self.run(url, proxy, headers, cookies, user_agent, timeout)
+        Path(filepath).write_text(md, encoding="utf-8")
+
+    def close(self):
+        """Close resources. Override in subclasses if needed."""
+        pass
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+
+class Pg2MdRequests(Pg2Md):
+    """Requests-based implementation."""
+
+    def fetch(
+        self,
+        url: str,
+        proxy: Optional[str] = None,
+        headers: Optional[dict] = None,
+        cookies: Optional[dict] = None,
+        user_agent: Optional[str] = None,
+        timeout: int = 30,
+    ) -> str:
+        session = Session()
+
+        final_headers = dict(headers) if headers else {}
+        if user_agent:
+            final_headers["User-Agent"] = user_agent
+
+        proxies = None
+        if proxy:
+            proxy_url = self._normalize_proxy(proxy)
+            proxies = {"http": proxy_url, "https": proxy_url}
+
+        resp = session.get(
+            url,
+            proxies=proxies,
+            headers=final_headers if final_headers else None,
+            cookies=cookies,
+            timeout=timeout,
+        )
+        resp.raise_for_status()
+        return resp.text
+
+    def _normalize_proxy(self, proxy: str) -> str:
+        """Normalize proxy to http://user:pass@host:port format."""
+        if proxy.startswith("http://") or proxy.startswith("https://"):
+            return proxy
+
+        parts = proxy.split(":")
+        if len(parts) == 4:
+            host, port, user, password = parts
+            return f"http://{user}:{password}@{host}:{port}"
+
+        return f"http://{proxy}"
+
+
+class Pg2MdPlaywright(Pg2Md):
+    """Playwright-based implementation with browser reuse."""
+
+    _shared_playwright = None
+    _shared_browsers: dict = {}
+
+    def __init__(
+        self,
+        browser: Optional[Browser] = None,
+        headless: bool = True,
+        with_image: bool = False,
+        with_link: bool = False,
+    ):
+        super().__init__(with_image, with_link)
+        self._browser = browser
+        self._headless = headless
+        self._owns_browser = browser is None
+
+    @classmethod
+    def _get_playwright(cls):
+        if cls._shared_playwright is None:
+            cls._shared_playwright = sync_playwright().start()
+        return cls._shared_playwright
+
+    @property
+    def browser(self) -> Browser:
+        """Get or create browser instance."""
+        if self._browser is None:
+            key = ("chromium", self._headless)
+            if key not in self._shared_browsers:
+                pw = self._get_playwright()
+                self._shared_browsers[key] = pw.chromium.launch(headless=self._headless)
+            self._browser = self._shared_browsers[key]
+        return self._browser
+
+    def fetch(
+        self,
+        url: str,
+        proxy: Optional[str] = None,
+        headers: Optional[dict] = None,
+        cookies: Optional[dict] = None,
+        user_agent: Optional[str] = None,
+        timeout: int = 30,
+    ) -> str:
+        context_opts = {}
+
+        if proxy:
+            context_opts["proxy"] = self._parse_proxy(proxy)
+        if user_agent:
+            context_opts["user_agent"] = user_agent
+
+        context = self.browser.new_context(**context_opts)
+
+        if headers:
+            context.set_extra_http_headers(headers)
+        if cookies:
+            parsed_url = urlparse(url)
+            domain = parsed_url.hostname
+            formatted_cookies = [
+                {"name": k, "value": v, "domain": domain} for k, v in cookies.items()
+            ]
+            context.add_cookies(formatted_cookies)
+
+        page = context.new_page()
+        page.goto(url, timeout=timeout * 1000)
+        html = page.content()
+
+        page.close()
+        context.close()
+
+        return html
+
+    def _parse_proxy(self, proxy: str) -> dict:
+        """Parse proxy string to Playwright format."""
+        if proxy.startswith("http://") or proxy.startswith("https://"):
+            parsed = urlparse(proxy)
+            result = {"server": f"{parsed.scheme}://{parsed.hostname}:{parsed.port}"}
+            if parsed.username and parsed.password:
+                result["username"] = parsed.username
+                result["password"] = parsed.password
+            return result
+
+        parts = proxy.split(":")
+        if len(parts) == 4:
+            host, port, user, password = parts
+            return {
+                "server": f"http://{host}:{port}",
+                "username": user,
+                "password": password,
+            }
+        elif len(parts) == 2:
+            host, port = parts
+            return {"server": f"http://{host}:{port}"}
+
+        return {"server": f"http://{proxy}"}
+
+    def close(self):
+        """Close browser if owned by this instance."""
+        if self._owns_browser:
+            key = ("chromium", self._headless)
+            if key in self._shared_browsers:
+                self._shared_browsers[key].close()
+                del self._shared_browsers[key]
+            self._browser = None
+
+    @classmethod
+    def close_all(cls):
+        """Close all shared browsers and playwright."""
+        for browser in cls._shared_browsers.values():
+            browser.close()
+        cls._shared_browsers.clear()
+        if cls._shared_playwright:
+            cls._shared_playwright.stop()
+            cls._shared_playwright = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()

pg2md-1.0.1/pg2md/html_to_md.py

@@ -0,0 +1,168 @@
+"""
+HtmlToMarkdown — standalone HTML to Markdown converter.
+
+No browser dependencies, just HTML -> Markdown conversion.
+
+Dependencies:
+    pip install html-to-markdown beautifulsoup4
+
+Usage:
+    converter = HtmlToMarkdown(with_image=False, with_link=False)
+    markdown = converter.convert(html_string)
+"""
+
+import re
+from typing import Optional
+
+from bs4 import BeautifulSoup
+from html_to_markdown import convert, ConversionOptions, PreprocessingOptions
+
+
+class HtmlToMarkdown:
+    """
+    Converts HTML to clean Markdown.
+
+    Steps:
+        1. Clean HTML (remove scripts, styles, optional images/links)
+        2. Convert to Markdown via html-to-markdown
+        3. Clean final Markdown (remove base64, excess newlines)
+
+    Args:
+        with_image: Include images in output. Default False.
+        with_link: Include links (href). Default True.
+                   False — links are replaced with their text.
+        heading_style: "atx" (#) or "setext" (underline). Default "atx".
+        strong_em_symbol: "*" or "_". Default "*".
+        bullets: Bullet character. Default "*".
+        escape_asterisks: Escape asterisks in text. Default False.
+        preprocessing_preset: "aggressive", "moderate", or "conservative". Default "aggressive".
+        remove_navigation: Remove navigation elements. Default True.
+        remove_forms: Remove form elements. Default True.
+    """
+
+    STRIP_TAGS = [
+        "script",
+        "style",
+        "noscript",
+        "svg",
+        "canvas",
+        "video",
+        "audio",
+        "iframe",
+        "object",
+        "embed",
+        "head",
+    ]
+
+    _BASE64_LINE = re.compile(r"^[A-Za-z0-9+/=]{40,}\s*$", re.MULTILINE)
+    _BINARY_GARBAGE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]")
+    _EXCESS_NEWLINES = re.compile(r"\n{3,}")
+    _MD_IMAGE = re.compile(r"!\[.*?\]\(.*?\)")
+
+    def __init__(
+        self,
+        with_image: bool = False,
+        with_link: bool = True,
+        heading_style: str = "atx",
+        strong_em_symbol: str = "*",
+        bullets: str = "*",
+        escape_asterisks: bool = False,
+        preprocessing_preset: str = "aggressive",
+        remove_navigation: bool = True,
+        remove_forms: bool = True,
+    ):
+        self.with_image = with_image
+        self.with_link = with_link
+        self.heading_style = heading_style
+        self.strong_em_symbol = strong_em_symbol
+        self.bullets = bullets
+        self.escape_asterisks = escape_asterisks
+        self.preprocessing_preset = preprocessing_preset
+        self.remove_navigation = remove_navigation
+        self.remove_forms = remove_forms
+
+    def convert(self, html: str) -> str:
+        """
+        Convert HTML to clean Markdown.
+
+        Args:
+            html: HTML string
+
+        Returns:
+            Clean Markdown string
+        """
+        clean_html = self._clean_html(html)
+        markdown = self._html_to_markdown_lib(clean_html)
+        markdown = self._clean_markdown(markdown)
+        return markdown
+
+    def _clean_html(self, html: str) -> str:
+        """Remove unwanted tags and attributes from HTML."""
+        soup = BeautifulSoup(html, "html.parser")
+
+        for tag in self.STRIP_TAGS:
+            for el in soup.find_all(tag):
+                el.decompose()
+
+        if not self.with_image:
+            for el in soup.find_all("img"):
+                el.decompose()
+        else:
+            for el in soup.find_all("img"):
+                src = el.get("src", "")
+                if isinstance(src, str) and (src.startswith("data:") or src.startswith("blob:")):
+                    el.decompose()
+
+        if not self.with_link:
+            for el in soup.find_all("a"):
+                el.replace_with(el.get_text())
+        else:
+            for el in soup.find_all("a"):
+                href = el.get("href", "")
+                if isinstance(href, str) and (href.startswith("data:") or href.startswith("blob:")):
+                    el["href"] = ""
+
+        for el in soup.find_all(True):
+            for attr in ("src", "href", "srcset", "poster", "background"):
+                val = el.get(attr, "")
+                if isinstance(val, str) and (val.startswith("data:") or val.startswith("blob:")):
+                    del el[attr]
+
+        return str(soup)
+
+    def _html_to_markdown_lib(self, html: str) -> str:
+        """Convert HTML to Markdown using html-to-markdown library."""
+        options = ConversionOptions(
+            heading_style=self.heading_style,
+            strong_em_symbol=self.strong_em_symbol,
+            bullets=self.bullets,
+            escape_asterisks=self.escape_asterisks,
+        )
+        preprocessing = PreprocessingOptions(
+            enabled=True,
+            preset=self.preprocessing_preset,
+            remove_navigation=self.remove_navigation,
+            remove_forms=self.remove_forms,
+        )
+        return convert(html, options, preprocessing)
+
+    def _clean_markdown(self, text: str) -> str:
+        """Final cleanup of Markdown text."""
+        text = self._BINARY_GARBAGE.sub("", text)
+        text = self._BASE64_LINE.sub("", text)
+
+        if not self.with_image:
+            text = self._MD_IMAGE.sub("", text)
+
+        text = self._EXCESS_NEWLINES.sub("\n\n", text)
+
+        return text.strip()
+
+
+if __name__ == "__main__":
+    import sys
+
+    html = sys.stdin.read() if not sys.argv[1:] else open(sys.argv[1]).read()
+
+    converter = HtmlToMarkdown(with_image=False, with_link=False)
+    print(converter.convert(html))

pg2md-1.0.1/pg2md.egg-info/PKG-INFO

@@ -0,0 +1,302 @@
+Metadata-Version: 2.4
+Name: pg2md
+Version: 1.0.1
+Summary: HTML to Markdown converter with Requests or Playwright backend
+Author-email: Your Name <your@email.com>
+License: MIT
+Project-URL: Homepage, https://github.com/yourname/pg2md
+Project-URL: Repository, https://github.com/yourname/pg2md
+Project-URL: Issues, https://github.com/yourname/pg2md/issues
+Keywords: html,markdown,converter,playwright,requests,scraper
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.28.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: html-to-markdown>=1.1.0
+Provides-Extra: playwright
+Requires-Dist: playwright>=1.40.0; extra == "playwright"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: build>=1.0.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+Dynamic: license-file
+
+# pg2md
+
+**HTML to Markdown converter** with Requests or Playwright backend.
+
+Convert any webpage to clean Markdown. Choose between fast `requests` or full browser `playwright` for JavaScript-rendered pages.
+
+## Features
+
+- **Two backends**: `Pg2MdRequests` (fast) or `Pg2MdPlaywright` (JS support)
+- **Browser reuse**: Playwright instances share a single browser
+- **Proxy support**: HTTP/HTTPS proxies with authentication
+- **Custom headers & cookies**: Full control over requests
+- **Clean output**: Optional removal of images and links
+- **Context manager**: Auto-cleanup with `with` statement
+
+## Installation
+
+```bash
+pip install pg2md
+
+# For Playwright backend:
+pip install pg2md[playwright]
+playwright install chromium
+```
+
+## Quick Start
+
+```python
+from pg2md import Pg2MdRequests, Pg2MdPlaywright
+
+# Simple usage with Requests
+pg = Pg2MdRequests()
+markdown = pg.run("https://example.com")
+print(markdown)
+
+# Playwright for JS-heavy sites
+pg = Pg2MdPlaywright()
+markdown = pg.run("https://spa-example.com")
+pg.close()
+```
+
+## Usage
+
+### Basic Conversion
+
+```python
+from pg2md import Pg2MdRequests
+
+pg = Pg2MdRequests(with_image=False, with_link=False)
+md = pg.run("https://news.ycombinator.com")
+```
+
+### With Proxy
+
+```python
+from pg2md import Pg2MdRequests, Pg2MdPlaywright
+
+# Format: http://user:password@host:port
+# Or: host:port:user:password
+proxy = "http://user:pass@proxy.example.com:8080"
+
+# Requests
+pg = Pg2MdRequests()
+md = pg.run("https://example.com", proxy=proxy)
+
+# Playwright
+pg = Pg2MdPlaywright()
+md = pg.run("https://example.com", proxy=proxy)
+pg.close()
+```
+
+### Custom Headers & User-Agent
+
+```python
+from pg2md import Pg2MdRequests
+
+pg = Pg2MdRequests()
+md = pg.run(
+    "https://api.example.com/data",
+    headers={
+        "X-API-Key": "secret123",
+        "Accept": "application/json",
+    },
+    user_agent="MyBot/1.0",
+)
+```
+
+### With Cookies
+
+```python
+from pg2md import Pg2MdRequests
+
+pg = Pg2MdRequests()
+md = pg.run(
+    "https://example.com/dashboard",
+    cookies={
+        "session": "abc123",
+        "auth_token": "xyz789",
+    },
+)
+```
+
+### Save to File
+
+```python
+from pg2md import Pg2MdRequests
+
+pg = Pg2MdRequests()
+pg.save("output.md", "https://example.com")
+
+# With options
+pg.save(
+    "article.md",
+    "https://blog.example.com/post",
+    proxy="http://user:pass@host:port",
+    user_agent="MyBot/1.0",
+)
+```
+
+### Context Manager
+
+```python
+from pg2md import Pg2MdPlaywright
+
+with Pg2MdPlaywright() as pg:
+    md1 = pg.run("https://site1.com")
+    md2 = pg.run("https://site2.com")
+    # Browser closed automatically
+```
+
+### Multiple Instances
+
+```python
+from pg2md import Pg2MdPlaywright
+
+# Both share the same browser (efficient)
+pg1 = Pg2MdPlaywright()
+pg2 = Pg2MdPlaywright()
+
+md1 = pg1.run("https://site1.com")
+md2 = pg2.run("https://site2.com")
+
+Pg2MdPlaywright.close_all()  # Close shared browser
+```
+
+## API Reference
+
+### Pg2MdRequests
+
+```python
+Pg2MdRequests(with_image=False, with_link=False)
+```
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `with_image` | bool | False | Include images in output |
+| `with_link` | bool | False | Include links in output |
+
+### Pg2MdPlaywright
+
+```python
+Pg2MdPlaywright(
+    browser=None,       # Custom Browser instance
+    headless=True,      # Headless mode
+    with_image=False,
+    with_link=False,
+)
+```
+
+### Methods
+
+#### `run(url, proxy=None, headers=None, cookies=None, user_agent=None, timeout=30)`
+
+Fetch URL and convert to Markdown.
+
+Returns: `str` (Markdown)
+
+#### `fetch(url, proxy=None, headers=None, cookies=None, user_agent=None, timeout=30)`
+
+Fetch HTML only.
+
+Returns: `str` (HTML)
+
+#### `convert(html)`
+
+Convert HTML to Markdown.
+
+Returns: `str` (Markdown)
+
+#### `save(filepath, url, **kwargs)`
+
+Fetch, convert, and save to file.
+
+#### `close()`
+
+Close browser (Playwright only).
+
+#### `close_all()` (classmethod, Playwright only)
+
+Close all shared browsers.
+
+## When to Use Which Backend?
+
+| Use Requests | Use Playwright |
+|--------------|----------------|
+| Static HTML pages | SPA / JavaScript apps |
+| Speed matters | Need rendered content |
+| Simple scraping | Bypass anti-bot (sometimes) |
+| Low memory | Modern web apps |
+
+## Examples
+
+### Scrape Multiple URLs
+
+```python
+from pg2md import Pg2MdRequests
+
+urls = [
+    "https://blog.example.com/post1",
+    "https://blog.example.com/post2",
+    "https://blog.example.com/post3",
+]
+
+pg = Pg2MdRequests(with_image=False, with_link=False)
+
+for i, url in enumerate(urls):
+    pg.save(f"post_{i+1}.md", url)
+    print(f"Saved: {url}")
+```
+
+### Batch with Proxies
+
+```python
+from pg2md import Pg2MdRequests
+
+urls = ["https://site1.com", "https://site2.com", "https://site3.com"]
+proxies = [
+    "http://user1:pass1@proxy1:8080",
+    "http://user2:pass2@proxy2:8080",
+]
+
+pg = Pg2MdRequests()
+
+for i, url in enumerate(urls):
+    proxy = proxies[i % len(proxies)]
+    md = pg.run(url, proxy=proxy)
+    print(f"[{i+1}] {len(md)} chars")
+```
+
+### Extract Article Content
+
+```python
+from pg2md import Pg2MdPlaywright
+
+with Pg2MdPlaywright() as pg:
+    md = pg.run(
+        "https://medium.com/some-article",
+        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+    )
+    
+    # Save clean text
+    with open("article.md", "w") as f:
+        f.write(md)
+```
+
+## License
+
+MIT

pg2md-1.0.1/pg2md.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+LICENSE
+README.md
+pyproject.toml
+pg2md/__init__.py
+pg2md/html_to_md.py
+pg2md.egg-info/PKG-INFO
+pg2md.egg-info/SOURCES.txt
+pg2md.egg-info/dependency_links.txt
+pg2md.egg-info/requires.txt
+pg2md.egg-info/top_level.txt

pg2md-1.0.1/pg2md.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

pg2md-1.0.1/pg2md.egg-info/requires.txt

@@ -0,0 +1,11 @@
+requests>=2.28.0
+beautifulsoup4>=4.12.0
+html-to-markdown>=1.1.0
+
+[dev]
+pytest>=7.0.0
+build>=1.0.0
+twine>=4.0.0
+
+[playwright]
+playwright>=1.40.0

pg2md-1.0.1/pg2md.egg-info/top_level.txt

@@ -0,0 +1 @@
+pg2md

pg2md-1.0.1/pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pg2md"
+version = "1.0.1"
+description = "HTML to Markdown converter with Requests or Playwright backend"
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.10"
+authors = [
+    {name = "Your Name", email = "your@email.com"}
+]
+keywords = ["html", "markdown", "converter", "playwright", "requests", "scraper"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Text Processing :: Markup :: HTML",
+    "Topic :: Text Processing :: Markup :: Markdown",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "requests>=2.28.0",
+    "beautifulsoup4>=4.12.0",
+    "html-to-markdown>=1.1.0",
+]
+
+[project.optional-dependencies]
+playwright = [
+    "playwright>=1.40.0",
+]
+dev = [
+    "pytest>=7.0.0",
+    "build>=1.0.0",
+    "twine>=4.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/yourname/pg2md"
+Repository = "https://github.com/yourname/pg2md"
+Issues = "https://github.com/yourname/pg2md/issues"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["pg2md"]

pg2md-1.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+