mdfetch 0.1.0__py3-none-any.whl

mdfetch/__init__.py

@@ -0,0 +1,35 @@
+"""mdfetch — extract article content from web platforms as clean Markdown."""
+
+from __future__ import annotations
+
+from mdfetch.exceptions import (
+    EmptyContentError,
+    FetchError,
+    HTTPStatusError,
+    InvalidURLError,
+    MdfetchError,
+    UnsupportedContentTypeError,
+    UnsupportedPlatformError,
+)
+from mdfetch.router import route
+
+__all__ = [
+    "extract",
+    "MdfetchError",
+    "InvalidURLError",
+    "UnsupportedPlatformError",
+    "UnsupportedContentTypeError",
+    "FetchError",
+    "HTTPStatusError",
+    "EmptyContentError",
+]
+
+
+def extract(url: str, *, retries: int = 3, retry_delay: float = 2.0) -> str:
+    """Extract article content from *url* and return it as Markdown.
+
+    On transient network failures (timeouts, connection errors, non-2xx responses)
+    the request is retried up to *retries* times with *retry_delay* seconds between
+    attempts. Set ``retries=1`` to disable retry behaviour.
+    """
+    return route(url).extract(url, retries=retries, retry_delay=retry_delay)

mdfetch/base.py

@@ -0,0 +1,93 @@
+"""Abstract base class for all platform extractors."""
+
+from __future__ import annotations
+
+import time
+from abc import ABC, abstractmethod
+
+import httpx
+from bs4 import BeautifulSoup
+from bs4.element import Tag
+
+from mdfetch.exceptions import FetchError, HTTPStatusError, MdfetchError
+
+_MAX_RESPONSE_BYTES = 10 * 1024 * 1024  # 10 MB — guard against runaway responses
+
+
+class BaseExtractor(ABC):
+    """Contract all platform-specific extractors must fulfil."""
+
+    DOMAINS: frozenset[str] = frozenset()
+
+    # FR-014: use a browser-like UA (no mdfetch-specific branding) so servers serve readable HTML
+    _USER_AGENT = (
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/120.0.0.0 Safari/537.36"
+    )
+
+    def fetch_html(self, url: str, *, retries: int = 3, retry_delay: float = 2.0) -> str:
+        """Fetch raw HTML from *url* using a 30-second timeout and a 10 MB size cap.
+
+        Retries up to *retries* times on transient :class:`FetchError` (network errors,
+        timeouts, non-2xx responses), waiting *retry_delay* seconds between attempts.
+        """
+        last_exc: FetchError | None = None
+        for attempt in range(max(1, retries)):
+            try:
+                return self._do_fetch(url)
+            except FetchError as exc:
+                last_exc = exc
+                if attempt < retries - 1:
+                    time.sleep(retry_delay)
+        assert last_exc is not None
+        raise last_exc
+
+    def _do_fetch(self, url: str) -> str:
+        """Single HTTP fetch attempt (no retry logic)."""
+        headers = {"User-Agent": self._USER_AGENT}
+        try:
+            with httpx.Client(timeout=30.0, follow_redirects=True) as client:
+                with client.stream("GET", url, headers=headers) as response:
+                    if not response.is_success:
+                        raise HTTPStatusError(
+                            f"HTTP {response.status_code} fetching {url}",
+                            status_code=response.status_code,
+                            url=url,
+                        )
+                    chunks: list[bytes] = []
+                    total = 0
+                    for chunk in response.iter_bytes():
+                        total += len(chunk)
+                        if total > _MAX_RESPONSE_BYTES:
+                            raise FetchError(
+                                f"Response from {url} exceeded "
+                                f"{_MAX_RESPONSE_BYTES // (1024 * 1024)} MB limit",
+                                url=url,
+                            )
+                        chunks.append(chunk)
+                    return b"".join(chunks).decode(response.encoding or "utf-8", errors="replace")
+        except httpx.TimeoutException as exc:
+            raise FetchError(f"Request timed out: {url}", url=url) from exc
+        except httpx.RequestError as exc:
+            raise FetchError(f"Network error fetching {url}: {exc}", url=url) from exc
+
+    @abstractmethod
+    def clean_html(self, soup: BeautifulSoup) -> Tag:
+        """Isolate the article body, strip non-content elements, and return the root Tag."""
+
+    @abstractmethod
+    def convert_to_markdown(self, tag: Tag) -> str:
+        """Convert the cleaned Tag to a Markdown string."""
+
+    def extract(self, url: str, *, retries: int = 3, retry_delay: float = 2.0) -> str:
+        """Orchestrate fetch → clean → convert and return Markdown."""
+        html = self.fetch_html(url, retries=retries, retry_delay=retry_delay)
+        soup = BeautifulSoup(html, "lxml")
+        try:
+            cleaned = self.clean_html(soup)
+            return self.convert_to_markdown(cleaned)
+        except MdfetchError as exc:
+            if exc.url is None:
+                exc.url = url
+            raise

mdfetch/exceptions.py

@@ -0,0 +1,40 @@
+"""Custom exception hierarchy for mdfetch."""
+
+from __future__ import annotations
+
+
+class MdfetchError(Exception):
+    """Base exception for all mdfetch errors."""
+
+    def __init__(self, message: str, url: str | None = None) -> None:
+        super().__init__(message)
+        self.message = message
+        self.url = url
+
+
+class InvalidURLError(MdfetchError):
+    """Raised when the supplied URL is syntactically invalid."""
+
+
+class UnsupportedPlatformError(MdfetchError):
+    """Raised when the URL domain has no registered provider."""
+
+
+class UnsupportedContentTypeError(MdfetchError):
+    """Raised when the domain is recognised but the page is not an extractable article."""
+
+
+class FetchError(MdfetchError):
+    """Raised when a network request fails (connection error, timeout)."""
+
+
+class HTTPStatusError(FetchError):
+    """Raised when the server returns a non-2xx HTTP status code."""
+
+    def __init__(self, message: str, status_code: int, url: str | None = None) -> None:
+        super().__init__(message, url)
+        self.status_code = status_code
+
+
+class EmptyContentError(MdfetchError):
+    """Raised when the article body yields no extractable text content."""

mdfetch/providers/medium.py

@@ -0,0 +1,67 @@
+"""Medium platform extractor."""
+
+from __future__ import annotations
+
+import re
+
+from bs4 import BeautifulSoup
+from bs4.element import Tag
+from markdownify import markdownify
+
+from mdfetch.base import BaseExtractor
+from mdfetch.exceptions import EmptyContentError, UnsupportedContentTypeError
+from mdfetch.router import register
+
+
+@register
+class MediumExtractor(BaseExtractor):
+    """Extracts article content from medium.com and its subdomains."""
+
+    DOMAINS: frozenset[str] = frozenset({"medium.com"})
+
+    def clean_html(self, soup: BeautifulSoup) -> Tag:
+        """Isolate the article body and strip all non-content elements."""
+        article = soup.find("article")
+        if not isinstance(article, Tag):
+            raise UnsupportedContentTypeError(
+                "URL is not an article page — no <article> element found",
+            )
+
+        for nav in article.find_all("nav"):
+            nav.decompose()
+
+        for button in article.find_all(
+            "button",
+            attrs={"aria-label": re.compile(r"clap|applaud", re.IGNORECASE)},
+        ):
+            button.decompose()
+
+        for element in article.find_all(attrs={"data-testid": "post-sidebar"}):
+            element.decompose()
+
+        for element in article.find_all(attrs={"data-testid": re.compile(r"share", re.IGNORECASE)}):
+            element.decompose()
+
+        for element in article.find_all(attrs={"data-testid": "post-footer"}):
+            element.decompose()
+
+        for section in article.find_all("section"):
+            raw = section.get("class")
+            classes: list[str] = raw if isinstance(raw, list) else ([raw] if raw else [])
+            if classes and any("author" in c.lower() or "bio" in c.lower() for c in classes):
+                section.decompose()
+
+        return article
+
+    def convert_to_markdown(self, tag: Tag) -> str:
+        """Convert cleaned article Tag to Markdown."""
+        md = markdownify(str(tag), heading_style="ATX", code_language="", strip=["script", "style"])
+        md = md.strip()
+        md = re.sub(r"\n{3,}", "\n\n", md)
+
+        if not md:
+            raise EmptyContentError(
+                "Article body contained no extractable text content",
+            )
+
+        return md

mdfetch/router.py

@@ -0,0 +1,65 @@
+"""Domain-to-provider routing."""
+
+from __future__ import annotations
+
+import importlib
+import pkgutil
+from urllib.parse import urlparse
+
+from mdfetch.base import BaseExtractor
+from mdfetch.exceptions import InvalidURLError, UnsupportedPlatformError
+
+_REGISTRY: dict[str, type[BaseExtractor]] = {}
+
+
+def register(provider_cls: type[BaseExtractor]) -> type[BaseExtractor]:
+    """Register *provider_cls* for each domain it declares."""
+    for domain in provider_cls.DOMAINS:
+        existing = _REGISTRY.get(domain)
+        if existing is not None and existing is not provider_cls:
+            raise ValueError(
+                f"Domain {domain!r} is already registered to {existing.__name__!r}; "
+                f"cannot re-register to {provider_cls.__name__!r}"
+            )
+        _REGISTRY[domain] = provider_cls
+    return provider_cls
+
+
+def route(url: str) -> BaseExtractor:
+    """Return a provider instance for *url*, raising typed errors on failure."""
+    parsed = urlparse(url)
+
+    # Use parsed.hostname (lowercased, port-stripped) for routing and validation.
+    # Checking hostname rather than netloc correctly rejects edge cases like
+    # "https://:80/" where netloc is non-empty but hostname is None/empty.
+    hostname = (parsed.hostname or "").lower()
+
+    if parsed.scheme not in ("http", "https") or not hostname:
+        raise InvalidURLError(f"Invalid URL: {url!r}", url=url)
+
+    # Exact match first; fall back to subdomain suffix check so any provider whose
+    # DOMAINS entry is a parent domain automatically handles its subdomains.
+    # Sort candidates by length descending so the most-specific suffix wins when
+    # multiple registered domains are suffixes of the same hostname.
+    provider_cls = _REGISTRY.get(hostname)
+    if provider_cls is None:
+        for domain in sorted(_REGISTRY, key=len, reverse=True):
+            if hostname.endswith(f".{domain}"):
+                provider_cls = _REGISTRY[domain]
+                break
+
+    if provider_cls is None:
+        raise UnsupportedPlatformError(f"No provider registered for domain {hostname!r}", url=url)
+
+    return provider_cls()
+
+
+def _autodiscover_providers() -> None:
+    """Import every module in mdfetch.providers; classes decorated with @register self-enrol."""
+    import mdfetch.providers as _providers_pkg  # noqa: PLC0415
+
+    for _, module_name, _ in pkgutil.iter_modules(_providers_pkg.__path__):
+        importlib.import_module(f"mdfetch.providers.{module_name}")
+
+
+_autodiscover_providers()

mdfetch-0.1.0.dist-info/METADATA

@@ -0,0 +1,105 @@
+Metadata-Version: 2.4
+Name: mdfetch
+Version: 0.1.0
+Summary: Extract article content from web platforms and return it as clean Markdown.
+Project-URL: Homepage, https://github.com/stn1slv/md-fetch
+Project-URL: Source, https://github.com/stn1slv/md-fetch
+Project-URL: Issues, https://github.com/stn1slv/md-fetch/issues
+Author-email: Stanislav Deviatov <devyatov@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: article,extraction,markdown,medium,scraping
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Requires-Python: >=3.12
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: httpx>=0.27
+Requires-Dist: lxml>=5.0
+Requires-Dist: markdownify>=0.13
+Provides-Extra: dev
+Requires-Dist: mypy>=1.9; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# mdfetch
+
+A Python library that extracts article content from web platforms and returns it as clean Markdown.
+
+## Install
+
+```bash
+pip install mdfetch
+```
+
+## Usage
+
+```python
+from mdfetch import extract
+
+markdown = extract("https://medium.com/some-publication/article-slug-abc123")
+print(markdown)
+```
+
+## Error handling
+
+```python
+from mdfetch import (
+    extract,
+    InvalidURLError,
+    UnsupportedPlatformError,
+    UnsupportedContentTypeError,
+    FetchError,
+    HTTPStatusError,
+    EmptyContentError,
+)
+
+url = "https://medium.com/some-publication/article-slug-abc123"
+
+try:
+    markdown = extract(url)
+except InvalidURLError as e:
+    print(f"Bad URL: {e.message}")
+except UnsupportedPlatformError as e:
+    print(f"Platform not supported: {e.message}")
+except UnsupportedContentTypeError as e:
+    print(f"Not an article page: {e.message}")
+except HTTPStatusError as e:
+    print(f"HTTP {e.status_code}: {e.message}")
+except FetchError as e:
+    print(f"Network error: {e.message}")
+except EmptyContentError as e:
+    print(f"No content: {e.message}")
+```
+
+## Supported platforms
+
+| Platform | Domains |
+|----------|---------|
+| Medium   | `medium.com`, `*.medium.com` |
+
+## Development
+
+Requires [uv](https://docs.astral.sh/uv/).
+
+```bash
+make setup        # install dependencies
+make test         # run unit tests
+make integration  # run integration tests (requires network access)
+make lint         # ruff check
+make format       # ruff format
+make build        # build wheel + sdist
+make upgrade-deps # upgrade all dependencies
+make clean        # remove build artifacts
+```
+
+## Requirements
+
+- Python 3.12+

mdfetch-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+mdfetch/__init__.py,sha256=jYK0fy0rIPkVrBethgIwFl4SdQohoCIPYGiZpyzg7pI,1014
+mdfetch/base.py,sha256=p71yd5FtsV0EPxr2f9QQ53qV-YmhRH_sV24ZWqcEuGY,3827
+mdfetch/exceptions.py,sha256=kMEgh6hTL3IICnmPmJ7c2WRmTbJ75-RLcXmk1ynUFfg,1180
+mdfetch/router.py,sha256=1g6vsFToMoruUkLcLKBJ2aQlHbgmP_ZnjzUBvQbW6Bg,2499
+mdfetch/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mdfetch/providers/medium.py,sha256=QF1HugoOH2b_Kg5YxuSbsKIZUjcxQTerCbJafPNjROQ,2242
+mdfetch-0.1.0.dist-info/METADATA,sha256=ykjqRi9cNSvMTNR6wC2o1xT9Fi7mI5e3KVJiNk4yHaE,2856
+mdfetch-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+mdfetch-0.1.0.dist-info/licenses/LICENSE,sha256=erwf-XsuvrFr0bi1_8aTbCk7wu7zuxcJWqTondbCvhA,1075
+mdfetch-0.1.0.dist-info/RECORD,,

mdfetch-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

mdfetch-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Stanislav Deviatov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.