mdfetch 0.5.1__tar.gz → 0.5.2__tar.gz

{mdfetch-0.5.1 → mdfetch-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mdfetch
-Version: 0.5.1
+Version: 0.5.2
 Summary: Extract article content from web platforms and return it as clean Markdown.
 Project-URL: Homepage, https://github.com/stn1slv/md-fetch
 Project-URL: Source, https://github.com/stn1slv/md-fetch

{mdfetch-0.5.1 → mdfetch-0.5.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "mdfetch"
-version = "0.5.1"
+version = "0.5.2"
 description = "Extract article content from web platforms and return it as clean Markdown."
 readme = "README.md"
 license = { text = "MIT" }

{mdfetch-0.5.1 → mdfetch-0.5.2}/src/mdfetch/base.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import re
 import time
 from abc import ABC, abstractmethod
+from collections.abc import Iterable
 from typing import Any
 
 import httpx
@@ -28,6 +29,10 @@ class BaseExtractor(ABC):
     """Contract all platform-specific extractors must fulfil."""
 
     DOMAINS: frozenset[str] = frozenset()
+    # When True, the router also matches hostnames that are subdomains of any
+    # entry in DOMAINS (e.g. ``foo.medium.com`` → MediumExtractor).  Leave as
+    # False for single-tenant sites where subdomains are not article URLs.
+    MATCH_SUBDOMAINS: bool = False
     _no_retry_status_codes: frozenset[int] = frozenset()
 
     # FR-014: use a browser-like UA (no mdfetch-specific branding) so servers serve readable HTML
@@ -139,17 +144,35 @@ class BaseExtractor(ABC):
 
         return md
 
-    @staticmethod
-    def _replace_iframes_with_links(container: Tag, soup: BeautifulSoup) -> None:
-        """Replace ``<iframe>`` elements inside *container* with plain anchor links."""
-        for iframe in container.find_all("iframe"):
-            src = str(iframe.get("src") or iframe.get("data-src") or "")
-            if src:
-                link = soup.new_tag("a", href=src)
-                link.string = src
-                iframe.replace_with(link)
+    _DEFAULT_EMBED_URL_ATTRS: tuple[str, ...] = ("src", "data-src", "data-url", "href")
+
+    @classmethod
+    def _replace_embeds_with_links(
+        cls,
+        embeds: Iterable[Tag],
+        soup: BeautifulSoup,
+        *,
+        attrs: tuple[str, ...] = _DEFAULT_EMBED_URL_ATTRS,
+    ) -> None:
+        """Replace each tag in *embeds* with an anchor pointing at its URL, or decompose."""
+        for embed in embeds:
+            url = ""
+            for attr in attrs:
+                val = embed.get(attr)
+                if val:
+                    url = str(val)
+                    break
+            if url:
+                link = soup.new_tag("a", href=url)
+                link.string = url
+                embed.replace_with(link)
             else:
-                iframe.decompose()
+                embed.decompose()
+
+    @classmethod
+    def _replace_iframes_with_links(cls, container: Tag, soup: BeautifulSoup) -> None:
+        """Replace ``<iframe>`` elements inside *container* with plain anchor links."""
+        cls._replace_embeds_with_links(container.find_all("iframe"), soup)
 
     def extract(self, url: str, *, retries: int = 3, retry_delay: float = 2.0) -> str:
         """Orchestrate fetch → clean → convert and return Markdown."""

{mdfetch-0.5.1 → mdfetch-0.5.2}/src/mdfetch/cli.py

@@ -4,7 +4,9 @@ Command-line interface for mdfetch.
 
 from __future__ import annotations
 
+import os
 import sys
+from urllib.parse import urlparse
 
 import click
 
@@ -36,8 +38,28 @@ from mdfetch.router import supported_domains
     show_default=True,
     help="Seconds to wait between retry attempts",
 )
-def main(url: str, output: str | None, retries: int, retry_delay: float) -> None:
+@click.option(
+    "-f",
+    "--force",
+    is_flag=True,
+    default=False,
+    help="Overwrite the output file if it already exists",
+)
+def main(
+    url: str,
+    output: str | None,
+    retries: int,
+    retry_delay: float,
+    force: bool,
+) -> None:
     """Fetch and extract Markdown from the given URL."""
+    if output and os.path.exists(output) and not force:
+        click.secho(
+            f"Error: '{output}' already exists. Use --force to overwrite.",
+            err=True,
+            fg="red",
+        )
+        sys.exit(1)
     try:
         content = extract(url, retries=retries, retry_delay=retry_delay)
 
@@ -47,8 +69,6 @@ def main(url: str, output: str | None, retries: int, retry_delay: float) -> None
         else:
             click.echo(content)
     except UnsupportedPlatformError as e:
-        from urllib.parse import urlparse
-
         domain = (urlparse(e.url).hostname if e.url else None) or str(e)
         domains = ", ".join(sorted(supported_domains()))
         click.secho(

{mdfetch-0.5.1 → mdfetch-0.5.2}/src/mdfetch/providers/devto.py

@@ -30,15 +30,15 @@ class DevToExtractor(BaseExtractor):
         # Replace iframes with plain anchor links (FR-008)
         self._replace_iframes_with_links(body, soup)
 
-        # Replace dev.to liquid-tag embeds with plain anchor links (FR-008)
-        for embed in body.find_all(class_=re.compile(r"ltag", re.IGNORECASE)):
-            src = str(embed.get("data-url") or embed.get("data-src") or embed.get("src") or "")
-            if src:
-                link = soup.new_tag("a", href=src)
-                link.string = src
-                embed.replace_with(link)
-            else:
-                embed.decompose()
+        # Replace dev.to liquid-tag embeds with plain anchor links (FR-008).
+        # Anchor at the start of the class name so unrelated classes that merely
+        # contain "ltag" (e.g. "ultraltagrelated") are not matched — bs4 matches
+        # each class independently via re.search.
+        self._replace_embeds_with_links(
+            body.find_all(class_=re.compile(r"^ltag(?:[-_]|$)", re.IGNORECASE)),
+            soup,
+            attrs=("data-url", "data-src", "src"),
+        )
 
         # Strip empty anchor-name links inserted before headings
         for anchor in body.find_all("a", attrs={"name": True}):

{mdfetch-0.5.1 → mdfetch-0.5.2}/src/mdfetch/providers/medium.py

@@ -22,6 +22,7 @@ class MediumExtractor(BaseExtractor):
     """Extracts article content from medium.com and its subdomains."""
 
     DOMAINS: frozenset[str] = frozenset({"medium.com"})
+    MATCH_SUBDOMAINS = True
     _FREEDIUM_BASE = "https://freedium-mirror.cfd/"
     _no_retry_status_codes: frozenset[int] = frozenset({403, 429})
     # Smart-quote characters that Medium serves but Freedium replaces with ASCII;
@@ -91,12 +92,19 @@ class MediumExtractor(BaseExtractor):
         return md
 
     def _parse_freedium(self, soup: BeautifulSoup) -> str:
-        """Parse Freedium mirror HTML, which uses div.main-content instead of <article>."""
-        content = soup.find("div", class_="main-content")
+        """Parse Freedium mirror HTML, whose Svelte rebuild holds the body in div.prose."""
+        # Scope to the article so a stray .prose block (e.g. a bio/summary) can't
+        # match first; fall back to a bare .prose if the <article> wrapper is absent.
+        content = soup.select_one("article .prose") or soup.find("div", class_="prose")
         if not isinstance(content, Tag):
             raise UnsupportedContentTypeError(
-                "Fallback page missing main-content element",
+                "Fallback page missing prose element",
             )
+        # Freedium's Shiki highlighter emits each code block twice — a light-theme
+        # and a dark-theme variant — so the visible text is duplicated. Drop the
+        # dark variant before conversion to avoid repeated fenced-code output.
+        for pre in content.select("pre.github-dark"):
+            pre.decompose()
         # Freedium renders section headings one level deeper than medium.com (h4 vs h3).
         # Remap so the output heading levels match the medium.com direct path.
         for level in (4, 5, 6):

{mdfetch-0.5.1 → mdfetch-0.5.2}/src/mdfetch/providers/substack.py

@@ -18,6 +18,7 @@ class SubstackExtractor(BaseExtractor):
     """Extracts article content from substack.com and its subdomains."""
 
     DOMAINS: frozenset[str] = frozenset({"substack.com"})
+    MATCH_SUBDOMAINS = True
 
     def clean_html(self, soup: BeautifulSoup) -> Tag:
         """Isolate the article body and strip all non-content elements."""
@@ -36,16 +37,15 @@ class SubstackExtractor(BaseExtractor):
 
         # Convert other Substack embed containers to plain anchor links (FR-011)
         _safe_components = {"SubscribeWidget", "Image2ToDOM"}
-        for embed in body.find_all(attrs={"data-component-name": True}):
-            if embed.get("data-component-name") in _safe_components:
-                continue
-            url = str(embed.get("href") or embed.get("data-url") or embed.get("src") or "")
-            if url:
-                link = soup.new_tag("a", href=url)
-                link.string = url
-                embed.replace_with(link)
-            else:
-                embed.decompose()
+        self._replace_embeds_with_links(
+            [
+                e
+                for e in body.find_all(attrs={"data-component-name": True})
+                if e.get("data-component-name") not in _safe_components
+            ],
+            soup,
+            attrs=("href", "data-url", "src"),
+        )
 
         # Prepend subtitle from post-header (FR-005)
         header = soup.find("div", class_="post-header")

{mdfetch-0.5.1 → mdfetch-0.5.2}/src/mdfetch/router.py

@@ -37,15 +37,16 @@ def route(url: str) -> BaseExtractor:
     if parsed.scheme not in ("http", "https") or not hostname:
         raise InvalidURLError(f"Invalid URL: {url!r}", url=url)
 
-    # Exact match first; fall back to subdomain suffix check so any provider whose
-    # DOMAINS entry is a parent domain automatically handles its subdomains.
-    # Sort candidates by length descending so the most-specific suffix wins when
-    # multiple registered domains are suffixes of the same hostname.
+    # Exact match first; fall back to a subdomain suffix check only for providers
+    # that opt in via MATCH_SUBDOMAINS=True (multi-tenant sites like Medium and
+    # Substack).  Sort candidates by length descending so the most-specific
+    # suffix wins when multiple registered domains are suffixes of the same host.
     provider_cls = _REGISTRY.get(hostname)
     if provider_cls is None:
         for domain in sorted(_REGISTRY, key=len, reverse=True):
-            if hostname.endswith(f".{domain}"):
-                provider_cls = _REGISTRY[domain]
+            candidate = _REGISTRY[domain]
+            if candidate.MATCH_SUBDOMAINS and hostname.endswith(f".{domain}"):
+                provider_cls = candidate
                 break
 
     if provider_cls is None:

{mdfetch-0.5.1 → mdfetch-0.5.2}/tests/unit/test_cli.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import pathlib
+
 import pytest
 import pytest_mock
 from click.testing import CliRunner
@@ -41,3 +43,26 @@ def test_unsupported_domain_error_message(runner: CliRunner) -> None:
     assert result.exit_code == 1
     assert "'google.com' is not a supported platform." in result.output
     assert "Supported domains:" in result.output
+
+
+def test_output_refuses_to_clobber_existing_file(
+    mocker: pytest_mock.MockerFixture, runner: CliRunner, tmp_path: pathlib.Path
+) -> None:
+    mocker.patch("mdfetch.cli.extract", return_value="# Test")
+    out = tmp_path / "out.md"
+    out.write_text("existing content")
+    result = runner.invoke(main, ["https://dev.to/test", "-o", str(out)])
+    assert result.exit_code == 1
+    assert "already exists" in result.output
+    assert out.read_text() == "existing content"
+
+
+def test_output_force_overwrites_existing_file(
+    mocker: pytest_mock.MockerFixture, runner: CliRunner, tmp_path: pathlib.Path
+) -> None:
+    mocker.patch("mdfetch.cli.extract", return_value="# Test")
+    out = tmp_path / "out.md"
+    out.write_text("existing content")
+    result = runner.invoke(main, ["https://dev.to/test", "-o", str(out), "--force"])
+    assert result.exit_code == 0
+    assert out.read_text() == "# Test"

{mdfetch-0.5.1 → mdfetch-0.5.2}/tests/unit/test_medium_extractor.py

@@ -174,27 +174,38 @@ class TestConvertToMarkdown:
         assert "line one  \nline two" in md
 
 
+# Mirrors the Svelte/Tailwind Freedium rebuild: body in div.prose, code blocks
+# rendered twice via Shiki (light + dark theme variants).
 _FREEDIUM_ARTICLE_HTML = """
 <html><body>
-<div class="main-content">
+<main>
+<article>
+<div class="prose max-w-none prose-external-links">
     <h4>Section One</h4>
     <p>First paragraph of the article.</p>
     <h4>Section Two</h4>
     <p>Second paragraph with more content.</p>
-    <pre><code>print("hello")</code></pre>
+    <div class="dark:hidden">
+        <pre class="shiki github-light"><code>print("hello")</code></pre>
+    </div>
+    <div class="hidden dark:block">
+        <pre class="shiki github-dark"><code>print("hello")</code></pre>
+    </div>
 </div>
+</article>
+</main>
 </body></html>
 """
 
 _FREEDIUM_NO_CONTENT_HTML = """
 <html><body>
-<div class="header">No main-content div here</div>
+<div class="header">No prose div here</div>
 </body></html>
 """
 
 
 class TestParseFreedium:
-    def test_returns_markdown_from_main_content(self, extractor: MediumExtractor) -> None:
+    def test_returns_markdown_from_prose(self, extractor: MediumExtractor) -> None:
         soup = BeautifulSoup(_FREEDIUM_ARTICLE_HTML, "lxml")
         md = extractor._parse_freedium(soup)
         assert "Section One" in md
@@ -206,9 +217,15 @@ class TestParseFreedium:
         assert "###" in md
         assert "####" not in md
 
-    def test_raises_when_main_content_missing(self, extractor: MediumExtractor) -> None:
+    def test_dark_theme_code_block_deduplicated(self, extractor: MediumExtractor) -> None:
+        """Shiki renders each code block twice (light + dark); only one must survive."""
+        soup = BeautifulSoup(_FREEDIUM_ARTICLE_HTML, "lxml")
+        md = extractor._parse_freedium(soup)
+        assert md.count('print("hello")') == 1
+
+    def test_raises_when_prose_missing(self, extractor: MediumExtractor) -> None:
         soup = BeautifulSoup(_FREEDIUM_NO_CONTENT_HTML, "lxml")
-        with pytest.raises(UnsupportedContentTypeError, match="main-content"):
+        with pytest.raises(UnsupportedContentTypeError, match="prose"):
             extractor._parse_freedium(soup)
 
 

{mdfetch-0.5.1 → mdfetch-0.5.2}/tests/unit/test_router.py

@@ -81,6 +81,26 @@ class TestUrlValidation:
         provider = route("https://dzone.com/articles/some-article")
         assert isinstance(provider, DZoneExtractor)
 
+    def test_rejects_subdomain_of_single_tenant_domain(self) -> None:
+        # dev.to does not opt in to MATCH_SUBDOMAINS, so foo.dev.to must not
+        # be routed to DevToExtractor — it should raise UnsupportedPlatformError.
+        with pytest.raises(UnsupportedPlatformError):
+            route("https://foo.dev.to/some-article")
+
+    def test_rejects_subdomain_of_dzone(self) -> None:
+        with pytest.raises(UnsupportedPlatformError):
+            route("https://blog.dzone.com/articles/some-article")
+
+    def test_rejects_subdomain_of_thenewstack(self) -> None:
+        with pytest.raises(UnsupportedPlatformError):
+            route("https://blog.thenewstack.io/some-article")
+
+    def test_routes_substack_subdomain(self) -> None:
+        from mdfetch.providers.substack import SubstackExtractor
+
+        provider = route("https://newsletter.substack.com/p/post")
+        assert isinstance(provider, SubstackExtractor)
+
 
 class TestSupportedDomains:
     def test_returns_frozenset(self) -> None:

{mdfetch-0.5.1 → mdfetch-0.5.2}/uv.lock

@@ -310,7 +310,7 @@ wheels = [
 
 [[package]]
 name = "mdfetch"
-version = "0.5.1"
+version = "0.5.2"
 source = { editable = "." }
 dependencies = [
     { name = "beautifulsoup4" },