mdfetch 0.9.2__tar.gz → 0.9.4__tar.gz

mdfetch-0.9.4/.github/dependabot.yml

@@ -0,0 +1,21 @@
+version: 2
+updates:
+  - package-ecosystem: uv
+    directory: /
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5
+    groups:
+      python-deps:
+        patterns:
+          - "*"
+
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5
+    groups:
+      actions:
+        patterns:
+          - "*"

{mdfetch-0.9.2 → mdfetch-0.9.4}/.github/workflows/ci.yml

@@ -12,9 +12,9 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"
@@ -36,9 +36,9 @@ jobs:
       matrix:
         python-version: ["3.12", "3.13", "3.14"]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"

{mdfetch-0.9.2 → mdfetch-0.9.4}/.github/workflows/integration.yml

@@ -14,10 +14,10 @@ jobs:
     runs-on: [self-hosted, Linux, ARM64]
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           enable-cache: true
           cache-dependency-glob: "uv.lock"
@@ -34,7 +34,7 @@ jobs:
 
       - name: Create issue on failure
         if: failure() && steps.integration.outcome == 'failure'
-        uses: actions/github-script@v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
             const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;

{mdfetch-0.9.2 → mdfetch-0.9.4}/.github/workflows/publish.yml

@@ -13,9 +13,9 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event.release.target_commitish == 'main'
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
       - name: Set up Python
         run: uv python pin 3.12
       - name: Validate version
@@ -29,7 +29,7 @@ jobs:
       - name: Build package
         run: make build
       - name: Store the distribution packages
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: python-package-distributions
           path: dist/
@@ -45,12 +45,12 @@ jobs:
       id-token: write
     steps:
       - name: Download all the dists
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: python-package-distributions
           path: dist/
       - name: Publish package distributions to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
         with:
           packages-dir: dist/
 
@@ -65,10 +65,10 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         
       - name: Set up Python
         run: uv python pin 3.12

{mdfetch-0.9.2 → mdfetch-0.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: mdfetch
-Version: 0.9.2
+Version: 0.9.4
 Summary: Extract article content from web platforms and return it as clean Markdown.
 Project-URL: Homepage, https://github.com/stn1slv/md-fetch
 Project-URL: Source, https://github.com/stn1slv/md-fetch
@@ -63,6 +63,10 @@ md-fetch https://medium.com/example/article
 # Fetch and save Markdown to a file
 md-fetch https://dev.to/example/article --output article.md
 
+# Fallback to Tavily API if the primary extractor fails (requires TAVILY_API_KEY)
+export TAVILY_API_KEY="tvly-..."
+md-fetch https://example.com/unsupported-blog --tavily-fallback
+
 # List all supported platforms (no URL or network needed)
 md-fetch --list-platforms
 ```
@@ -80,6 +84,9 @@ markdown = extract("https://thenewstack.io/article-slug")
 markdown = extract("https://dzone.com/articles/article-slug")
 markdown = extract("https://boomi.com/blog/article-slug")
 markdown = extract("https://konghq.com/blog/category/article-slug")
+
+# Fallback to Tavily API if the primary extractor fails (requires TAVILY_API_KEY env var)
+markdown = extract("https://example.com/unsupported-blog", tavily_fallback=True)
 print(markdown)
 ```
 
@@ -94,6 +101,7 @@ from mdfetch import (
     FetchError,
     HTTPStatusError,
     EmptyContentError,
+    MissingAPIKeyError,
 )
 
 url = "https://medium.com/some-publication/article-slug-abc123"
@@ -112,6 +120,8 @@ except FetchError as e:
     print(f"Network error: {e.message}")
 except EmptyContentError as e:
     print(f"No content: {e.message}")
+except MissingAPIKeyError as e:
+    print(f"Missing API Key: {e.message}")
 ```
 
 ## Supported platforms

{mdfetch-0.9.2 → mdfetch-0.9.4}/README.md

@@ -29,6 +29,10 @@ md-fetch https://medium.com/example/article
 # Fetch and save Markdown to a file
 md-fetch https://dev.to/example/article --output article.md
 
+# Fallback to Tavily API if the primary extractor fails (requires TAVILY_API_KEY)
+export TAVILY_API_KEY="tvly-..."
+md-fetch https://example.com/unsupported-blog --tavily-fallback
+
 # List all supported platforms (no URL or network needed)
 md-fetch --list-platforms
 ```
@@ -46,6 +50,9 @@ markdown = extract("https://thenewstack.io/article-slug")
 markdown = extract("https://dzone.com/articles/article-slug")
 markdown = extract("https://boomi.com/blog/article-slug")
 markdown = extract("https://konghq.com/blog/category/article-slug")
+
+# Fallback to Tavily API if the primary extractor fails (requires TAVILY_API_KEY env var)
+markdown = extract("https://example.com/unsupported-blog", tavily_fallback=True)
 print(markdown)
 ```
 
@@ -60,6 +67,7 @@ from mdfetch import (
     FetchError,
     HTTPStatusError,
     EmptyContentError,
+    MissingAPIKeyError,
 )
 
 url = "https://medium.com/some-publication/article-slug-abc123"
@@ -78,6 +86,8 @@ except FetchError as e:
     print(f"Network error: {e.message}")
 except EmptyContentError as e:
     print(f"No content: {e.message}")
+except MissingAPIKeyError as e:
+    print(f"Missing API Key: {e.message}")
 ```
 
 ## Supported platforms

{mdfetch-0.9.2 → mdfetch-0.9.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "mdfetch"
-version = "0.9.2"
+version = "0.9.4"
 description = "Extract article content from web platforms and return it as clean Markdown."
 readme = "README.md"
 license = { text = "MIT" }

mdfetch-0.9.4/src/mdfetch/fallback.py

@@ -0,0 +1,53 @@
+"""Fallback extraction mechanisms for mdfetch."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from mdfetch.exceptions import EmptyContentError, FetchError, MissingAPIKeyError
+
+
+def _extract(client: Any, url: str, depth: str, timeout: float) -> str:
+    response = client.extract(urls=[url], extract_depth=depth, timeout=timeout)
+    results = response.get("results", [])
+    if not results:
+        raise EmptyContentError(f"Tavily returned no results for {url}", url=url)
+
+    raw_content = results[0].get("raw_content")
+    if not raw_content:
+        raise EmptyContentError(f"Tavily returned empty content for {url}", url=url)
+
+    return str(raw_content)
+
+
+def tavily_extract(url: str, timeout: float = 30.0) -> str:
+    """Extract content using the Tavily API fallback.
+
+    Tries the cheaper "basic" depth first. If it fails or returns nothing, retries
+    once with "advanced". Empty content from the retry raises EmptyContentError;
+    any other failure raises FetchError.
+    """
+    try:
+        from tavily import TavilyClient  # type: ignore[import-untyped]
+    except ImportError as e:
+        raise MissingAPIKeyError(
+            "tavily-python package is not installed. Please install it "
+            f"to use the fallback feature. {e}"
+        ) from e
+
+    try:
+        client = TavilyClient()
+    except Exception as e:
+        raise MissingAPIKeyError(str(e)) from e
+
+    try:
+        return _extract(client, url, "basic", timeout)
+    except Exception:
+        pass  # any basic-depth failure is retried with advanced below
+
+    try:
+        return _extract(client, url, "advanced", timeout)
+    except EmptyContentError:
+        raise
+    except Exception as e:
+        raise FetchError(f"Tavily fallback failed: {e}", url=url) from e

{mdfetch-0.9.2 → mdfetch-0.9.4}/tests/unit/test_fallback.py

@@ -7,7 +7,7 @@ from unittest.mock import MagicMock, patch
 import pytest
 
 from mdfetch import extract
-from mdfetch.exceptions import FetchError, MissingAPIKeyError
+from mdfetch.exceptions import EmptyContentError, FetchError, MissingAPIKeyError
 from mdfetch.fallback import tavily_extract
 
 
@@ -21,7 +21,7 @@ class TestTavilyExtract:
 
         assert result == "# Hello"
         mock_instance.extract.assert_called_once_with(
-            urls=["https://example.com"], extract_depth="basic"
+            urls=["https://example.com"], extract_depth="basic", timeout=30.0
         )
 
     @patch("tavily.TavilyClient")
@@ -38,7 +38,7 @@ class TestTavilyExtract:
         assert result == "# Advanced"
         assert mock_instance.extract.call_count == 2
         mock_instance.extract.assert_any_call(
-            urls=["https://example.com"], extract_depth="advanced"
+            urls=["https://example.com"], extract_depth="advanced", timeout=30.0
         )
 
     @patch("tavily.TavilyClient")
@@ -63,6 +63,30 @@ class TestTavilyExtract:
         with pytest.raises(FetchError):
             tavily_extract("https://example.com")
 
+    @patch("tavily.TavilyClient")
+    def test_empty_advanced_after_basic_error_raises_empty_content(
+        self, mock_tavily_client: MagicMock
+    ) -> None:
+        mock_instance = mock_tavily_client.return_value
+        mock_instance.extract.side_effect = [
+            Exception("API Error"),
+            {"results": []},
+        ]
+
+        with pytest.raises(EmptyContentError):
+            tavily_extract("https://example.com")
+
+    @patch("tavily.TavilyClient")
+    def test_timeout_is_passed_to_tavily(self, mock_tavily_client: MagicMock) -> None:
+        mock_instance = mock_tavily_client.return_value
+        mock_instance.extract.return_value = {"results": [{"raw_content": "# Hello"}]}
+
+        tavily_extract("https://example.com", timeout=5.0)
+
+        mock_instance.extract.assert_called_once_with(
+            urls=["https://example.com"], extract_depth="basic", timeout=5.0
+        )
+
 
 class TestExtractRouting:
     @patch("mdfetch.tavily_extract")

{mdfetch-0.9.2 → mdfetch-0.9.4}/uv.lock

@@ -8,15 +8,33 @@ resolution-markers = [
 
 [[package]]
 name = "anyio"
-version = "4.13.0"
+version = "4.14.2"
 source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.15'",
+]
+dependencies = [
+    { name = "idna" },
+    { name = "typing-extensions", marker = "python_full_version < '3.13' or python_full_version >= '3.15'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
+]
+
+[[package]]
+name = "anyio"
+version = "4.15.1"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version >= '3.15'",
+]
 dependencies = [
     { name = "idna" },
-    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" },
+    { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" },
 ]
 
 [[package]]
@@ -188,7 +206,8 @@ name = "httpx"
 version = "0.28.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "anyio" },
+    { name = "anyio", version = "4.14.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" },
+    { name = "anyio", version = "4.15.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" },
     { name = "certifi" },
     { name = "httpcore" },
     { name = "idna" },
@@ -371,7 +390,7 @@ wheels = [
 
 [[package]]
 name = "mdfetch"
-version = "0.9.2"
+version = "0.9.4"
 source = { editable = "." }
 dependencies = [
     { name = "beautifulsoup4" },
@@ -661,11 +680,11 @@ wheels = [
 
 [[package]]
 name = "soupsieve"
-version = "2.8.4"
+version = "2.10"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/47/2c/0a5f6f8ee0d5589e48c7640213ed5175d52cf540a06725b628cc1a45d6ce/soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e", size = 121110, upload-time = "2026-05-24T13:55:57.154Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/71/c3/1b817965ac12dc002d7c9cd7dfffdd7d4fbf9b45763ef2ffe7b86ee94670/soupsieve-2.10.tar.gz", hash = "sha256:49e9380d7d2905463583bafe285e818c7366a9ed7b3aee221c1ac79c905d8bc0", size = 124337, upload-time = "2026-09-24T02:36:35.048Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" },
+    { url = "https://files.pythonhosted.org/packages/66/87/5ed59e1d0290564e2027ed066c52b059892c4637741337fc0967183c8d4d/soupsieve-2.10-py3-none-any.whl", hash = "sha256:8596eb8967d744174820280fa62b4542a2e955bfaccca73ed8a13c6eb8e9b502", size = 38019, upload-time = "2026-09-24T02:36:33.709Z" },
 ]
 
 [[package]]

mdfetch-0.9.2/.vscode/settings.json

@@ -1,10 +0,0 @@
-{
-    "files.exclude": {
-        "**/.git": true,
-        "**/.svn": true,
-        "**/.hg": true,
-        "**/.DS_Store": true,
-        "**/Thumbs.db": true,
-        ".mule": true
-    }
-}

mdfetch-0.9.2/src/mdfetch/fallback.py

@@ -1,64 +0,0 @@
-"""Fallback extraction mechanisms for mdfetch."""
-
-from __future__ import annotations
-
-from mdfetch.exceptions import EmptyContentError, FetchError, MissingAPIKeyError
-
-
-def tavily_extract(url: str, timeout: float = 30.0) -> str:
-    """Extract content using the Tavily API fallback."""
-    try:
-        from tavily import TavilyClient  # type: ignore[import-untyped]
-    except ImportError as e:
-        raise MissingAPIKeyError(
-            "tavily-python package is not installed. Please install it "
-            f"to use the fallback feature. {e}"
-        ) from e
-
-    try:
-        client = TavilyClient()
-    except Exception as e:
-        raise MissingAPIKeyError(str(e)) from e
-
-    try:
-        response = client.extract(urls=[url], extract_depth="basic")
-        results = response.get("results", [])
-        if not results:
-            raise EmptyContentError(f"Tavily returned no results for {url}", url=url)
-
-        raw_content = results[0].get("raw_content")
-        if not raw_content:
-            raise EmptyContentError(f"Tavily returned empty content for {url}", url=url)
-
-        return str(raw_content)
-    except EmptyContentError:
-        try:
-            response = client.extract(urls=[url], extract_depth="advanced")
-            results = response.get("results", [])
-            if not results:
-                raise EmptyContentError(f"Tavily returned no results for {url}", url=url)
-
-            raw_content = results[0].get("raw_content")
-            if not raw_content:
-                raise EmptyContentError(f"Tavily returned empty content for {url}", url=url)
-
-            return str(raw_content)
-        except Exception as e:
-            if isinstance(e, EmptyContentError):
-                raise
-            raise FetchError(f"Tavily fallback failed: {e}", url=url) from e
-    except Exception:
-        # If basic threw an API error, fallback to advanced as well
-        try:
-            response = client.extract(urls=[url], extract_depth="advanced")
-            results = response.get("results", [])
-            if not results:
-                raise EmptyContentError(f"Tavily returned no results for {url}", url=url)
-
-            raw_content = results[0].get("raw_content")
-            if not raw_content:
-                raise EmptyContentError(f"Tavily returned empty content for {url}", url=url)
-
-            return str(raw_content)
-        except Exception as e:
-            raise FetchError(f"Tavily fallback failed: {e}", url=url) from e

mdfetch-0.9.2/tests/integration_log.txt

@@ -1 +0,0 @@
-[{"path":".github","blob_href":"https://github.com/stn1slv/md-fetch/blob/9104266a16d8dbd63c1bb011df63a137f53987b5/.github","start_line":2,"start_column":null,"end_line":2,"end_column":null,"annotation_level":"warning","title":"","message":"Node.js 20 is deprecated. The following actions target Node.js 20 but are being forced to run on Node.js 24: actions/checkout@v4, actions/github-script@v7, astral-sh/setup-uv@v4. For more information see: https://github.blog/changelog/2025-09-19-deprecation-of-node-20-on-github-actions-runners/","raw_details":""},{"path":".github","blob_href":"https://github.com/stn1slv/md-fetch/blob/9104266a16d8dbd63c1bb011df63a137f53987b5/.github","start_line":133,"start_column":null,"end_line":133,"end_column":null,"annotation_level":"failure","title":"","message":"Process completed with exit code 2.","raw_details":""},{"path":".github","blob_href":"https://github.com/stn1slv/md-fetch/blob/9104266a16d8dbd63c1bb011df63a137f53987b5/.github","start_line":22,"start_column":null,"end_line":22,"end_column":null,"annotation_level":"warning","title":"","message":"Failed to restore: Cache service responded with 400","raw_details":""}]