content-core 2.2.0__tar.gz → 2.2.1__tar.gz

{content_core-2.2.0 → content_core-2.2.1}/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "content-core",
-  "version": "2.2.0",
+  "version": "2.2.1",
   "description": "Extract text content from URLs, documents, YouTube videos, Reddit posts, and audio/video files — via CLI (uvx) or MCP.",
   "author": {
     "name": "Luis Novo",

{content_core-2.2.0 → content_core-2.2.1}/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "content-core",
-  "version": "2.2.0",
+  "version": "2.2.1",
   "description": "Extract text content from URLs, documents, YouTube videos, Reddit posts, and audio/video files — via CLI (uvx) or MCP.",
   "skills": "./skills/"
 }

{content_core-2.2.0 → content_core-2.2.1}/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.2.1] - 2026-11-03
+
+### Security
+- URL fetches made directly by content-core (the `simple` engine, the MIME-type probe, remote file downloads and Reddit posts) refuse link-local and cloud-metadata addresses, including on every redirect hop and for hostnames that resolve to them. Private and loopback addresses are still allowed. A refused URL raises `InvalidInputError` (`BlockedAddressError`).
+
 ## [2.2.0] - 2026-10-03
 
 ### Added

{content_core-2.2.0 → content_core-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: content-core
-Version: 2.2.0
+Version: 2.2.1
 Summary: Extract what matters from any media source. Available as Python Library, macOS Service, CLI and MCP Server
 Author-email: LUIS NOVO <lfnovo@gmail.com>
 License-File: LICENSE

{content_core-2.2.0 → content_core-2.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "content-core"
-version = "2.2.0"
+version = "2.2.1"
 description = "Extract what matters from any media source. Available as Python Library, macOS Service, CLI and MCP Server"
 readme = "README.md"
 homepage = "https://github.com/lfnovo/content-core"

content_core-2.2.1/src/content_core/common/http.py

@@ -0,0 +1,131 @@
+"""HTTP sessions for fetching user-supplied URLs.
+
+Callers (Open Notebook, the MCP server) pass URLs they did not choose, and the
+fetch runs from inside their network. ``guarded_session()`` refuses to connect
+to link-local addresses (169.254.0.0/16, fe80::/10, which includes cloud
+metadata services such as 169.254.169.254) and to AWS's IPv6 metadata address,
+on the first request and on every redirect hop:
+
+- hostnames are checked by the connector's resolver, on the addresses the
+  connection will actually use (no re-resolution between check and connect);
+- IP literals skip the resolver, so they are checked before the request and
+  on each redirect ``Location``.
+
+Private and loopback addresses stay allowed: self-hosted deployments
+legitimately fetch from their own network. When the session goes through an
+HTTP(S) proxy (``trust_env``), the target hostname is resolved by the proxy,
+so only IP literals are checked locally.
+"""
+
+import ipaddress
+import socket
+from types import SimpleNamespace
+from typing import Any
+
+import aiohttp
+from aiohttp.abc import AbstractResolver, ResolveResult
+from yarl import URL
+
+from content_core.common.exceptions import InvalidInputError
+
+# AWS IMDSv6 is a Unique Local Address, not link-local.
+_AWS_IMDS_V6 = ipaddress.ip_address("fd00:ec2::254")
+
+IPAddress = ipaddress.IPv4Address | ipaddress.IPv6Address
+
+
+class BlockedAddressError(InvalidInputError):
+    """Raised when a URL, or a redirect it leads to, targets a blocked address."""
+
+
+def is_blocked_address(ip: IPAddress) -> bool:
+    """True for link-local and cloud-metadata addresses."""
+    if ip.is_link_local:
+        return True
+    if isinstance(ip, ipaddress.IPv6Address):
+        mapped = ip.ipv4_mapped
+        if mapped is not None and mapped.is_link_local:
+            return True
+        # Compare without scope id so fd00:ec2::254%eth0 is caught too.
+        if int(ip) == int(_AWS_IMDS_V6):
+            return True
+    return False
+
+
+def _parse_ip(host: str) -> IPAddress | None:
+    try:
+        return ipaddress.ip_address(host.strip("[]").split("%", 1)[0])
+    except ValueError:
+        return None
+
+
+def check_url_host(url: str | URL) -> None:
+    """Raise BlockedAddressError if ``url``'s host is a blocked IP literal."""
+    host = URL(str(url)).host or ""
+    ip = _parse_ip(host)
+    if ip is not None and is_blocked_address(ip):
+        raise BlockedAddressError(
+            f"Refusing to fetch {url}: link-local and cloud-metadata "
+            "addresses are not allowed"
+        )
+
+
+class GuardedResolver(AbstractResolver):
+    """Resolver that drops blocked addresses and fails if none remain."""
+
+    def __init__(self) -> None:
+        self._resolver = aiohttp.DefaultResolver()
+
+    async def resolve(
+        self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET
+    ) -> list[ResolveResult]:
+        results = await self._resolver.resolve(host, port, family)
+        allowed = [
+            r
+            for r in results
+            if not ((ip := _parse_ip(r["host"])) and is_blocked_address(ip))
+        ]
+        if not allowed:
+            raise BlockedAddressError(
+                f"Refusing to fetch from {host}: it resolves to a link-local or "
+                "cloud-metadata address"
+            )
+        return allowed
+
+    async def close(self) -> None:
+        await self._resolver.close()
+
+
+async def _on_request_redirect(
+    session: aiohttp.ClientSession,
+    ctx: SimpleNamespace,
+    params: aiohttp.TraceRequestRedirectParams,
+) -> None:
+    location = params.response.headers.get("Location") or params.response.headers.get(
+        "URI"
+    )
+    if location:
+        check_url_host(params.response.url.join(URL(location)))
+
+
+def guarded_session(**kwargs: Any) -> aiohttp.ClientSession:
+    """A ClientSession that refuses blocked addresses on every hop.
+
+    Use it for any request whose URL comes from the caller. The initial URL's
+    IP literal is checked by ``guarded_request_url``; pass the URL through it
+    before the request.
+    """
+    trace = aiohttp.TraceConfig()
+    trace.on_request_redirect.append(_on_request_redirect)
+    kwargs.setdefault("trust_env", True)
+    return aiohttp.ClientSession(
+        connector=aiohttp.TCPConnector(resolver=GuardedResolver()),
+        trace_configs=[trace],
+        **kwargs,
+    )
+
+
+def guarded_request_url(url: str) -> str:
+    """Check ``url``'s host before the first request; returns it unchanged."""
+    check_url_host(url)
+    return url

{content_core-2.2.0 → content_core-2.2.1}/src/content_core/extraction.py

@@ -5,8 +5,6 @@ import os
 import tempfile
 from urllib.parse import urlparse
 
-import aiohttp
-
 from content_core.common.exceptions import (
     ConfigurationError,
     ContentCoreError,
@@ -15,6 +13,7 @@ from content_core.common.exceptions import (
     UnsupportedTypeException,
 )
 from content_core.common.messages import DOCLING_MISSING_MESSAGE
+from content_core.common.http import guarded_request_url, guarded_session
 from content_core.common.retry import retry_download
 from content_core.config import ContentCoreConfig, get_default_config
 from content_core.logging import logger
@@ -307,8 +306,8 @@ async def _extract_file(
 @retry_download()
 async def _fetch_remote_file(url: str) -> tuple:
     """Download a remote file. Wrapped with retry logic."""
-    async with aiohttp.ClientSession(trust_env=True) as session:
-        async with session.get(url) as resp:
+    async with guarded_session() as session:
+        async with session.get(guarded_request_url(url)) as resp:
             resp.raise_for_status()
             mime = resp.headers.get("content-type", "").split(";", 1)[0]
             content = await resp.read()

{content_core-2.2.0 → content_core-2.2.1}/src/content_core/processors/url/__init__.py

@@ -12,6 +12,7 @@ from content_core.common.exceptions import (
     NetworkError,
     NotFoundError,
 )
+from content_core.common.http import guarded_request_url, guarded_session
 from content_core.common.retry import retry_url_network
 from content_core.common.types import UrlEngine
 from content_core.config import ContentCoreConfig
@@ -35,8 +36,10 @@ from content_core.processors.url.crawl4ai import extract_url_crawl4ai
 @retry_url_network()
 async def _fetch_url_mime_type(url: str) -> str:
     """Internal function to fetch URL MIME type - wrapped with retry logic."""
-    async with aiohttp.ClientSession(trust_env=True) as session:
-        async with session.head(url, timeout=10, allow_redirects=True) as resp:
+    async with guarded_session() as session:
+        async with session.head(
+            guarded_request_url(url), timeout=10, allow_redirects=True
+        ) as resp:
             mime = resp.headers.get("content-type", "").split(";", 1)[0]
             logger.debug(f"MIME type for {url}: {mime}")
             return mime

{content_core-2.2.0 → content_core-2.2.1}/src/content_core/processors/url/bs4.py

@@ -1,7 +1,7 @@
-import aiohttp
 from bs4 import BeautifulSoup
 from readability import Document
 
+from content_core.common.http import guarded_request_url, guarded_session
 from content_core.common.retry import retry_url_network
 from content_core.logging import logger
 
@@ -9,8 +9,8 @@ from content_core.logging import logger
 @retry_url_network()
 async def _fetch_url_html(url: str) -> str:
     """Internal function to fetch URL HTML content - wrapped with retry logic."""
-    async with aiohttp.ClientSession(trust_env=True) as session:
-        async with session.get(url, timeout=10) as response:
+    async with guarded_session() as session:
+        async with session.get(guarded_request_url(url), timeout=10) as response:
             # Raise ClientResponseError so retry logic can inspect status code
             # (5xx and 429 will be retried, 4xx will not)
             response.raise_for_status()

{content_core-2.2.0 → content_core-2.2.1}/src/content_core/processors/url/reddit.py

@@ -4,6 +4,7 @@ from urllib.parse import urlparse, urlunparse
 
 import aiohttp
 
+from content_core.common.http import guarded_request_url, guarded_session
 from content_core.common.retry import retry_url_network
 from content_core.config import ContentCoreConfig
 from content_core.logging import logger
@@ -28,9 +29,9 @@ async def _fetch_reddit_json(url: str) -> dict:
     clean_url = urlunparse((parsed.scheme, parsed.netloc, clean_path, "", "", ""))
     json_url = clean_url + ".json"
 
-    async with aiohttp.ClientSession(trust_env=True) as session:
+    async with guarded_session() as session:
         async with session.get(
-            json_url,
+            guarded_request_url(json_url),
             headers={"User-Agent": "content-core/2.0"},
             timeout=aiohttp.ClientTimeout(total=15),
         ) as resp:

content_core-2.2.1/tests/unit/test_guarded_http.py

@@ -0,0 +1,170 @@
+"""Unit tests for the guarded HTTP session (common/http.py).
+
+A local aiohttp server stands in for the remote site, so no network is needed.
+"""
+
+import ipaddress
+import socket
+
+import pytest
+from aiohttp import web
+
+from content_core.common.exceptions import InvalidInputError
+from content_core.common.http import (
+    BlockedAddressError,
+    GuardedResolver,
+    check_url_host,
+    guarded_request_url,
+    guarded_session,
+    is_blocked_address,
+)
+from content_core.processors.url.bs4 import _fetch_url_html
+
+
+@pytest.fixture
+async def site():
+    """Local server: /ok answers, /redirect?to=<target> redirects to <target>."""
+
+    async def ok(request):
+        return web.Response(text="<html><body>fine</body></html>")
+
+    async def redirect(request):
+        raise web.HTTPFound(request.query["to"])
+
+    app = web.Application()
+    app.router.add_get("/ok", ok)
+    app.router.add_get("/redirect", redirect)
+    runner = web.AppRunner(app)
+    await runner.setup()
+    tcp = web.TCPSite(runner, "127.0.0.1", 0)
+    await tcp.start()
+    port = tcp._server.sockets[0].getsockname()[1]
+    yield f"http://127.0.0.1:{port}"
+    await runner.cleanup()
+
+
+@pytest.mark.parametrize(
+    "address,blocked",
+    [
+        ("169.254.169.254", True),
+        ("169.254.0.1", True),
+        ("fe80::1", True),
+        ("::ffff:169.254.169.254", True),
+        ("fd00:ec2::254", True),
+        ("127.0.0.1", False),
+        ("10.0.0.5", False),
+        ("192.168.1.10", False),
+        ("8.8.8.8", False),
+        ("::1", False),
+    ],
+)
+def test_is_blocked_address(address, blocked):
+    assert is_blocked_address(ipaddress.ip_address(address)) is blocked
+
+
+def test_check_url_host_rejects_metadata_literal():
+    with pytest.raises(BlockedAddressError):
+        check_url_host("http://169.254.169.254/latest/meta-data/")
+    with pytest.raises(BlockedAddressError):
+        check_url_host("http://[fd00:ec2::254]/latest/")
+
+
+def test_check_url_host_allows_hostnames_and_private_ips():
+    check_url_host("https://example.com/page")
+    check_url_host("http://192.168.1.10:11434/")
+
+
+def test_blocked_address_error_is_invalid_input():
+    # Typed as invalid input, so retry logic treats it as permanent.
+    assert issubclass(BlockedAddressError, InvalidInputError)
+
+
+async def test_guarded_session_fetches_allowed_url(site):
+    async with guarded_session() as session:
+        async with session.get(guarded_request_url(f"{site}/ok")) as resp:
+            assert resp.status == 200
+
+
+async def test_guarded_session_follows_allowed_redirect(site):
+    async with guarded_session() as session:
+        async with session.get(f"{site}/redirect", params={"to": "/ok"}) as resp:
+            assert resp.status == 200
+            assert "fine" in await resp.text()
+
+
+async def test_redirect_to_metadata_address_is_refused(site):
+    async with guarded_session() as session:
+        with pytest.raises(BlockedAddressError):
+            async with session.get(
+                f"{site}/redirect",
+                params={"to": "http://169.254.169.254/latest/meta-data/"},
+            ):
+                pass
+
+
+async def test_bs4_fetch_refuses_redirect_to_metadata(site):
+    with pytest.raises(BlockedAddressError):
+        await _fetch_url_html(
+            f"{site}/redirect?to=http://169.254.169.254/latest/meta-data/"
+        )
+
+
+async def test_bs4_fetch_refuses_metadata_literal():
+    with pytest.raises(BlockedAddressError):
+        await _fetch_url_html("http://169.254.169.254/latest/meta-data/")
+
+
+class _FakeResolver:
+    def __init__(self, addresses):
+        self.addresses = addresses
+
+    async def resolve(self, host, port=0, family=socket.AF_INET):
+        return [
+            {
+                "hostname": host,
+                "host": a,
+                "port": port,
+                "family": family,
+                "proto": 0,
+                "flags": 0,
+            }
+            for a in self.addresses
+        ]
+
+    async def close(self):
+        pass
+
+
+async def test_resolver_refuses_host_resolving_only_to_link_local():
+    resolver = GuardedResolver()
+    resolver._resolver = _FakeResolver(["169.254.169.254"])
+    with pytest.raises(BlockedAddressError):
+        await resolver.resolve("metadata.attacker.test", 80)
+
+
+async def test_resolver_drops_link_local_and_keeps_the_rest():
+    resolver = GuardedResolver()
+    resolver._resolver = _FakeResolver(["169.254.169.254", "93.184.216.34"])
+    results = await resolver.resolve("mixed.test", 80)
+    assert [r["host"] for r in results] == ["93.184.216.34"]
+
+
+async def test_redirect_to_hostname_resolving_to_link_local_is_refused(
+    site, monkeypatch
+):
+    real_resolve = GuardedResolver.resolve
+
+    async def resolve(self, host, port=0, family=socket.AF_INET):
+        if host == "metadata.attacker.test":
+            self._resolver = _FakeResolver(["169.254.169.254"])
+        return await real_resolve(self, host, port, family)
+
+    monkeypatch.setattr(GuardedResolver, "resolve", resolve)
+    # No proxy: the hostname must be resolved locally for this check.
+    async with guarded_session(trust_env=False) as session:
+        with pytest.raises(BlockedAddressError):
+            async with session.get(
+                f"{site}/redirect",
+                params={"to": "http://metadata.attacker.test/latest/"},
+            ):
+                pass

{content_core-2.2.0 → content_core-2.2.1}/uv.lock

@@ -940,7 +940,7 @@ wheels = [
 
 [[package]]
 name = "content-core"
-version = "2.2.0"
+version = "2.2.1"
 source = { editable = "." }
 dependencies = [
     { name = "ai-prompter" },