fastfeedparser 0.6.0__tar.gz → 0.6.1__tar.gz

{fastfeedparser-0.6.0/src/fastfeedparser.egg-info → fastfeedparser-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fastfeedparser
-Version: 0.6.0
+Version: 0.6.1
 Summary: High performance RSS, Atom, JSON and RDF feed parser in Python
 Home-page: https://github.com/kagisearch/fastfeedparser
 Author: Vladimir Prelovac
@@ -122,6 +122,38 @@ Feedparser: 11 entries in 0.339s
 Speedup: 50.1x
 ```
 
+And publish a full report looking like this
+```
+Summary:
+--------------------------------------------------
+Total wall-clock time: 38.70s
+Successfully tested 200/200 feeds
+
+FastFeedParser:
+  Total entries: 6600
+  Total parsing time: 0.46s
+  Average per feed: 0.002s
+  Feeds/sec: 439.0
+
+Feedparser:
+  Total entries: 6555
+  Total parsing time: 12.31s
+  Average per feed: 0.062s
+  Feeds/sec: 16.2
+
+Speedup: FastFeedParser is 27.0x faster
+
+OUTLIERS: Entry Count Mismatches (2 feeds)
+--------------------------------------------------
+  https://dylanharris.org/feed-me.rss
+    FastFeedParser: 35 entries
+    Feedparser: 0 entries
+    Difference: +35
+  https://humanwhocodes.com/feeds/all.json
+    FastFeedParser: 10 entries
+    Feedparser: 0 entries
+    Difference: +10
+```
 
 ## Key Features
 

{fastfeedparser-0.6.0 → fastfeedparser-0.6.1}/README.md

@@ -91,6 +91,38 @@ Feedparser: 11 entries in 0.339s
 Speedup: 50.1x
 ```
 
+And publish a full report looking like this
+```
+Summary:
+--------------------------------------------------
+Total wall-clock time: 38.70s
+Successfully tested 200/200 feeds
+
+FastFeedParser:
+  Total entries: 6600
+  Total parsing time: 0.46s
+  Average per feed: 0.002s
+  Feeds/sec: 439.0
+
+Feedparser:
+  Total entries: 6555
+  Total parsing time: 12.31s
+  Average per feed: 0.062s
+  Feeds/sec: 16.2
+
+Speedup: FastFeedParser is 27.0x faster
+
+OUTLIERS: Entry Count Mismatches (2 feeds)
+--------------------------------------------------
+  https://dylanharris.org/feed-me.rss
+    FastFeedParser: 35 entries
+    Feedparser: 0 entries
+    Difference: +35
+  https://humanwhocodes.com/feeds/all.json
+    FastFeedParser: 10 entries
+    Feedparser: 0 entries
+    Difference: +10
+```
 
 ## Key Features
 

{fastfeedparser-0.6.0 → fastfeedparser-0.6.1}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = fastfeedparser
-version = 0.6.0
+version = 0.6.1
 author = Vladimir Prelovac
 author_email = vlad@kagi.com
 description = High performance RSS, Atom, JSON and RDF feed parser in Python

{fastfeedparser-0.6.0 → fastfeedparser-0.6.1}/src/fastfeedparser/__init__.py

@@ -1,4 +1,4 @@
 from .main import parse, FastFeedParserDict
 
-__version__ = "0.6.0"
+__version__ = "0.6.1"
 __all__ = ["parse", "FastFeedParserDict"]

{fastfeedparser-0.6.0 → fastfeedparser-0.6.1}/src/fastfeedparser/main.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import datetime
 from email.utils import parsedate_to_datetime
-import gzip
 import html as _html_mod
 import json
 import re
@@ -23,7 +22,7 @@ try:
 except ImportError:
     _json_loads = json.loads
 from typing import Any, Callable, Optional, Protocol, TYPE_CHECKING, Literal
-from urllib.parse import urljoin
+from urllib.parse import urljoin, urlsplit
 from urllib.request import (
     HTTPErrorProcessor,
     HTTPRedirectHandler,
@@ -440,7 +439,100 @@ def _parse_json_feed(
     return feed
 
 
+_ALLOWED_URL_SCHEMES = frozenset(("http", "https"))
+
+
+def _is_http_url(url: str) -> bool:
+    """True only for http(s) URLs.
+
+    urlsplit lowercases the scheme and rejects one that is not
+    ``[a-zA-Z][a-zA-Z0-9+.-]*``, so "FILE://" and "+file://" both fail here.
+    urllib.request derives Request.type the same way (lowercased text before
+    the first colon), so a URL that passes this check cannot dispatch to
+    FileHandler, FTPHandler or DataHandler.
+    """
+    return urlsplit(url).scheme in _ALLOWED_URL_SCHEMES
+
+
+class _SchemeRestrictedRedirectHandler(HTTPRedirectHandler):
+    """Refuse 30x redirects that leave http(s).
+
+    urllib's default handler permits redirecting to ftp:// as well, which
+    would let a server reach a non-http scheme through the same fetch.
+    """
+
+    def redirect_request(
+        self,
+        req: Any,
+        fp: Any,
+        code: int,
+        msg: str,
+        headers: Any,
+        newurl: str,
+    ) -> Any:
+        if not _is_http_url(newurl):
+            raise ValueError(f"refusing redirect to non-http(s) URL: {newurl[:100]}")
+        return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+# Cap on both the bytes read off the wire and the bytes a compressed response
+# is allowed to expand into. A ~1.7KB brotli body can otherwise inflate to 1GB.
+_MAX_CONTENT_BYTES = 32 * 1024 * 1024
+_BROTLI_INPUT_CHUNK = 64 * 1024
+
+
+def _inflate_bounded(data: bytes, wbits: int, limit: int) -> bytes:
+    """Inflate `data`, stopping once more than `limit` bytes are produced.
+
+    Returns up to limit + 1 bytes so the caller can detect the overflow.
+    `wbits` selects the container: 16 + MAX_WBITS for gzip, -MAX_WBITS for raw
+    deflate. gzip bodies may concatenate members, which gzip.decompress joined,
+    so a finished stream with trailing bytes is resumed rather than truncated.
+    """
+    parts: list[bytes] = []
+    produced = 0
+    while True:
+        # max_length of 0 means "unlimited" to zlib; the produced > limit break
+        # below keeps limit + 1 - produced at 1 or more.
+        decompressor = zlib.decompressobj(wbits)
+        parts.append(decompressor.decompress(data, limit + 1 - produced))
+        produced += len(parts[-1])
+        if produced > limit:
+            break
+        if not decompressor.eof:
+            # Under the limit with the stream unfinished means it ended early.
+            # The one-shot decompressors raised on truncated input (gzip with
+            # EOFError, zlib with zlib.error); keep failing rather than
+            # returning a partial body as if it were the whole feed.
+            raise zlib.error("incomplete or truncated stream")
+        data = decompressor.unused_data
+        if not data:
+            break
+    return b"".join(parts)
+
+
+def _brotli_decompress_bounded(data: bytes, limit: int) -> bytes:
+    """Brotli-decompress `data`, producing at most limit + 1 bytes."""
+    try:
+        return brotli.Decompressor().process(data, output_buffer_limit=limit + 1)
+    except TypeError:
+        # brotli < 1.2.0 has no output cap, so feed the input in slices and
+        # check the running total instead.
+        pass
+    decompressor = brotli.Decompressor()
+    parts: list[bytes] = []
+    produced = 0
+    for start in range(0, len(data), _BROTLI_INPUT_CHUNK):
+        parts.append(decompressor.process(data[start : start + _BROTLI_INPUT_CHUNK]))
+        produced += len(parts[-1])
+        if produced > limit:
+            break
+    return b"".join(parts)
+
+
 def _fetch_url_content(url: str) -> str | bytes:
+    if not _is_http_url(url):
+        raise ValueError(f"refusing to fetch non-http(s) URL: {url[:100]}")
     accept_encoding = "gzip, deflate, br" if HAS_BROTLI else "gzip, deflate"
     request = Request(
         url,
@@ -450,20 +542,25 @@ def _fetch_url_content(url: str) -> str | bytes:
             "User-Agent": "fastfeedparser (+https://github.com/kagisearch/fastfeedparser)",
         },
     )
-    opener = build_opener(HTTPRedirectHandler(), HTTPErrorProcessor())
+    opener = build_opener(_SchemeRestrictedRedirectHandler(), HTTPErrorProcessor())
     with opener.open(request, timeout=30) as response:
-        content: bytes = response.read()
+        limit = _MAX_CONTENT_BYTES
+        content: bytes = response.read(limit + 1)
+        if len(content) > limit:
+            raise ValueError(f"response body exceeds {limit} bytes")
         content_encoding = response.headers.get("Content-Encoding")
         if content_encoding == "gzip":
-            content = gzip.decompress(content)
+            content = _inflate_bounded(content, 16 + zlib.MAX_WBITS, limit)
         elif content_encoding == "deflate":
-            content = zlib.decompress(content, -zlib.MAX_WBITS)
+            content = _inflate_bounded(content, -zlib.MAX_WBITS, limit)
         elif content_encoding == "br":
             if not HAS_BROTLI:
                 raise ValueError(
                     "Received brotli-compressed response but 'brotli' is not installed"
                 )
-            content = brotli.decompress(content)
+            content = _brotli_decompress_bounded(content, limit)
+        if len(content) > limit:
+            raise ValueError(f"decompressed response exceeds {limit} bytes")
         content_charset = response.headers.get_content_charset()
         if content_charset:
             try:
@@ -653,7 +750,10 @@ def _extract_meta_refresh_url(content: str | bytes, base_url: str) -> str | None
             match = _RE_META_REFRESH_URL.search(meta.get("content", ""))
             if match:
                 url = urljoin(base_url, match.group(1))
-                if url != base_url:
+                # urljoin keeps an absolute reference's own scheme, so an
+                # attacker page can name file:// or ftp:// here. Apply the
+                # same allow-list parse() applies to its source.
+                if url != base_url and _is_http_url(url):
                     return url
     return None
 

{fastfeedparser-0.6.0 → fastfeedparser-0.6.1/src/fastfeedparser.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fastfeedparser
-Version: 0.6.0
+Version: 0.6.1
 Summary: High performance RSS, Atom, JSON and RDF feed parser in Python
 Home-page: https://github.com/kagisearch/fastfeedparser
 Author: Vladimir Prelovac
@@ -122,6 +122,38 @@ Feedparser: 11 entries in 0.339s
 Speedup: 50.1x
 ```
 
+And publish a full report looking like this
+```
+Summary:
+--------------------------------------------------
+Total wall-clock time: 38.70s
+Successfully tested 200/200 feeds
+
+FastFeedParser:
+  Total entries: 6600
+  Total parsing time: 0.46s
+  Average per feed: 0.002s
+  Feeds/sec: 439.0
+
+Feedparser:
+  Total entries: 6555
+  Total parsing time: 12.31s
+  Average per feed: 0.062s
+  Feeds/sec: 16.2
+
+Speedup: FastFeedParser is 27.0x faster
+
+OUTLIERS: Entry Count Mismatches (2 feeds)
+--------------------------------------------------
+  https://dylanharris.org/feed-me.rss
+    FastFeedParser: 35 entries
+    Feedparser: 0 entries
+    Difference: +35
+  https://humanwhocodes.com/feeds/all.json
+    FastFeedParser: 10 entries
+    Feedparser: 0 entries
+    Difference: +10
+```
 
 ## Key Features
 

{fastfeedparser-0.6.0 → fastfeedparser-0.6.1}/src/fastfeedparser.egg-info/SOURCES.txt

@@ -9,6 +9,8 @@ src/fastfeedparser.egg-info/SOURCES.txt
 src/fastfeedparser.egg-info/dependency_links.txt
 src/fastfeedparser.egg-info/requires.txt
 src/fastfeedparser.egg-info/top_level.txt
+tests/test_decompression_limits.py
 tests/test_encoding.py
 tests/test_feed_image.py
-tests/test_integration.py
+tests/test_integration.py
+tests/test_url_scheme.py

fastfeedparser-0.6.1/tests/test_decompression_limits.py

@@ -0,0 +1,172 @@
+"""Bounded-decompression tests for _fetch_url_content.
+
+A compressed response body must not be allowed to expand without limit:
+~1.7KB of brotli inflates to 1GB, which is a cheap remote memory-exhaustion.
+"""
+
+import gzip
+import tracemalloc
+import zlib
+
+import pytest
+
+import fastfeedparser.main as main
+from fastfeedparser.main import HAS_BROTLI, _fetch_url_content, _inflate_bounded
+
+FEED = (
+    b'<?xml version="1.0"?><rss version="2.0"><channel><title>ok</title>'
+    b"<item><title>one</title></item></channel></rss>"
+)
+
+
+class _FakeHeaders(dict):
+    def get_content_charset(self):
+        return self.get("charset")
+
+
+class _FakeResponse:
+    def __init__(self, body: bytes, headers: dict):
+        self._body = body
+        self.headers = _FakeHeaders(headers)
+
+    def read(self, amt=None):
+        return self._body if amt is None else self._body[:amt]
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        return False
+
+
+class _FakeOpener:
+    def __init__(self, response):
+        self._response = response
+
+    def open(self, request, timeout=None):
+        return self._response
+
+
+def _serve(monkeypatch, body: bytes, headers: dict):
+    """Point _fetch_url_content at a canned response instead of the network."""
+    monkeypatch.setattr(
+        main, "build_opener", lambda *a, **kw: _FakeOpener(_FakeResponse(body, headers))
+    )
+
+
+def _raw_deflate(data: bytes) -> bytes:
+    compressor = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS)
+    return compressor.compress(data) + compressor.flush()
+
+
+def test_gzip_bomb_is_refused(monkeypatch):
+    monkeypatch.setattr(main, "_MAX_CONTENT_BYTES", 1024 * 1024)
+    _serve(monkeypatch, gzip.compress(b"\0" * (64 * 1024 * 1024)), {"Content-Encoding": "gzip"})
+
+    with pytest.raises(ValueError, match="decompressed response exceeds"):
+        _fetch_url_content("https://attacker.example/feed")
+
+
+def test_deflate_bomb_is_refused(monkeypatch):
+    monkeypatch.setattr(main, "_MAX_CONTENT_BYTES", 1024 * 1024)
+    _serve(monkeypatch, _raw_deflate(b"\0" * (64 * 1024 * 1024)), {"Content-Encoding": "deflate"})
+
+    with pytest.raises(ValueError, match="decompressed response exceeds"):
+        _fetch_url_content("https://attacker.example/feed")
+
+
+@pytest.mark.skipif(not HAS_BROTLI, reason="brotli not installed")
+def test_brotli_bomb_is_refused(monkeypatch):
+    import brotli
+
+    monkeypatch.setattr(main, "_MAX_CONTENT_BYTES", 1024 * 1024)
+    _serve(
+        monkeypatch,
+        brotli.compress(b"\0" * (128 * 1024 * 1024), quality=5),
+        {"Content-Encoding": "br"},
+    )
+
+    with pytest.raises(ValueError, match="decompressed response exceeds"):
+        _fetch_url_content("https://attacker.example/feed")
+
+
+@pytest.mark.skipif(not HAS_BROTLI, reason="brotli not installed")
+def test_brotli_bomb_does_not_materialize_in_memory(monkeypatch):
+    """The point of the cap: refusing must not require inflating the bomb first."""
+    import brotli
+
+    cap = 1024 * 1024
+    monkeypatch.setattr(main, "_MAX_CONTENT_BYTES", cap)
+    bomb = brotli.compress(b"\0" * (512 * 1024 * 1024), quality=5)
+    _serve(monkeypatch, bomb, {"Content-Encoding": "br"})
+
+    tracemalloc.start()
+    try:
+        with pytest.raises(ValueError, match="decompressed response exceeds"):
+            _fetch_url_content("https://attacker.example/feed")
+        _, peak = tracemalloc.get_traced_memory()
+    finally:
+        tracemalloc.stop()
+
+    # 512MB bomb, 1MB cap: peak must stay near the cap, not near the payload.
+    assert peak < 16 * 1024 * 1024, f"peak allocation was {peak} bytes"
+
+
+def test_uncompressed_body_over_limit_is_refused(monkeypatch):
+    monkeypatch.setattr(main, "_MAX_CONTENT_BYTES", 4096)
+    _serve(monkeypatch, b"x" * 8192, {})
+
+    with pytest.raises(ValueError, match="response body exceeds"):
+        _fetch_url_content("https://attacker.example/feed")
+
+
+def test_body_exactly_at_limit_is_allowed(monkeypatch):
+    monkeypatch.setattr(main, "_MAX_CONTENT_BYTES", 4096)
+    _serve(monkeypatch, b"x" * 4096, {})
+
+    assert _fetch_url_content("https://example.com/feed") == b"x" * 4096
+
+
+@pytest.mark.parametrize("encoding", ["gzip", "deflate", "br", None])
+def test_normal_feed_still_round_trips(monkeypatch, encoding):
+    if encoding == "gzip":
+        body = gzip.compress(FEED)
+    elif encoding == "deflate":
+        body = _raw_deflate(FEED)
+    elif encoding == "br":
+        if not HAS_BROTLI:
+            pytest.skip("brotli not installed")
+        import brotli
+
+        body = brotli.compress(FEED)
+    else:
+        body = FEED
+
+    _serve(monkeypatch, body, {"Content-Encoding": encoding} if encoding else {})
+    assert _fetch_url_content("https://example.com/feed") == FEED
+
+
+def test_multi_member_gzip_is_not_truncated(monkeypatch):
+    """gzip.decompress joined concatenated members; the bounded path must too."""
+    _serve(
+        monkeypatch,
+        gzip.compress(b"<rss>one</rss>") + gzip.compress(b"<rss>two</rss>"),
+        {"Content-Encoding": "gzip"},
+    )
+    assert _fetch_url_content("https://example.com/feed") == b"<rss>one</rss><rss>two</rss>"
+
+
+def test_truncated_gzip_is_not_accepted_as_partial_body(monkeypatch):
+    """A cut-off stream must fail, not return a truncated feed as if complete."""
+    _serve(monkeypatch, gzip.compress(FEED)[:-8], {"Content-Encoding": "gzip"})
+
+    # gzip.decompress raised EOFError here; the bounded path raises zlib.error.
+    with pytest.raises((zlib.error, EOFError)):
+        _fetch_url_content("https://example.com/feed")
+
+
+def test_inflate_bounded_returns_one_byte_over_limit():
+    """The overflow signal the caller checks: limit+1 bytes, never a silent cut."""
+    data = gzip.compress(b"\0" * 5000)
+    assert len(_inflate_bounded(data, 16 + zlib.MAX_WBITS, 1000)) == 1001
+    assert len(_inflate_bounded(data, 16 + zlib.MAX_WBITS, 5000)) == 5000

fastfeedparser-0.6.1/tests/test_url_scheme.py

@@ -0,0 +1,108 @@
+"""Regression tests for GHSA-xwxr-vfq4-mq54.
+
+A meta-refresh tag in an attacker-controlled HTML response must not be able
+to steer the follow-up fetch at a non-http(s) scheme (file://, ftp://, data:).
+"""
+
+import pytest
+
+import fastfeedparser.main as main
+from fastfeedparser import parse
+from fastfeedparser.main import (
+    _extract_meta_refresh_url,
+    _fetch_url_content,
+    _SchemeRestrictedRedirectHandler,
+)
+
+
+def _meta_refresh_html(target: str) -> str:
+    return (
+        "<!doctype html><html><head>"
+        f'<meta http-equiv="refresh" content="0; url={target}">'
+        "</head><body>not a feed</body></html>"
+    )
+
+
+@pytest.mark.parametrize(
+    "target",
+    [
+        "file:///etc/hosts",
+        "FILE:///etc/hosts",
+        "ftp://attacker.example/payload",
+        "data:text/html,<html></html>",
+        "jar:file:///etc/hosts!/",
+    ],
+)
+def test_meta_refresh_rejects_non_http_scheme(target):
+    html = _meta_refresh_html(target)
+    assert _extract_meta_refresh_url(html, "http://attacker.example/x") is None
+
+
+def test_meta_refresh_still_follows_http_targets():
+    html = _meta_refresh_html("https://example.com/feed.xml")
+    assert (
+        _extract_meta_refresh_url(html, "http://attacker.example/x")
+        == "https://example.com/feed.xml"
+    )
+
+
+def test_meta_refresh_still_follows_relative_targets():
+    html = _meta_refresh_html("/index.xml")
+    assert (
+        _extract_meta_refresh_url(html, "https://example.com/feed/")
+        == "https://example.com/index.xml"
+    )
+
+
+def test_fetch_url_content_refuses_file_scheme(tmp_path):
+    secret = tmp_path / "secret.txt"
+    secret.write_text("SENTINEL-do-not-leak")
+
+    with pytest.raises(ValueError, match="non-http"):
+        _fetch_url_content(secret.as_uri())
+
+
+@pytest.mark.parametrize(
+    "url",
+    ["ftp://attacker.example/payload", "data:text/plain,hello", "/etc/hosts"],
+)
+def test_fetch_url_content_refuses_other_non_http_schemes(url):
+    with pytest.raises(ValueError, match="non-http"):
+        _fetch_url_content(url)
+
+
+def test_redirect_handler_refuses_non_http_location():
+    handler = _SchemeRestrictedRedirectHandler()
+    with pytest.raises(ValueError, match="non-http"):
+        handler.redirect_request(
+            None, None, 302, "Found", {}, "ftp://attacker.example/payload"
+        )
+
+
+def test_parse_does_not_leak_local_file_via_meta_refresh(tmp_path, monkeypatch):
+    """End-to-end PoC from the advisory: the second fetch must not read the file."""
+    secret = tmp_path / "secret.txt"
+    secret.write_text("SENTINEL-do-not-leak")
+
+    real_fetch = main._fetch_url_content
+    calls = []
+
+    def fake_fetch(url: str):
+        calls.append(url)
+        if len(calls) == 1:
+            # Stands in for the attacker's web server, not for the library.
+            return _meta_refresh_html(secret.as_uri())
+        return real_fetch(url)
+
+    monkeypatch.setattr(main, "_fetch_url_content", fake_fetch)
+
+    with pytest.raises(ValueError) as excinfo:
+        parse("http://attacker.example/feed")
+
+    assert calls == ["http://attacker.example/feed"]
+    chain = []
+    exc = excinfo.value
+    while exc is not None:
+        chain.append(str(exc))
+        exc = exc.__cause__ or exc.__context__
+    assert "SENTINEL-do-not-leak" not in "\n".join(chain)