fastfeedparser 0.4.8__tar.gz → 0.5.0__tar.gz

{fastfeedparser-0.4.8/src/fastfeedparser.egg-info → fastfeedparser-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fastfeedparser
-Version: 0.4.8
+Version: 0.5.0
 Summary: High performance RSS, Atom, JSON and RDF feed parser in Python
 Home-page: https://github.com/kagisearch/fastfeedparser
 Author: Vladimir Prelovac

{fastfeedparser-0.4.8 → fastfeedparser-0.5.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = fastfeedparser
-version = 0.4.8
+version = 0.5.0
 author = Vladimir Prelovac
 author_email = vlad@kagi.com
 description = High performance RSS, Atom, JSON and RDF feed parser in Python

{fastfeedparser-0.4.8 → fastfeedparser-0.5.0}/src/fastfeedparser/__init__.py

@@ -1,4 +1,4 @@
 from .main import parse, FastFeedParserDict
 
-__version__ = "0.4.4"
+__version__ = "0.5.0"
 __all__ = ["parse", "FastFeedParserDict"]

{fastfeedparser-0.4.8 → fastfeedparser-0.5.0}/src/fastfeedparser/main.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import datetime
 from email.utils import parsedate_to_datetime
 import gzip
+import html as _html_mod
 import json
 import re
 import zlib
@@ -15,6 +16,7 @@ try:
 except ImportError:
     HAS_BROTLI = False
 from typing import Any, Callable, Optional, TYPE_CHECKING, Literal
+from urllib.parse import urljoin
 from urllib.request import (
     HTTPErrorProcessor,
     HTTPRedirectHandler,
@@ -58,8 +60,8 @@ _RE_UNCLOSED_LINK_BYTES = re.compile(
     br"<link([^>]*[^/])>\s*(?=\n\s*<(?!/link\s*>))", re.MULTILINE
 )
 _RE_FEB29 = re.compile(r"(\d{4})-02-29")
+_RE_HTML_TAGS = re.compile(r"<[^>]+>")
 _RE_WHITESPACE = re.compile(r"\s+")
-_RE_ISO_LIKE = re.compile(r"^\d{4}-\d{2}-\d{2}")
 _RE_ISO_TZ_NO_COLON = re.compile(r"([+-]\d{2})(\d{2})$")
 _RE_ISO_TZ_HOUR_ONLY = re.compile(r"([+-]\d{2})$")
 _RE_ISO_FRACTION = re.compile(r"\.(\d{7,})(?=(?:[+-]\d{2}:?\d{2}|Z|$))", re.IGNORECASE)
@@ -594,7 +596,31 @@ def _raise_for_non_feed_root(
         raise ValueError(
             "Received XML sitemap instead of feed (sitemap is for search engines, not a feed)"
         )
-    raise ValueError(f"Not a valid feed: {root_tag_local} element found - {error_msg[:100]}")
+
+
+_RE_META_REFRESH_URL = re.compile(
+    r'url\s*=\s*["\']?\s*([^"\'>\s]+)', re.IGNORECASE
+)
+
+
+def _extract_meta_refresh_url(content: str | bytes, base_url: str) -> str | None:
+    """Extract redirect URL from an HTML meta-refresh tag."""
+    html_bytes = content.encode("utf-8") if isinstance(content, str) else content
+    try:
+        doc = etree.fromstring(html_bytes, parser=etree.HTMLParser())
+    except Exception:
+        return None
+    if doc is None:
+        return None
+
+    for meta in doc.iter("meta"):
+        if (meta.get("http-equiv") or "").lower() == "refresh":
+            match = _RE_META_REFRESH_URL.search(meta.get("content", ""))
+            if match:
+                url = urljoin(base_url, match.group(1))
+                if url != base_url:
+                    return url
+    return None
 
 
 def _detect_feed_structure(
@@ -704,24 +730,26 @@ def _detect_feed_structure(
     raise ValueError(f"Unknown feed type: {root.tag}")
 
 
-def parse(source: str | bytes) -> FastFeedParserDict:
-    """Parse a feed from a URL or XML content.
+def _should_parse_media_content(root: _Element, xml_content: bytes) -> bool:
+    """Check if feed likely contains Media RSS fields."""
+    ns_values = root.nsmap.values() if root.nsmap else ()
+    for ns_value in ns_values:
+        if not ns_value:
+            continue
+        if "search.yahoo.com/mrss" in ns_value:
+            return True
 
-    Args:
-        source: URL string or XML content string/bytes
+    # Fallback for feeds with undeclared/late namespace usage.
+    return b"search.yahoo.com/mrss" in xml_content or b"<media:" in xml_content
 
-    Returns:
-        FastFeedParserDict containing parsed feed data
 
-    Raises:
-        ValueError: If content is empty or invalid
-        HTTPError: If URL fetch fails
-    """
-    if isinstance(source, str) and source.startswith(("http://", "https://")):
-        xml_content = _fetch_url_content(source)
-    else:
-        xml_content = source
+def _should_parse_enclosures(feed_type: _FeedType, xml_content: bytes) -> bool:
+    """Check if feed likely contains RSS enclosure elements."""
+    return feed_type == "rss" and b"<enclosure" in xml_content
+
 
+def _parse_content(xml_content: str | bytes) -> FastFeedParserDict:
+    """Parse feed content (XML or JSON) that has already been fetched."""
     json_feed = _maybe_parse_json_feed(xml_content)
     if json_feed is not None:
         return json_feed
@@ -734,6 +762,8 @@ def parse(source: str | bytes) -> FastFeedParserDict:
     feed_type, channel, items, atom_namespace = _detect_feed_structure(
         root, xml_content, root_tag_local
     )
+    parse_media_content = _should_parse_media_content(root, xml_content)
+    parse_enclosures = _should_parse_enclosures(feed_type, xml_content)
 
     feed = _parse_feed_info(channel, feed_type, atom_namespace)
 
@@ -741,7 +771,13 @@ def parse(source: str | bytes) -> FastFeedParserDict:
     entries: list[FastFeedParserDict] = []
     feed["entries"] = entries
     for item in items:
-        entry = _parse_feed_entry(item, feed_type, atom_namespace)
+        entry = _parse_feed_entry(
+            item,
+            feed_type,
+            atom_namespace,
+            parse_media_content=parse_media_content,
+            parse_enclosures=parse_enclosures,
+        )
         # Ensure that titles and descriptions are always present
         entry["title"] = entry.get("title", "").strip()
         entry["description"] = entry.get("description", "").strip()
@@ -750,6 +786,39 @@ def parse(source: str | bytes) -> FastFeedParserDict:
     return feed
 
 
+def parse(source: str | bytes) -> FastFeedParserDict:
+    """Parse a feed from a URL or XML content.
+
+    Args:
+        source: URL string or XML content string/bytes
+
+    Returns:
+        FastFeedParserDict containing parsed feed data
+
+    Raises:
+        ValueError: If content is empty or invalid
+        HTTPError: If URL fetch fails
+    """
+    is_url = isinstance(source, str) and source.startswith(("http://", "https://"))
+    if is_url:
+        content = _fetch_url_content(source)
+    else:
+        content = source
+
+    try:
+        return _parse_content(content)
+    except ValueError as e:
+        if not is_url:
+            raise
+        err_msg = str(e)
+        if "HTML" not in err_msg and "not a valid RSS/Atom feed" not in err_msg:
+            raise
+        redirect_url = _extract_meta_refresh_url(content, source)
+        if redirect_url is None:
+            raise
+        return parse(redirect_url)
+
+
 def _parse_feed_info(
     channel: _Element, feed_type: _FeedType, atom_namespace: Optional[str] = None
 ) -> FastFeedParserDict:
@@ -1054,16 +1123,9 @@ def _populate_entry_content(
         content_value = entry["content"][0]["value"]
         if content_value:
             if "<" in content_value:
-                try:
-                    html_content = etree.HTML(content_value)
-                    if html_content is not None:
-                        content_text = html_content.xpath("string()")
-                        if isinstance(content_text, str):
-                            content_value = _RE_WHITESPACE.sub(" ", content_text)
-                except etree.ParserError:
-                    pass
-            else:
-                content_value = _RE_WHITESPACE.sub(" ", content_value)
+                content_value = _RE_HTML_TAGS.sub(" ", content_value[:2048])
+                content_value = _html_mod.unescape(content_value)
+            content_value = _RE_WHITESPACE.sub(" ", content_value).strip()
         entry["description"] = content_value[:512]
 
 
@@ -1154,12 +1216,133 @@ def _parse_enclosures(item: _Element) -> list[dict[str, Any]] | None:
     return enclosures or None
 
 
+def _build_rss_item_text_maps(item: _Element) -> tuple[dict[str, Optional[str]], dict[str, Optional[str]]]:
+    by_local: dict[str, Optional[str]] = {}
+    by_full: dict[str, Optional[str]] = {}
+    for child in item:
+        tag = child.tag
+        if not isinstance(tag, str):
+            continue
+        text_value = child.text.strip() if child.text else None
+        if tag not in by_full:
+            by_full[tag] = text_value
+        local = tag.rsplit("}", 1)[-1].lower()
+        if ":" in local:
+            local = local.split(":", 1)[1]
+        if local not in by_local:
+            by_local[local] = text_value
+    return by_local, by_full
+
+
+def _first_non_empty(mapping: dict[str, Optional[str]], keys: tuple[str, ...]) -> Optional[str]:
+    for key in keys:
+        value = mapping.get(key)
+        if value:
+            return value
+    return None
+
+
+def _parse_rss_feed_entry_fast(
+    item: _Element,
+    atom_ns: str,
+    parse_media_content: bool = True,
+    parse_enclosures: bool = True,
+) -> FastFeedParserDict:
+    text_by_local, text_by_full = _build_rss_item_text_maps(item)
+
+    entry = FastFeedParserDict()
+    atom_id = text_by_full.get(f"{{{atom_ns}}}id")
+    rss_guid = text_by_local.get("guid")
+    rdf_about = item.get("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about")
+    entry_id: Optional[str] = atom_id or rss_guid or rdf_about
+    if entry_id:
+        entry["id"] = entry_id.strip()
+
+    title = text_by_local.get("title")
+    if title:
+        entry["title"] = title
+
+    description = _first_non_empty(text_by_local, ("description", "summary"))
+    if description:
+        entry["description"] = description
+
+    link = text_by_local.get("link")
+    if link:
+        entry["link"] = link
+
+    published_source = _first_non_empty(text_by_local, ("pubdate", "published", "issued", "date"))
+    if published_source:
+        published = _parse_date(published_source)
+        if published:
+            entry["published"] = published
+
+    updated_source = _first_non_empty(text_by_local, ("lastbuilddate", "updated", "modified"))
+    if updated_source:
+        updated = _parse_date(updated_source)
+        if updated:
+            entry["updated"] = updated
+
+    if "published" not in entry and rss_guid:
+        guid_date = _parse_date(rss_guid)
+        if guid_date:
+            entry["published"] = guid_date
+
+    if "updated" in entry and "published" not in entry:
+        entry["published"] = entry["updated"]
+
+    _populate_entry_links(entry, item, atom_ns)
+    if "id" not in entry and "link" in entry:
+        entry["id"] = entry["link"]
+
+    _populate_entry_content(entry, item, "rss", atom_ns)
+
+    if parse_media_content:
+        media_contents = _parse_media_content(item)
+        if media_contents:
+            entry["media_content"] = media_contents
+
+    if parse_enclosures:
+        enclosures = _parse_enclosures(item)
+        if enclosures:
+            entry["enclosures"] = enclosures
+
+    author = _first_non_empty(text_by_local, ("author", "creator"))
+    if not author:
+        atom_author = item.find(f"{{{atom_ns}}}author/{{{atom_ns}}}name")
+        author = atom_author.text.strip() if atom_author is not None and atom_author.text else None
+    if author:
+        entry["author"] = author
+
+    comments = text_by_local.get("comments")
+    if comments:
+        entry["comments"] = comments
+
+    tags = _parse_tags(item, "rss", atom_ns)
+    if tags:
+        entry["tags"] = tags
+
+    return entry
+
+
 def _parse_feed_entry(
-    item: _Element, feed_type: _FeedType, atom_namespace: Optional[str] = None
+    item: _Element,
+    feed_type: _FeedType,
+    atom_namespace: Optional[str] = None,
+    *,
+    parse_media_content: bool = True,
+    parse_enclosures: bool = True,
 ) -> FastFeedParserDict:
     # Use dynamic atom namespace or fallback to default
     atom_ns = atom_namespace or "http://www.w3.org/2005/Atom"
 
+    if feed_type == "rss":
+        return _parse_rss_feed_entry_fast(
+            item,
+            atom_ns,
+            parse_media_content=parse_media_content,
+            parse_enclosures=parse_enclosures,
+        )
+
     # Check if this is Atom 0.3 to use different date field names
     is_atom_03 = atom_ns == "http://purl.org/atom/ns#"
 
@@ -1272,38 +1455,27 @@ def _parse_feed_entry(
 
     _populate_entry_content(entry, item, feed_type, atom_ns)
 
-    media_contents = _parse_media_content(item)
-    if media_contents:
-        entry["media_content"] = media_contents
-
-    enclosures = _parse_enclosures(item)
-    if enclosures:
-        entry["enclosures"] = enclosures
-
-    author = (
-        get_field_value(
-            "author",
-            f"{{{atom_ns}}}author/{{{atom_ns}}}name",
-            "{http://purl.org/dc/elements/1.1/}creator",
-            False,
-        )
-        or get_field_value(
-            "{http://purl.org/dc/elements/1.1/}creator",
-            "{http://purl.org/dc/elements/1.1/}creator",
-            "{http://purl.org/dc/elements/1.1/}creator",
-            False,
-        )
-        or element_get("{http://purl.org/dc/elements/1.1/}creator")
-        or element_get("author")
+    if parse_media_content:
+        media_contents = _parse_media_content(item)
+        if media_contents:
+            entry["media_content"] = media_contents
+
+    if parse_enclosures:
+        enclosures = _parse_enclosures(item)
+        if enclosures:
+            entry["enclosures"] = enclosures
+
+    author = get_field_value(
+        "author",
+        f"{{{atom_ns}}}author/{{{atom_ns}}}name",
+        "{http://purl.org/dc/elements/1.1/}creator",
+        False,
     )
+    if not author:
+        author = element_get("{http://purl.org/dc/elements/1.1/}creator") or element_get("author")
     if author:
         entry["author"] = author
 
-    if feed_type == "rss":
-        comments = element_get("comments")
-        if comments:
-            entry["comments"] = comments
-
     # Parse entry-level tags/categories
     tags = _parse_tags(item, feed_type, atom_ns)
     if tags:
@@ -1456,7 +1628,7 @@ def _normalize_iso_datetime_string(value: str) -> str:
     if cleaned.endswith(("Z", "z")):
         cleaned = cleaned[:-1] + "+00:00"
 
-    if " " in cleaned and "T" not in cleaned[:11] and _RE_ISO_LIKE.match(cleaned):
+    if " " in cleaned and "T" not in cleaned[:11] and len(cleaned) >= 10 and cleaned[4] == "-" and cleaned[0:4].isdigit():
         date_part, rest = cleaned.split(" ", 1)
         if rest and rest[0].isdigit():
             cleaned = f"{date_part}T{rest}"
@@ -1572,18 +1744,20 @@ def _parse_date(date_str: str) -> Optional[str]:
     if not date_str:
         return None
 
-    candidate = _RE_WHITESPACE.sub(" ", date_str.strip())
+    candidate = date_str.strip()
     if not candidate:
         return None
+    if "\n" in candidate or "\r" in candidate or "\t" in candidate or "  " in candidate:
+        candidate = _RE_WHITESPACE.sub(" ", candidate)
 
     # Fix invalid leap year dates (Feb 29 in non-leap years)
     # This handles feeds with incorrect dates like "2023-02-29"
-    year_match = _RE_FEB29.match(candidate)
-    if year_match:
-        year = int(year_match.group(1))
-        if not ((year % 4 == 0 and year % 100 != 0) or (year % 400 == 0)):
-            # Not a leap year, change Feb 29 to Feb 28
-            candidate = candidate.replace(f"{year}-02-29", f"{year}-02-28")
+    if "-02-29" in candidate:
+        year_match = _RE_FEB29.match(candidate)
+        if year_match:
+            year = int(year_match.group(1))
+            if not ((year % 4 == 0 and year % 100 != 0) or (year % 400 == 0)):
+                candidate = candidate.replace(f"{year}-02-29", f"{year}-02-28")
 
     if "24:00" in candidate:
         candidate = candidate.replace("24:00:00", "00:00:00").replace(
@@ -1592,7 +1766,8 @@ def _parse_date(date_str: str) -> Optional[str]:
 
     dt: Optional[datetime.datetime] = None
 
-    if _RE_ISO_LIKE.match(candidate):
+    is_iso_like = len(candidate) >= 10 and candidate[4] == "-" and candidate[0:4].isdigit()
+    if is_iso_like:
         iso_candidate = _normalize_iso_datetime_string(candidate)
         try:
             dt = datetime.datetime.fromisoformat(iso_candidate)

{fastfeedparser-0.4.8 → fastfeedparser-0.5.0/src/fastfeedparser.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fastfeedparser
-Version: 0.4.8
+Version: 0.5.0
 Summary: High performance RSS, Atom, JSON and RDF feed parser in Python
 Home-page: https://github.com/kagisearch/fastfeedparser
 Author: Vladimir Prelovac

fastfeedparser-0.5.0/tests/test_encoding.py

@@ -0,0 +1,53 @@
+from fastfeedparser import parse
+from fastfeedparser.main import _extract_meta_refresh_url
+
+
+def test_parse_str_with_non_utf8_xml_declaration():
+    xml = (
+        '<?xml version="1.0" encoding="iso-8859-1"?>'
+        '<rss version="2.0">'
+        "<channel>"
+        "<title>café</title>"
+        "<item><title>café</title></item>"
+        "</channel>"
+        "</rss>"
+    )
+    feed = parse(xml)
+    assert feed.feed.title == "café"
+    assert feed.entries[0].title == "café"
+
+
+def test_parse_bytes_with_non_utf8_encoding():
+    xml_bytes = (
+        b'<?xml version="1.0" encoding="iso-8859-1"?>'
+        b'<rss version="2.0">'
+        b"<channel>"
+        b"<title>caf\xe9</title>"
+        b"<item><title>caf\xe9</title></item>"
+        b"</channel>"
+        b"</rss>"
+    )
+    feed = parse(xml_bytes)
+    assert feed.feed.title == "café"
+    assert feed.entries[0].title == "café"
+
+
+def test_meta_refresh_extraction():
+    html = '<!doctype html><html><head><meta http-equiv=refresh content="0; url=https://example.com/feed.xml"></head></html>'
+    assert _extract_meta_refresh_url(html, "https://example.com/feed/") == "https://example.com/feed.xml"
+
+
+def test_meta_refresh_relative_url():
+    html = b'<html><head><meta http-equiv="refresh" content="0;url=/index.xml"></head></html>'
+    assert _extract_meta_refresh_url(html, "https://example.com/feed/") == "https://example.com/index.xml"
+
+
+def test_meta_refresh_none_when_missing():
+    html = "<html><head><title>Hello</title></head><body></body></html>"
+    assert _extract_meta_refresh_url(html, "https://example.com/") is None
+
+
+def test_meta_refresh_none_when_same_url():
+    html = '<html><head><meta http-equiv="refresh" content="0; url=https://example.com/"></head></html>'
+    assert _extract_meta_refresh_url(html, "https://example.com/") is None
+

fastfeedparser-0.4.8/tests/test_encoding.py

@@ -1,32 +0,0 @@
-from fastfeedparser import parse
-
-
-def test_parse_str_with_non_utf8_xml_declaration():
-    xml = (
-        '<?xml version="1.0" encoding="iso-8859-1"?>'
-        '<rss version="2.0">'
-        "<channel>"
-        "<title>café</title>"
-        "<item><title>café</title></item>"
-        "</channel>"
-        "</rss>"
-    )
-    feed = parse(xml)
-    assert feed.feed.title == "café"
-    assert feed.entries[0].title == "café"
-
-
-def test_parse_bytes_with_non_utf8_encoding():
-    xml_bytes = (
-        b'<?xml version="1.0" encoding="iso-8859-1"?>'
-        b'<rss version="2.0">'
-        b"<channel>"
-        b"<title>caf\xe9</title>"
-        b"<item><title>caf\xe9</title></item>"
-        b"</channel>"
-        b"</rss>"
-    )
-    feed = parse(xml_bytes)
-    assert feed.feed.title == "café"
-    assert feed.entries[0].title == "café"
-