fastfeedparser 0.5.0__tar.gz → 0.5.2__tar.gz

{fastfeedparser-0.5.0/src/fastfeedparser.egg-info → fastfeedparser-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fastfeedparser
-Version: 0.5.0
+Version: 0.5.2
 Summary: High performance RSS, Atom, JSON and RDF feed parser in Python
 Home-page: https://github.com/kagisearch/fastfeedparser
 Author: Vladimir Prelovac

{fastfeedparser-0.5.0 → fastfeedparser-0.5.2}/pyproject.toml

@@ -2,3 +2,6 @@
 requires = ["setuptools~=67.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+

{fastfeedparser-0.5.0 → fastfeedparser-0.5.2}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = fastfeedparser
-version = 0.5.0
+version = 0.5.2
 author = Vladimir Prelovac
 author_email = vlad@kagi.com
 description = High performance RSS, Atom, JSON and RDF feed parser in Python

{fastfeedparser-0.5.0 → fastfeedparser-0.5.2}/src/fastfeedparser/__init__.py

@@ -1,4 +1,4 @@
 from .main import parse, FastFeedParserDict
 
-__version__ = "0.5.0"
+__version__ = "0.5.1"
 __all__ = ["parse", "FastFeedParserDict"]

{fastfeedparser-0.5.0 → fastfeedparser-0.5.2}/src/fastfeedparser/main.py

@@ -41,21 +41,12 @@ _RE_XML_DECL_ENCODING = re.compile(
 _RE_XML_DECL_ENCODING_BYTES = re.compile(
     br'(<\?xml[^>]*encoding=["\'])([^"\']+)(["\'][^>]*\?>)', re.IGNORECASE
 )
-_RE_DOUBLE_XML_DECL = re.compile(r"<\?xml\?xml\s+", re.IGNORECASE)
 _RE_DOUBLE_XML_DECL_BYTES = re.compile(br"<\?xml\?xml\s+", re.IGNORECASE)
-_RE_DOUBLE_CLOSE = re.compile(r"\?\?>\s*")
 _RE_DOUBLE_CLOSE_BYTES = re.compile(br"\?\?>\s*")
-_RE_UNQUOTED_ATTR = re.compile(r'(\s+[\w:]+)=([^\s>"\']+)')
 _RE_UNQUOTED_ATTR_BYTES = re.compile(br'(\s+[\w:]+)=([^\s>"\']+)')
-_RE_UTF16_ENCODING = re.compile(
-    r'(<\?xml[^>]*encoding=["\'])utf-16(-le|-be)?(["\'][^>]*\?>)', re.IGNORECASE
-)
 _RE_UTF16_ENCODING_BYTES = re.compile(
     br'(<\?xml[^>]*encoding=["\'])utf-16(-le|-be)?(["\'][^>]*\?>)', re.IGNORECASE
 )
-_RE_UNCLOSED_LINK = re.compile(
-    r"<link([^>]*[^/])>\s*(?=\n\s*<(?!/link\s*>))", re.MULTILINE
-)
 _RE_UNCLOSED_LINK_BYTES = re.compile(
     br"<link([^>]*[^/])>\s*(?=\n\s*<(?!/link\s*>))", re.MULTILINE
 )
@@ -112,36 +103,6 @@ def _ensure_utf8_xml_declaration(content: str) -> str:
     return _RE_XML_DECL_ENCODING.sub(r"\1utf-8\3", content, count=1)
 
 
-def _clean_feed_text(content: str) -> str:
-    """Clean feed text by extracting the XML document (if it's embedded in junk)."""
-    stripped_content = content.lstrip()
-    stripped_lower = stripped_content[:2000].lower()
-    if stripped_lower.startswith(("<?xml", "<rss", "<feed", "<rdf")):
-        return stripped_content
-
-    if stripped_lower.startswith("<!doctype html") or stripped_lower.startswith("<html"):
-        raise ValueError("Content appears to be HTML, not a valid RSS/Atom feed")
-
-    xml_start_patterns = (
-        "<?xml",
-        "<rss",
-        "<feed",
-        "<rdf:rdf",
-        "<?xml-stylesheet",
-    )
-
-    content_lines = content.splitlines()
-    for i, line in enumerate(content_lines):
-        line_stripped = line.strip().lower()
-        if any(line_stripped.startswith(pattern) for pattern in xml_start_patterns):
-            return "\n".join(content_lines[i:])
-
-    if "<script>" in stripped_lower or "<body>" in stripped_lower:
-        raise ValueError("Content appears to be HTML, not a valid RSS/Atom feed")
-
-    return content
-
-
 def _clean_feed_bytes(content: bytes) -> bytes:
     """Clean feed bytes by extracting the XML document (if it's embedded in junk)."""
     stripped_content = content.lstrip()
@@ -224,60 +185,9 @@ def _prepare_xml_bytes(xml_content: str | bytes) -> bytes:
             cleaned = _fix_malformed_xml_bytes(cleaned, actual_encoding=actual_encoding)
         return cleaned
 
-    cleaned_text = _clean_feed_text(xml_content)
-    if not cleaned_text.strip():
-        raise ValueError("Empty content")
-
-    needs_fixing = (
-        "?xml?xml" in cleaned_text[:200]
-        or "??>" in cleaned_text[:200]
-        or (
-            "rss:" in cleaned_text[:500] and "xmlns:rss" not in cleaned_text[:1000]
-        )
-        or ("utf-16" in cleaned_text[:200].lower())
-    )
-    if needs_fixing:
-        cleaned_text = _fix_malformed_xml(cleaned_text, actual_encoding="utf-8")
-
-    cleaned_text = _ensure_utf8_xml_declaration(cleaned_text)
-    return cleaned_text.encode("utf-8", errors="replace")
-
-
-def _fix_malformed_xml(content: str, actual_encoding: str = "utf-8") -> str:
-    """Fix common malformed XML issues in feeds.
-
-    Some feeds have malformed XML like unclosed link tags or other issues
-    that can be automatically corrected.
-
-    Args:
-        content: The XML content as a string
-        actual_encoding: The actual encoding used (default: utf-8)
-    """
-    # Fix double XML declarations like "<?xml?xml version="1.0"?>"
-    # This is found in dylanharris.org feed
-    content = _RE_DOUBLE_XML_DECL.sub(r"<?xml ", content)
-
-    # Fix double closing ?> in XML declaration like "??>>"
-    content = _RE_DOUBLE_CLOSE.sub(r"?>", content)
-
-    # Fix malformed attribute syntax like rss:version=2.0 (missing quotes)
-    # This is found in dylanharris.org feed
-    content = _RE_UNQUOTED_ATTR.sub(r'\1="\2"', content)
-
-    # Update encoding in XML declaration to match actual encoding
-    # This handles cases where content was transcoded from UTF-16 to UTF-8
-    if actual_encoding.lower() != "utf-16":
-        content = _RE_UTF16_ENCODING.sub(rf"\1{actual_encoding}\3", content)
-
-    # Fix unclosed link tags - common in Atom feeds
-    # Pattern: <link ...> followed by whitespace and another tag (not </link>)
-    # should be <link .../>
-    # Only fix link tags that are clearly malformed:
-    # - End with > instead of />
-    # - Are followed by whitespace and another tag (not a closing </link>)
-    content = _RE_UNCLOSED_LINK.sub(r"<link\1/>", content)
-
-    return content
+    # Str input: fix encoding declaration, encode to bytes, then use bytes path.
+    xml_content = _ensure_utf8_xml_declaration(xml_content)
+    return _prepare_xml_bytes(xml_content.encode("utf-8", errors="replace"))
 
 
 def _parse_json_feed(json_data: dict) -> FastFeedParserDict:
@@ -556,46 +466,31 @@ def _extract_error_message(root: _Element, raw_bytes: Optional[bytes] = None) ->
     return error_msg
 
 
+_NON_FEED_MESSAGES: dict[str, str] = {
+    "html": "Received HTML page instead of feed",
+    "div": "Received HTML fragment instead of feed",
+    "body": "Received HTML fragment instead of feed",
+    "br": "Received HTML fragment instead of feed",
+    "status": "Feed server returned status message",
+    "error": "Feed server returned error",
+    "opml": "Received OPML document instead of feed (OPML is an outline format, not a feed)",
+    "urlset": "Received XML sitemap instead of feed (sitemap is for search engines, not a feed)",
+    "sitemapindex": "Received XML sitemap instead of feed (sitemap is for search engines, not a feed)",
+}
+
+
 def _raise_for_non_feed_root(
     root: _Element, root_tag_local: str, raw_bytes: Optional[bytes] = None
 ) -> None:
-    non_feed_tags = {
-        "status", "error", "html", "opml", "br", "div", "body",
-        "urlset", "sitemapindex",
-    }
-    if root_tag_local not in non_feed_tags:
+    base_msg = _NON_FEED_MESSAGES.get(root_tag_local)
+    if base_msg is None:
         return
 
     error_msg = _extract_error_message(root, raw_bytes).strip()[:300] or "No error message"
 
-    if root_tag_local == "html":
-        if error_msg != "No error message" and len(error_msg) > 10:
-            raise ValueError(f"Received HTML page instead of feed: {error_msg[:150]}")
-        raise ValueError(
-            "Received HTML page instead of feed (possible redirect, 404, or server error)"
-        )
-    if root_tag_local in {"div", "body"}:
-        if error_msg != "No error message" and len(error_msg) > 10:
-            raise ValueError(f"Received HTML fragment instead of feed: {error_msg[:150]}")
-        raise ValueError("Received HTML fragment instead of feed")
-    if root_tag_local == "br":
-        if error_msg != "No error message" and len(error_msg) > 10:
-            raise ValueError(f"Received HTML error instead of feed: {error_msg[:150]}")
-        raise ValueError("Received HTML fragment instead of feed")
-    if root_tag_local == "status":
-        raise ValueError(f"Feed server returned status message: {error_msg}")
-    if root_tag_local == "error":
-        if error_msg != "No error message":
-            raise ValueError(f"Feed server returned error: {error_msg}")
-        raise ValueError("Feed server returned error (no details provided)")
-    if root_tag_local == "opml":
-        raise ValueError(
-            "Received OPML document instead of feed (OPML is an outline format, not a feed)"
-        )
-    if root_tag_local in {"urlset", "sitemapindex"}:
-        raise ValueError(
-            "Received XML sitemap instead of feed (sitemap is for search engines, not a feed)"
-        )
+    if error_msg != "No error message" and len(error_msg) > 10:
+        raise ValueError(f"{base_msg}: {error_msg[:150]}")
+    raise ValueError(base_msg)
 
 
 _RE_META_REFRESH_URL = re.compile(
@@ -730,24 +625,6 @@ def _detect_feed_structure(
     raise ValueError(f"Unknown feed type: {root.tag}")
 
 
-def _should_parse_media_content(root: _Element, xml_content: bytes) -> bool:
-    """Check if feed likely contains Media RSS fields."""
-    ns_values = root.nsmap.values() if root.nsmap else ()
-    for ns_value in ns_values:
-        if not ns_value:
-            continue
-        if "search.yahoo.com/mrss" in ns_value:
-            return True
-
-    # Fallback for feeds with undeclared/late namespace usage.
-    return b"search.yahoo.com/mrss" in xml_content or b"<media:" in xml_content
-
-
-def _should_parse_enclosures(feed_type: _FeedType, xml_content: bytes) -> bool:
-    """Check if feed likely contains RSS enclosure elements."""
-    return feed_type == "rss" and b"<enclosure" in xml_content
-
-
 def _parse_content(xml_content: str | bytes) -> FastFeedParserDict:
     """Parse feed content (XML or JSON) that has already been fetched."""
     json_feed = _maybe_parse_json_feed(xml_content)
@@ -762,8 +639,6 @@ def _parse_content(xml_content: str | bytes) -> FastFeedParserDict:
     feed_type, channel, items, atom_namespace = _detect_feed_structure(
         root, xml_content, root_tag_local
     )
-    parse_media_content = _should_parse_media_content(root, xml_content)
-    parse_enclosures = _should_parse_enclosures(feed_type, xml_content)
 
     feed = _parse_feed_info(channel, feed_type, atom_namespace)
 
@@ -775,8 +650,6 @@ def _parse_content(xml_content: str | bytes) -> FastFeedParserDict:
             item,
             feed_type,
             atom_namespace,
-            parse_media_content=parse_media_content,
-            parse_enclosures=parse_enclosures,
         )
         # Ensure that titles and descriptions are always present
         entry["title"] = entry.get("title", "").strip()
@@ -1223,7 +1096,7 @@ def _build_rss_item_text_maps(item: _Element) -> tuple[dict[str, Optional[str]],
         tag = child.tag
         if not isinstance(tag, str):
             continue
-        text_value = child.text.strip() if child.text else None
+        text_value = child.text or None
         if tag not in by_full:
             by_full[tag] = text_value
         local = tag.rsplit("}", 1)[-1].lower()
@@ -1245,8 +1118,6 @@ def _first_non_empty(mapping: dict[str, Optional[str]], keys: tuple[str, ...]) -
 def _parse_rss_feed_entry_fast(
     item: _Element,
     atom_ns: str,
-    parse_media_content: bool = True,
-    parse_enclosures: bool = True,
 ) -> FastFeedParserDict:
     text_by_local, text_by_full = _build_rss_item_text_maps(item)
 
@@ -1268,7 +1139,7 @@ def _parse_rss_feed_entry_fast(
 
     link = text_by_local.get("link")
     if link:
-        entry["link"] = link
+        entry["link"] = link.strip()
 
     published_source = _first_non_empty(text_by_local, ("pubdate", "published", "issued", "date"))
     if published_source:
@@ -1282,7 +1153,7 @@ def _parse_rss_feed_entry_fast(
         if updated:
             entry["updated"] = updated
 
-    if "published" not in entry and rss_guid:
+    if "published" not in entry and rss_guid and not rss_guid.startswith(("http://", "https://")):
         guid_date = _parse_date(rss_guid)
         if guid_date:
             entry["published"] = guid_date
@@ -1296,26 +1167,24 @@ def _parse_rss_feed_entry_fast(
 
     _populate_entry_content(entry, item, "rss", atom_ns)
 
-    if parse_media_content:
-        media_contents = _parse_media_content(item)
-        if media_contents:
-            entry["media_content"] = media_contents
+    media_contents = _parse_media_content(item)
+    if media_contents:
+        entry["media_content"] = media_contents
 
-    if parse_enclosures:
-        enclosures = _parse_enclosures(item)
-        if enclosures:
-            entry["enclosures"] = enclosures
+    enclosures = _parse_enclosures(item)
+    if enclosures:
+        entry["enclosures"] = enclosures
 
     author = _first_non_empty(text_by_local, ("author", "creator"))
     if not author:
         atom_author = item.find(f"{{{atom_ns}}}author/{{{atom_ns}}}name")
         author = atom_author.text.strip() if atom_author is not None and atom_author.text else None
     if author:
-        entry["author"] = author
+        entry["author"] = author.strip()
 
     comments = text_by_local.get("comments")
     if comments:
-        entry["comments"] = comments
+        entry["comments"] = comments.strip()
 
     tags = _parse_tags(item, "rss", atom_ns)
     if tags:
@@ -1324,25 +1193,114 @@ def _parse_rss_feed_entry_fast(
     return entry
 
 
+def _parse_atom_feed_entry_fast(
+    item: _Element,
+    atom_ns: str,
+) -> FastFeedParserDict:
+    ns = f"{{{atom_ns}}}"
+    entry = FastFeedParserDict()
+
+    # ID
+    el = item.find(ns + "id")
+    if el is not None and el.text:
+        entry["id"] = el.text.strip()
+
+    # Title
+    el = item.find(ns + "title")
+    if el is not None and el.text:
+        entry["title"] = el.text.strip()
+
+    # Description (summary)
+    el = item.find(ns + "summary")
+    if el is not None and el.text:
+        entry["description"] = el.text.strip()
+
+    # Link (href attribute)
+    el = item.find(ns + "link")
+    if el is not None:
+        href = el.get("href")
+        if href:
+            entry["link"] = href.strip()
+
+    # Dates: Atom 1.0 uses published/updated, Atom 0.3 uses issued/modified
+    is_atom_03 = atom_ns == "http://purl.org/atom/ns#"
+    pub_tag = "issued" if is_atom_03 else "published"
+    upd_tag = "modified" if is_atom_03 else "updated"
+    pub_fallback_tag = "published" if is_atom_03 else "issued"
+    upd_fallback_tag = "updated" if is_atom_03 else "modified"
+
+    el = item.find(ns + pub_tag)
+    if el is not None and el.text:
+        published = _parse_date(el.text)
+        if published:
+            entry["published"] = published
+
+    el = item.find(ns + upd_tag)
+    if el is not None and el.text:
+        updated = _parse_date(el.text)
+        if updated:
+            entry["updated"] = updated
+
+    # Fallback date fields for mixed namespace scenarios
+    if "published" not in entry:
+        el = item.find(ns + pub_fallback_tag)
+        if el is not None and el.text:
+            published = _parse_date(el.text)
+            if published:
+                entry["published"] = published
+
+    if "updated" not in entry:
+        el = item.find(ns + upd_fallback_tag)
+        if el is not None and el.text:
+            updated = _parse_date(el.text)
+            if updated:
+                entry["updated"] = updated
+
+    if "updated" in entry and "published" not in entry:
+        entry["published"] = entry["updated"]
+
+    _populate_entry_links(entry, item, atom_ns)
+
+    if "id" not in entry and "link" in entry:
+        entry["id"] = entry["link"]
+
+    _populate_entry_content(entry, item, "atom", atom_ns)
+
+    media_contents = _parse_media_content(item)
+    if media_contents:
+        entry["media_content"] = media_contents
+
+    enclosures = _parse_enclosures(item)
+    if enclosures:
+        entry["enclosures"] = enclosures
+
+    # Author
+    el = item.find(ns + "author/" + ns + "name")
+    if el is not None and el.text:
+        entry["author"] = el.text.strip()
+
+    tags = _parse_tags(item, "atom", atom_ns)
+    if tags:
+        entry["tags"] = tags
+
+    return entry
+
+
 def _parse_feed_entry(
     item: _Element,
     feed_type: _FeedType,
     atom_namespace: Optional[str] = None,
-    *,
-    parse_media_content: bool = True,
-    parse_enclosures: bool = True,
 ) -> FastFeedParserDict:
     # Use dynamic atom namespace or fallback to default
     atom_ns = atom_namespace or "http://www.w3.org/2005/Atom"
 
     if feed_type == "rss":
-        return _parse_rss_feed_entry_fast(
-            item,
-            atom_ns,
-            parse_media_content=parse_media_content,
-            parse_enclosures=parse_enclosures,
-        )
+        return _parse_rss_feed_entry_fast(item, atom_ns)
 
+    if feed_type == "atom":
+        return _parse_atom_feed_entry_fast(item, atom_ns)
+
+    # RDF path uses the generic field machinery
     # Check if this is Atom 0.3 to use different date field names
     is_atom_03 = atom_ns == "http://purl.org/atom/ns#"
 
@@ -1434,8 +1392,7 @@ def _parse_feed_entry(
             entry["updated"] = _parse_date(fallback_updated)
 
     # Try to extract date from GUID as final fallback
-    if "published" not in entry and rss_guid:
-        # Check if GUID contains date information
+    if "published" not in entry and rss_guid and not rss_guid.startswith(("http://", "https://")):
         guid_date = _parse_date(rss_guid)
         if guid_date:
             entry["published"] = guid_date
@@ -1455,15 +1412,13 @@ def _parse_feed_entry(
 
     _populate_entry_content(entry, item, feed_type, atom_ns)
 
-    if parse_media_content:
-        media_contents = _parse_media_content(item)
-        if media_contents:
-            entry["media_content"] = media_contents
+    media_contents = _parse_media_content(item)
+    if media_contents:
+        entry["media_content"] = media_contents
 
-    if parse_enclosures:
-        enclosures = _parse_enclosures(item)
-        if enclosures:
-            entry["enclosures"] = enclosures
+    enclosures = _parse_enclosures(item)
+    if enclosures:
+        entry["enclosures"] = enclosures
 
     author = get_field_value(
         "author",
@@ -1618,6 +1573,14 @@ def _normalize_iso_datetime_string(value: str) -> str:
     if not cleaned:
         return cleaned
 
+    # Fast path: 'Z' suffix (most common in Atom feeds)
+    if cleaned[-1] in ("Z", "z"):
+        return cleaned[:-1] + "+00:00"
+
+    # Fast path: already has proper +HH:MM or -HH:MM timezone
+    if len(cleaned) > 6 and cleaned[-6] in ("+", "-") and cleaned[-3] == ":":
+        return cleaned
+
     upper_cleaned = cleaned.upper()
     for suffix in (" UTC", " GMT", " Z"):
         if upper_cleaned.endswith(suffix):
@@ -1664,7 +1627,7 @@ def _parsedate_to_utc(value: str) -> Optional[datetime.datetime]:
     return _ensure_utc(parsed)
 
 
-custom_tzinfos: dict[str, int] = {
+_custom_tzinfos: dict[str, int] = {
     "UTC": 0,
     "UT": 0,
     "GMT": 0,
@@ -1715,7 +1678,7 @@ _DATEPARSER_SETTINGS = {
 @lru_cache(maxsize=512)
 def _slow_dateutil_parse(value: str) -> Optional[datetime.datetime]:
     try:
-        return dateutil_parser.parse(value, tzinfos=custom_tzinfos, ignoretz=False)
+        return dateutil_parser.parse(value, tzinfos=_custom_tzinfos, ignoretz=False)
     except (ValueError, TypeError, OverflowError):
         return None
 
@@ -1727,7 +1690,7 @@ def _slow_dateparser(value: str) -> Optional[datetime.datetime]:
     except ImportError:
         return None
     try:
-        return _dateparser.parse(value, languages=["en"], settings=_DATEPARSER_SETTINGS)
+        return _dateparser.parse(value, languages=["en"], settings={**_DATEPARSER_SETTINGS})
     except (ValueError, TypeError):
         return None
 
@@ -1747,6 +1710,27 @@ def _parse_date(date_str: str) -> Optional[str]:
     candidate = date_str.strip()
     if not candidate:
         return None
+
+    # Fast path: clean ISO-8601 (covers >90% of Atom/modern RSS dates)
+    clen = len(candidate)
+    if clen >= 20 and candidate[4] == "-" and candidate[0:4].isdigit():
+        last = candidate[-1]
+        # Most common: ends with 'Z' (e.g., 2024-01-15T10:30:00Z)
+        if last in ("Z", "z"):
+            try:
+                dt = datetime.datetime.fromisoformat(candidate[:-1] + "+00:00")
+                return dt.isoformat()
+            except ValueError:
+                pass  # Fall through to full parsing
+        # Second most common: ends with +HH:MM (e.g., 2024-01-15T10:30:00+00:00)
+        elif clen > 6 and candidate[-6] in ("+", "-") and candidate[-3] == ":":
+            try:
+                dt = datetime.datetime.fromisoformat(candidate)
+                utc_dt = dt.replace(tzinfo=_UTC) if dt.tzinfo is None else dt.astimezone(_UTC)
+                return utc_dt.isoformat()
+            except (ValueError, OverflowError):
+                pass  # Fall through to full parsing
+
     if "\n" in candidate or "\r" in candidate or "\t" in candidate or "  " in candidate:
         candidate = _RE_WHITESPACE.sub(" ", candidate)
 

{fastfeedparser-0.5.0 → fastfeedparser-0.5.2/src/fastfeedparser.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fastfeedparser
-Version: 0.5.0
+Version: 0.5.2
 Summary: High performance RSS, Atom, JSON and RDF feed parser in Python
 Home-page: https://github.com/kagisearch/fastfeedparser
 Author: Vladimir Prelovac