fastfeedparser 0.5.1__tar.gz → 0.5.3__tar.gz

{fastfeedparser-0.5.1/src/fastfeedparser.egg-info → fastfeedparser-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fastfeedparser
-Version: 0.5.1
+Version: 0.5.3
 Summary: High performance RSS, Atom, JSON and RDF feed parser in Python
 Home-page: https://github.com/kagisearch/fastfeedparser
 Author: Vladimir Prelovac

{fastfeedparser-0.5.1 → fastfeedparser-0.5.3}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = fastfeedparser
-version = 0.5.1
+version = 0.5.3
 author = Vladimir Prelovac
 author_email = vlad@kagi.com
 description = High performance RSS, Atom, JSON and RDF feed parser in Python

{fastfeedparser-0.5.1 → fastfeedparser-0.5.3}/src/fastfeedparser/main.py

@@ -56,6 +56,18 @@ _RE_WHITESPACE = re.compile(r"\s+")
 _RE_ISO_TZ_NO_COLON = re.compile(r"([+-]\d{2})(\d{2})$")
 _RE_ISO_TZ_HOUR_ONLY = re.compile(r"([+-]\d{2})$")
 _RE_ISO_FRACTION = re.compile(r"\.(\d{7,})(?=(?:[+-]\d{2}:?\d{2}|Z|$))", re.IGNORECASE)
+_RE_RFC822 = re.compile(
+    r"(?:\w{3},\s+)?(\d{1,2})\s+(\w{3})\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})\s+([+-]\d{4}|[A-Z]{2,5})"
+)
+_MONTHS_RFC822: dict[str, int] = {
+    "jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
+    "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12,
+}
+_TZ_OFFSETS_RFC822: dict[str, int] = {
+    "GMT": 0, "UTC": 0, "UT": 0,
+    "EST": -18000, "EDT": -14400, "CST": -21600, "CDT": -18000,
+    "MST": -25200, "MDT": -21600, "PST": -28800, "PDT": -25200,
+}
 
 
 class FastFeedParserDict(dict):
@@ -382,24 +394,26 @@ def _maybe_parse_json_feed(content: str | bytes) -> FastFeedParserDict | None:
     return None
 
 
+_STRICT_XML_PARSER = etree.XMLParser(
+    ns_clean=True,
+    recover=False,
+    collect_ids=False,
+    resolve_entities=False,
+)
+_RECOVER_XML_PARSER = etree.XMLParser(
+    ns_clean=True,
+    recover=True,
+    collect_ids=False,
+    resolve_entities=False,
+)
+
+
 def _parse_xml_root(xml_content: bytes) -> _Element:
     try:
-        strict_parser = etree.XMLParser(
-            ns_clean=True,
-            recover=False,
-            collect_ids=False,
-            resolve_entities=False,
-        )
-        root = etree.fromstring(xml_content, parser=strict_parser)
+        root = etree.fromstring(xml_content, parser=_STRICT_XML_PARSER)
     except etree.XMLSyntaxError:
-        recover_parser = etree.XMLParser(
-            ns_clean=True,
-            recover=True,
-            collect_ids=False,
-            resolve_entities=False,
-        )
         try:
-            root = etree.fromstring(xml_content, parser=recover_parser)
+            root = etree.fromstring(xml_content, parser=_RECOVER_XML_PARSER)
         except etree.XMLSyntaxError as e:
             raise ValueError(f"Failed to parse XML content: {str(e)}")
 
@@ -642,6 +656,9 @@ def _parse_content(xml_content: str | bytes) -> FastFeedParserDict:
 
     feed = _parse_feed_info(channel, feed_type, atom_namespace)
 
+    # Detect once whether media namespace is used anywhere in the document
+    has_media_ns = b"search.yahoo.com/mrss" in xml_content if isinstance(xml_content, bytes) else "search.yahoo.com/mrss" in xml_content
+
     # Parse entries
     entries: list[FastFeedParserDict] = []
     feed["entries"] = entries
@@ -650,6 +667,7 @@ def _parse_content(xml_content: str | bytes) -> FastFeedParserDict:
             item,
             feed_type,
             atom_namespace,
+            has_media_ns,
         )
         # Ensure that titles and descriptions are always present
         entry["title"] = entry.get("title", "").strip()
@@ -998,7 +1016,10 @@ def _populate_entry_content(
             if "<" in content_value:
                 content_value = _RE_HTML_TAGS.sub(" ", content_value[:2048])
                 content_value = _html_mod.unescape(content_value)
-            content_value = _RE_WHITESPACE.sub(" ", content_value).strip()
+            if "  " in content_value or "\n" in content_value or "\t" in content_value or "\r" in content_value:
+                content_value = _RE_WHITESPACE.sub(" ", content_value).strip()
+            else:
+                content_value = content_value.strip()
         entry["description"] = content_value[:512]
 
 
@@ -1096,12 +1117,16 @@ def _build_rss_item_text_maps(item: _Element) -> tuple[dict[str, Optional[str]],
         tag = child.tag
         if not isinstance(tag, str):
             continue
-        text_value = child.text.strip() if child.text else None
+        text_value = child.text or None
         if tag not in by_full:
             by_full[tag] = text_value
-        local = tag.rsplit("}", 1)[-1].lower()
-        if ":" in local:
-            local = local.split(":", 1)[1]
+        # Fast path: ~80% of RSS tags have no namespace or colon prefix
+        if "{" in tag:
+            local = tag.rsplit("}", 1)[1].lower()
+        elif ":" in tag:
+            local = tag.split(":", 1)[1].lower()
+        else:
+            local = tag.lower()
         if local not in by_local:
             by_local[local] = text_value
     return by_local, by_full
@@ -1118,6 +1143,7 @@ def _first_non_empty(mapping: dict[str, Optional[str]], keys: tuple[str, ...]) -
 def _parse_rss_feed_entry_fast(
     item: _Element,
     atom_ns: str,
+    has_media_ns: bool = True,
 ) -> FastFeedParserDict:
     text_by_local, text_by_full = _build_rss_item_text_maps(item)
 
@@ -1139,7 +1165,7 @@ def _parse_rss_feed_entry_fast(
 
     link = text_by_local.get("link")
     if link:
-        entry["link"] = link
+        entry["link"] = link.strip()
 
     published_source = _first_non_empty(text_by_local, ("pubdate", "published", "issued", "date"))
     if published_source:
@@ -1161,15 +1187,26 @@ def _parse_rss_feed_entry_fast(
     if "updated" in entry and "published" not in entry:
         entry["published"] = entry["updated"]
 
-    _populate_entry_links(entry, item, atom_ns)
+    # Inline link population for RSS (avoids redundant findall/find for 98.8% of entries)
+    atom_links = item.findall(f"{{{atom_ns}}}link")
+    if atom_links:
+        # Has atom:link elements - use full logic
+        _populate_entry_links(entry, item, atom_ns)
+    else:
+        # Common RSS case: no atom:link elements
+        entry["links"] = []
+        if "link" not in entry and rss_guid and rss_guid.startswith(("http://", "https://")):
+            entry["link"] = rss_guid
+
     if "id" not in entry and "link" in entry:
         entry["id"] = entry["link"]
 
     _populate_entry_content(entry, item, "rss", atom_ns)
 
-    media_contents = _parse_media_content(item)
-    if media_contents:
-        entry["media_content"] = media_contents
+    if has_media_ns:
+        media_contents = _parse_media_content(item)
+        if media_contents:
+            entry["media_content"] = media_contents
 
     enclosures = _parse_enclosures(item)
     if enclosures:
@@ -1180,11 +1217,11 @@ def _parse_rss_feed_entry_fast(
         atom_author = item.find(f"{{{atom_ns}}}author/{{{atom_ns}}}name")
         author = atom_author.text.strip() if atom_author is not None and atom_author.text else None
     if author:
-        entry["author"] = author
+        entry["author"] = author.strip()
 
     comments = text_by_local.get("comments")
     if comments:
-        entry["comments"] = comments
+        entry["comments"] = comments.strip()
 
     tags = _parse_tags(item, "rss", atom_ns)
     if tags:
@@ -1193,17 +1230,117 @@ def _parse_rss_feed_entry_fast(
     return entry
 
 
+def _parse_atom_feed_entry_fast(
+    item: _Element,
+    atom_ns: str,
+    has_media_ns: bool = True,
+) -> FastFeedParserDict:
+    ns = f"{{{atom_ns}}}"
+    entry = FastFeedParserDict()
+
+    # ID
+    el = item.find(ns + "id")
+    if el is not None and el.text:
+        entry["id"] = el.text.strip()
+
+    # Title
+    el = item.find(ns + "title")
+    if el is not None and el.text:
+        entry["title"] = el.text.strip()
+
+    # Description (summary)
+    el = item.find(ns + "summary")
+    if el is not None and el.text:
+        entry["description"] = el.text.strip()
+
+    # Link (href attribute)
+    el = item.find(ns + "link")
+    if el is not None:
+        href = el.get("href")
+        if href:
+            entry["link"] = href.strip()
+
+    # Dates: Atom 1.0 uses published/updated, Atom 0.3 uses issued/modified
+    is_atom_03 = atom_ns == "http://purl.org/atom/ns#"
+    pub_tag = "issued" if is_atom_03 else "published"
+    upd_tag = "modified" if is_atom_03 else "updated"
+    pub_fallback_tag = "published" if is_atom_03 else "issued"
+    upd_fallback_tag = "updated" if is_atom_03 else "modified"
+
+    el = item.find(ns + pub_tag)
+    if el is not None and el.text:
+        published = _parse_date(el.text)
+        if published:
+            entry["published"] = published
+
+    el = item.find(ns + upd_tag)
+    if el is not None and el.text:
+        updated = _parse_date(el.text)
+        if updated:
+            entry["updated"] = updated
+
+    # Fallback date fields for mixed namespace scenarios
+    if "published" not in entry:
+        el = item.find(ns + pub_fallback_tag)
+        if el is not None and el.text:
+            published = _parse_date(el.text)
+            if published:
+                entry["published"] = published
+
+    if "updated" not in entry:
+        el = item.find(ns + upd_fallback_tag)
+        if el is not None and el.text:
+            updated = _parse_date(el.text)
+            if updated:
+                entry["updated"] = updated
+
+    if "updated" in entry and "published" not in entry:
+        entry["published"] = entry["updated"]
+
+    _populate_entry_links(entry, item, atom_ns)
+
+    if "id" not in entry and "link" in entry:
+        entry["id"] = entry["link"]
+
+    _populate_entry_content(entry, item, "atom", atom_ns)
+
+    if has_media_ns:
+        media_contents = _parse_media_content(item)
+        if media_contents:
+            entry["media_content"] = media_contents
+
+    enclosures = _parse_enclosures(item)
+    if enclosures:
+        entry["enclosures"] = enclosures
+
+    # Author
+    el = item.find(ns + "author/" + ns + "name")
+    if el is not None and el.text:
+        entry["author"] = el.text.strip()
+
+    tags = _parse_tags(item, "atom", atom_ns)
+    if tags:
+        entry["tags"] = tags
+
+    return entry
+
+
 def _parse_feed_entry(
     item: _Element,
     feed_type: _FeedType,
     atom_namespace: Optional[str] = None,
+    has_media_ns: bool = True,
 ) -> FastFeedParserDict:
     # Use dynamic atom namespace or fallback to default
     atom_ns = atom_namespace or "http://www.w3.org/2005/Atom"
 
     if feed_type == "rss":
-        return _parse_rss_feed_entry_fast(item, atom_ns)
+        return _parse_rss_feed_entry_fast(item, atom_ns, has_media_ns)
 
+    if feed_type == "atom":
+        return _parse_atom_feed_entry_fast(item, atom_ns, has_media_ns)
+
+    # RDF path uses the generic field machinery
     # Check if this is Atom 0.3 to use different date field names
     is_atom_03 = atom_ns == "http://purl.org/atom/ns#"
 
@@ -1315,9 +1452,10 @@ def _parse_feed_entry(
 
     _populate_entry_content(entry, item, feed_type, atom_ns)
 
-    media_contents = _parse_media_content(item)
-    if media_contents:
-        entry["media_content"] = media_contents
+    if has_media_ns:
+        media_contents = _parse_media_content(item)
+        if media_contents:
+            entry["media_content"] = media_contents
 
     enclosures = _parse_enclosures(item)
     if enclosures:
@@ -1476,6 +1614,14 @@ def _normalize_iso_datetime_string(value: str) -> str:
     if not cleaned:
         return cleaned
 
+    # Fast path: 'Z' suffix (most common in Atom feeds)
+    if cleaned[-1] in ("Z", "z"):
+        return cleaned[:-1] + "+00:00"
+
+    # Fast path: already has proper +HH:MM or -HH:MM timezone
+    if len(cleaned) > 6 and cleaned[-6] in ("+", "-") and cleaned[-3] == ":":
+        return cleaned
+
     upper_cleaned = cleaned.upper()
     for suffix in (" UTC", " GMT", " Z"):
         if upper_cleaned.endswith(suffix):
@@ -1511,8 +1657,54 @@ def _ensure_utc(dt: datetime.datetime) -> Optional[datetime.datetime]:
         return None
 
 
+def _fast_rfc822_to_iso(value: str) -> Optional[str]:
+    """Fast RFC-822 date to ISO string, bypassing datetime objects for UTC dates."""
+    m = _RE_RFC822.match(value)
+    if not m:
+        return None
+    day, mon_str, year, hour, minute, second, tz = m.groups()
+    month = _MONTHS_RFC822.get(mon_str.lower())
+    if month is None:
+        return None
+    if tz[0] in "+-":
+        tz_offset_seconds = (int(tz[1:3]) * 3600 + int(tz[3:5]) * 60) * (
+            1 if tz[0] == "+" else -1
+        )
+    else:
+        tz_offset_seconds = _TZ_OFFSETS_RFC822.get(tz)
+        if tz_offset_seconds is None:
+            return None  # Unknown tz name, fall through to full parser
+    # Python requires offset strictly between -24h and +24h
+    if not (-86400 < tz_offset_seconds < 86400):
+        return None
+    d = int(day)
+    h = int(hour)
+    mi = int(minute)
+    s = int(second)
+    # Hour 24 is invalid (even ISO only allows 24:00:00); roll to next day at 00:mm:ss
+    if h == 24:
+        base = datetime.date(int(year), month, d) + datetime.timedelta(days=1)
+        h = 0
+        if tz_offset_seconds == 0:
+            return f"{base.year:04d}-{base.month:02d}-{base.day:02d}T{h:02d}:{mi:02d}:{s:02d}+00:00"
+        dt = datetime.datetime(
+            base.year, base.month, base.day, h, mi, s,
+            tzinfo=datetime.timezone(datetime.timedelta(seconds=tz_offset_seconds)),
+        )
+        utc = dt.astimezone(_UTC)
+        return f"{utc.year:04d}-{utc.month:02d}-{utc.day:02d}T{utc.hour:02d}:{utc.minute:02d}:{utc.second:02d}+00:00"
+    if tz_offset_seconds == 0:
+        return f"{year}-{month:02d}-{d:02d}T{hour}:{minute}:{second}+00:00"
+    dt = datetime.datetime(
+        int(year), month, d, h, mi, s,
+        tzinfo=datetime.timezone(datetime.timedelta(seconds=tz_offset_seconds)),
+    )
+    utc = dt.astimezone(_UTC)
+    return f"{utc.year:04d}-{utc.month:02d}-{utc.day:02d}T{utc.hour:02d}:{utc.minute:02d}:{utc.second:02d}+00:00"
+
+
 def _parsedate_to_utc(value: str) -> Optional[datetime.datetime]:
-    """Fast RFC-822 / RFC-2822 parsing via email.utils."""
+    """RFC-822 / RFC-2822 parsing via email.utils (fallback)."""
     try:
         parsed = parsedate_to_datetime(value)
     except (TypeError, ValueError, IndexError):
@@ -1585,7 +1777,7 @@ def _slow_dateparser(value: str) -> Optional[datetime.datetime]:
     except ImportError:
         return None
     try:
-        return _dateparser.parse(value, languages=["en"], settings=_DATEPARSER_SETTINGS)
+        return _dateparser.parse(value, languages=["en"], settings={**_DATEPARSER_SETTINGS})
     except (ValueError, TypeError):
         return None
 
@@ -1605,6 +1797,30 @@ def _parse_date(date_str: str) -> Optional[str]:
     candidate = date_str.strip()
     if not candidate:
         return None
+
+    # Fast path: clean ISO-8601 (covers >90% of Atom/modern RSS dates)
+    clen = len(candidate)
+    if clen >= 20 and candidate[4] == "-" and candidate[0:4].isdigit():
+        last = candidate[-1]
+        # Most common: ends with 'Z' (e.g., 2024-01-15T10:30:00Z)
+        if last in ("Z", "z"):
+            iso = candidate[:-1] + "+00:00"
+            try:
+                dt = datetime.datetime.fromisoformat(iso)
+                return dt.isoformat()
+            except ValueError:
+                pass  # Fall through to full parsing
+        # Second most common: ends with +HH:MM (e.g., 2024-01-15T10:30:00+00:00)
+        elif clen > 6 and candidate[-6] in ("+", "-") and candidate[-3] == ":":
+            try:
+                dt = datetime.datetime.fromisoformat(candidate)
+                if dt.tzinfo is _UTC:
+                    return dt.isoformat()
+                utc_dt = dt.astimezone(_UTC)
+                return utc_dt.isoformat()
+            except (ValueError, OverflowError):
+                pass  # Fall through to full parsing
+
     if "\n" in candidate or "\r" in candidate or "\t" in candidate or "  " in candidate:
         candidate = _RE_WHITESPACE.sub(" ", candidate)
 
@@ -1617,10 +1833,13 @@ def _parse_date(date_str: str) -> Optional[str]:
             if not ((year % 4 == 0 and year % 100 != 0) or (year % 400 == 0)):
                 candidate = candidate.replace(f"{year}-02-29", f"{year}-02-28")
 
-    if "24:00" in candidate:
-        candidate = candidate.replace("24:00:00", "00:00:00").replace(
-            " 24:00", " 00:00"
-        )
+    if "T24:" in candidate or " 24:" in candidate:
+        m24 = re.search(r"(\d{4}-\d{2}-\d{2})[T ]24:(\d{2}):(\d{2})", candidate)
+        if m24:
+            base = datetime.date.fromisoformat(m24.group(1))
+            mins, secs = int(m24.group(2)), int(m24.group(3))
+            next_day = base + datetime.timedelta(days=1)
+            candidate = candidate[:m24.start()] + f"{next_day}T00:{mins:02d}:{secs:02d}" + candidate[m24.end():]
 
     dt: Optional[datetime.datetime] = None
 
@@ -1636,6 +1855,10 @@ def _parse_date(date_str: str) -> Optional[str]:
             if utc_dt is not None:
                 return utc_dt.isoformat()
 
+    rfc822_result = _fast_rfc822_to_iso(candidate)
+    if rfc822_result is not None:
+        return rfc822_result
+
     dt = _parsedate_to_utc(candidate)
     if dt is not None:
         return dt.isoformat()

{fastfeedparser-0.5.1 → fastfeedparser-0.5.3/src/fastfeedparser.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fastfeedparser
-Version: 0.5.1
+Version: 0.5.3
 Summary: High performance RSS, Atom, JSON and RDF feed parser in Python
 Home-page: https://github.com/kagisearch/fastfeedparser
 Author: Vladimir Prelovac