fastfeedparser 0.5.5__tar.gz → 0.5.6__tar.gz

{fastfeedparser-0.5.5/src/fastfeedparser.egg-info → fastfeedparser-0.5.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fastfeedparser
-Version: 0.5.5
+Version: 0.5.6
 Summary: High performance RSS, Atom, JSON and RDF feed parser in Python
 Home-page: https://github.com/kagisearch/fastfeedparser
 Author: Vladimir Prelovac
@@ -25,6 +25,7 @@ Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 Provides-Extra: dateparser
 Provides-Extra: brotli
+Provides-Extra: orjson
 Provides-Extra: full
 License-File: LICENSE
 
@@ -146,7 +147,7 @@ Speedup: 50.1x
 
 ### Main Functions
 
-- `parse(source)`: Parse feed from a source that can be URL or a string
+- `parse(source, *, include_content=True, include_tags=True, include_media=True, include_enclosures=True)`: Parse feed from a URL/XML/JSON source, with optional field extraction toggles for faster parsing.
 
 
 ### Feed Object Structure

{fastfeedparser-0.5.5 → fastfeedparser-0.5.6}/README.md

@@ -116,7 +116,7 @@ Speedup: 50.1x
 
 ### Main Functions
 
-- `parse(source)`: Parse feed from a source that can be URL or a string
+- `parse(source, *, include_content=True, include_tags=True, include_media=True, include_enclosures=True)`: Parse feed from a URL/XML/JSON source, with optional field extraction toggles for faster parsing.
 
 
 ### Feed Object Structure

{fastfeedparser-0.5.5 → fastfeedparser-0.5.6}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = fastfeedparser
-version = 0.5.5
+version = 0.5.6
 author = Vladimir Prelovac
 author_email = vlad@kagi.com
 description = High performance RSS, Atom, JSON and RDF feed parser in Python
@@ -40,9 +40,12 @@ dateparser =
 	dateparser
 brotli = 
 	brotli
+orjson = 
+	orjson
 full = 
 	dateparser
 	brotli
+	orjson
 
 [options.packages.find]
 where = src

{fastfeedparser-0.5.5 → fastfeedparser-0.5.6}/src/fastfeedparser/main.py

@@ -15,7 +15,14 @@ try:
     HAS_BROTLI = True
 except ImportError:
     HAS_BROTLI = False
-from typing import Any, Callable, Optional, TYPE_CHECKING, Literal
+
+try:
+    import orjson
+
+    _json_loads = orjson.loads
+except ImportError:
+    _json_loads = json.loads
+from typing import Any, Callable, Optional, Protocol, TYPE_CHECKING, Literal
 from urllib.parse import urljoin
 from urllib.request import (
     HTTPErrorProcessor,
@@ -81,6 +88,44 @@ _MONTHS_RFC822: dict[str, int] = {
     "dec": 12,
 }
 
+_XML_NS = "{http://www.w3.org/XML/1998/namespace}"
+_XML_LANG_ATTR = _XML_NS + "lang"
+_XML_BASE_ATTR = _XML_NS + "base"
+_RDF_ABOUT_ATTR = "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about"
+_RSS_CONTENT_ENCODED_TAG = "{http://purl.org/rss/1.0/modules/content/}encoded"
+_DC_SUBJECT_TAG = "{http://purl.org/dc/elements/1.1/}subject"
+_MEDIA_CONTENT_TAG = "{http://search.yahoo.com/mrss/}content"
+_MEDIA_THUMBNAIL_TAG = "{http://search.yahoo.com/mrss/}thumbnail"
+_MEDIA_TITLE_TAG = "{http://search.yahoo.com/mrss/}title"
+_MEDIA_TEXT_TAG = "{http://search.yahoo.com/mrss/}text"
+_MEDIA_DESCRIPTION_TAG = "{http://search.yahoo.com/mrss/}description"
+_MEDIA_CREDIT_TAG = "{http://search.yahoo.com/mrss/}credit"
+
+
+@lru_cache(maxsize=4)
+def _atom_ns_tags(atom_ns: str) -> dict[str, str]:
+    """Pre-compute namespace-prefixed tag strings once per unique namespace.
+
+    Avoids thousands of redundant f-string / concatenation operations when
+    parsing feeds with many entries.
+    """
+    ns = f"{{{atom_ns}}}"
+    is_atom_03 = atom_ns == "http://purl.org/atom/ns#"
+    return {
+        "ns": ns,
+        "id": ns + "id",
+        "title": ns + "title",
+        "summary": ns + "summary",
+        "link": ns + "link",
+        "content": ns + "content",
+        "author_name": ns + "author/" + ns + "name",
+        "category": ns + "category",
+        "published": ns + ("issued" if is_atom_03 else "published"),
+        "updated": ns + ("modified" if is_atom_03 else "updated"),
+        "pub_fallback": ns + ("published" if is_atom_03 else "issued"),
+        "upd_fallback": ns + ("updated" if is_atom_03 else "modified"),
+    }
+
 
 class FastFeedParserDict(dict):
     """A dictionary that allows access to its keys as attributes."""
@@ -154,11 +199,18 @@ def _clean_feed_bytes(content: bytes) -> bytes:
         b"<?xml-stylesheet",
     )
 
-    lines = content.splitlines()
-    for i, line in enumerate(lines):
-        line_stripped = line.strip().lower()
-        if any(line_stripped.startswith(pattern) for pattern in xml_start_patterns):
-            return b"\n".join(lines[i:])
+    # Search for XML start patterns without splitting entire content into lines.
+    # For large feeds (multi-MB), splitlines() creates thousands of byte string
+    # objects; find() scans in-place with zero allocations.
+    search_limit = min(len(content), 8192)
+    search_chunk = content[:search_limit].lower()
+    earliest = -1
+    for pattern in xml_start_patterns:
+        idx = search_chunk.find(pattern)
+        if idx != -1 and (earliest == -1 or idx < earliest):
+            earliest = idx
+    if earliest != -1:
+        return content[earliest:]
 
     if b"<script>" in preview_lower or b"<body>" in preview_lower:
         raise ValueError("Content appears to be HTML, not a valid RSS/Atom feed")
@@ -167,21 +219,30 @@ def _clean_feed_bytes(content: bytes) -> bytes:
 
 
 def _fix_malformed_xml_bytes(content: bytes, actual_encoding: str = "utf-8") -> bytes:
+    # XML declarations and encoding definitions live at the top of the file.
+    # Run declaration-fixing regexes only on the first 2 KB to avoid scanning
+    # multi-megabyte payloads with patterns that can only match the header.
+    header = content[:2048]
+    tail = content[2048:]
+
     # Fix double XML declarations like "<?xml?xml version="1.0"?>"
-    content = _RE_DOUBLE_XML_DECL_BYTES.sub(b"<?xml ", content)
+    header = _RE_DOUBLE_XML_DECL_BYTES.sub(b"<?xml ", header)
 
     # Fix double closing ?> in XML declaration like "??>>"
-    content = _RE_DOUBLE_CLOSE_BYTES.sub(b"?>", content)
-
-    # Fix malformed attribute syntax like rss:version=2.0 (missing quotes)
-    content = _RE_UNQUOTED_ATTR_BYTES.sub(rb'\1="\2"', content)
+    header = _RE_DOUBLE_CLOSE_BYTES.sub(b"?>", header)
 
     # Update encoding in XML declaration to match actual encoding when a feed was transcoded.
     if actual_encoding.lower() != "utf-16":
         replacement = (
             rb"\1" + actual_encoding.encode("ascii", errors="replace") + rb"\3"
         )
-        content = _RE_UTF16_ENCODING_BYTES.sub(replacement, content)
+        header = _RE_UTF16_ENCODING_BYTES.sub(replacement, header)
+
+    # Reassemble before running body-wide fixes
+    content = header + tail
+
+    # Fix malformed attribute syntax like rss:version=2.0 (missing quotes)
+    content = _RE_UNQUOTED_ATTR_BYTES.sub(rb'\1="\2"', content)
 
     # Fix unclosed link tags - common in Atom feeds
     content = _RE_UNCLOSED_LINK_BYTES.sub(rb"<link\1/>", content)
@@ -225,7 +286,13 @@ def _prepare_xml_bytes(xml_content: str | bytes) -> bytes:
     return _prepare_xml_bytes(xml_content.encode("utf-8", errors="replace"))
 
 
-def _parse_json_feed(json_data: dict) -> FastFeedParserDict:
+def _parse_json_feed(
+    json_data: dict,
+    *,
+    include_content: bool = True,
+    include_tags: bool = True,
+    include_enclosures: bool = True,
+) -> FastFeedParserDict:
     """Parse a JSON Feed and convert to FastFeedParserDict format.
 
     JSON Feed spec: https://jsonfeed.org/
@@ -283,10 +350,12 @@ def _parse_json_feed(json_data: dict) -> FastFeedParserDict:
         summary = item.get("summary", "")
 
         if content_html:
-            entry["content"] = [{"type": "text/html", "value": content_html}]
+            if include_content:
+                entry["content"] = [{"type": "text/html", "value": content_html}]
             entry["description"] = summary
         elif content_text:
-            entry["content"] = [{"type": "text/plain", "value": content_text}]
+            if include_content:
+                entry["content"] = [{"type": "text/plain", "value": content_text}]
             entry["description"] = summary or content_text[:512]
         else:
             entry["description"] = summary
@@ -319,14 +388,14 @@ def _parse_json_feed(json_data: dict) -> FastFeedParserDict:
 
         # Add tags
         tags = item.get("tags")
-        if tags:
+        if include_tags and tags:
             entry["tags"] = [
                 {"term": tag, "scheme": None, "label": None} for tag in tags
             ]
 
         # Add attachments as enclosures
         attachments = item.get("attachments")
-        if attachments:
+        if include_enclosures and attachments:
             enclosures = []
             for attachment in attachments:
                 url = attachment.get("url", "")
@@ -389,19 +458,23 @@ def _fetch_url_content(url: str) -> str | bytes:
         return content.decode(content_charset) if content_charset else content
 
 
-def _maybe_parse_json_feed(content: str | bytes) -> FastFeedParserDict | None:
+def _maybe_parse_json_feed(
+    content: str | bytes,
+    *,
+    include_content: bool = True,
+    include_tags: bool = True,
+    include_enclosures: bool = True,
+) -> FastFeedParserDict | None:
     if isinstance(content, bytes):
         if not content.lstrip().startswith(b"{"):
             return None
-        json_str = content.decode("utf-8", errors="replace")
     else:
         if not content.lstrip().startswith("{"):
             return None
-        json_str = content
 
     try:
-        json_data = json.loads(json_str)
-    except (json.JSONDecodeError, ValueError):
+        json_data = _json_loads(content)
+    except Exception:
         return None
 
     if not isinstance(json_data, dict):
@@ -409,10 +482,20 @@ def _maybe_parse_json_feed(content: str | bytes) -> FastFeedParserDict | None:
 
     version = json_data.get("version")
     if isinstance(version, str) and "jsonfeed.org" in version:
-        return _parse_json_feed(json_data)
+        return _parse_json_feed(
+            json_data,
+            include_content=include_content,
+            include_tags=include_tags,
+            include_enclosures=include_enclosures,
+        )
 
     if isinstance(json_data.get("items"), list):
-        return _parse_json_feed(json_data)
+        return _parse_json_feed(
+            json_data,
+            include_content=include_content,
+            include_tags=include_tags,
+            include_enclosures=include_enclosures,
+        )
 
     return None
 
@@ -667,9 +750,21 @@ def _detect_feed_structure(
     raise ValueError(f"Unknown feed type: {root.tag}")
 
 
-def _parse_content(xml_content: str | bytes) -> FastFeedParserDict:
+def _parse_content(
+    xml_content: str | bytes,
+    *,
+    include_content: bool = True,
+    include_tags: bool = True,
+    include_media: bool = True,
+    include_enclosures: bool = True,
+) -> FastFeedParserDict:
     """Parse feed content (XML or JSON) that has already been fetched."""
-    json_feed = _maybe_parse_json_feed(xml_content)
+    json_feed = _maybe_parse_json_feed(
+        xml_content,
+        include_content=include_content,
+        include_tags=include_tags,
+        include_enclosures=include_enclosures,
+    )
     if json_feed is not None:
         return json_feed
 
@@ -682,7 +777,9 @@ def _parse_content(xml_content: str | bytes) -> FastFeedParserDict:
         root, xml_content, root_tag_local
     )
 
-    feed = _parse_feed_info(channel, feed_type, atom_namespace)
+    feed = _parse_feed_info(
+        channel, feed_type, atom_namespace, include_tags=include_tags
+    )
 
     # Detect once whether media namespace is used anywhere in the document
     has_media_ns = (
@@ -694,34 +791,41 @@ def _parse_content(xml_content: str | bytes) -> FastFeedParserDict:
     # Parse entries — resolve parser once per feed instead of per entry
     entries: list[FastFeedParserDict] = []
     feed["entries"] = entries
-    atom_ns = atom_namespace or "http://www.w3.org/2005/Atom"
-    if feed_type == "rss":
-        for item in items:
-            entry = _parse_rss_feed_entry_fast(item, atom_ns, has_media_ns)
-            entry.setdefault("title", "")
-            entry.setdefault("description", "")
-            entries.append(entry)
-    elif feed_type == "atom":
-        for item in items:
-            entry = _parse_atom_feed_entry_fast(item, atom_ns, has_media_ns)
-            entry.setdefault("title", "")
-            entry.setdefault("description", "")
-            entries.append(entry)
-    else:
-        for item in items:
-            entry = _parse_feed_entry(item, feed_type, atom_namespace, has_media_ns)
-            entry["title"] = entry.get("title", "").strip()
-            entry["description"] = entry.get("description", "").strip()
-            entries.append(entry)
+    for item in items:
+        entry = _parse_feed_entry(
+            item,
+            feed_type,
+            atom_namespace,
+            has_media_ns,
+            include_content=include_content,
+            include_tags=include_tags,
+            include_media=include_media,
+            include_enclosures=include_enclosures,
+        )
+        # Ensure that titles and descriptions are always present
+        entry["title"] = entry.get("title", "").strip()
+        entry["description"] = entry.get("description", "").strip()
+        entries.append(entry)
 
     return feed
 
 
-def parse(source: str | bytes) -> FastFeedParserDict:
+def parse(
+    source: str | bytes,
+    *,
+    include_content: bool = True,
+    include_tags: bool = True,
+    include_media: bool = True,
+    include_enclosures: bool = True,
+) -> FastFeedParserDict:
     """Parse a feed from a URL or XML content.
 
     Args:
         source: URL string or XML content string/bytes
+        include_content: Include per-entry content blobs and synthesized descriptions
+        include_tags: Include feed and entry tags/categories
+        include_media: Include media namespace content (media:content/media:thumbnail)
+        include_enclosures: Include RSS enclosures and JSON-feed attachments
 
     Returns:
         FastFeedParserDict containing parsed feed data
@@ -738,7 +842,13 @@ def parse(source: str | bytes) -> FastFeedParserDict:
         content = source
 
     try:
-        return _parse_content(content)
+        return _parse_content(
+            content,
+            include_content=include_content,
+            include_tags=include_tags,
+            include_media=include_media,
+            include_enclosures=include_enclosures,
+        )
     except ValueError as e:
         if not is_url:
             raise
@@ -749,11 +859,21 @@ def parse(source: str | bytes) -> FastFeedParserDict:
         redirect_url = _extract_meta_refresh_url(content, source)
         if redirect_url is None:
             raise
-        return parse(redirect_url)
+        return parse(
+            redirect_url,
+            include_content=include_content,
+            include_tags=include_tags,
+            include_media=include_media,
+            include_enclosures=include_enclosures,
+        )
 
 
 def _parse_feed_info(
-    channel: _Element, feed_type: _FeedType, atom_namespace: Optional[str] = None
+    channel: _Element,
+    feed_type: _FeedType,
+    atom_namespace: Optional[str] = None,
+    *,
+    include_tags: bool = True,
 ) -> FastFeedParserDict:
     # Use dynamic atom namespace or fallback to default
     atom_ns = atom_namespace or "http://www.w3.org/2005/Atom"
@@ -824,8 +944,8 @@ def _parse_feed_info(
         if value:
             feed[field[0]] = value
 
-    feed_lang = channel.get("{http://www.w3.org/XML/1998/namespace}lang")
-    feed_base = channel.get("{http://www.w3.org/XML/1998/namespace}base")
+    feed_lang = channel.get(_XML_LANG_ATTR)
+    feed_base = channel.get(_XML_BASE_ATTR)
     feed["language"] = feed_lang
 
     # Add title_detail and subtitle_detail
@@ -896,9 +1016,10 @@ def _parse_feed_info(
             feed["author"] = managing_editor
 
     # Parse feed-level tags/categories
-    tags = _parse_tags(channel, feed_type, atom_ns)
-    if tags:
-        feed["tags"] = tags
+    if include_tags:
+        tags = _parse_tags(channel, feed_type, atom_ns)
+        if tags:
+            feed["tags"] = tags
 
     return FastFeedParserDict(feed=feed)
 
@@ -917,14 +1038,14 @@ def _parse_tags(
                     {"term": term, "scheme": cat.get("domain"), "label": None}
                 )
         # RSS might also use <dc:subject>
-        for subject in element.findall("{http://purl.org/dc/elements/1.1/}subject"):
+        for subject in element.findall(_DC_SUBJECT_TAG):
             term = subject.text.strip() if subject.text else None
             if term:
                 tags_list.append({"term": term, "scheme": None, "label": None})
     elif feed_type == "atom":
         # Atom uses <category> elements with attributes
         atom_ns = atom_namespace or "http://www.w3.org/2005/Atom"
-        for cat in element.findall(f"{{{atom_ns}}}category"):
+        for cat in element.findall(_atom_ns_tags(atom_ns)["category"]):
             term = cat.get("term")
             if term:
                 tags_list.append(
@@ -936,7 +1057,7 @@ def _parse_tags(
                 )
     elif feed_type == "rdf":
         # RDF uses <dc:subject> or <taxo:topic>
-        for subject in element.findall("{http://purl.org/dc/elements/1.1/}subject"):
+        for subject in element.findall(_DC_SUBJECT_TAG):
             term = subject.text.strip() if subject.text else None
             if term:
                 tags_list.append({"term": term, "scheme": None, "label": None})
@@ -972,12 +1093,16 @@ def _coerce_int_fields(mapping: dict[str, Any], fields: tuple[str, ...]) -> None
             mapping.pop(field, None)
 
 
-def _populate_entry_links(
-    entry: FastFeedParserDict, item: _Element, atom_ns: str
+def _populate_entry_links_from_elements(
+    entry: FastFeedParserDict,
+    atom_links: list[_Element],
+    *,
+    guid_text: Optional[str] = None,
+    guid_is_permalink: bool = False,
 ) -> None:
     entry_links: list[dict[str, Optional[str]]] = []
     alternate_link: Optional[dict[str, Optional[str]]] = None
-    for link in item.findall(f"{{{atom_ns}}}link"):
+    for link in atom_links:
         rel = link.get("rel")
         href = link.get("href") or link.get("link")
         if not href:
@@ -993,8 +1118,6 @@ def _populate_entry_links(
         elif rel not in {"edit", "self"}:
             entry_links.append(link_dict)
 
-    guid = item.find("guid")
-    guid_text = guid.text.strip() if guid is not None and guid.text else None
     is_guid_url = guid_text is not None and guid_text.startswith(
         ("http://", "https://")
     )
@@ -1008,44 +1131,55 @@ def _populate_entry_links(
     elif alternate_link:
         entry["link"] = alternate_link["href"]
         entry_links.insert(0, alternate_link)
-    elif (
-        ("link" not in entry)
-        and (guid is not None)
-        and guid.get("isPermaLink") == "true"
-    ):
+    elif ("link" not in entry) and guid_is_permalink:
         entry["link"] = guid_text
 
     entry["links"] = entry_links
 
 
-def _populate_entry_content(
-    entry: FastFeedParserDict,
-    item: _Element,
-    feed_type: _FeedType,
-    atom_ns: str,
-    rss_text_by_full: Optional[dict[str, Optional[str]]] = None,
+def _populate_entry_links(
+    entry: FastFeedParserDict, item: _Element, atom_ns: str
 ) -> None:
-    content_el = None
-    if feed_type == "rss":
-        # Fast path: check pre-built text map before doing tree searches
-        if rss_text_by_full is not None:
-            content_encoded_text = rss_text_by_full.get(
-                "{http://purl.org/rss/1.0/modules/content/}encoded"
-            )
-            if content_encoded_text is not None:
-                # We have content:encoded text — still need the element for type/lang/base attrs
-                content_el = item.find("{http://purl.org/rss/1.0/modules/content/}encoded")
-            else:
-                content_text = rss_text_by_full.get("content")
-                if content_text is not None:
-                    content_el = item.find("content")
+    tags = _atom_ns_tags(atom_ns)
+    guid = item.find("guid")
+    guid_text = guid.text.strip() if guid is not None and guid.text else None
+    _populate_entry_links_from_elements(
+        entry,
+        item.findall(tags["link"]),
+        guid_text=guid_text,
+        guid_is_permalink=guid is not None and guid.get("isPermaLink") == "true",
+    )
+
+
+def _synthesize_entry_description(entry: FastFeedParserDict) -> None:
+    if "description" in entry or "content" not in entry:
+        return
+
+    content_value = entry["content"][0]["value"]
+    if content_value:
+        if "<" in content_value and ">" in content_value:
+            content_value = _RE_HTML_TAGS.sub(" ", content_value[:2048])
+            if "&" in content_value:
+                content_value = _html_mod.unescape(content_value)
+        if (
+            "  " in content_value
+            or "\n" in content_value
+            or "\t" in content_value
+            or "\r" in content_value
+        ):
+            content_value = _RE_WHITESPACE.sub(" ", content_value).strip()
         else:
-            content_el = item.find("{http://purl.org/rss/1.0/modules/content/}encoded")
-            if content_el is None:
-                content_el = item.find("content")
-    elif feed_type == "atom":
-        content_el = item.find(f"{{{atom_ns}}}content")
+            content_value = content_value.strip()
+    entry["description"] = content_value[:512]
 
+
+def _populate_entry_content_preparsed(
+    entry: FastFeedParserDict,
+    item: _Element,
+    *,
+    content_el: Optional[_Element],
+    rss_description_text: Optional[str],
+) -> None:
     if content_el is not None:
         content_type = content_el.get("type", "text/html")
         if content_type in {"xhtml", "application/xhtml+xml"}:
@@ -1055,48 +1189,51 @@ def _populate_entry_content(
         entry["content"] = [
             {
                 "type": content_type,
-                "language": content_el.get(
-                    "{http://www.w3.org/XML/1998/namespace}lang"
-                ),
-                "base": content_el.get("{http://www.w3.org/XML/1998/namespace}base"),
+                "language": content_el.get(_XML_LANG_ATTR),
+                "base": content_el.get(_XML_BASE_ATTR),
                 "value": content_value,
             }
         ]
+    elif rss_description_text:
+        entry["content"] = [
+            {
+                "type": "text/html",
+                "language": item.get(_XML_LANG_ATTR),
+                "base": item.get(_XML_BASE_ATTR),
+                "value": rss_description_text,
+            }
+        ]
+
+    _synthesize_entry_description(entry)
+
 
-    if "content" not in entry:
+def _populate_entry_content(
+    entry: FastFeedParserDict, item: _Element, feed_type: _FeedType, atom_ns: str
+) -> None:
+    content_el: Optional[_Element] = None
+    rss_description_text: Optional[str] = None
+    if feed_type == "rss":
+        content_el = item.find(_RSS_CONTENT_ENCODED_TAG)
+        if content_el is None:
+            content_el = item.find("content")
         description = item.find("description")
-        if description is not None and description.text:
-            entry["content"] = [
-                {
-                    "type": "text/html",
-                    "language": item.get("{http://www.w3.org/XML/1998/namespace}lang"),
-                    "base": item.get("{http://www.w3.org/XML/1998/namespace}base"),
-                    "value": description.text,
-                }
-            ]
+        if description is not None:
+            rss_description_text = description.text
+    elif feed_type == "atom":
+        content_el = item.find(_atom_ns_tags(atom_ns)["content"])
 
-    if "description" not in entry and "content" in entry:
-        content_value = entry["content"][0]["value"]
-        if content_value:
-            if "<" in content_value:
-                content_value = _RE_HTML_TAGS.sub(" ", content_value[:1024])
-                content_value = _html_mod.unescape(content_value)
-            if (
-                "  " in content_value
-                or "\n" in content_value
-                or "\t" in content_value
-                or "\r" in content_value
-            ):
-                content_value = _RE_WHITESPACE.sub(" ", content_value).strip()
-            else:
-                content_value = content_value.strip()
-        entry["description"] = content_value[:512]
+    _populate_entry_content_preparsed(
+        entry,
+        item,
+        content_el=content_el,
+        rss_description_text=rss_description_text,
+    )
 
 
 def _parse_media_content(item: _Element) -> list[dict[str, Any]] | None:
     media_contents: list[dict[str, Any]] = []
 
-    for media in item.findall(".//{http://search.yahoo.com/mrss/}content"):
+    for media in item.findall(f".//{_MEDIA_CONTENT_TAG}"):
         media_item: dict[str, str | int | None] = {
             "url": media.get("url"),
             "type": media.get("type"),
@@ -1106,32 +1243,32 @@ def _parse_media_content(item: _Element) -> list[dict[str, Any]] | None:
         }
         _coerce_int_fields(media_item, ("width", "height"))
 
-        title = media.find("{http://search.yahoo.com/mrss/}title")
+        title = media.find(_MEDIA_TITLE_TAG)
         if title is not None and title.text:
             media_item["title"] = title.text.strip()
 
-        text = media.find("{http://search.yahoo.com/mrss/}text")
+        text = media.find(_MEDIA_TEXT_TAG)
         if text is not None and text.text:
             media_item["text"] = text.text.strip()
 
-        desc = media.find("{http://search.yahoo.com/mrss/}description")
+        desc = media.find(_MEDIA_DESCRIPTION_TAG)
         if desc is None:
             parent = media.getparent()
             if parent is not None:
-                desc = parent.find("{http://search.yahoo.com/mrss/}description")
+                desc = parent.find(_MEDIA_DESCRIPTION_TAG)
         if desc is not None and desc.text:
             media_item["description"] = desc.text.strip()
 
-        credit = media.find("{http://search.yahoo.com/mrss/}credit")
+        credit = media.find(_MEDIA_CREDIT_TAG)
         if credit is None:
             parent = media.getparent()
             if parent is not None:
-                credit = parent.find("{http://search.yahoo.com/mrss/}credit")
+                credit = parent.find(_MEDIA_CREDIT_TAG)
         if credit is not None and credit.text:
             media_item["credit"] = credit.text.strip()
             media_item["credit_scheme"] = credit.get("scheme")
 
-        thumbnail = media.find("{http://search.yahoo.com/mrss/}thumbnail")
+        thumbnail = media.find(_MEDIA_THUMBNAIL_TAG)
         if thumbnail is not None:
             media_item["thumbnail_url"] = thumbnail.get("url")
 
@@ -1140,9 +1277,9 @@ def _parse_media_content(item: _Element) -> list[dict[str, Any]] | None:
             media_contents.append(cleaned)
 
     if not media_contents:
-        for thumbnail in item.findall(".//{http://search.yahoo.com/mrss/}thumbnail"):
+        for thumbnail in item.findall(f".//{_MEDIA_THUMBNAIL_TAG}"):
             parent = thumbnail.getparent()
-            if parent is None or parent.tag == "{http://search.yahoo.com/mrss/}content":
+            if parent is None or parent.tag == _MEDIA_CONTENT_TAG:
                 continue
             thumb_item: dict[str, str | int | None] = {
                 "url": thumbnail.get("url"),
@@ -1161,47 +1298,26 @@ def _parse_media_content(item: _Element) -> list[dict[str, Any]] | None:
 def _parse_enclosures(item: _Element) -> list[dict[str, Any]] | None:
     enclosures: list[dict[str, Any]] = []
     for enclosure in item.findall("enclosure"):
-        enc_item: dict[str, str | int | None] = {
-            "url": enclosure.get("url"),
-            "type": enclosure.get("type"),
-            "length": enclosure.get("length"),
-        }
-        length = enc_item.get("length")
-        if length:
-            try:
-                enc_item["length"] = int(length)
-            except (ValueError, TypeError):
-                enc_item.pop("length", None)
-
-        cleaned = _drop_none_values(enc_item)
+        cleaned = _parse_enclosure_element(enclosure)
         if cleaned.get("url"):
             enclosures.append(cleaned)
 
     return enclosures or None
 
 
-def _build_rss_item_text_maps(
-    item: _Element,
-) -> tuple[dict[str, Optional[str]], dict[str, Optional[str]]]:
-    by_local: dict[str, Optional[str]] = {}
-    by_full: dict[str, Optional[str]] = {}
-    for child in item:
-        tag = child.tag
-        if not isinstance(tag, str):
-            continue
-        text_value = child.text or None
-        if tag not in by_full:
-            by_full[tag] = text_value
-        # Fast path: ~80% of RSS tags have no namespace or colon prefix
-        if "{" in tag:
-            local = tag.rsplit("}", 1)[1].lower()
-        elif ":" in tag:
-            local = tag.split(":", 1)[1].lower()
-        else:
-            local = tag.lower()
-        if local not in by_local:
-            by_local[local] = text_value
-    return by_local, by_full
+def _parse_enclosure_element(enclosure: _Element) -> dict[str, Any]:
+    enc_item: dict[str, str | int | None] = {
+        "url": enclosure.get("url"),
+        "type": enclosure.get("type"),
+        "length": enclosure.get("length"),
+    }
+    length = enc_item.get("length")
+    if length:
+        try:
+            enc_item["length"] = int(length)
+        except (ValueError, TypeError):
+            enc_item.pop("length", None)
+    return _drop_none_values(enc_item)
 
 
 def _first_non_empty(
@@ -1218,13 +1334,78 @@ def _parse_rss_feed_entry_fast(
     item: _Element,
     atom_ns: str,
     has_media_ns: bool = True,
+    *,
+    include_content: bool = True,
+    include_tags: bool = True,
+    include_media: bool = True,
+    include_enclosures: bool = True,
 ) -> FastFeedParserDict:
-    text_by_local, text_by_full = _build_rss_item_text_maps(item)
+    atom_tags = _atom_ns_tags(atom_ns)
+    text_by_local: dict[str, Optional[str]] = {}
+    text_by_full: dict[str, Optional[str]] = {}
+    atom_links: list[_Element] = []
+    guid_element: Optional[_Element] = None
+    encoded_content_el: Optional[_Element] = None
+    raw_content_el: Optional[_Element] = None
+    rss_description_text: Optional[str] = None
+    tag_categories: list[dict[str, str | None]] = []
+    tag_subjects: list[dict[str, str | None]] = []
+    enclosures: list[dict[str, Any]] = []
+
+    for child in item:
+        tag = child.tag
+        if not isinstance(tag, str):
+            continue
+
+        text_value = child.text or None
+        if tag not in text_by_full:
+            text_by_full[tag] = text_value
+
+        if "{" in tag:
+            local = tag.rsplit("}", 1)[1].lower()
+        elif ":" in tag:
+            local = tag.split(":", 1)[1].lower()
+        else:
+            local = tag.lower()
+        if local not in text_by_local:
+            text_by_local[local] = text_value
+
+        if tag == atom_tags["link"]:
+            atom_links.append(child)
+        elif tag == "guid":
+            if guid_element is None:
+                guid_element = child
+        elif tag == _RSS_CONTENT_ENCODED_TAG:
+            if encoded_content_el is None:
+                encoded_content_el = child
+        elif tag == "content":
+            if raw_content_el is None:
+                raw_content_el = child
+        elif tag == "description":
+            if rss_description_text is None:
+                rss_description_text = text_value
+
+        if include_enclosures and tag == "enclosure":
+            cleaned = _parse_enclosure_element(child)
+            if cleaned.get("url"):
+                enclosures.append(cleaned)
+
+        if include_tags:
+            if local == "category":
+                term = text_value.strip() if text_value else None
+                if term:
+                    tag_categories.append(
+                        {"term": term, "scheme": child.get("domain"), "label": None}
+                    )
+            elif tag == _DC_SUBJECT_TAG:
+                term = text_value.strip() if text_value else None
+                if term:
+                    tag_subjects.append({"term": term, "scheme": None, "label": None})
 
     entry = FastFeedParserDict()
-    atom_id = text_by_full.get(f"{{{atom_ns}}}id")
+    atom_id = text_by_full.get(atom_tags["id"])
     rss_guid = text_by_local.get("guid")
-    rdf_about = item.get("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about")
+    rdf_about = item.get(_RDF_ABOUT_ATTR)
     entry_id: Optional[str] = atom_id or rss_guid or rdf_about
     if entry_id:
         entry["id"] = entry_id.strip()
@@ -1269,13 +1450,20 @@ def _parse_rss_feed_entry_fast(
     if "updated" in entry and "published" not in entry:
         entry["published"] = entry["updated"]
 
-    # Inline link population for RSS (avoids redundant findall/find for 98.8% of entries)
-    atom_links = item.findall(f"{{{atom_ns}}}link")
     if atom_links:
-        # Has atom:link elements - use full logic
-        _populate_entry_links(entry, item, atom_ns)
+        guid_text = (
+            guid_element.text.strip()
+            if guid_element is not None and guid_element.text
+            else None
+        )
+        _populate_entry_links_from_elements(
+            entry,
+            atom_links,
+            guid_text=guid_text,
+            guid_is_permalink=guid_element is not None
+            and guid_element.get("isPermaLink") == "true",
+        )
     else:
-        # Common RSS case: no atom:link elements
         entry["links"] = []
         if (
             "link" not in entry
@@ -1287,20 +1475,27 @@ def _parse_rss_feed_entry_fast(
     if "id" not in entry and "link" in entry:
         entry["id"] = entry["link"]
 
-    _populate_entry_content(entry, item, "rss", atom_ns, rss_text_by_full=text_by_full)
+    if include_content:
+        _populate_entry_content_preparsed(
+            entry,
+            item,
+            content_el=(
+                encoded_content_el if encoded_content_el is not None else raw_content_el
+            ),
+            rss_description_text=rss_description_text,
+        )
 
-    if has_media_ns:
+    if include_media and has_media_ns:
         media_contents = _parse_media_content(item)
         if media_contents:
             entry["media_content"] = media_contents
 
-    enclosures = _parse_enclosures(item)
-    if enclosures:
+    if include_enclosures and enclosures:
         entry["enclosures"] = enclosures
 
     author = _first_non_empty(text_by_local, ("author", "creator"))
     if not author:
-        atom_author = item.find(f"{{{atom_ns}}}author/{{{atom_ns}}}name")
+        atom_author = item.find(atom_tags["author_name"])
         author = (
             atom_author.text.strip()
             if atom_author is not None and atom_author.text
@@ -1313,9 +1508,8 @@ def _parse_rss_feed_entry_fast(
     if comments:
         entry["comments"] = comments.strip()
 
-    tags = _parse_tags(item, "rss", atom_ns)
-    if tags:
-        entry["tags"] = tags
+    if include_tags and (tag_categories or tag_subjects):
+        entry["tags"] = tag_categories + tag_subjects
 
     return entry
 
@@ -1324,93 +1518,134 @@ def _parse_atom_feed_entry_fast(
     item: _Element,
     atom_ns: str,
     has_media_ns: bool = True,
+    *,
+    include_content: bool = True,
+    include_tags: bool = True,
+    include_media: bool = True,
+    include_enclosures: bool = True,
 ) -> FastFeedParserDict:
-    ns = f"{{{atom_ns}}}"
+    t = _atom_ns_tags(atom_ns)
+    atom_link_tag = t["link"]
+    atom_author_tag = t["ns"] + "author"
+    atom_name_tag = t["ns"] + "name"
+    atom_links: list[_Element] = []
+    atom_categories: list[dict[str, str | None]] = []
+    enclosures: list[dict[str, Any]] = []
+    content_el: Optional[_Element] = None
+    author_name: Optional[str] = None
+    first_link_href: Optional[str] = None
+    published_source: Optional[str] = None
+    updated_source: Optional[str] = None
+    published_fallback_source: Optional[str] = None
+    updated_fallback_source: Optional[str] = None
+
     entry = FastFeedParserDict()
+    for child in item:
+        tag = child.tag
+        if not isinstance(tag, str):
+            continue
 
-    # ID
-    el = item.find(ns + "id")
-    if el is not None and el.text:
-        entry["id"] = el.text.strip()
-
-    # Title
-    el = item.find(ns + "title")
-    if el is not None and el.text:
-        entry["title"] = el.text.strip()
-
-    # Description (summary)
-    el = item.find(ns + "summary")
-    if el is not None and el.text:
-        entry["description"] = el.text.strip()
-
-    # Link (href attribute)
-    el = item.find(ns + "link")
-    if el is not None:
-        href = el.get("href")
-        if href:
-            entry["link"] = href.strip()
-
-    # Dates: Atom 1.0 uses published/updated, Atom 0.3 uses issued/modified
-    is_atom_03 = atom_ns == "http://purl.org/atom/ns#"
-    pub_tag = "issued" if is_atom_03 else "published"
-    upd_tag = "modified" if is_atom_03 else "updated"
-    pub_fallback_tag = "published" if is_atom_03 else "issued"
-    upd_fallback_tag = "updated" if is_atom_03 else "modified"
-
-    el = item.find(ns + pub_tag)
-    if el is not None and el.text:
-        published = _parse_date(el.text)
+        text_value = child.text
+        if tag == t["id"] and "id" not in entry and text_value:
+            entry["id"] = text_value.strip()
+        elif tag == t["title"] and "title" not in entry and text_value:
+            entry["title"] = text_value.strip()
+        elif tag == t["summary"] and "description" not in entry and text_value:
+            entry["description"] = text_value.strip()
+        elif tag == t["published"] and published_source is None and text_value:
+            published_source = text_value
+        elif tag == t["updated"] and updated_source is None and text_value:
+            updated_source = text_value
+        elif (
+            tag == t["pub_fallback"]
+            and published_fallback_source is None
+            and text_value
+        ):
+            published_fallback_source = text_value
+        elif (
+            tag == t["upd_fallback"] and updated_fallback_source is None and text_value
+        ):
+            updated_fallback_source = text_value
+        elif tag == atom_link_tag:
+            atom_links.append(child)
+            href = child.get("href")
+            if href and first_link_href is None:
+                first_link_href = href.strip()
+        elif include_content and tag == t["content"] and content_el is None:
+            content_el = child
+        elif tag == atom_author_tag and author_name is None:
+            author_name_el = child.find(atom_name_tag)
+            if author_name_el is not None and author_name_el.text:
+                author_name = author_name_el.text.strip()
+
+        if include_tags and tag == t["category"]:
+            term = child.get("term")
+            if term:
+                atom_categories.append(
+                    {
+                        "term": term,
+                        "scheme": child.get("scheme"),
+                        "label": child.get("label"),
+                    }
+                )
+
+        if include_enclosures and tag == "enclosure":
+            cleaned = _parse_enclosure_element(child)
+            if cleaned.get("url"):
+                enclosures.append(cleaned)
+
+    if first_link_href:
+        entry["link"] = first_link_href
+
+    if published_source:
+        published = _parse_date(published_source)
         if published:
             entry["published"] = published
 
-    el = item.find(ns + upd_tag)
-    if el is not None and el.text:
-        updated = _parse_date(el.text)
+    if updated_source:
+        updated = _parse_date(updated_source)
         if updated:
             entry["updated"] = updated
 
-    # Fallback date fields for mixed namespace scenarios
-    if "published" not in entry:
-        el = item.find(ns + pub_fallback_tag)
-        if el is not None and el.text:
-            published = _parse_date(el.text)
-            if published:
-                entry["published"] = published
+    if "published" not in entry and published_fallback_source:
+        published = _parse_date(published_fallback_source)
+        if published:
+            entry["published"] = published
 
-    if "updated" not in entry:
-        el = item.find(ns + upd_fallback_tag)
-        if el is not None and el.text:
-            updated = _parse_date(el.text)
-            if updated:
-                entry["updated"] = updated
+    if "updated" not in entry and updated_fallback_source:
+        updated = _parse_date(updated_fallback_source)
+        if updated:
+            entry["updated"] = updated
 
     if "updated" in entry and "published" not in entry:
         entry["published"] = entry["updated"]
 
-    _populate_entry_links(entry, item, atom_ns)
+    _populate_entry_links_from_elements(entry, atom_links)
 
     if "id" not in entry and "link" in entry:
         entry["id"] = entry["link"]
 
-    _populate_entry_content(entry, item, "atom", atom_ns)
+    if include_content:
+        _populate_entry_content_preparsed(
+            entry,
+            item,
+            content_el=content_el,
+            rss_description_text=None,
+        )
 
-    if has_media_ns:
+    if include_media and has_media_ns:
         media_contents = _parse_media_content(item)
         if media_contents:
             entry["media_content"] = media_contents
 
-    enclosures = _parse_enclosures(item)
-    if enclosures:
+    if include_enclosures and enclosures:
         entry["enclosures"] = enclosures
 
-    # Author
-    el = item.find(ns + "author/" + ns + "name")
-    if el is not None and el.text:
-        entry["author"] = el.text.strip()
+    if author_name:
+        entry["author"] = author_name
 
-    tags = _parse_tags(item, "atom", atom_ns)
-    if tags:
-        entry["tags"] = tags
+    if include_tags and atom_categories:
+        entry["tags"] = atom_categories
 
     return entry
 
@@ -1420,15 +1655,36 @@ def _parse_feed_entry(
     feed_type: _FeedType,
     atom_namespace: Optional[str] = None,
     has_media_ns: bool = True,
+    *,
+    include_content: bool = True,
+    include_tags: bool = True,
+    include_media: bool = True,
+    include_enclosures: bool = True,
 ) -> FastFeedParserDict:
     # Use dynamic atom namespace or fallback to default
     atom_ns = atom_namespace or "http://www.w3.org/2005/Atom"
 
     if feed_type == "rss":
-        return _parse_rss_feed_entry_fast(item, atom_ns, has_media_ns)
+        return _parse_rss_feed_entry_fast(
+            item,
+            atom_ns,
+            has_media_ns,
+            include_content=include_content,
+            include_tags=include_tags,
+            include_media=include_media,
+            include_enclosures=include_enclosures,
+        )
 
     if feed_type == "atom":
-        return _parse_atom_feed_entry_fast(item, atom_ns, has_media_ns)
+        return _parse_atom_feed_entry_fast(
+            item,
+            atom_ns,
+            has_media_ns,
+            include_content=include_content,
+            include_tags=include_tags,
+            include_media=include_media,
+            include_enclosures=include_enclosures,
+        )
 
     # RDF path uses the generic field machinery
     # Check if this is Atom 0.3 to use different date field names
@@ -1544,16 +1800,18 @@ def _parse_feed_entry(
     if "id" not in entry and "link" in entry:
         entry["id"] = entry["link"]
 
-    _populate_entry_content(entry, item, feed_type, atom_ns)
+    if include_content:
+        _populate_entry_content(entry, item, feed_type, atom_ns)
 
-    if has_media_ns:
+    if include_media and has_media_ns:
         media_contents = _parse_media_content(item)
         if media_contents:
             entry["media_content"] = media_contents
 
-    enclosures = _parse_enclosures(item)
-    if enclosures:
-        entry["enclosures"] = enclosures
+    if include_enclosures:
+        enclosures = _parse_enclosures(item)
+        if enclosures:
+            entry["enclosures"] = enclosures
 
     author = get_field_value(
         "author",
@@ -1569,9 +1827,10 @@ def _parse_feed_entry(
         entry["author"] = author
 
     # Parse entry-level tags/categories
-    tags = _parse_tags(item, feed_type, atom_ns)
-    if tags:
-        entry["tags"] = tags
+    if include_tags:
+        tags = _parse_tags(item, feed_type, atom_ns)
+        if tags:
+            entry["tags"] = tags
 
     return entry
 
@@ -1902,6 +2161,7 @@ def _slow_dateparser(value: str) -> Optional[datetime.datetime]:
         return None
 
 
+@lru_cache(maxsize=8192)
 def _parse_date(date_str: str) -> Optional[str]:
     """Parse date string and return as an ISO 8601 formatted UTC string.
 

{fastfeedparser-0.5.5 → fastfeedparser-0.5.6/src/fastfeedparser.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fastfeedparser
-Version: 0.5.5
+Version: 0.5.6
 Summary: High performance RSS, Atom, JSON and RDF feed parser in Python
 Home-page: https://github.com/kagisearch/fastfeedparser
 Author: Vladimir Prelovac
@@ -25,6 +25,7 @@ Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 Provides-Extra: dateparser
 Provides-Extra: brotli
+Provides-Extra: orjson
 Provides-Extra: full
 License-File: LICENSE
 
@@ -146,7 +147,7 @@ Speedup: 50.1x
 
 ### Main Functions
 
-- `parse(source)`: Parse feed from a source that can be URL or a string
+- `parse(source, *, include_content=True, include_tags=True, include_media=True, include_enclosures=True)`: Parse feed from a URL/XML/JSON source, with optional field extraction toggles for faster parsing.
 
 
 ### Feed Object Structure

{fastfeedparser-0.5.5 → fastfeedparser-0.5.6}/src/fastfeedparser.egg-info/requires.txt

@@ -10,3 +10,7 @@ dateparser
 [full]
 dateparser
 brotli
+orjson
+
+[orjson]
+orjson

{fastfeedparser-0.5.5 → fastfeedparser-0.5.6}/tests/test_encoding.py

@@ -56,3 +56,49 @@ def test_meta_refresh_none_when_missing():
 def test_meta_refresh_none_when_same_url():
     html = '<html><head><meta http-equiv="refresh" content="0; url=https://example.com/"></head></html>'
     assert _extract_meta_refresh_url(html, "https://example.com/") is None
+
+
+def test_parse_optional_field_flags():
+    xml = """<?xml version="1.0" encoding="utf-8"?>
+<rss version="2.0"
+     xmlns:content="http://purl.org/rss/1.0/modules/content/"
+     xmlns:media="http://search.yahoo.com/mrss/">
+  <channel>
+    <title>Example Feed</title>
+    <category>feed-tag</category>
+    <item>
+      <title>Example Item</title>
+      <link>https://example.com/item</link>
+      <pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
+      <description>Summary</description>
+      <content:encoded><![CDATA[<p>Body</p>]]></content:encoded>
+      <category>entry-tag</category>
+      <enclosure url="https://example.com/audio.mp3" type="audio/mpeg" length="123" />
+      <media:content url="https://example.com/image.jpg" type="image/jpeg" />
+    </item>
+  </channel>
+</rss>
+"""
+    full = parse(xml)
+    entry_full = full.entries[0]
+    assert "content" in entry_full
+    assert "tags" in entry_full
+    assert "enclosures" in entry_full
+    assert "media_content" in entry_full
+    assert "tags" in full.feed
+
+    trimmed = parse(
+        xml,
+        include_content=False,
+        include_tags=False,
+        include_media=False,
+        include_enclosures=False,
+    )
+    entry_trimmed = trimmed.entries[0]
+    assert "content" not in entry_trimmed
+    assert "tags" not in entry_trimmed
+    assert "enclosures" not in entry_trimmed
+    assert "media_content" not in entry_trimmed
+    assert "tags" not in trimmed.feed
+    assert entry_trimmed["title"] == "Example Item"
+    assert entry_trimmed["link"] == "https://example.com/item"