pulse-engine 0.2.0__py3-none-any.whl

pulse_engine/adapters/digital_news_metadata.py

@@ -0,0 +1,536 @@
+"""Discovers news article URLs and metadata from Google News sitemaps."""
+
+import re
+import xml.etree.ElementTree as ET
+from datetime import UTC, datetime, timedelta
+
+import httpx
+import structlog
+
+from .exceptions import SitemapDiscoveryError
+from .models import NewsArticleMetadata
+
+logger = structlog.get_logger(__name__)
+
+# Regex to extract a year-month from date-encoded sitemap URL filenames.
+# Matches patterns like: 2026-April-1.xml, 2026-03-1.xml, 2025-December-2.xml
+_SITEMAP_DATE_RE = re.compile(
+    r"(\d{4})[-_]"
+    r"(January|February|March|April|May|June|July|August|September|October|November|December|"
+    r"01|02|03|04|05|06|07|08|09|10|11|12)"
+    r"(?:[-_]\d+)?\.xml",
+    re.IGNORECASE,
+)
+
+_MONTH_NAME_TO_NUM = {
+    "january": 1,
+    "february": 2,
+    "march": 3,
+    "april": 4,
+    "may": 5,
+    "june": 6,
+    "july": 7,
+    "august": 8,
+    "september": 9,
+    "october": 10,
+    "november": 11,
+    "december": 12,
+}
+
+_FALLBACK_PATHS = ["/news-sitemap.xml", "/sitemap_news.xml", "/sitemap-news.xml"]
+
+_DEFAULT_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (compatible; PulseBot/1.0)",
+}
+
+# HTTP status codes that indicate bot-blocking; trigger curl_cffi fallback
+_BLOCKED_STATUS_CODES = frozenset([403, 429, 503])
+
+
+class DigitalNewsMetadataAdapter:
+    """Discovers news article URLs and metadata from a domain's Google News sitemaps.
+
+    Two usage modes:
+
+    1. Domain-based discovery (auto-discovers sitemaps via robots.txt / probes):
+       ``DigitalNewsMetadataAdapter("hindustantimes.com")``
+
+    2. Direct sitemap URLs (skips discovery entirely):
+       ``DigitalNewsMetadataAdapter(sitemap_urls=["https://example.com/news-sitemap.xml"])``
+       ``DigitalNewsMetadataAdapter("example.com", sitemap_urls=["https://..."])``
+
+    When ``sitemap_urls`` is provided, ``domain`` is optional and used only for logging.
+
+    Parent sitemap index URLs are fully supported in both modes. If a provided (or
+    discovered) URL points to a ``<sitemapindex>``, all child sitemaps are fetched and
+    parsed recursively. Example:
+
+    .. code-block:: python
+
+        # ET index → resolves sitemap-today.xml, sitemap-yesterday.xml, … automatically
+        adapter = DigitalNewsMetadataAdapter(
+            sitemap_urls=[
+                "https://economictimes.indiatimes.com/etstatic/sitemaps/et/news/sitemap-index.xml"
+            ]
+        )
+        articles = await adapter.fetch(since=timedelta(days=7))
+    """
+
+    def __init__(
+        self,
+        domain: str | None = None,
+        *,
+        sitemap_urls: list[str] | None = None,
+    ) -> None:
+        if domain is None and not sitemap_urls:
+            raise ValueError("Provide domain, sitemap_urls, or both.")
+        self._sitemap_urls = sitemap_urls
+        # Infer domain from first sitemap URL when not explicitly given
+        if domain is None:
+            from urllib.parse import urlparse
+
+            self.domain = urlparse(sitemap_urls[0]).netloc  # type: ignore[index]
+        else:
+            self.domain = domain
+        self._base_url = f"https://{self.domain}"
+
+    async def fetch(
+        self,
+        since: timedelta | None = None,
+        keywords: list[str] | None = None,
+    ) -> list[NewsArticleMetadata]:
+        """Discover and return filtered news article metadata."""
+        logger.info(
+            "fetch_started", domain=self.domain, since=str(since), keywords=keywords
+        )
+        if self._sitemap_urls is not None:
+            sitemap_urls = self._sitemap_urls
+            logger.info(
+                "sitemaps_provided",
+                domain=self.domain,
+                count=len(sitemap_urls),
+                urls=sitemap_urls,
+            )
+        else:
+            sitemap_urls = await self._discover_sitemap_urls()
+            logger.info(
+                "sitemaps_discovered",
+                domain=self.domain,
+                count=len(sitemap_urls),
+                urls=sitemap_urls,
+            )
+        cutoff = (datetime.now(UTC) - since) if since is not None else None
+        entries = await self._resolve_and_parse(sitemap_urls, cutoff=cutoff)
+        logger.info("entries_parsed", domain=self.domain, total_entries=len(entries))
+        filtered = self._apply_filters(entries, since=since, keywords=keywords)
+        logger.info(
+            "entries_filtered",
+            domain=self.domain,
+            before=len(entries),
+            after=len(filtered),
+        )
+        return filtered
+
+    # --- Discovery ---
+
+    async def _discover_sitemap_urls(self) -> list[str]:
+        all_urls: list[str] = []
+        robots_url = f"{self._base_url}/robots.txt"
+        logger.debug("fetching_robots_txt", url=robots_url)
+        try:
+            status, text = await _fetch_url(robots_url)
+            logger.debug("robots_txt_response", status=status)
+            if 200 <= status < 300:
+                for line in text.splitlines():
+                    stripped = line.strip()
+                    if stripped.lower().startswith("sitemap:"):
+                        url = stripped.split(":", 1)[1].strip()
+                        all_urls.append(url)
+                if all_urls:
+                    news_urls = _filter_news_sitemaps(all_urls)
+                    news_urls = _prioritize_news_sitemaps(news_urls)
+                    logger.info(
+                        "sitemap_directives_found",
+                        domain=self.domain,
+                        total=len(all_urls),
+                        news=len(news_urls),
+                        urls=news_urls,
+                    )
+                    if news_urls:
+                        return news_urls
+                logger.info("no_news_sitemap_directives", domain=self.domain)
+        except Exception as exc:
+            logger.warning(
+                "robots_txt_fetch_failed", domain=self.domain, error=str(exc)
+            )
+
+        # Fallback probes — only news-related paths
+        logger.info("trying_fallback_probes", domain=self.domain, paths=_FALLBACK_PATHS)
+        found: list[str] = []
+        for path in _FALLBACK_PATHS:
+            probe_url = f"{self._base_url}{path}"
+            try:
+                status, _ = await _fetch_url(probe_url)
+                if 200 <= status < 300:
+                    logger.info("fallback_probe_success", url=probe_url)
+                    found.append(probe_url)
+                else:
+                    logger.debug("fallback_probe_miss", url=probe_url, status=status)
+            except Exception:
+                logger.debug("fallback_probe_failed", url=probe_url)
+
+        if not found:
+            raise SitemapDiscoveryError(
+                self.domain,
+                "No news sitemaps found in robots.txt and all fallback probes failed",
+            )
+        return found
+
+    # --- XML Parsing ---
+
+    def _parse_sitemap_index(self, xml_text: str) -> tuple[bool, list[str]]:
+        try:
+            root = ET.fromstring(xml_text)  # noqa: S314
+        except ET.ParseError:
+            return False, []
+
+        tag = root.tag
+        if "}" in tag:
+            tag = tag.split("}", 1)[1]
+
+        if tag != "sitemapindex":
+            return False, []
+
+        ns = _detect_sm_ns(root)
+        urls: list[str] = []
+        for sitemap_el in root.findall(f"{ns}sitemap"):
+            loc = sitemap_el.find(f"{ns}loc")
+            if loc is not None and loc.text:
+                urls.append(loc.text.strip())
+        return True, urls
+
+    def _parse_plain_entries(self, xml_text: str) -> list[NewsArticleMetadata]:
+        """Fallback parser for plain sitemaps without <news:news> elements.
+
+        Handles the format used by TOI, ET, and similar publishers:
+        ``<url><loc>URL</loc><lastmod>DATE</lastmod></url>``
+        Title is unavailable at this stage; keyword filtering falls back to URL slug
+        matching.
+        """
+        try:
+            root = ET.fromstring(xml_text)  # noqa: S314
+        except ET.ParseError:
+            return []
+
+        sm_ns = _detect_sm_ns(root)
+        entries: list[NewsArticleMetadata] = []
+        for url_el in root.findall(f"{sm_ns}url"):
+            loc_el = url_el.find(f"{sm_ns}loc")
+            if loc_el is None or not loc_el.text:
+                continue
+            lastmod_el = url_el.find(f"{sm_ns}lastmod")
+            pub_date = (
+                _parse_date(lastmod_el.text.strip())
+                if lastmod_el is not None and lastmod_el.text
+                else None
+            )
+            entries.append(
+                NewsArticleMetadata(
+                    url=loc_el.text.strip(),
+                    title=None,
+                    publication_name=None,
+                    language=None,
+                    publication_date=pub_date,
+                    keywords=[],
+                )
+            )
+        return entries
+
+    def _parse_news_entries(self, xml_text: str) -> list[NewsArticleMetadata]:
+        try:
+            root = ET.fromstring(xml_text)  # noqa: S314
+        except ET.ParseError:
+            return []
+
+        sm_ns = _detect_sm_ns(root)
+        news_ns = "{http://www.google.com/schemas/sitemap-news/0.9}"
+
+        entries: list[NewsArticleMetadata] = []
+        for url_el in root.findall(f"{sm_ns}url"):
+            loc_el = url_el.find(f"{sm_ns}loc")
+            if loc_el is None or not loc_el.text:
+                continue
+
+            news_el = url_el.find(f"{news_ns}news")
+            if news_el is None:
+                continue
+
+            url = loc_el.text.strip()
+            title = _text(news_el.find(f"{news_ns}title"))
+
+            pub_el = news_el.find(f"{news_ns}publication")
+            pub_name = (
+                _text(pub_el.find(f"{news_ns}name")) if pub_el is not None else None
+            )
+            language = (
+                _text(pub_el.find(f"{news_ns}language")) if pub_el is not None else None
+            )
+
+            date_text = _text(news_el.find(f"{news_ns}publication_date"))
+            pub_date = _parse_date(date_text) if date_text else None
+
+            kw_text = _text(news_el.find(f"{news_ns}keywords"))
+            keywords = (
+                [k.strip() for k in kw_text.split(",") if k.strip()] if kw_text else []
+            )
+
+            entries.append(
+                NewsArticleMetadata(
+                    url=url,
+                    title=title,
+                    publication_name=pub_name,
+                    language=language,
+                    publication_date=pub_date,
+                    keywords=keywords,
+                )
+            )
+        return entries
+
+    async def _resolve_and_parse(
+        self,
+        sitemap_urls: list[str],
+        cutoff: datetime | None = None,
+    ) -> list[NewsArticleMetadata]:
+        all_entries: list[NewsArticleMetadata] = []
+        for url in sitemap_urls:
+            # Early termination: if URL encodes a year-month older than cutoff, skip.
+            if cutoff is not None and _sitemap_url_older_than(url, cutoff):
+                logger.debug("sitemap_skipped_too_old", url=url)
+                continue
+
+            logger.debug("fetching_sitemap", url=url)
+            try:
+                status, xml_text = await _fetch_url(url)
+                if not (200 <= status < 300):
+                    logger.warning("sitemap_fetch_failed", url=url, status=status)
+                    continue
+            except Exception as exc:
+                logger.warning("sitemap_fetch_failed", url=url, error=str(exc))
+                continue
+
+            is_index, nested = self._parse_sitemap_index(xml_text)
+            if is_index:
+                logger.info("sitemap_index_found", url=url, nested_count=len(nested))
+                nested_entries = await self._resolve_and_parse(nested, cutoff=cutoff)
+                all_entries.extend(nested_entries)
+            else:
+                try:
+                    entries = self._parse_news_entries(xml_text)
+                    if not entries:
+                        # No <news:news> elements — try plain <loc>+<lastmod> format
+                        entries = self._parse_plain_entries(xml_text)
+                        if entries:
+                            logger.info(
+                                "plain_sitemap_parsed", url=url, entries=len(entries)
+                            )
+                        else:
+                            logger.info("sitemap_parsed_empty", url=url)
+                    else:
+                        logger.info(
+                            "sitemap_parsed", url=url, news_entries=len(entries)
+                        )
+                    all_entries.extend(entries)
+                except Exception as exc:
+                    logger.warning("sitemap_parse_failed", url=url, error=str(exc))
+        return all_entries
+
+    # --- Filtering ---
+
+    def _apply_filters(
+        self,
+        entries: list[NewsArticleMetadata],
+        since: timedelta | None = None,
+        keywords: list[str] | None = None,
+        now: datetime | None = None,
+    ) -> list[NewsArticleMetadata]:
+        if now is None:
+            now = datetime.now(UTC)
+
+        result = entries
+
+        if since is not None:
+            cutoff = now - since
+            result = [
+                e
+                for e in result
+                if e.publication_date is None or e.publication_date >= cutoff
+            ]
+
+        if keywords is not None:
+            kw_lower = [k.lower() for k in keywords]
+            result = [e for e in result if _matches_keywords(e, kw_lower)]
+
+        return result
+
+
+def _detect_sm_ns(root: ET.Element) -> str:
+    """Return the sitemap namespace prefix (with braces) detected from the root tag.
+
+    Handles both the standard http:// and non-standard https:// variants used by
+    some publishers (e.g. Jagran uses https://www.sitemaps.org/...).
+    Falls back to the standard http:// prefix if undetectable.
+    """
+    tag = root.tag
+    if tag.startswith("{"):
+        return tag[: tag.index("}") + 1]
+    return "{http://www.sitemaps.org/schemas/sitemap/0.9}"
+
+
+def _sitemap_url_older_than(url: str, cutoff: datetime) -> bool:
+    """Return True if the URL encodes a year-month that is entirely before *cutoff*.
+
+    Matches filename patterns like:
+      2026-April-1.xml   2026-April-2.xml   2025-December-5.xml
+      2026-03-1.xml      2024-09.xml
+    Only skips when the *entire month* is before the cutoff — i.e., when the last
+    day of that month is still before cutoff.  This avoids skipping the current
+    month even when we are only a few days in.
+    """
+    m = _SITEMAP_DATE_RE.search(url)
+    if not m:
+        return False
+    year = int(m.group(1))
+    month_raw = m.group(2).lower()
+    month_num = _MONTH_NAME_TO_NUM.get(month_raw) or int(month_raw)
+
+    # Last day of the detected month
+    if month_num == 12:
+        last_day = datetime(year + 1, 1, 1, tzinfo=UTC) - timedelta(seconds=1)
+    else:
+        last_day = datetime(year, month_num + 1, 1, tzinfo=UTC) - timedelta(seconds=1)
+
+    return last_day < cutoff
+
+
+_LANGUAGE_PATH_SEGMENTS = frozenset(
+    [
+        "telugu",
+        "bangla",
+        "hindi",
+        "marathi",
+        "urdu",
+        "tamil",
+        "kannada",
+        "malayalam",
+        "gujarati",
+        "punjabi",
+        "odia",
+    ]
+)
+
+
+async def _fetch_url(url: str) -> tuple[int, str]:
+    """Fetch URL with httpx; retry via curl_cffi Chrome impersonation on bot-block."""
+    try:
+        async with httpx.AsyncClient(
+            follow_redirects=True, headers=_DEFAULT_HEADERS, timeout=15
+        ) as client:
+            resp = await client.get(url)
+            if resp.status_code not in _BLOCKED_STATUS_CODES:
+                return resp.status_code, resp.text
+            logger.debug(
+                "httpx_blocked_retrying_curl_cffi",
+                url=url,
+                status=resp.status_code,
+            )
+    except httpx.HTTPError as exc:
+        logger.debug("httpx_error_retrying_curl_cffi", url=url, error=str(exc))
+
+    from curl_cffi.requests import AsyncSession
+
+    # Do NOT pass _DEFAULT_HEADERS here — curl_cffi sets Chrome-native headers
+    # as part of impersonation; overriding User-Agent breaks the fingerprint.
+    async with AsyncSession(impersonate="chrome120") as session:
+        resp = await session.get(url)
+        logger.debug("curl_cffi_response", url=url, status=resp.status_code)
+        return resp.status_code, resp.text
+
+
+def _filter_news_sitemaps(urls: list[str]) -> list[str]:
+    """Keep only sitemap URLs with 'news' in the path."""
+    return [u for u in urls if "news" in u.lower().split("//", 1)[-1]]
+
+
+def _url_filename(url: str) -> str:
+    return url.rstrip("/").split("/")[-1].lower()
+
+
+def _prioritize_news_sitemaps(urls: list[str]) -> list[str]:
+    """Apply priority rules when multiple news sitemaps are discovered.
+
+    Rules (applied in order, short-circuit on match):
+    1. Strip language-specific paths (e.g. /telugu/, /bangla/).
+    2. If a sitemap-index URL exists, keep only index URLs — they recursively
+       subsume child sitemaps like sitemap-today.xml / sitemap-yesterday.xml.
+    3. If an "all-" prefixed sitemap exists, prefer it over plain variants.
+    """
+    if len(urls) <= 1:
+        return urls
+
+    # Rule 1: language filter
+    lang_filtered = [
+        u
+        for u in urls
+        if not any(
+            seg in _LANGUAGE_PATH_SEGMENTS
+            for seg in u.lower().split("//", 1)[-1].split("/")
+        )
+    ]
+    working = lang_filtered if lang_filtered else urls
+
+    if len(working) <= 1:
+        return working
+
+    # Rule 2: prefer index over individual child sitemaps
+    index_urls = [u for u in working if "index" in _url_filename(u)]
+    if index_urls:
+        return index_urls
+
+    # Rule 3: prefer "all-" prefixed sitemap
+    all_urls = [u for u in working if _url_filename(u).startswith("all-")]
+    if all_urls:
+        return all_urls
+
+    return working
+
+
+def _text(el: ET.Element | None) -> str | None:
+    if el is None or not el.text:
+        return None
+    return el.text.strip()
+
+
+def _parse_date(text: str) -> datetime | None:
+    try:
+        cleaned = text.replace("Z", "+00:00")
+        return datetime.fromisoformat(cleaned)
+    except (ValueError, TypeError):
+        return None
+
+
+def _matches_keywords(entry: NewsArticleMetadata, kw_lower: list[str]) -> bool:
+    title_lower = (entry.title or "").lower()
+    entry_kw_lower = [k.lower() for k in entry.keywords]
+    url_lower = entry.url.lower()
+    for kw in kw_lower:
+        if kw in title_lower:
+            return True
+        if any(kw in ek for ek in entry_kw_lower):
+            return True
+        # Plain sitemaps have no title/keywords — match via URL slug (ASCII only).
+        # e.g. "rahul gandhi" → "rahul-gandhi" in URL path
+        if kw.isascii():
+            slug = kw.replace(" ", "-")
+            if slug in url_lower:
+                return True
+    return False

pulse_engine/adapters/exceptions.py

@@ -0,0 +1,10 @@
+"""Custom exceptions for digital news adapters."""
+
+
+class SitemapDiscoveryError(Exception):
+    """Raised when sitemap discovery fails for a domain."""
+
+    def __init__(self, domain: str, reason: str) -> None:
+        self.domain = domain
+        self.reason = reason
+        super().__init__(f"Sitemap discovery failed for {domain}: {reason}")

pulse_engine/adapters/models.py

@@ -0,0 +1,134 @@
+"""Data models for digital news, speech, YouTube, and Twitter adapters."""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+
+# ---------------------------------------------------------------------------
+# Digital news models
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class NewsArticleMetadata:
+    url: str
+    title: str | None
+    publication_name: str | None
+    language: str | None
+    publication_date: datetime | None
+    keywords: list[str]
+
+
+@dataclass
+class ArticleContent:
+    title: str
+    content: str
+
+
+@dataclass
+class ArticleResult:
+    url: str
+    content: ArticleContent | None
+    error: str | None
+
+
+# ---------------------------------------------------------------------------
+# Speech models
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SpeechMetadata:
+    url: str
+    title: str | None
+    speaker: str | None
+    speech_date: datetime | None
+    pdf_url: str | None
+    youtube_url: str | None
+    language: str | None
+
+
+@dataclass
+class SpeechContent:
+    text: str
+    language: str
+    pdf_url: str | None
+
+
+@dataclass
+class SpeechResult:
+    url: str
+    content: SpeechContent | None
+    error: str | None
+
+
+# ---------------------------------------------------------------------------
+# Twitter models
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class TweetMetadata:
+    tweet_id: str
+    url: str
+    text: str
+    screen_name: str
+    created_at: datetime
+    language: str
+    hashtags: list[str]
+    embedded_urls: list[str]
+    views: int
+    retweet_count: int
+    like_count: int
+    reply_count: int
+    has_images: bool
+    has_videos: bool
+
+
+@dataclass
+class TwitterUserResult:
+    screen_name: str
+    tweets: list[TweetMetadata]
+    error: str | None
+
+
+# ---------------------------------------------------------------------------
+# YouTube models
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class VideoMetadata:
+    video_id: str
+    url: str
+    title: str
+    channel_id: str
+    channel_name: str
+    published_at: datetime
+    description: str
+    tags: list[str]
+    duration_seconds: int
+    view_count: str | None
+    like_count: str | None
+    comment_count: str | None
+    thumbnail_url: str | None = None
+
+
+@dataclass
+class TranscriptSegment:
+    start: float
+    end: float
+    text: str
+
+
+@dataclass
+class VideoAudio:
+    transcript: str
+    language: str
+    segments: list[TranscriptSegment] = field(default_factory=list)
+
+
+@dataclass
+class VideoResult:
+    video_id: str
+    audio: VideoAudio | None
+    error: str | None