pulse-engine 0.2.0__py3-none-any.whl

pulse_engine/adapters/opensearch_storage.py

@@ -0,0 +1,160 @@
+"""OpenSearchStorageAdapter — upsert documents into an OpenSearch index.
+
+Schema (index name, field mappings, id field) is defined at call time so
+products can configure it entirely from pipeline.yaml args.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# Maps pipeline.yaml type names → OpenSearch mapping types
+_TYPE_MAP: dict[str, dict[str, Any]] = {
+    "text":    {"type": "text"},
+    "keyword": {"type": "keyword"},
+    "date":    {"type": "date"},
+    "integer": {"type": "integer"},
+    "long":    {"type": "long"},
+    "float":   {"type": "float"},
+    "boolean": {"type": "boolean"},
+    "object":  {"type": "object", "enabled": False},  # stored but not indexed
+}
+
+
+class OpenSearchStorageAdapter:
+    """Upsert documents into an OpenSearch index.
+
+    Args:
+        url: OpenSearch endpoint URL.
+        username: HTTP basic auth username (optional).
+        password: HTTP basic auth password (optional).
+        use_ssl: Whether to use SSL. Default True.
+        verify_certs: Whether to verify SSL certificates. Default True.
+        index_prefix: Prefix prepended to index name. Default "".
+
+    Usage::
+
+        adapter = OpenSearchStorageAdapter(url="https://...")
+        await adapter.ensure_index(
+            index="pulse_videos",
+            id_field="video_id",
+            mappings={"video_id": "keyword", "title": "text", ...},
+        )
+        result = await adapter.upsert_batch(
+            index="pulse_videos",
+            id_field="video_id",
+            documents=[{"video_id": "abc", "title": "...", ...}],
+        )
+    """
+
+    def __init__(
+        self,
+        url: str,
+        username: str = "",
+        password: str = "",
+        use_ssl: bool = True,
+        verify_certs: bool = True,
+        index_prefix: str = "",
+    ) -> None:
+        self._url = url.rstrip("/")
+        self._username = username
+        self._password = password
+        self._use_ssl = use_ssl
+        self._verify_certs = verify_certs
+        self._index_prefix = index_prefix
+
+    def _client(self) -> Any:
+        from opensearchpy import AsyncOpenSearch
+
+        kwargs: dict[str, Any] = {
+            "hosts": [self._url],
+            "use_ssl": self._use_ssl,
+            "verify_certs": self._verify_certs,
+        }
+        if self._username and self._password:
+            kwargs["http_auth"] = (self._username, self._password)
+        return AsyncOpenSearch(**kwargs)
+
+    def _index_name(self, index: str) -> str:
+        return f"{self._index_prefix}{index}" if self._index_prefix else index
+
+    @staticmethod
+    def _build_properties(mappings: dict[str, str]) -> dict[str, Any]:
+        properties: dict[str, Any] = {}
+        for field, type_name in mappings.items():
+            properties[field] = _TYPE_MAP.get(type_name, {"type": "keyword"})
+        return properties
+
+    async def ensure_index(
+        self,
+        index: str,
+        id_field: str,
+        mappings: dict[str, str],
+    ) -> None:
+        """Create index with mappings if it doesn't exist."""
+        name = self._index_name(index)
+        body = {
+            "settings": {"number_of_shards": 1, "number_of_replicas": 1},
+            "mappings": {"properties": self._build_properties(mappings)},
+        }
+        async with self._client() as client:
+            exists = await client.indices.exists(index=name)
+            if not exists:
+                await client.indices.create(index=name, body=body)
+                logger.info("Created OpenSearch index '%s'", name)
+            else:
+                logger.debug("OpenSearch index '%s' already exists", name)
+
+    async def upsert_batch(
+        self,
+        index: str,
+        id_field: str,
+        documents: list[dict[str, Any]],
+    ) -> dict[str, Any]:
+        """Bulk upsert documents. Uses id_field value as document _id.
+
+        Returns:
+            {"indexed": N, "errors": [...]}
+        """
+        from opensearchpy import AsyncOpenSearch
+        from opensearchpy.helpers import async_bulk
+
+        name = self._index_name(index)
+        actions = [
+            {
+                "_op_type": "index",
+                "_index": name,
+                "_id": str(doc[id_field]),
+                "_source": doc,
+            }
+            for doc in documents
+            if doc.get(id_field)
+        ]
+
+        skipped = len(documents) - len(actions)
+        if skipped:
+            logger.warning("Skipped %d documents missing id_field '%s'", skipped, id_field)
+
+        errors: list[Any] = []
+        async with self._client() as client:
+            success, failed = await async_bulk(client, actions, raise_on_error=False, stats_only=False)
+            if failed:
+                errors = failed
+                logger.warning("OpenSearch bulk errors: %d failed", len(failed))
+
+        logger.info("Upserted %d documents to index '%s'", success, name)
+        return {"indexed": success, "errors": errors}
+
+    async def store(
+        self,
+        index: str,
+        id_field: str,
+        mappings: dict[str, str],
+        documents: list[dict[str, Any]],
+    ) -> dict[str, Any]:
+        """Ensure index exists then upsert all documents."""
+        await self.ensure_index(index=index, id_field=id_field, mappings=mappings)
+        return await self.upsert_batch(index=index, id_field=id_field, documents=documents)

pulse_engine/adapters/speech_content.py

@@ -0,0 +1,130 @@
+"""Fetches and extracts text content from speech PDFs."""
+
+from __future__ import annotations
+
+import asyncio
+import re
+
+import httpx
+import structlog
+
+from .models import SpeechContent, SpeechMetadata, SpeechResult
+
+logger = structlog.get_logger(__name__)
+
+_DEFAULT_HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; PulseBot/1.0)"}
+_DEVANAGARI_RE = re.compile(r"[\u0900-\u097F]")
+
+
+def _detect_language(text: str) -> str:
+    deva = len(_DEVANAGARI_RE.findall(text))
+    total = len(text.replace(" ", ""))
+    if not total:
+        return "en"
+    ratio = deva / total
+    if ratio > 0.4:
+        return "hi"
+    if ratio > 0.1:
+        return "mixed"
+    return "en"
+
+
+class SpeechContentAdapter:
+    """Fetches PDF text for speeches discovered by ``SpeechMetadataAdapter``.
+
+    Mirrors the ``DigitalNewsAdapter`` interface: call ``fetch()`` for a single
+    speech or ``fetch_many()`` for a batch.
+
+    Requires ``pymupdf`` (``fitz``) to be installed for PDF extraction.
+
+    Usage::
+
+        adapter = SpeechContentAdapter()
+        content = await adapter.fetch(metadata)
+
+        results = await adapter.fetch_many(metadata_list, concurrency=4)
+    """
+
+    def __init__(self, concurrency: int = 4, timeout: float = 60.0) -> None:
+        self._default_concurrency = concurrency
+        self._timeout = timeout
+
+    async def fetch(self, speech: SpeechMetadata) -> SpeechContent:
+        """Download and extract text from a single speech PDF. Raises on failure."""
+        if not speech.pdf_url:
+            raise ValueError(f"No PDF URL for speech: {speech.url}")
+
+        logger.info("speech_content_fetch_started", url=speech.url, pdf=speech.pdf_url)
+        text = await self._extract_pdf(speech.pdf_url)
+        if not text:
+            raise RuntimeError(
+                f"PDF extraction returned empty text for {speech.pdf_url}"
+            )
+
+        language = _detect_language(text)
+        logger.info(
+            "speech_content_fetch_completed",
+            url=speech.url,
+            chars=len(text),
+            language=language,
+        )
+        return SpeechContent(text=text, language=language, pdf_url=speech.pdf_url)
+
+    async def fetch_many(
+        self,
+        speeches: list[SpeechMetadata],
+        concurrency: int | None = None,
+    ) -> list[SpeechResult]:
+        """Extract content from multiple speeches concurrently.
+
+        Speeches without a PDF URL are returned immediately as errors.
+        Preserves input order.
+        """
+        if not speeches:
+            return []
+
+        sem = asyncio.Semaphore(concurrency or self._default_concurrency)
+
+        async def _guarded(speech: SpeechMetadata) -> SpeechResult:
+            async with sem:
+                try:
+                    content = await self.fetch(speech)
+                    return SpeechResult(url=speech.url, content=content, error=None)
+                except Exception as exc:
+                    logger.warning(
+                        "speech_content_fetch_failed",
+                        url=speech.url,
+                        error=str(exc),
+                    )
+                    return SpeechResult(url=speech.url, content=None, error=str(exc))
+
+        tasks = [_guarded(s) for s in speeches]
+        return list(await asyncio.gather(*tasks))
+
+    # ------------------------------------------------------------------
+    # Private helpers
+    # ------------------------------------------------------------------
+
+    async def _extract_pdf(self, pdf_url: str) -> str:
+        """Download PDF bytes and extract plain text via pymupdf."""
+        try:
+            import fitz  # pymupdf
+        except ImportError:
+            raise RuntimeError(
+                "pymupdf is required for PDF extraction: pip install pymupdf"
+            )
+
+        async with httpx.AsyncClient(
+            follow_redirects=True, headers=_DEFAULT_HEADERS, timeout=self._timeout
+        ) as client:
+            resp = await client.get(pdf_url)
+            resp.raise_for_status()
+            pdf_bytes = resp.content
+
+        def _parse() -> str:
+            doc = fitz.open(stream=pdf_bytes, filetype="pdf")
+            pages = [page.get_text() for page in doc if page.get_text().strip()]
+            doc.close()
+            return "\n\n".join(pages).strip()
+
+        return await asyncio.to_thread(_parse)

pulse_engine/adapters/speech_metadata.py

@@ -0,0 +1,374 @@
+"""Discovers speech metadata from inc.in listing pages."""
+
+from __future__ import annotations
+
+import json
+import re
+from datetime import UTC, datetime, timedelta
+
+import httpx
+import structlog
+
+from .models import SpeechMetadata
+
+logger = structlog.get_logger(__name__)
+
+_INC_BASE = "https://www.inc.in/media/speeches"
+_MEDIA_PATH = "/media/speeches/"
+_MAX_PAGES = 200
+
+_DATE_RE = re.compile(r"\b(\d{1,2})\s+([A-Za-z]{3,9})\s+(\d{4})\b")
+_YT_RE = re.compile(
+    r"(?:https?://)?(?:www\.)?(?:youtube\.com/(?:watch\?v=|embed/|shorts/)|youtu\.be/)"
+    r"([\w\-]{11})",
+    re.I,
+)
+_PDF_RE = re.compile(r"https?://[^\s'\"]+?\.pdf(?:\?[^'\"\s]*)?", re.I)
+_DATE_FORMATS = [
+    "%d %B %Y",
+    "%d %b %Y",
+    "%B %d, %Y",
+    "%b %d, %Y",
+    "%Y-%m-%d",
+    "%d/%m/%Y",
+]
+
+_DEFAULT_HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; PulseBot/1.0)"}
+
+
+def _parse_date(text: str | None) -> datetime | None:
+    if not text:
+        return None
+    m = _DATE_RE.search(text)
+    if m:
+        text = f"{m.group(1)} {m.group(2)} {m.group(3)}"
+    for fmt in _DATE_FORMATS:
+        try:
+            return datetime.strptime(text.strip(), fmt).replace(tzinfo=UTC)
+        except ValueError:
+            continue
+    return None
+
+
+def _scan_for_media(obj: object) -> tuple[str | None, str | None]:
+    """Recursively scan a JSON object for PDF and YouTube URLs."""
+    pdf: str | None = None
+    yt: str | None = None
+
+    def walk(node: object) -> None:
+        nonlocal pdf, yt
+        if pdf and yt:
+            return
+        if isinstance(node, str):
+            if not pdf and ".pdf" in node.lower():
+                m = _PDF_RE.search(node)
+                if m:
+                    pdf = m.group(0)
+            if not yt:
+                m = _YT_RE.search(node)
+                if m:
+                    yt = f"https://www.youtube.com/watch?v={m.group(1)}"
+        elif isinstance(node, dict):
+            if not pdf:
+                u = node.get("url", "")
+                if isinstance(u, str) and ".pdf" in u.lower() and u.startswith("http"):
+                    pdf = u
+            if not yt:
+                _yt_keys = (
+                    "youtube_url",
+                    "youtube",
+                    "video_url",
+                    "video_link",
+                    "embed_url",
+                )
+                for k in _yt_keys:
+                    v = node.get(k)
+                    if isinstance(v, str):
+                        m = _YT_RE.search(v)
+                        if m:
+                            yt = f"https://www.youtube.com/watch?v={m.group(1)}"
+                            break
+            for v in node.values():
+                walk(v)
+        elif isinstance(node, list | tuple):
+            for v in node:
+                walk(v)
+
+    walk(obj)
+    return pdf, yt
+
+
+class SpeechMetadataAdapter:
+    """Discovers speech metadata from the inc.in speeches listing.
+
+    Mirrors the ``DigitalNewsMetadataAdapter`` interface: construct with
+    optional filters, call ``fetch()`` to get a list of ``SpeechMetadata``.
+
+    Each entry contains the speech URL, title, speaker, date, PDF URL, and
+    YouTube URL (when present). Content extraction (PDF text) is handled
+    separately by ``SpeechContentAdapter``.
+
+    Usage::
+
+        adapter = SpeechMetadataAdapter()
+        speeches = await adapter.fetch(since=timedelta(days=7), speaker="rahul")
+    """
+
+    def __init__(self, base_url: str = _INC_BASE) -> None:
+        self._base_url = base_url.rstrip("/")
+
+    async def fetch(
+        self,
+        since: timedelta | None = None,
+        speaker: str | None = None,
+    ) -> list[SpeechMetadata]:
+        """Discover and return filtered speech metadata.
+
+        Args:
+            since: Only return speeches published within this window.
+            speaker: Case-insensitive substring match against speaker name.
+
+        Returns:
+            List of ``SpeechMetadata``, ordered as discovered from listing.
+        """
+        logger.info(
+            "speech_metadata_fetch_started",
+            since=str(since),
+            speaker=speaker,
+        )
+        cutoff = (datetime.now(UTC) - since) if since is not None else None
+        urls = await self._discover_urls()
+        logger.info("speech_urls_discovered", count=len(urls))
+
+        entries = await self._parse_pages(urls)
+        logger.info("speech_pages_parsed", total=len(entries))
+
+        filtered = self._apply_filters(entries, cutoff=cutoff, speaker=speaker)
+        logger.info(
+            "speech_metadata_fetch_completed",
+            before=len(entries),
+            after=len(filtered),
+        )
+        return filtered
+
+    # ------------------------------------------------------------------
+    # Discovery
+    # ------------------------------------------------------------------
+
+    async def _discover_urls(self) -> list[str]:
+        seen: set[str] = set()
+
+        async with httpx.AsyncClient(
+            follow_redirects=True, headers=_DEFAULT_HEADERS, timeout=30
+        ) as client:
+            # Page 1 — also extract buildId for JSON API route.
+            build_id = await self._scrape_html_page(client, self._base_url, seen)
+
+            if isinstance(build_id, str):
+                # JSON API is complete — no need for HTML pagination.
+                await self._scrape_json_pages(client, build_id, seen)
+            else:
+                # JSON unavailable — fall back to HTML pagination.
+                # Break as soon as a page returns no speech links at all.
+                for page in range(2, _MAX_PAGES + 1):
+                    url = f"{self._base_url}/page/{page}"
+                    had_links = await self._scrape_html_page(
+                        client, url, seen, return_had_links=True
+                    )
+                    if not had_links:
+                        break
+
+        return list(seen)
+
+    async def _scrape_html_page(
+        self,
+        client: httpx.AsyncClient,
+        url: str,
+        seen: set[str],
+        return_had_links: bool = False,
+    ) -> str | None | bool:
+        """Scrape one HTML listing page, add speech URLs to *seen*.
+
+        When *return_had_links* is True, returns a bool indicating whether any
+        speech links were found on the page (used to detect end of pagination).
+        Otherwise returns the Next.js buildId if found, else None.
+        """
+        try:
+            resp = await client.get(url)
+            if resp.status_code != 200:
+                return False if return_had_links else None
+        except httpx.HTTPError:
+            return False if return_had_links else None
+
+        from bs4 import BeautifulSoup
+
+        soup = BeautifulSoup(resp.text, "html.parser")
+        had_links = False
+        for a in soup.find_all("a", href=True):
+            href = str(a["href"])
+            if href.startswith(_MEDIA_PATH) and href.count("/") >= 3:
+                seen.add(f"https://www.inc.in{href}")
+                had_links = True
+
+        if return_had_links:
+            return had_links
+
+        build_id: str | None = None
+        script = soup.find("script", id="__NEXT_DATA__", type="application/json")
+        if script:
+            try:
+                data = json.loads(script.string or script.get_text())
+                build_id = data.get("buildId")
+                self._collect_urls_from_obj(data, seen)
+            except (json.JSONDecodeError, AttributeError):
+                pass
+        return build_id
+
+    async def _scrape_json_pages(
+        self,
+        client: httpx.AsyncClient,
+        build_id: str,
+        seen: set[str],
+    ) -> None:
+        base = f"https://www.inc.in/_next/data/{build_id}/media/speeches.json"
+        no_progress = 0
+        for page in range(1, _MAX_PAGES + 1):
+            try:
+                resp = await client.get(f"{base}?page={page}")
+                if resp.status_code != 200:
+                    break
+                before = len(seen)
+                self._collect_urls_from_obj(resp.json(), seen)
+                added = len(seen) - before
+                logger.debug("json_page", page=page, added=added, total=len(seen))
+                no_progress = 0 if added else no_progress + 1
+                if no_progress >= 2:
+                    break
+            except Exception:
+                logger.debug("json_page_failed", page=page, exc_info=True)
+                break
+
+    @staticmethod
+    def _collect_urls_from_obj(obj: object, seen: set[str]) -> None:
+        if isinstance(obj, dict):
+            uid = obj.get("uid")
+            if isinstance(uid, str) and uid and "/" not in uid.strip("/"):
+                seen.add(f"https://www.inc.in{_MEDIA_PATH}{uid.strip('/')}")
+            for v in obj.values():
+                SpeechMetadataAdapter._collect_urls_from_obj(v, seen)
+        elif isinstance(obj, list | tuple):
+            for v in obj:
+                SpeechMetadataAdapter._collect_urls_from_obj(v, seen)
+        elif isinstance(obj, str) and obj.strip().startswith(_MEDIA_PATH):
+            seen.add(f"https://www.inc.in{obj.strip()}")
+
+    # ------------------------------------------------------------------
+    # Page parsing
+    # ------------------------------------------------------------------
+
+    async def _parse_pages(self, urls: list[str]) -> list[SpeechMetadata]:
+        async with httpx.AsyncClient(
+            follow_redirects=True, headers=_DEFAULT_HEADERS, timeout=30
+        ) as client:
+            import asyncio
+
+            tasks = [self._parse_one(client, url) for url in urls]
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        entries: list[SpeechMetadata] = []
+        for url, result in zip(urls, results):
+            if isinstance(result, BaseException):
+                logger.warning("speech_page_parse_failed", url=url, error=str(result))
+            elif result is not None:
+                entries.append(result)
+        return entries
+
+    async def _parse_one(
+        self, client: httpx.AsyncClient, url: str
+    ) -> SpeechMetadata | None:
+        try:
+            resp = await client.get(url)
+            if resp.status_code != 200:
+                return None
+        except httpx.HTTPError as exc:
+            logger.debug("speech_page_http_error", url=url, error=str(exc))
+            return None
+
+        from bs4 import BeautifulSoup
+
+        soup = BeautifulSoup(resp.text, "html.parser")
+        title = speaker = date_raw = pdf_url = yt_url = None
+
+        script = soup.find("script", id="__NEXT_DATA__", type="application/json")
+        if script:
+            try:
+                data = json.loads(script.string or script.get_text())
+                pp = (data.get("props") or {}).get("pageProps") or {}
+                obj = (
+                    pp.get("objspeeches")
+                    or pp.get("objSpeeches")
+                    or pp.get("speech")
+                    or pp
+                )
+                if isinstance(obj, dict):
+                    speaker = obj.get("title") or obj.get("speaker")
+                    title = obj.get("expert") or obj.get("title") or obj.get("heading")
+                    date_raw = (
+                        obj.get("publishdate")
+                        or obj.get("publishedate")
+                        or obj.get("date")
+                    )
+                    speech_doc = obj.get("SpeechDocument")
+                    if isinstance(speech_doc, dict):
+                        pdf_url = speech_doc.get("url")
+                    yt_url = obj.get("video_link")
+                _pdf, _yt = _scan_for_media(data)
+                pdf_url = pdf_url or _pdf
+                yt_url = yt_url or _yt
+            except (json.JSONDecodeError, AttributeError):
+                pass
+
+        if not title:
+            h = soup.find(["h1", "h2"])
+            title = h.get_text(strip=True) if h else None
+        if not date_raw:
+            m = _DATE_RE.search(soup.get_text(" ", strip=True))
+            if m:
+                date_raw = f"{m.group(1)} {m.group(2)} {m.group(3)}"
+
+        return SpeechMetadata(
+            url=url,
+            title=title,
+            speaker=speaker,
+            speech_date=_parse_date(date_raw),
+            pdf_url=pdf_url,
+            youtube_url=yt_url,
+            language=None,  # detected after content extraction
+        )
+
+    # ------------------------------------------------------------------
+    # Filtering
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _apply_filters(
+        entries: list[SpeechMetadata],
+        cutoff: datetime | None,
+        speaker: str | None,
+    ) -> list[SpeechMetadata]:
+        result = entries
+
+        if cutoff is not None:
+            result = [
+                e for e in result if e.speech_date is None or e.speech_date >= cutoff
+            ]
+
+        if speaker is not None:
+            kw = speaker.lower()
+            result = [
+                e
+                for e in result
+                if kw in (e.speaker or "").lower() or kw in (e.title or "").lower()
+            ]
+
+        return result