pulse-engine 0.2.0__py3-none-any.whl

pulse_engine/adapters/__init__.py

@@ -0,0 +1,58 @@
+"""Pulse Engine adapters — digital news, YouTube, speech, and Twitter."""
+
+from .audio_transcription import AudioTranscriptionAdapter
+from .batcher import BatcherAdapter
+from .digital_news import DigitalNewsAdapter
+from .opensearch_storage import OpenSearchStorageAdapter
+from .digital_news_metadata import DigitalNewsMetadataAdapter
+from .exceptions import SitemapDiscoveryError
+from .models import (
+    ArticleContent,
+    ArticleResult,
+    NewsArticleMetadata,
+    SpeechContent,
+    SpeechMetadata,
+    SpeechResult,
+    TranscriptSegment,
+    TweetMetadata,
+    TwitterUserResult,
+    VideoAudio,
+    VideoMetadata,
+    VideoResult,
+)
+from .speech_content import SpeechContentAdapter
+from .speech_metadata import SpeechMetadataAdapter
+from .twitter import TwitterAdapter
+from .youtube_downloader import YouTubeDownloader
+from .youtube_metadata import YouTubeMetadataAdapter
+
+__all__ = [
+    # Utilities
+    "BatcherAdapter",
+    "OpenSearchStorageAdapter",
+    # Digital news
+    "DigitalNewsMetadataAdapter",
+    "DigitalNewsAdapter",
+    "NewsArticleMetadata",
+    "ArticleContent",
+    "ArticleResult",
+    "SitemapDiscoveryError",
+    # YouTube
+    "YouTubeMetadataAdapter",
+    "YouTubeDownloader",
+    "AudioTranscriptionAdapter",
+    "VideoMetadata",
+    "VideoAudio",
+    "VideoResult",
+    "TranscriptSegment",
+    # Speeches
+    "SpeechMetadataAdapter",
+    "SpeechContentAdapter",
+    "SpeechMetadata",
+    "SpeechContent",
+    "SpeechResult",
+    # Twitter
+    "TwitterAdapter",
+    "TweetMetadata",
+    "TwitterUserResult",
+]

pulse_engine/adapters/audio_transcription.py

@@ -0,0 +1,167 @@
+"""Audio transcription adapter using OpenAI Whisper."""
+
+from __future__ import annotations
+
+import asyncio
+import glob
+import os
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from .models import TranscriptSegment, VideoAudio
+
+logger = structlog.get_logger(__name__)
+
+_WHISPER_MAX_BYTES = 24 * 1024 * 1024  # stay under Whisper's 25 MB hard limit
+_WHISPER_CHUNK_SECS = 10 * 60  # split large files into 10-min chunks
+
+
+class AudioTranscriptionAdapter:
+    """Transcribes an audio file using OpenAI Whisper, returning a VideoAudio.
+
+    Large files (>24 MB) are automatically split into 10-minute chunks via
+    ffmpeg and stitched back together with adjusted timestamps.
+
+    Usage::
+
+        adapter = AudioTranscriptionAdapter(openai_api_key="sk-...")
+        audio = await adapter.transcribe(Path("/tmp/abc123.mp3"), language="en")
+    """
+
+    def __init__(self, openai_api_key: str) -> None:
+        self._openai_api_key = openai_api_key
+
+    async def transcribe(self, audio_path: Path, language: str = "en") -> VideoAudio:
+        """Transcribe *audio_path* and return a :class:`VideoAudio`.
+
+        Args:
+            audio_path: Path to an mp3 (or other audio) file.
+            language:   BCP-47 language tag, or ``"auto"`` for Whisper auto-detect.
+        """
+        logger.info(
+            "audio_transcription_started",
+            audio_path=str(audio_path),
+            language=language,
+        )
+        transcript, segments = await self._transcribe(audio_path, language=language)
+        logger.info(
+            "audio_transcription_completed",
+            audio_path=str(audio_path),
+            segments=len(segments),
+            chars=len(transcript),
+        )
+        return VideoAudio(transcript=transcript, language=language, segments=segments)
+
+    # ------------------------------------------------------------------
+    # Private helpers
+    # ------------------------------------------------------------------
+
+    async def _transcribe(
+        self, audio_path: Path, language: str
+    ) -> tuple[str, list[TranscriptSegment]]:
+        """Send audio to Whisper, returning (full_text, segments).
+
+        Large files (>24 MB) are split into 10-minute chunks via ffmpeg and
+        stitched back together with adjusted timestamps.
+        """
+        # Empty language string signals Whisper auto-detect.
+        whisper_language = "" if language.lower() == "auto" else language
+
+        try:
+            from openai import AsyncOpenAI
+
+            client = AsyncOpenAI(api_key=self._openai_api_key)
+
+            def _transcribe_file(path: Path, offset: float) -> list[TranscriptSegment]:
+                import asyncio as _asyncio
+
+                loop = _asyncio.new_event_loop()
+                try:
+                    with open(path, "rb") as fobj:
+                        kwargs: dict[str, Any] = {
+                            "model": "whisper-1",
+                            "file": fobj,
+                            "response_format": "verbose_json",
+                            "timestamp_granularities": ["segment"],
+                        }
+                        if whisper_language:
+                            kwargs["language"] = whisper_language
+                        resp = loop.run_until_complete(
+                            client.audio.transcriptions.create(**kwargs)
+                        )
+                finally:
+                    loop.close()
+
+                raw = getattr(resp, "segments", None) or []
+                return [
+                    TranscriptSegment(
+                        start=float(
+                            s.get("start", 0) if isinstance(s, dict) else s.start
+                        )
+                        + offset,
+                        end=float(s.get("end", 0) if isinstance(s, dict) else s.end)
+                        + offset,
+                        text=(
+                            s.get("text", "") if isinstance(s, dict) else s.text
+                        ).strip(),
+                    )
+                    for s in raw
+                ]
+
+            file_size = audio_path.stat().st_size
+
+            if file_size <= _WHISPER_MAX_BYTES:
+                segments = await asyncio.to_thread(_transcribe_file, audio_path, 0.0)
+            else:
+                logger.info(
+                    "audio_whisper_splitting_large_file",
+                    size_mb=round(file_size / (1024 * 1024), 1),
+                )
+
+                def _split_and_transcribe() -> list[TranscriptSegment]:
+                    with tempfile.TemporaryDirectory(prefix="whisper_chunks_") as td:
+                        chunk_pattern = os.path.join(td, "chunk_%04d.mp3")
+                        subprocess.run(
+                            [
+                                "ffmpeg",
+                                "-i",
+                                str(audio_path),
+                                "-f",
+                                "segment",
+                                "-segment_time",
+                                str(_WHISPER_CHUNK_SECS),
+                                "-c",
+                                "copy",
+                                "-reset_timestamps",
+                                "1",
+                                chunk_pattern,
+                            ],
+                            check=True,
+                            capture_output=True,
+                        )
+                        chunk_files = sorted(glob.glob(os.path.join(td, "chunk_*.mp3")))
+                        all_segs: list[TranscriptSegment] = []
+                        offset = 0.0
+                        for chunk_path in chunk_files:
+                            segs = _transcribe_file(Path(chunk_path), offset)
+                            if segs:
+                                offset = segs[-1].end
+                            all_segs.extend(segs)
+                    return all_segs
+
+                segments = await asyncio.to_thread(_split_and_transcribe)
+
+            full_text = " ".join(s.text for s in segments)
+            return full_text, segments
+
+        except Exception:
+            logger.warning(
+                "audio_whisper_transcription_failed",
+                audio_path=str(audio_path),
+                exc_info=True,
+            )
+            return "", []

pulse_engine/adapters/batcher.py

@@ -0,0 +1,36 @@
+"""BatcherAdapter — splits a list of items into fixed-size batches."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class BatcherAdapter:
+    """Split a flat list of items into batches of a fixed size.
+
+    Usage::
+
+        adapter = BatcherAdapter(batch_size=50)
+        result = adapter.batch(items)
+        # result == {"batches": [[...], [...]]}
+    """
+
+    def __init__(self, batch_size: int = 50) -> None:
+        if batch_size < 1:
+            raise ValueError(f"batch_size must be >= 1, got {batch_size}")
+        self.batch_size = batch_size
+
+    def batch(self, items: list[Any]) -> dict[str, list[list[Any]]]:
+        """Split items into batches.
+
+        Args:
+            items: Flat list of items to batch.
+
+        Returns:
+            {"batches": [[item, ...], ...]}
+        """
+        batches = [
+            items[i : i + self.batch_size]
+            for i in range(0, len(items), self.batch_size)
+        ]
+        return {"batches": batches}

pulse_engine/adapters/digital_news.py

@@ -0,0 +1,128 @@
+"""Fetches article content via Cloudflare Browser Rendering and LLM extraction."""
+
+import asyncio
+import os
+from typing import Any
+
+import httpx
+import structlog
+from langchain_core.messages import HumanMessage, SystemMessage
+from pydantic import BaseModel, Field
+
+from .models import ArticleContent, ArticleResult
+
+logger = structlog.get_logger(__name__)
+
+_CF_MARKDOWN_URL = (
+    "https://api.cloudflare.com/client/v4/accounts"
+    "/{account_id}/browser-rendering/markdown"
+)
+
+_EXTRACTION_PROMPT = (
+    "Extract the article title and the main body content verbatim. "
+    "Ignore ads, navigation, comments, and unrelated sections. "
+    "Return clean readable text."
+)
+
+
+class _ArticleSchema(BaseModel):
+    """Schema for structured LLM output."""
+
+    title: str = Field(description="The main headline of the article")
+    content: str = Field(description="The full main article text content")
+
+
+class DigitalNewsAdapter:
+    """Fetches article content via Cloudflare Browser Rendering + LLM.
+
+    Accepts any LangChain BaseChatModel — works with OpenAI, Anthropic,
+    Ollama, Gemini, or any provider LangChain supports.
+
+    Usage:
+        # With explicit LLM
+        from langchain_openai import ChatOpenAI
+        llm = ChatOpenAI(model="gpt-5.4-nano")
+        adapter = DigitalNewsAdapter(llm=llm)
+
+        # With project default (from get_llm())
+        from pulse_engine.processor.llm.provider import get_llm
+        adapter = DigitalNewsAdapter(llm=get_llm())
+
+        # Ollama
+        from langchain_ollama import ChatOllama
+        adapter = DigitalNewsAdapter(llm=ChatOllama(model="llama3"))
+    """
+
+    def __init__(
+        self,
+        llm: Any,
+        cloudflare_timeout: float = 60.0,
+    ) -> None:
+        self._llm = llm
+        self._cloudflare_timeout = cloudflare_timeout
+
+        self._account_id = os.environ.get("CLOUDFLARE_ACCOUNT_ID")
+        if not self._account_id:
+            raise OSError("CLOUDFLARE_ACCOUNT_ID environment variable is required")
+
+        self._bearer_token = os.environ.get("CLOUDFLARE_BEARER_TOKEN")
+        if not self._bearer_token:
+            raise OSError("CLOUDFLARE_BEARER_TOKEN environment variable is required")
+
+    async def fetch(self, url: str) -> ArticleContent:
+        """Fetch and extract content from a single article URL."""
+        markdown = await self._fetch_markdown(url)
+        return await self._extract_content(markdown)
+
+    async def fetch_many(
+        self,
+        urls: list[str],
+        concurrency: int = 5,
+    ) -> list[ArticleResult]:
+        """Fetch multiple articles concurrently, preserving input order."""
+        if not urls:
+            return []
+
+        semaphore = asyncio.Semaphore(concurrency)
+
+        async def _guarded(u: str) -> ArticleResult:
+            async with semaphore:
+                try:
+                    content = await self.fetch(u)
+                    return ArticleResult(url=u, content=content, error=None)
+                except Exception as exc:
+                    logger.warning("fetch_failed", url=u, error=str(exc))
+                    return ArticleResult(url=u, content=None, error=str(exc))
+
+        tasks = [_guarded(u) for u in urls]
+        return list(await asyncio.gather(*tasks))
+
+    async def _fetch_markdown(self, url: str) -> str:
+        cf_url = _CF_MARKDOWN_URL.format(account_id=self._account_id)
+        async with httpx.AsyncClient(
+            timeout=self._cloudflare_timeout, follow_redirects=True
+        ) as client:
+            resp = await client.post(
+                cf_url,
+                headers={
+                    "Authorization": f"Bearer {self._bearer_token}",
+                    "Content-Type": "application/json",
+                },
+                json={"url": url},
+            )
+            resp.raise_for_status()
+
+        data = resp.json()
+        markdown: str = data.get("result", "")
+        if not markdown or not markdown.strip():
+            raise RuntimeError(f"Empty markdown returned by Cloudflare for {url}")
+        return markdown
+
+    async def _extract_content(self, markdown: str) -> ArticleContent:
+        structured_llm = self._llm.with_structured_output(_ArticleSchema)
+        messages = [
+            SystemMessage(content=_EXTRACTION_PROMPT),
+            HumanMessage(content=markdown),
+        ]
+        result: _ArticleSchema = await structured_llm.ainvoke(messages)
+        return ArticleContent(title=result.title, content=result.content)