pulse-engine 0.2.0__py3-none-any.whl

pulse_engine/adapters/youtube_metadata.py

@@ -0,0 +1,261 @@
+"""YouTube Data API v3 channel metadata adapter."""
+
+from __future__ import annotations
+
+import re
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+import httpx
+import structlog
+
+from .models import VideoMetadata
+
+logger = structlog.get_logger(__name__)
+
+_YOUTUBE_API_BASE = "https://www.googleapis.com/youtube/v3"
+_MAX_PAGE_SIZE = 50
+
+_ISO8601_DURATION_RE = re.compile(r"PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?")
+
+
+def _parse_duration(duration: str) -> int:
+    """Parse an ISO 8601 duration string (e.g. ``PT15M33S``) to seconds."""
+    m = _ISO8601_DURATION_RE.search(duration)
+    if not m:
+        return 0
+    hours = int(m.group(1) or 0)
+    minutes = int(m.group(2) or 0)
+    seconds = int(m.group(3) or 0)
+    return hours * 3600 + minutes * 60 + seconds
+
+
+def _is_live_or_upcoming(video: dict[str, Any]) -> bool:
+    lbc = (video.get("snippet") or {}).get("liveBroadcastContent") or "none"
+    return lbc in ("live", "upcoming")
+
+
+class YouTubeMetadataAdapter:
+    """Discovers video metadata from a YouTube channel via the Data API v3.
+
+    Mirrors the ``DigitalNewsMetadataAdapter`` interface: construct with
+    connection config, call ``fetch()`` to get a list of ``VideoMetadata``.
+
+    Usage::
+
+        adapter = YouTubeMetadataAdapter(
+            channel_name="@mkbhd",
+            api_key="YOUR_YT_API_KEY",
+        )
+        videos = await adapter.fetch(since=timedelta(days=7), max_videos=50)
+    """
+
+    def __init__(self, channel_name: str, api_key: str) -> None:
+        self._channel_name = channel_name
+        self._api_key = api_key
+
+    async def fetch(
+        self,
+        since: timedelta | None = None,
+        max_videos: int = 50,
+        min_duration_seconds: int = 0,
+        skip_live_streams: bool = True,
+    ) -> list[VideoMetadata]:
+        """Fetch video metadata from the channel.
+
+        Args:
+            since: Only return videos published within this window.
+            max_videos: Upper bound on videos returned.
+            min_duration_seconds: Skip videos shorter than this threshold.
+            skip_live_streams: Drop live and scheduled broadcasts.
+
+        Returns:
+            List of ``VideoMetadata`` ordered newest-first from the API.
+        """
+        published_after: str | None = None
+        if since is not None:
+            cutoff = datetime.now(UTC) - since
+            published_after = cutoff.isoformat()
+
+        logger.info(
+            "youtube_metadata_fetch_started",
+            channel=self._channel_name,
+            since=str(since),
+            max_videos=max_videos,
+        )
+
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            channel_id = await self._resolve_channel_id(client)
+            playlist_id = await self._get_uploads_playlist_id(client, channel_id)
+            video_ids = await self._list_playlist_video_ids(
+                client,
+                playlist_id,
+                max_videos=max_videos,
+                published_after=published_after,
+            )
+            raw_videos = await self._get_video_details(client, video_ids)
+
+        videos = self._apply_filters(
+            raw_videos,
+            min_duration_seconds=min_duration_seconds,
+            skip_live_streams=skip_live_streams,
+        )
+
+        result = [self._to_model(v) for v in videos]
+        logger.info(
+            "youtube_metadata_fetch_completed",
+            channel=self._channel_name,
+            returned=len(result),
+        )
+        return result
+
+    # ------------------------------------------------------------------
+    # Private helpers
+    # ------------------------------------------------------------------
+
+    async def _resolve_channel_id(self, client: httpx.AsyncClient) -> str:
+        """Resolve a channel handle or display name to a channel ID."""
+        handle = self._channel_name.lstrip("@")
+        resp = await client.get(
+            f"{_YOUTUBE_API_BASE}/channels",
+            params={"part": "id", "forHandle": handle, "key": self._api_key},
+        )
+        resp.raise_for_status()
+        items = resp.json().get("items", [])
+        if items:
+            return str(items[0]["id"])
+
+        # Fall back to search by display name.
+        resp = await client.get(
+            f"{_YOUTUBE_API_BASE}/search",
+            params={
+                "part": "snippet",
+                "type": "channel",
+                "q": self._channel_name,
+                "maxResults": 1,
+                "key": self._api_key,
+            },
+        )
+        resp.raise_for_status()
+        items = resp.json().get("items", [])
+        if not items:
+            raise ValueError(f"YouTube channel not found: '{self._channel_name}'")
+        return str(items[0]["snippet"]["channelId"])
+
+    async def _get_uploads_playlist_id(
+        self, client: httpx.AsyncClient, channel_id: str
+    ) -> str:
+        resp = await client.get(
+            f"{_YOUTUBE_API_BASE}/channels",
+            params={"part": "contentDetails", "id": channel_id, "key": self._api_key},
+        )
+        resp.raise_for_status()
+        items = resp.json().get("items", [])
+        if not items:
+            raise ValueError(f"Channel not found: {channel_id}")
+        return str(items[0]["contentDetails"]["relatedPlaylists"]["uploads"])
+
+    async def _list_playlist_video_ids(
+        self,
+        client: httpx.AsyncClient,
+        playlist_id: str,
+        max_videos: int,
+        published_after: str | None,
+    ) -> list[str]:
+        video_ids: list[str] = []
+        page_token: str | None = None
+
+        while len(video_ids) < max_videos:
+            page_size = min(_MAX_PAGE_SIZE, max_videos - len(video_ids))
+            params: dict[str, Any] = {
+                "part": "contentDetails,snippet",
+                "playlistId": playlist_id,
+                "maxResults": page_size,
+                "key": self._api_key,
+            }
+            if page_token:
+                params["pageToken"] = page_token
+
+            resp = await client.get(f"{_YOUTUBE_API_BASE}/playlistItems", params=params)
+            resp.raise_for_status()
+            data = resp.json()
+
+            for item in data.get("items", []):
+                vid_id = item["contentDetails"]["videoId"]
+                if published_after:
+                    published = item["snippet"].get("publishedAt", "")
+                    if published < published_after:
+                        # Playlist is newest-first; older video means we're done.
+                        return video_ids
+                video_ids.append(vid_id)
+
+            page_token = data.get("nextPageToken")
+            if not page_token:
+                break
+
+        return video_ids
+
+    async def _get_video_details(
+        self, client: httpx.AsyncClient, video_ids: list[str]
+    ) -> list[dict[str, Any]]:
+        if not video_ids:
+            return []
+        details: list[dict[str, Any]] = []
+        for i in range(0, len(video_ids), _MAX_PAGE_SIZE):
+            batch = video_ids[i : i + _MAX_PAGE_SIZE]
+            resp = await client.get(
+                f"{_YOUTUBE_API_BASE}/videos",
+                params={
+                    "part": "snippet,contentDetails,statistics",
+                    "id": ",".join(batch),
+                    "key": self._api_key,
+                },
+            )
+            resp.raise_for_status()
+            details.extend(resp.json().get("items", []))
+        return details
+
+    def _apply_filters(
+        self,
+        videos: list[dict[str, Any]],
+        min_duration_seconds: int,
+        skip_live_streams: bool,
+    ) -> list[dict[str, Any]]:
+        result: list[dict[str, Any]] = []
+        for v in videos:
+            if skip_live_streams and _is_live_or_upcoming(v):
+                continue
+            if min_duration_seconds > 0:
+                details = v.get("contentDetails", {})
+                dur = _parse_duration(details.get("duration", "PT0S"))
+                if dur < min_duration_seconds:
+                    continue
+            result.append(v)
+        return result
+
+    @staticmethod
+    def _to_model(v: dict[str, Any]) -> VideoMetadata:
+        snippet = v["snippet"]
+        pub_at = datetime.fromisoformat(snippet["publishedAt"].replace("Z", "+00:00"))
+        thumbnails = snippet.get("thumbnails", {})
+        thumbnail_url = (
+            (thumbnails.get("maxres") or thumbnails.get("high") or thumbnails.get("medium") or {})
+            .get("url")
+        )
+        return VideoMetadata(
+            video_id=v["id"],
+            url=f"https://www.youtube.com/watch?v={v['id']}",
+            title=snippet["title"],
+            channel_id=snippet["channelId"],
+            channel_name=snippet["channelTitle"],
+            published_at=pub_at,
+            description=snippet.get("description", ""),
+            tags=snippet.get("tags", []),
+            duration_seconds=_parse_duration(
+                v.get("contentDetails", {}).get("duration", "PT0S")
+            ),
+            view_count=v.get("statistics", {}).get("viewCount"),
+            like_count=v.get("statistics", {}).get("likeCount"),
+            comment_count=v.get("statistics", {}).get("commentCount"),
+            thumbnail_url=thumbnail_url,
+        )

pulse_engine/api/v1/auth.py

@@ -0,0 +1,91 @@
+"""Authentication endpoint — exchanges email/password for Cognito tokens."""
+
+import base64
+import hashlib
+import hmac
+
+import boto3
+import structlog
+from botocore.exceptions import ClientError
+from fastapi import APIRouter, HTTPException, Request
+from jose import jwt as jose_jwt
+from pydantic import BaseModel, EmailStr, Field
+
+from pulse_engine.config import get_settings
+from pulse_engine.middleware.rate_limit import check_rate_limit
+
+logger = structlog.get_logger()
+
+router = APIRouter(prefix="/auth", tags=["Auth"])
+
+
+class LoginRequest(BaseModel):
+    email: EmailStr
+    password: str = Field(min_length=8, max_length=128)
+
+
+class LoginResponse(BaseModel):
+    id_token: str
+    access_token: str
+    refresh_token: str
+    expires_in: int
+    token_type: str = "Bearer"
+    tenant_id: str
+    email: str
+
+
+def _compute_secret_hash(email: str, client_id: str, client_secret: str) -> str:
+    message = email + client_id
+    digest = hmac.new(
+        client_secret.encode("utf-8"),
+        message.encode("utf-8"),
+        hashlib.sha256,
+    ).digest()
+    return base64.b64encode(digest).decode("utf-8")
+
+
+@router.post("/login", response_model=LoginResponse)
+async def login(request: Request, body: LoginRequest) -> LoginResponse:
+    """Authenticate with email and password, returns JWT tokens."""
+    # Strict per-IP rate limit: 5 attempts per 60 seconds
+    check_rate_limit(request, scope="login", max_requests=5, window_seconds=60)
+    settings = get_settings()
+
+    client_id = settings.cognito_app_client_id
+    client_secret = settings.cognito_app_client_secret
+
+    auth_params: dict[str, str] = {
+        "USERNAME": body.email,
+        "PASSWORD": body.password,
+    }
+
+    if client_secret:
+        auth_params["SECRET_HASH"] = _compute_secret_hash(
+            body.email, client_id, client_secret
+        )
+
+    try:
+        cognito = boto3.client("cognito-idp", region_name=settings.aws_region)
+        result = cognito.initiate_auth(
+            ClientId=client_id,
+            AuthFlow="USER_PASSWORD_AUTH",
+            AuthParameters=auth_params,
+        )
+    except ClientError as e:
+        code = e.response["Error"]["Code"]
+        if code in ("NotAuthorizedException", "UserNotFoundException"):
+            raise HTTPException(status_code=401, detail="Invalid email or password")
+        logger.error("cognito_auth_error", error=str(e), code=code)
+        raise HTTPException(status_code=500, detail="Authentication service error")
+
+    auth = result["AuthenticationResult"]
+    # Decode id_token (without verification) to extract tenant_id
+    claims = jose_jwt.get_unverified_claims(auth["IdToken"])
+    return LoginResponse(
+        id_token=auth["IdToken"],
+        access_token=auth["AccessToken"],
+        refresh_token=auth["RefreshToken"],
+        expires_in=auth["ExpiresIn"],
+        tenant_id=claims.get("custom:tenant_id", ""),
+        email=claims.get("email", body.email),
+    )

pulse_engine/api/v1/health.py

@@ -0,0 +1,62 @@
+import httpx
+import structlog
+from fastapi import APIRouter, Depends, Request
+from pydantic import BaseModel
+
+from pulse_engine.config import Settings, get_settings
+from pulse_engine.services.opensearch import OpenSearchService
+
+router = APIRouter()
+logger = structlog.get_logger()
+
+
+class ServiceHealth(BaseModel):
+    opensearch: str
+    prefect: str
+
+
+class HealthResponse(BaseModel):
+    status: str
+    version: str
+    environment: str
+    services: ServiceHealth
+
+
+async def _check_opensearch(opensearch: OpenSearchService) -> str:
+    if await opensearch.ping():
+        return "up"
+    return "down"
+
+
+async def _check_prefect(url: str) -> str:
+    try:
+        async with httpx.AsyncClient(timeout=5.0) as client:
+            resp = await client.get(f"{url}/health")
+            resp.raise_for_status()
+            return "up"
+    except Exception:
+        logger.warning("prefect_health_check_failed", url=url)
+        return "down"
+
+
+@router.get("/health", response_model=HealthResponse)
+async def health_check(
+    request: Request,
+    settings: Settings = Depends(get_settings),
+) -> HealthResponse:
+    opensearch: OpenSearchService = request.app.state.opensearch
+    opensearch_status = await _check_opensearch(opensearch)
+    prefect_status = await _check_prefect(settings.prefect_api_url)
+
+    all_up = opensearch_status == "up" and prefect_status == "up"
+    overall = "ok" if all_up else "degraded"
+
+    return HealthResponse(
+        status=overall,
+        version=settings.app_version,
+        environment=settings.app_env,
+        services=ServiceHealth(
+            opensearch=opensearch_status,
+            prefect=prefect_status,
+        ),
+    )

pulse_engine/api/v1/router.py

@@ -0,0 +1,16 @@
+from fastapi import APIRouter
+
+from pulse_engine.api.v1.auth import router as auth_router
+from pulse_engine.api.v1.health import router as health_router
+from pulse_engine.deployment.router import router as deployments_router
+from pulse_engine.extractor.router import router as jobs_router
+from pulse_engine.processor.router import router as process_router
+from pulse_engine.storage.router import router as kb_router
+
+v1_router = APIRouter(prefix="/api/v1")
+v1_router.include_router(auth_router)
+v1_router.include_router(health_router)
+v1_router.include_router(kb_router)
+v1_router.include_router(jobs_router)
+v1_router.include_router(process_router)
+v1_router.include_router(deployments_router)

pulse_engine/chain_recovery.py

@@ -0,0 +1,131 @@
+"""Background task that monitors Prefect for stalled or failed chained flows."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING
+
+import structlog
+
+if TYPE_CHECKING:
+    from pulse_engine.config import Settings
+    from pulse_engine.deployment.job_launcher import JobLauncher
+    from pulse_engine.extractor.orchestrator.base import BaseOrchestratorAdapter
+    from pulse_engine.extractor.repository import JobRepository
+    from pulse_engine.extractor.stage_repository import StageRepository
+
+logger = structlog.get_logger(__name__)
+
+_NEXT_STAGE = {
+    "extraction": ("processor", "processing"),
+    "processing": ("storage", "storage"),
+}
+
+
+class ChainRecoveryTask:
+    """Polls Prefect for stalled chained flows and auto-recovers."""
+
+    def __init__(
+        self,
+        stage_repo: StageRepository,
+        job_repo: JobRepository,
+        job_launcher: JobLauncher,
+        orchestrator: BaseOrchestratorAdapter,
+        settings: Settings,
+    ) -> None:
+        self._stage_repo = stage_repo
+        self._job_repo = job_repo
+        self._job_launcher = job_launcher
+        self._orchestrator = orchestrator
+        self._grace_seconds = settings.pulse_chain_grace_period_seconds
+
+    async def check_once(self) -> None:
+        """Check running stages against Prefect for failures."""
+        stages = await self._stage_repo.get_running_stages()
+        for stage in stages:
+            if not stage.prefect_flow_run_id:
+                continue
+            try:
+                run_status = await self._orchestrator.get_run_status(
+                    stage.prefect_flow_run_id
+                )
+                if run_status.status in ("failed", "cancelled"):
+                    await self._stage_repo.update_status(
+                        stage.id,
+                        run_status.status,
+                    )
+                    job = await self._job_repo.get_by_id(stage.job_id)
+                    if job:
+                        await self._job_repo.update_status(
+                            stage.job_id, job.tenant_id, "failed"
+                        )
+                    logger.warning(
+                        "chain_recovery_stage_failed",
+                        job_id=stage.job_id,
+                        stage=stage.stage,
+                        flow_run_status=run_status.status,
+                    )
+            except Exception:
+                logger.warning(
+                    "chain_recovery_check_error",
+                    stage_id=stage.id,
+                    exc_info=True,
+                )
+
+    async def check_stalled_chains(self) -> None:
+        """Auto-trigger next stage for completed stages where chain stalled."""
+        cutoff = datetime.now(UTC) - timedelta(
+            seconds=self._grace_seconds,
+        )
+        stages = await self._stage_repo.get_completed_unchained_stages()
+
+        for stage in stages:
+            if stage.completed_at and stage.completed_at < cutoff:
+                next_info = _NEXT_STAGE.get(stage.stage)
+                if not next_info:
+                    continue
+
+                next_stage_key, next_stage_name = next_info
+                job = await self._job_repo.get_by_id(stage.job_id)
+                if not job:
+                    continue
+
+                params = getattr(job, "parameters", {}) or {}
+                if not params.get("chain", False):
+                    continue
+
+                orchestrator = params.get("orchestrator", "prefect")
+                compute = params.get("compute", "ecs")
+
+                try:
+                    result = await self._job_launcher.launch(
+                        job_id=job.job_id,
+                        tenant_id=job.tenant_id,
+                        product=job.product,
+                        stage=next_stage_key,
+                        orchestrator=orchestrator,
+                        compute=compute,
+                        chain=True,
+                        config=params.get("config", {}),
+                    )
+
+                    await self._stage_repo.create(
+                        job_id=job.job_id,
+                        stage=next_stage_name,
+                        prefect_flow_run_id=result.flow_run_id,
+                    )
+
+                    logger.info(
+                        "chain_recovery_triggered_next_stage",
+                        job_id=job.job_id,
+                        from_stage=stage.stage,
+                        to_stage=next_stage_name,
+                        flow_run_id=result.flow_run_id,
+                    )
+                except Exception:
+                    logger.warning(
+                        "chain_recovery_trigger_error",
+                        job_id=stage.job_id,
+                        stage=stage.stage,
+                        exc_info=True,
+                    )