pulse-engine 0.2.20__tar.gz → 0.2.22__tar.gz

{pulse_engine-0.2.20 → pulse_engine-0.2.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pulse-engine
-Version: 0.2.20
+Version: 0.2.22
 Summary: Pulse Engine — Hybrid framework for building Pulse products
 Author: Pulse Team
 Requires-Python: >=3.11,<3.13

{pulse_engine-0.2.20 → pulse_engine-0.2.22}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pulse-engine"
-version = "0.2.20"
+version = "0.2.22"
 description = "Pulse Engine — Hybrid framework for building Pulse products"
 authors = ["Pulse Team"]
 readme = "README.md"
@@ -122,7 +122,7 @@ module = ["alembic.*", "alembic.versions.*", "alembic.env"]
 ignore_errors = true
 
 [[tool.mypy.overrides]]
-module = ["pulse_engine.database"]
+module = ["pulse_engine.database", "pulse_engine.sandbox.lambda_microvm"]
 disallow_subclassing_any = false
 
 [[tool.mypy.overrides]]

{pulse_engine-0.2.20 → pulse_engine-0.2.22}/src/pulse_engine/adapters/__init__.py

@@ -4,11 +4,13 @@ from .audio_transcription import AudioTranscriptionAdapter
 from .batcher import BatcherAdapter
 from .digital_news import DigitalNewsAdapter
 from .digital_news_metadata import DigitalNewsMetadataAdapter
+from .document_event_campaign_mapper import DocumentEventCampaignMapper
 from .exceptions import SitemapDiscoveryError
 from .mart_export import MartExportAdapter
 from .models import (
     ArticleContent,
     ArticleResult,
+    DocumentMapping,
     NewsArticleMetadata,
     SpeechContent,
     SpeechMetadata,
@@ -37,6 +39,9 @@ __all__ = [
     "OpenSearchSqlAdapter",
     "MartExportAdapter",
     "ReportGeneratorAdapter",
+    # Document → event/campaign mapping
+    "DocumentEventCampaignMapper",
+    "DocumentMapping",
     # Digital news
     "DigitalNewsMetadataAdapter",
     "DigitalNewsAdapter",

pulse_engine-0.2.22/src/pulse_engine/adapters/document_event_campaign_mapper.py

@@ -0,0 +1,489 @@
+"""Maps documents to event code(s) + campaign id(s) via date-windowed LLM matching.
+
+For each document the adapter pre-filters the events/campaigns catalog by date
+window, batches date-adjacent documents together (one shared candidate list per
+batch), and asks the LLM which candidates each document is actually about. Events
+and campaigns are read from the media_coverage Postgres tables using a caller-
+supplied ``database_url``; the LLM is resolved via the engine's ``get_llm``
+provider (OpenAI by default, per ``Settings``).
+
+Usage::
+
+    adapter = DocumentEventCampaignMapper(
+        database_url="postgresql+asyncpg://user:pass@host:5432/media_coverage",
+    )
+    mappings = await adapter.map(documents)
+    # [DocumentMapping(doc_id="123", event_codes=["SM_2026_131"],
+    #  campaign_ids=["3"]), ...]
+
+Provider/model/api_key/temperature override the engine defaults from ``Settings``::
+
+    adapter = DocumentEventCampaignMapper(
+        database_url=url, provider="openai", model="gpt-5-mini",
+    )
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from datetime import date, datetime, timedelta
+from typing import Any
+
+import structlog
+from langchain_core.messages import HumanMessage, SystemMessage
+from pydantic import BaseModel, Field
+from sqlalchemy import text
+
+from pulse_engine.processor.llm.provider import get_llm
+
+from .models import DocumentMapping
+from .prompts import (
+    DOCUMENT_BLOCK,
+    MAPPING_PROMPT,
+    MAX_CONTENT_CHARS,
+)
+
+logger = structlog.get_logger(__name__)
+
+
+class _DocPrediction(BaseModel):
+    """One document's predicted event/campaign matches."""
+
+    doc_id: str = Field(description="The document id, echoed back")
+    event_codes: list[str] = Field(
+        default_factory=list, description="Matched event codes, or empty list"
+    )
+    campaign_ids: list[str] = Field(
+        default_factory=list, description="Matched campaign ids, or empty list"
+    )
+
+
+class _BatchPrediction(BaseModel):
+    """Structured LLM output: one prediction per document in the batch."""
+
+    predictions: list[_DocPrediction] = Field(default_factory=list)
+
+
+class DocumentEventCampaignMapper:
+    """Map documents to event codes and campaign ids using date-windowed LLM matching.
+
+    Args:
+        database_url: Async Postgres URL for the media_coverage DB, e.g.
+            ``postgresql+asyncpg://user:pass@host:5432/media_coverage``. Events and
+            campaigns are read from its ``events`` / ``campaigns`` tables.
+        provider: LLM provider override (defaults to ``Settings.pulse_llm_provider``).
+        model: LLM model override (defaults to ``"gpt-5-mini"``).
+        api_key: LLM API key override (defaults to ``Settings.pulse_llm_api_key``).
+        temperature: LLM temperature override (defaults to ``Settings``).
+        seed: OpenAI request seed for reproducible output (default 42). The gpt-5
+            family only supports temperature 1, so a fixed seed is what makes
+            identical inputs return identical results. Pass ``None`` to disable.
+        event_window_days: Days before a document's date to include events
+            (default 3). An event is a candidate when its date falls in
+            ``[doc_date - window, doc_date]``.
+        campaign_window_days: Days after a campaign start to keep it eligible
+            (default 30).
+        batch_size: Max documents per LLM call (default 10).
+        batch_gap_days: Start a new batch when consecutive documents are more than this
+            many days apart, so a batch never mixes far-apart documents (default 3).
+        concurrency: Max batches processed concurrently (default 4).
+    """
+
+    def __init__(
+        self,
+        database_url: str,
+        *,
+        provider: str | None = None,
+        model: str | None = "gpt-5-mini",
+        api_key: str | None = None,
+        temperature: float | None = None,
+        seed: int | None = 42,
+        event_window_days: int = 3,
+        campaign_window_days: int = 30,
+        batch_size: int = 10,
+        batch_gap_days: int = 3,
+        concurrency: int = 4,
+    ) -> None:
+        if not database_url:
+            raise ValueError("database_url is required")
+        self._database_url = database_url
+        self._provider = provider
+        self._model = model
+        self._api_key = api_key
+        self._temperature = temperature
+        self._seed = seed
+        self._event_window_days = event_window_days
+        self._campaign_window_days = campaign_window_days
+        self._batch_size = batch_size
+        self._batch_gap_days = batch_gap_days
+        self._concurrency = concurrency
+
+    async def map(self, documents: list[dict[str, Any]]) -> list[DocumentMapping]:
+        """Map each document to its matching event codes and campaign ids.
+
+        ``documents`` are dicts, each either flat (id/content/title/timestamp) or an
+        OpenSearch ``_source``-style dict (content, metadata.title, timestamp,
+        identifier). Returns one :class:`DocumentMapping` per input document, in input
+        order (empty lists when nothing matched).
+        """
+        if not documents:
+            return []
+
+        docs = [_normalize_document(d, i) for i, d in enumerate(documents)]
+
+        events, campaigns = await self._load_catalog()
+        valid_event_codes = {e["event_code"] for e in events}
+        valid_campaign_ids = {c["campaign_id"] for c in campaigns}
+
+        batches = _build_batches(docs, self._batch_size, self._batch_gap_days)
+        logger.info(
+            "mapping_started",
+            documents=len(docs),
+            batches=len(batches),
+            event_window_days=self._event_window_days,
+            campaign_window_days=self._campaign_window_days,
+            batch_size=self._batch_size,
+            batch_gap_days=self._batch_gap_days,
+            concurrency=self._concurrency,
+        )
+
+        llm = get_llm(
+            provider=self._provider,
+            model=self._model,
+            api_key=self._api_key,
+            temperature=self._temperature,
+        )
+        # The gpt-5 family only supports temperature 1 (non-deterministic), so pin a
+        # seed to make identical inputs return identical output across runs. The seed
+        # is bound per-call (not on the shared get_llm singleton).
+        if self._seed is not None:
+            llm = llm.bind(seed=self._seed)
+        structured_llm = llm.with_structured_output(_BatchPrediction)
+
+        semaphore = asyncio.Semaphore(self._concurrency)
+
+        async def _guarded(batch: list[dict[str, Any]]) -> dict[str, DocumentMapping]:
+            async with semaphore:
+                return await self._process_batch(
+                    batch,
+                    structured_llm,
+                    events,
+                    campaigns,
+                    valid_event_codes,
+                    valid_campaign_ids,
+                )
+
+        per_batch = await asyncio.gather(*(_guarded(b) for b in batches))
+
+        merged: dict[str, DocumentMapping] = {}
+        for batch_result in per_batch:
+            merged.update(batch_result)
+
+        # One entry per input document, in input order.
+        return [
+            merged.get(d["doc_id"], DocumentMapping(doc_id=d["doc_id"])) for d in docs
+        ]
+
+    # -- catalog loading ----------------------------------------------------
+
+    async def _load_catalog(
+        self,
+    ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+        """Read events and campaigns from the media_coverage DB (read-only)."""
+        from pulse_engine.database import build_async_engine, build_session_factory
+
+        engine = build_async_engine(self._database_url)
+        factory = build_session_factory(engine)
+        try:
+            async with factory() as session:
+                event_rows = (
+                    await session.execute(
+                        text(
+                            "SELECT event_id, event_type, description, key_issue, "
+                            "date_of_event FROM events"
+                        )
+                    )
+                ).all()
+                campaign_rows = (
+                    await session.execute(
+                        text(
+                            "SELECT id, campaign_name, description, "
+                            "campaign_start_date FROM campaigns"
+                        )
+                    )
+                ).all()
+        except Exception:
+            logger.error("catalog_load_failed", exc_info=True)
+            raise
+        finally:
+            await engine.dispose()
+
+        events = [
+            {
+                "event_code": r.event_id,
+                "event_type": r.event_type or "",
+                "description": r.description or "",
+                "key_issue": r.key_issue or "",
+                "date": _coerce_date(r.date_of_event),
+            }
+            for r in event_rows
+        ]
+        campaigns = [
+            {
+                "campaign_id": str(r.id),
+                "campaign_name": r.campaign_name or "",
+                "description": r.description or "",
+                "start_date": _coerce_date(r.campaign_start_date),
+            }
+            for r in campaign_rows
+        ]
+        logger.info("catalog_loaded", events=len(events), campaigns=len(campaigns))
+        return events, campaigns
+
+    # -- per-batch mapping --------------------------------------------------
+
+    async def _process_batch(
+        self,
+        batch: list[dict[str, Any]],
+        structured_llm: Any,
+        events: list[dict[str, Any]],
+        campaigns: list[dict[str, Any]],
+        valid_event_codes: set[str],
+        valid_campaign_ids: set[str],
+    ) -> dict[str, DocumentMapping]:
+        """Filter candidates for a batch, call the LLM, return per-doc mappings."""
+        for doc in batch:
+            doc["cand_events"] = _candidate_events(
+                doc["timestamp"], events, self._event_window_days
+            )
+            doc["cand_campaigns"] = _candidate_campaigns(
+                doc["timestamp"], campaigns, self._campaign_window_days
+            )
+
+        # Documents with no date-eligible candidates skip the LLM entirely.
+        llm_docs = [d for d in batch if d["cand_events"] or d["cand_campaigns"]]
+
+        predictions: dict[str, _DocPrediction] = {}
+        if llm_docs:
+            shared_events = _dedup_by(
+                (e for d in llm_docs for e in d["cand_events"]),
+                "event_code",
+                "date",
+            )
+            shared_campaigns = _dedup_by(
+                (c for d in llm_docs for c in d["cand_campaigns"]),
+                "campaign_id",
+                "start_date",
+            )
+            try:
+                parsed = await self._call_llm(
+                    structured_llm, llm_docs, shared_events, shared_campaigns
+                )
+                predictions = {p.doc_id.strip(): p for p in parsed.predictions}
+            except Exception as exc:  # noqa: BLE001 — one batch failing is non-fatal
+                logger.warning(
+                    "batch_failed",
+                    size=len(llm_docs),
+                    doc_ids=[d["doc_id"] for d in llm_docs],
+                    error=str(exc),
+                    exc_info=True,
+                )
+
+        result: dict[str, DocumentMapping] = {}
+        for doc in batch:
+            p = predictions.get(doc["doc_id"])
+            result[doc["doc_id"]] = DocumentMapping(
+                doc_id=doc["doc_id"],
+                event_codes=_valid_ids(p.event_codes if p else [], valid_event_codes),
+                campaign_ids=_valid_ids(
+                    p.campaign_ids if p else [], valid_campaign_ids
+                ),
+            )
+        return result
+
+    async def _call_llm(
+        self,
+        structured_llm: Any,
+        batch: list[dict[str, Any]],
+        shared_events: list[dict[str, Any]],
+        shared_campaigns: list[dict[str, Any]],
+    ) -> _BatchPrediction:
+        """Send one batch against its candidate lists; return structured output."""
+        events_json = json.dumps(
+            [
+                {
+                    "event_code": e["event_code"],
+                    "event_type": e["event_type"],
+                    "description": e["description"],
+                    "key_issue": e["key_issue"],
+                }
+                for e in shared_events
+            ],
+            ensure_ascii=False,
+        )
+        campaigns_json = json.dumps(
+            [
+                {
+                    "campaign_id": c["campaign_id"],
+                    "campaign_name": c["campaign_name"],
+                    "description": c["description"],
+                }
+                for c in shared_campaigns
+            ],
+            ensure_ascii=False,
+        )
+        documents_text = "\n".join(
+            f"--- Document {i + 1} ---\n{self._render_document(doc)}"
+            for i, doc in enumerate(batch)
+        )
+
+        system = MAPPING_PROMPT.format(
+            candidate_events=events_json,
+            candidate_campaigns=campaigns_json,
+        )
+        messages = [
+            SystemMessage(content=system),
+            HumanMessage(content=f"DOCUMENTS:\n{documents_text}"),
+        ]
+        result: _BatchPrediction = await structured_llm.ainvoke(messages)
+        return result
+
+    def _render_document(self, doc: dict[str, Any]) -> str:
+        title_line = f"title: {doc['title']}\n" if doc["title"] else ""
+        timestamp = (
+            doc["timestamp"].date().isoformat() if doc["timestamp"] else "unknown"
+        )
+        return DOCUMENT_BLOCK.format(
+            doc_id=doc["doc_id"],
+            title_line=title_line,
+            content=doc["content"][:MAX_CONTENT_CHARS],
+            timestamp=timestamp,
+            event_window_days=self._event_window_days,
+            campaign_window_days=self._campaign_window_days,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Pure helpers
+# ---------------------------------------------------------------------------
+
+
+def _parse_timestamp(value: Any) -> datetime | None:
+    """Parse an ISO date/datetime string (or date/datetime) to a naive datetime."""
+    if value is None or value == "":
+        return None
+    if isinstance(value, datetime):
+        return value.replace(tzinfo=None)
+    if isinstance(value, date):
+        return datetime(value.year, value.month, value.day)
+    try:
+        return datetime.fromisoformat(str(value).replace("Z", "+00:00")).replace(
+            tzinfo=None
+        )
+    except (ValueError, TypeError):
+        return None
+
+
+def _coerce_date(value: Any) -> datetime | None:
+    """Coerce a DB date/datetime column value to a naive datetime for comparison."""
+    return _parse_timestamp(value)
+
+
+def _normalize_document(doc: dict[str, Any], index: int) -> dict[str, Any]:
+    """Coerce an input document into the internal shape the mapper works with."""
+    doc_id = (
+        doc.get("id")
+        or doc.get("doc_id")
+        or doc.get("identifier")
+        or doc.get("sn")
+        or doc.get("SN")
+        or index
+    )
+    metadata = doc.get("metadata") or {}
+    title = doc.get("title") or metadata.get("title") or ""
+    content = doc.get("content") or ""
+    return {
+        "doc_id": str(doc_id).strip(),
+        "content": str(content),
+        "title": str(title),
+        "timestamp": _parse_timestamp(doc.get("timestamp")),
+    }
+
+
+def _candidate_events(
+    doc_ts: datetime | None, events: list[dict[str, Any]], window_days: int
+) -> list[dict[str, Any]]:
+    if doc_ts is None:
+        return []
+    # Events dated in [doc_ts - window_days, doc_ts]: the window_days before the
+    # document, up to and including the document's own date.
+    lo, hi = doc_ts - timedelta(days=window_days), doc_ts
+    return [e for e in events if e["date"] is not None and lo <= e["date"] <= hi]
+
+
+def _candidate_campaigns(
+    doc_ts: datetime | None, campaigns: list[dict[str, Any]], window_days: int
+) -> list[dict[str, Any]]:
+    if doc_ts is None:
+        return []
+    return [
+        c
+        for c in campaigns
+        if c["start_date"] is not None
+        and c["start_date"] <= doc_ts <= c["start_date"] + timedelta(days=window_days)
+    ]
+
+
+def _dedup_by(items: Any, key: str, key_date: str) -> list[dict[str, Any]]:
+    """Dedup dicts by ``key`` (keep first seen), sorted by ``key_date`` (None last)."""
+    seen: dict[Any, dict[str, Any]] = {}
+    for item in items:
+        seen.setdefault(item[key], item)
+    return sorted(
+        seen.values(),
+        key=lambda x: (x.get(key_date) is None, x.get(key_date) or datetime.min),
+    )
+
+
+def _build_batches(
+    docs: list[dict[str, Any]], batch_size: int, gap_days: int
+) -> list[list[dict[str, Any]]]:
+    """Sort docs by timestamp, then split into batches of date-adjacent documents.
+
+    A new batch starts whenever the batch is full (``batch_size``) or the gap between
+    consecutive documents' timestamps exceeds ``gap_days`` — so a batch never mixes
+    documents far apart in time. Undated documents sort last and group by size only.
+    """
+    ordered = sorted(
+        docs, key=lambda d: (d["timestamp"] is None, d["timestamp"] or datetime.min)
+    )
+    batches: list[list[dict[str, Any]]] = []
+    current: list[dict[str, Any]] = []
+    for doc in ordered:
+        if current:
+            prev_ts, ts = current[-1]["timestamp"], doc["timestamp"]
+            gap_too_large = (
+                prev_ts is not None
+                and ts is not None
+                and (ts - prev_ts) > timedelta(days=gap_days)
+            ) or (prev_ts is None) != (ts is None)
+            if len(current) >= batch_size or gap_too_large:
+                batches.append(current)
+                current = []
+        current.append(doc)
+    if current:
+        batches.append(current)
+    return batches
+
+
+def _valid_ids(values: list[str], valid: set[str]) -> list[str]:
+    """Keep only known, non-empty ids, deduped, preserving order."""
+    out: list[str] = []
+    for v in values:
+        if v in ("", "null", "NONE", None):
+            continue
+        v = str(v).strip()
+        if v in valid and v not in out:
+            out.append(v)
+    return out

{pulse_engine-0.2.20 → pulse_engine-0.2.22}/src/pulse_engine/adapters/models.py

@@ -154,3 +154,17 @@ class VideoResult:
     video_id: str
     audio: VideoAudio | None
     error: str | None
+
+
+# ---------------------------------------------------------------------------
+# Document → event/campaign mapping models
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class DocumentMapping:
+    """Event codes and campaign ids matched to a single document."""
+
+    doc_id: str
+    event_codes: list[str] = field(default_factory=list)
+    campaign_ids: list[str] = field(default_factory=list)

pulse_engine-0.2.22/src/pulse_engine/adapters/prompts.py

@@ -0,0 +1,32 @@
+MAX_CONTENT_CHARS = 4000
+
+# System prompt: the matching rules, with the shared candidate lists injected.
+MAPPING_PROMPT = (
+    "For each document, pick its event_code(s) and campaign_id(s) from the shared\n"
+    "candidate lists below (already date-filtered). Judge by content only.\n\n"
+    "candidate_events:\n{candidate_events}\n\n"
+    "candidate_campaigns:\n{candidate_campaigns}\n\n"
+    "Event rules:\n"
+    "- Match an event only if the document is actually reporting on or depicting\n"
+    "  that specific event. Match what it describes, not just its date: live event\n"
+    "  if it's happening now, the aftermath/follow-up event if it's looking back.\n"
+    "- A shared theme, location, date, or a mentioned participant is NOT enough.\n"
+    "  When in doubt, don't match — a wrong match is worse than none.\n\n"
+    "Campaign rules — match a campaign only if BOTH:\n"
+    "1. The document names the campaign (or something unique to it) or directly\n"
+    "   addresses its specific subject — not just a shared theme.\n"
+    "2. That campaign is the document's main subject, not a passing mention.\n\n"
+    "Return an empty list when nothing clearly matches. A document may match more\n"
+    "than one event or campaign."
+)
+
+# Per-document block rendered into the human message for each batch.
+DOCUMENT_BLOCK = (
+    "doc_id: {doc_id}\n"
+    "{title_line}"
+    "content: {content}\n"
+    "candidate_events: events dated within {event_window_days} days before this "
+    "document's timestamp {timestamp}, up to and including that date\n"
+    "candidate_campaigns: campaigns whose start date is within the "
+    "{campaign_window_days} day window before this document's timestamp {timestamp}"
+)

{pulse_engine-0.2.20 → pulse_engine-0.2.22}/src/pulse_engine/config.py

@@ -54,7 +54,7 @@ class Settings(BaseSettings):
     # Fleet-wide cap on concurrently-running flow runs for the ECS work pool.
     # Acts as the queue throttle: scheduled runs beyond this wait in Prefect's
     # queue instead of all launching at once (and OOM-ing the workers).
-    prefect_work_pool_concurrency_limit: int = 15
+    prefect_work_pool_concurrency_limit: int = 50
     # ECR image used to run pipeline_flow in ECS work pool
     prefect_engine_image: str = ""
     # Override compute backend for all pipeline steps ("ecs", "local", etc.)