pulse-engine 0.2.39__tar.gz → 0.2.42__tar.gz

{pulse_engine-0.2.39 → pulse_engine-0.2.42}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.1
 Name: pulse-engine
-Version: 0.2.39
+Version: 0.2.42
 Summary: Pulse Engine — Hybrid framework for building Pulse products
 Author: Pulse Team
 Requires-Python: >=3.11,<3.13
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: sheets
 Provides-Extra: worker
 Requires-Dist: alembic (>=1.14.0,<2.0.0)
 Requires-Dist: apprise (==1.11.0)
@@ -23,6 +24,8 @@ Requires-Dist: deepagents (>=0.6,<0.7)
 Requires-Dist: deepgram-sdk (>=3.7.0,<4.0.0)
 Requires-Dist: email-validator (>=2.3.0,<3.0.0)
 Requires-Dist: fastapi (>=0.115.0,<0.116.0)
+Requires-Dist: google-auth (>=2.35,<3.0) ; extra == "sheets"
+Requires-Dist: gspread (>=6.1,<7.0) ; extra == "sheets"
 Requires-Dist: httpx (>=0.28.0,<0.29.0)
 Requires-Dist: importlib-metadata (>=9.0.0,<10.0.0)
 Requires-Dist: langchain (>=1.2.13,<2.0.0)

{pulse_engine-0.2.39 → pulse_engine-0.2.42}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pulse-engine"
-version = "0.2.39"
+version = "0.2.42"
 description = "Pulse Engine — Hybrid framework for building Pulse products"
 authors = ["Pulse Team"]
 readme = "README.md"
@@ -53,10 +53,16 @@ langchain-agentcore-codeinterpreter = "^0.0.5"
 bedrock-agentcore = "^1.17.0"
 yt-dlp = "^2026.7.4"
 openpyxl = "^3.1"
+# Optional: only GoogleSheetsAdapter needs these, and it imports them lazily, so an
+# install that never reads a spreadsheet does not pay for them. Enable with the
+# "sheets" extra.
+gspread = {version = "^6.1", optional = true}
+google-auth = {version = "^2.35", optional = true}
 
 
 [tool.poetry.extras]
 worker = ["prefect"]
+sheets = ["gspread", "google-auth"]
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.3.0"
@@ -111,9 +117,15 @@ module = ["langdetect.*", "tiktoken.*", "bs4.*", "jose", "jose.*", "boto3", "bot
 ignore_missing_imports = true
 
 [[tool.mypy.overrides]]
-module = ["prefect", "prefect.*", "entrypoint", "fitz", "twikit", "twikit.*", "yt_dlp", "yt_dlp.*", "google", "google.*", "apprise", "apprise.*", "deepagents", "deepagents.*", "langchain_agentcore_codeinterpreter", "langchain_agentcore_codeinterpreter.*", "bedrock_agentcore", "bedrock_agentcore.*", "openpyxl", "openpyxl.*"]
+module = ["prefect", "prefect.*", "entrypoint", "fitz", "twikit", "twikit.*", "yt_dlp", "yt_dlp.*", "google", "google.*", "apprise", "apprise.*", "deepagents", "deepagents.*", "langchain_agentcore_codeinterpreter", "langchain_agentcore_codeinterpreter.*", "bedrock_agentcore", "bedrock_agentcore.*", "openpyxl", "openpyxl.*", "gspread", "gspread.*"]
 ignore_missing_imports = true
 
+[[tool.mypy.overrides]]
+# google-auth ships no annotations for `from_service_account_info`, so the one call this
+# adapter makes into it is untyped. Scoped to this module rather than pragma'd inline.
+module = ["pulse_engine.adapters.google_sheets"]
+disallow_untyped_calls = false
+
 [[tool.mypy.overrides]]
 module = ["scripts.*"]
 ignore_errors = true

{pulse_engine-0.2.39 → pulse_engine-0.2.42}/src/pulse_engine/adapters/digital_news.py

@@ -44,6 +44,11 @@ class _ArticleSchema(BaseModel):
     content: str = Field(description="The full main article text content")
 
 
+class TransientFetchError(Exception):
+    """Raised for any failure worth retrying: network issues, rate limits,
+    server errors, or a 404 that may be a stale CDN edge-cache response."""
+
+
 class DigitalNewsAdapter:
     """Fetches article content via Cloudflare Browser Rendering + LLM.
 
@@ -65,13 +70,19 @@ class DigitalNewsAdapter:
         adapter = DigitalNewsAdapter(llm=ChatOllama(model="llama3"))
     """
 
+    _RETRYABLE_TARGET_STATUSES = [404, 429, 500, 502, 503, 504]
+
     def __init__(
         self,
         llm: Any,
         cloudflare_timeout: float = 60.0,
+        max_retries: int = 3,
+        retry_delay: float = 5.0,
     ) -> None:
         self._llm = llm
         self._cloudflare_timeout = cloudflare_timeout
+        self._max_retries = max_retries
+        self._retry_delay = retry_delay
 
         self._account_id = os.environ.get("CLOUDFLARE_ACCOUNT_ID")
         if not self._account_id:
@@ -83,8 +94,30 @@ class DigitalNewsAdapter:
 
     async def fetch(self, url: str) -> ArticleContent:
         """Fetch and extract content from a single article URL."""
-        markdown = await self._fetch_markdown(url)
-        return await self._extract_content(markdown)
+        url = url.strip()
+        last_exc: Exception | None = None
+
+        for attempt in range(1, self._max_retries + 1):
+            try:
+                markdown = await self._fetch_markdown(url)
+                return await self._extract_content(markdown)
+            except TransientFetchError as exc:
+                last_exc = exc
+                if attempt < self._max_retries:
+                    logger.warning(
+                        "fetch_retry", url=url, attempt=attempt, reason=str(exc)
+                    )
+                    await asyncio.sleep(self._retry_delay)
+                else:
+                    logger.warning(
+                        "fetch_retries_exhausted",
+                        url=url,
+                        attempts=attempt,
+                        reason=str(exc),
+                    )
+
+        assert last_exc is not None
+        raise last_exc
 
     async def fetch_many(
         self,
@@ -114,20 +147,38 @@ class DigitalNewsAdapter:
         async with httpx.AsyncClient(
             timeout=self._cloudflare_timeout, follow_redirects=True
         ) as client:
-            resp = await client.post(
-                cf_url,
-                headers={
-                    "Authorization": f"Bearer {self._bearer_token}",
-                    "Content-Type": "application/json",
-                },
-                json={"url": url},
-            )
+            try:
+                resp = await client.post(
+                    cf_url,
+                    headers={
+                        "Authorization": f"Bearer {self._bearer_token}",
+                        "Content-Type": "application/json",
+                    },
+                    json={"url": url},
+                )
+            except (httpx.TimeoutException, httpx.TransportError) as exc:
+                raise TransientFetchError(
+                    f"network error contacting Cloudflare for {url}: {exc}"
+                ) from exc
+
+            # Rate limiting or server-side failure from Cloudflare's own API.
+            if resp.status_code == 429 or resp.status_code >= 500:
+                raise TransientFetchError(
+                    f"Cloudflare API returned {resp.status_code} for {url}"
+                )
             resp.raise_for_status()
 
         data = resp.json()
+
+        target_status = (data.get("meta") or {}).get("status")
+        if target_status in self._RETRYABLE_TARGET_STATUSES:
+            raise TransientFetchError(
+                f"target page returned status {target_status} for {url}"
+            )
+
         markdown: str = data.get("result", "")
         if not markdown or not markdown.strip():
-            raise RuntimeError(f"Empty markdown returned by Cloudflare for {url}")
+            raise TransientFetchError(f"Empty markdown returned by Cloudflare: {url}")
         return markdown
 
     async def _extract_content(self, markdown: str) -> ArticleContent:

pulse_engine-0.2.42/src/pulse_engine/adapters/google_sheets.py

@@ -0,0 +1,292 @@
+"""GoogleSheetsAdapter — read-only access to a Google Sheets workbook.
+
+Auth, quota handling and the API's rough edges live here; what the rows *mean* stays
+with the product. A caller supplies a service-account key and a workbook id and gets
+rows back — it never sees gspread, a credentials object, or a 429.
+
+Two of those rough edges are worth naming, because both were learned from a running
+system and neither is visible in the API:
+
+* **Only rate limits are retried.** A quota rejection is transient and worth waiting
+  out; a missing tab or a revoked key is not, and retrying it six times turns a clear
+  failure into a slow one that reports the wrong cause. Anything that is not a rate
+  limit propagates on the first attempt.
+* **Tab names drift.** The same logical tab is "Channel Codes" in one workbook and
+  "channel codes" in the next, because a human typed it. An exact match is tried
+  first, then a case- and whitespace-insensitive one, so a renamed tab does not fail
+  a run.
+
+``gspread`` is imported lazily and declared under the ``sheets`` extra. It is a
+dependency of this module, not of the engine, and an installation that never reads a
+spreadsheet should not pay for it.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# Read-only: this adapter has no write path, and the narrower scope means a leaked key
+# cannot be used to modify a workbook.
+READONLY_SCOPES = (
+    "https://www.googleapis.com/auth/spreadsheets.readonly",
+    "https://www.googleapis.com/auth/drive.readonly",
+)
+
+_RATE_LIMIT_MARKERS = ("429", "quota exceeded", "rate limit")
+
+
+class SheetError(Exception):
+    """A spreadsheet could not be read: bad credentials, missing tab, bad workbook
+    id."""
+
+
+class SheetRateLimitError(SheetError):
+    """Quota rejected the read and the retries were exhausted.
+
+    Distinct from `SheetError` so a caller can tell "come back later" from "this will
+    never work", and alert differently on each.
+    """
+
+
+def _is_rate_limit(exc: Exception) -> bool:
+    message = str(exc).lower()
+    return any(marker in message for marker in _RATE_LIMIT_MARKERS)
+
+
+@dataclass(frozen=True)
+class TabScan:
+    """One worksheet seen during a scan.
+
+    `title` and `first_cell` are read as the scan reaches the tab; `values()` is
+    deferred, because the caller that needs this — a workbook whose tabs *are* the
+    records, keyed by a cell rather than by name — decides from `first_cell` whether
+    the rows are wanted. Reading every tab eagerly would spend a request on each one
+    it skips.
+    """
+
+    title: str
+    first_cell: str
+    _values: Callable[[], list[list[Any]]]
+    error: str | None = None
+    """Why this tab could not be read, or None.
+
+    Set instead of raising, because a workbook whose tabs are records is exactly the
+    place where one malformed tab must not cost the whole walk. A quota rejection is
+    different — that is workbook-wide, so it still propagates.
+    """
+
+    def values(self) -> list[list[Any]]:
+        """The tab's rows, fetched on first ask."""
+        return self._values()
+
+
+class GoogleSheetsAdapter:
+    """Read worksheets from a Google Sheets workbook.
+
+    Args:
+        credentials_json: Service-account key as a JSON string.
+        max_retries: Attempts before giving up on a rate-limited read. Default 6.
+        backoff_base_seconds: Base for exponential backoff between retries. Default 2.0,
+            giving 2s, 4s, 8s … — about two minutes over six attempts, which is
+            the order of a Sheets per-minute quota window.
+        sleep: Injected so tests can drive the backoff without waiting. Production
+        callers
+            leave this alone.
+        client: A pre-built gspread client, for tests. When given, `credentials_json` is
+            not used and no authorisation is attempted.
+
+    Usage::
+
+        adapter = GoogleSheetsAdapter(credentials_json=key)
+        rows = adapter.read_records(spreadsheet_id="1abc…", tab="Master")
+    """
+
+    def __init__(
+        self,
+        credentials_json: str = "",
+        *,
+        max_retries: int = 6,
+        backoff_base_seconds: float = 2.0,
+        sleep: Callable[[float], None] = time.sleep,
+        client: Any | None = None,
+    ) -> None:
+        if max_retries < 1:
+            raise ValueError(f"max_retries must be at least 1, got {max_retries}")
+        self._credentials_json = credentials_json
+        self._max_retries = max_retries
+        self._backoff_base_seconds = backoff_base_seconds
+        self._sleep = sleep
+        self._client = client
+
+    # -- connection ----------------------------------------------------------------
+
+    @property
+    def client(self) -> Any:
+        """The authorised gspread client, built once on first use."""
+        if self._client is None:
+            self._client = self._authorize(self._credentials_json)
+        return self._client
+
+    @staticmethod
+    def _authorize(credentials_json: str) -> Any:
+        # Validated before gspread is imported: a blank or malformed key is a config
+        # error, and reporting it should not depend on the optional library being
+        # installed. It also fails faster, on the cheaper check.
+        if not credentials_json.strip():
+            raise SheetError("credentials_json is required to read a spreadsheet")
+        try:
+            info = json.loads(credentials_json)
+        except json.JSONDecodeError as exc:
+            raise SheetError(f"credentials_json is not valid JSON: {exc}") from exc
+
+        import gspread
+        from google.oauth2.service_account import Credentials
+
+        credentials = Credentials.from_service_account_info(
+            info, scopes=list(READONLY_SCOPES)
+        )
+        return gspread.authorize(credentials)
+
+    def _worksheet(self, spreadsheet_id: str, tab: str) -> Any:
+        try:
+            spreadsheet = self.client.open_by_key(spreadsheet_id)
+        except Exception as exc:
+            if _is_rate_limit(exc):
+                raise
+            raise SheetError(f"cannot open workbook {spreadsheet_id!r}: {exc}") from exc
+
+        try:
+            return spreadsheet.worksheet(tab)
+        except Exception:
+            # Tab names drift in case and spacing between workbooks; a relaxed match
+            # before giving up keeps a renamed "Channel Codes" from failing the run.
+            wanted = tab.strip().lower()
+            for worksheet in spreadsheet.worksheets():
+                if str(worksheet.title).strip().lower() == wanted:
+                    return worksheet
+            raise SheetError(f"tab {tab!r} not found in the workbook") from None
+
+    def _with_retry(self, what: str, call: Callable[[], Any]) -> Any:
+        """Run `call`, waiting out rate limits and nothing else."""
+        last_error: Exception | None = None
+        for attempt in range(self._max_retries):
+            try:
+                return call()
+            except Exception as exc:
+                if not _is_rate_limit(exc):
+                    raise
+                last_error = exc
+                delay = self._backoff_base_seconds * (2**attempt)
+                logger.warning(
+                    "sheets rate limited on %s (attempt %d/%d), sleeping %.2fs",
+                    what,
+                    attempt + 1,
+                    self._max_retries,
+                    delay,
+                )
+                self._sleep(delay)
+
+        raise SheetRateLimitError(
+            f"quota rejected the read of {what} after {self._max_retries} attempts: "
+            f"{last_error}"
+        )
+
+    # -- reads ---------------------------------------------------------------------
+
+    def read_records(self, spreadsheet_id: str, tab: str) -> list[dict[str, Any]]:
+        """One worksheet as a list of row dicts, keyed by the header row.
+
+        `expected_headers=[]` disables gspread's duplicate-header check: a tracker
+        maintained by hand routinely has two blank or repeated column titles, and the
+        caller decides which columns it cares about anyway.
+        """
+
+        def _read() -> list[dict[str, Any]]:
+            worksheet = self._worksheet(spreadsheet_id, tab)
+            rows: list[dict[str, Any]] = worksheet.get_all_records(expected_headers=[])
+            return rows
+
+        return list(self._with_retry(repr(tab), _read))
+
+    def read_values(self, spreadsheet_id: str, tab: str) -> list[list[Any]]:
+        """One worksheet as raw rows, header included and nothing coerced."""
+
+        def _read() -> list[list[Any]]:
+            worksheet = self._worksheet(spreadsheet_id, tab)
+            values: list[list[Any]] = worksheet.get_all_values()
+            return values
+
+        return list(self._with_retry(repr(tab), _read))
+
+    def read_first_cell(self, spreadsheet_id: str, tab: str) -> str:
+        """A1 of one worksheet, empty string when the sheet is blank.
+
+        Cheap enough to probe a tab's identity without pulling every row.
+        """
+
+        def _read() -> str:
+            worksheet = self._worksheet(spreadsheet_id, tab)
+            return str(worksheet.acell("A1").value or "")
+
+        return str(self._with_retry(repr(tab), _read))
+
+    def list_worksheet_titles(self, spreadsheet_id: str) -> list[str]:
+        """Every tab title in the workbook, in sheet order."""
+
+        def _read() -> list[str]:
+            spreadsheet = self.client.open_by_key(spreadsheet_id)
+            return [str(w.title) for w in spreadsheet.worksheets()]
+
+        return list(self._with_retry(f"workbook {spreadsheet_id!r}", _read))
+
+    def scan_tabs(self, spreadsheet_id: str) -> Iterator[TabScan]:
+        """Walk every tab in the workbook from a single open.
+
+        For a workbook that keys its records by a cell rather than by tab name, the
+        tab name is not known ahead of time, so `read_records` cannot address it.
+        Addressing tabs by name would also re-open the workbook per call, turning one
+        open into roughly 2N for N tabs — the wrong trade against a read quota.
+
+        Yields lazily so a caller can stop early, and defers the row read so a tab it
+        skips costs one cell rather than a whole sheet.
+        """
+        spreadsheet = self._with_retry(
+            f"workbook {spreadsheet_id!r}",
+            lambda: self.client.open_by_key(spreadsheet_id),
+        )
+
+        for worksheet in spreadsheet.worksheets():
+            title = str(worksheet.title)
+
+            def _first(w: Any = worksheet, t: str = title) -> str:
+                return str(self._with_retry(repr(t), lambda: w.acell("A1").value or ""))
+
+            def _values(w: Any = worksheet, t: str = title) -> list[list[Any]]:
+                rows: list[list[Any]] = list(
+                    self._with_retry(repr(t), lambda: w.get_all_values())
+                )
+                return rows
+
+            # A tab that cannot be read is reported, not raised. Raising would end
+            # the generator and take every remaining tab with it — and a generator
+            # cannot be resumed past an exception, so the caller could not skip one
+            # bad tab even if it wanted to. A rate limit is not a per-tab problem and
+            # still propagates from `_with_retry` once its retries are spent.
+            try:
+                first_cell, error = _first(), None
+            except SheetRateLimitError:
+                raise
+            except Exception as exc:  # noqa: BLE001 — reported per tab, not swallowed
+                logger.warning("sheets could not read A1 of %r: %s", title, exc)
+                first_cell, error = "", str(exc)
+
+            yield TabScan(
+                title=title, first_cell=first_cell, _values=_values, error=error
+            )

{pulse_engine-0.2.39 → pulse_engine-0.2.42}/src/pulse_engine/adapters/transcription/providers/deepgram.py

@@ -53,9 +53,7 @@ class DeepgramProvider(TranscriptionProvider):
         # back at the caller and discard the one useful output of detection —
         # which is what lets a caller assert the source language was plausible.
         detected = str(
-            alt.get("language")
-            or channel.get("detected_language")
-            or language
+            alt.get("language") or channel.get("detected_language") or language
         )
 
         utterances = results.get("utterances") or []

{pulse_engine-0.2.39 → pulse_engine-0.2.42}/src/pulse_engine/runners/prefect_pipeline_flow.py

@@ -40,9 +40,34 @@ logger = logging.getLogger(__name__)
 
 # Runner script injected as the container CMD for both ECS and VM dispatch.
 # Bypasses awslambdaric (Lambda-only) and runs the product entrypoint directly.
+# Steps on ECS, Fargate and MicroVMs run the script below under `python -c`, so they
+# never reach a product's Lambda handler and inherit no logging setup. Without this the
+# root logger sits at its default WARNING with no handler, and every logger.info a
+# module writes is discarded by logging.lastResort, which emits WARNING and above only.
+#
+# stderr, explicitly: the runner prints the step's JSON result on stdout and the MicroVM
+# path captures that stream, so a log line written there would land inside the payload.
+#
+# Configures only a process that configured nothing. An absent handler is what marks
+# the default state, so it is also the signal that no one has chosen a verbosity: a
+# caller who called basicConfig(level=ERROR) meant it, and raising that to INFO would
+# overrule the decision while a second handler duplicated every line.
+#
+# The Lambda handler in each product reads AWS_LAMBDA_LOG_LEVEL for the same purpose,
+# because awslambdaric always installs a handler and absence cannot be the signal there.
+_LOGGING_PREAMBLE = (
+    "import logging as _logging, sys as _sys\n"
+    "_root = _logging.getLogger()\n"
+    "if not _root.handlers:\n"
+    "    _root.addHandler(_logging.StreamHandler(_sys.stderr))\n"
+    "    if _root.getEffectiveLevel() > _logging.INFO:\n"
+    "        _root.setLevel(_logging.INFO)\n"
+)
+
 _VM_RUNNER_SCRIPT = (
     "import os, json, sys, traceback, boto3\n"
-    "sys.path.insert(0, '/app')\n"
+    + _LOGGING_PREAMBLE
+    + "sys.path.insert(0, '/app')\n"
     "sys.path.insert(0, '/app/src')\n"
     "try:\n"
     "    from botocore.exceptions import ClientError as _CE\n"

{pulse_engine-0.2.39 → pulse_engine-0.2.42}/src/pulse_engine/sandbox/backend.py

@@ -85,7 +85,15 @@ def make_lambda_microvm_backend(
     if client is None:  # pragma: no cover — real AWS path
         import boto3
 
-        client = boto3.client("lambda-microvms", region_name=region)  # type: ignore[call-overload]
+        # `lambda-microvms` is newer than the pinned boto3-stubs, which type
+        # `client()` as an overload set over known service names. The pinned stubs
+        # reject the literal; a newer installed set accepts it. A `type: ignore` is
+        # therefore wrong in one direction or the other: absent, CI fails on the
+        # unknown overload; present, it fails locally as an unused ignore, because
+        # warn_unused_ignores is on. Calling through an untyped reference sidesteps
+        # overload resolution entirely and holds under both.
+        make_client: Any = boto3.client
+        client = make_client("lambda-microvms", region_name=region)
 
     # Ship the resumed VM's stdout to CloudWatch. Without a `logging` config the
     # microVM's runtime output goes nowhere (no log group is auto-created), so the