pulse-engine 0.2.14__tar.gz → 0.2.15__tar.gz

{pulse_engine-0.2.14 → pulse_engine-0.2.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pulse-engine
-Version: 0.2.14
+Version: 0.2.15
 Summary: Pulse Engine — Hybrid framework for building Pulse products
 Author: Pulse Team
 Requires-Python: >=3.11,<3.13

{pulse_engine-0.2.14 → pulse_engine-0.2.15}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pulse-engine"
-version = "0.2.14"
+version = "0.2.15"
 description = "Pulse Engine — Hybrid framework for building Pulse products"
 authors = ["Pulse Team"]
 readme = "README.md"
@@ -107,7 +107,7 @@ module = ["langdetect.*", "tiktoken.*", "bs4.*", "jose", "jose.*", "boto3", "bot
 ignore_missing_imports = true
 
 [[tool.mypy.overrides]]
-module = ["prefect", "prefect.*", "entrypoint", "fitz", "twikit", "twikit.*", "yt_dlp", "google", "google.*", "apprise", "apprise.*"]
+module = ["prefect", "prefect.*", "entrypoint", "fitz", "twikit", "twikit.*", "yt_dlp", "yt_dlp.*", "google", "google.*", "apprise", "apprise.*"]
 ignore_missing_imports = true
 
 [[tool.mypy.overrides]]

{pulse_engine-0.2.14 → pulse_engine-0.2.15}/src/pulse_engine/adapters/youtube_downloader.py

@@ -21,6 +21,8 @@ _DEFAULT_PLAYER_CLIENTS = ["tv_embedded", "web"]
 def _build_yt_dlp_opts(
     output_template: str,
     extra: dict[str, Any] | None = None,
+    clip_start: float | None = None,
+    clip_end: float | None = None,
 ) -> dict[str, Any]:
     """Build yt-dlp option dict, merging env defaults with caller overrides."""
     raw_clients = os.environ.get("YT_DLP_PLAYER_CLIENTS", "").strip()
@@ -53,23 +55,28 @@ def _build_yt_dlp_opts(
         },
     }
 
-    if not extra:
-        return opts
-
-    override = dict(extra)
-    # cookies_content is handled separately — materialised to a temp file.
-    override.pop("cookies_content", None)
-    ea = override.pop("extractor_args", None)
-    opts.update(override)
-    if isinstance(ea, dict):
-        existing = opts.get("extractor_args") or {}
-        merged: dict[str, Any] = {**existing}
-        for k, v in ea.items():
-            if isinstance(v, dict) and isinstance(merged.get(k), dict):
-                merged[k] = {**merged[k], **v}
-            else:
-                merged[k] = v
-        opts["extractor_args"] = merged
+    if extra:
+        override = dict(extra)
+        # cookies_content is handled separately — materialised to a temp file.
+        override.pop("cookies_content", None)
+        ea = override.pop("extractor_args", None)
+        opts.update(override)
+        if isinstance(ea, dict):
+            existing = opts.get("extractor_args") or {}
+            merged: dict[str, Any] = {**existing}
+            for k, v in ea.items():
+                if isinstance(v, dict) and isinstance(merged.get(k), dict):
+                    merged[k] = {**merged[k], **v}
+                else:
+                    merged[k] = v
+            opts["extractor_args"] = merged
+
+    if clip_start is not None or clip_end is not None:
+        from yt_dlp.utils import download_range_func
+
+        start = clip_start if clip_start is not None else 0.0
+        end = clip_end if clip_end is not None else float("inf")
+        opts["download_ranges"] = download_range_func(None, [(start, end)])
 
     return opts
 
@@ -91,7 +98,13 @@ class YouTubeDownloader:
         self._aws_region: str = os.environ.get("AWS_REGION", "us-east-1").strip()
         self._cookies_cache: str | None = None  # lazily populated
 
-    async def download(self, video_id: str, output_dir: str) -> Path | None:
+    async def download(
+        self,
+        video_id: str,
+        output_dir: str,
+        start: float | None = None,
+        end: float | None = None,
+    ) -> Path | None:
         """Download audio for *video_id* into *output_dir*, returning path to mp3.
 
         Returns ``None`` if the download fails for any reason.
@@ -99,7 +112,7 @@ class YouTubeDownloader:
         url = f"https://www.youtube.com/watch?v={video_id}"
         logger.info("youtube_download_started", video_id=video_id)
 
-        result = await self._download_audio(url, output_dir)
+        result = await self._download_audio(url, output_dir, start=start, end=end)
         if result is None:
             logger.warning("youtube_download_failed", video_id=video_id)
         else:
@@ -152,7 +165,13 @@ class YouTubeDownloader:
         )
         return cookies
 
-    async def _download_audio(self, url: str, output_dir: str) -> Path | None:
+    async def _download_audio(
+        self,
+        url: str,
+        output_dir: str,
+        start: float | None = None,
+        end: float | None = None,
+    ) -> Path | None:
         """Download audio from a YouTube URL to *output_dir* via yt-dlp."""
         try:
             import yt_dlp
@@ -161,7 +180,9 @@ class YouTubeDownloader:
             return None
 
         output_template = str(Path(output_dir) / "%(id)s.%(ext)s")
-        ydl_opts = _build_yt_dlp_opts(output_template, extra=self._yt_dlp_opts)
+        ydl_opts = _build_yt_dlp_opts(
+            output_template, extra=self._yt_dlp_opts, clip_start=start, clip_end=end
+        )
 
         # Resolve cookies (priority: Secrets Manager > env var).
         cookies_content = await self._resolve_cookies()

{pulse_engine-0.2.14 → pulse_engine-0.2.15}/src/pulse_engine/api/v1/auth.py

@@ -47,8 +47,7 @@ def _compute_secret_hash(email: str, client_id: str, client_secret: str) -> str:
 @router.post("/login", response_model=LoginResponse)
 async def login(request: Request, body: LoginRequest) -> LoginResponse:
     """Authenticate with email and password, returns JWT tokens."""
-    # Strict per-IP rate limit: 5 attempts per 60 seconds
-    check_rate_limit(request, scope="login", max_requests=5, window_seconds=60)
+    check_rate_limit(request, scope="login", max_requests=500, window_seconds=60)
     settings = get_settings()
 
     client_id = settings.cognito_app_client_id

{pulse_engine-0.2.14 → pulse_engine-0.2.15}/src/pulse_engine/config.py

@@ -51,6 +51,10 @@ class Settings(BaseSettings):
 
     # Prefect ECS backend
     prefect_ecs_work_pool_name: str = "pipeline-ecs"
+    # Fleet-wide cap on concurrently-running flow runs for the ECS work pool.
+    # Acts as the queue throttle: scheduled runs beyond this wait in Prefect's
+    # queue instead of all launching at once (and OOM-ing the workers).
+    prefect_work_pool_concurrency_limit: int = 100
     # ECR image used to run pipeline_flow in ECS work pool
     prefect_engine_image: str = ""
     # Override compute backend for all pipeline steps ("ecs", "local", etc.)

{pulse_engine-0.2.14 → pulse_engine-0.2.15}/src/pulse_engine/main.py

@@ -117,7 +117,7 @@ def create_app(
     )
     app.add_middleware(
         RateLimitMiddleware,
-        max_requests=100,
+        max_requests=10000,
         window_seconds=60,
     )
 

{pulse_engine-0.2.14 → pulse_engine-0.2.15}/src/pulse_engine/pipeline/translators/prefect_translator.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import base64
+import logging
 from typing import Any
 
 import httpx
@@ -8,6 +9,8 @@ import httpx
 from pulse_engine.pipeline.schemas import PipelineConfig, SecretRef
 from pulse_engine.pipeline.translators.base import BaseTranslator
 
+logger = logging.getLogger(__name__)
+
 
 class PrefectTranslator(BaseTranslator):
     """Translates a PipelineConfig into a Prefect flow run submission."""
@@ -128,6 +131,32 @@ class PrefectTranslator(BaseTranslator):
         deploy_resp.raise_for_status()
         return str(deploy_resp.json()["id"])
 
+    async def ensure_work_pool_concurrency_limit(self, limit: int) -> None:
+        """Set the work pool's concurrency limit (idempotent).
+
+        This is the fleet-wide queue throttle: Prefect holds scheduled runs
+        beyond ``limit`` in its queue rather than dispatching them all at once,
+        which is what prevents workers from over-grabbing and OOM-ing. Failures
+        are swallowed (logged) so a Prefect hiccup can't block app startup.
+        """
+        client = self._get_client()
+        try:
+            resp = await client.patch(
+                f"{self._api_url}/work_pools/{self._work_pool_name}",
+                json={"concurrency_limit": limit},
+            )
+            resp.raise_for_status()
+        except Exception as exc:  # noqa: BLE001 - best-effort, non-fatal at startup
+            logger.warning(
+                "Failed to set concurrency limit %s on work pool %s: %s",
+                limit,
+                self._work_pool_name,
+                exc,
+            )
+        finally:
+            if self._client is None:
+                await client.aclose()
+
     async def submit(
         self,
         pipeline_run_id: str,

{pulse_engine-0.2.14 → pulse_engine-0.2.15}/src/pulse_engine/runners/prefect_pipeline_flow.py

@@ -25,6 +25,12 @@ from pulse_engine.secrets import fetch_secret
 
 _STDERR_LIMIT = int(os.getenv("PULSE_VM_STDERR_LIMIT", "4000"))
 _VM_DEBUG = os.getenv("PULSE_VM_DEBUG", "0") == "1"
+# Safety cap applied to fan-out steps that omit `max_concurrency`, so a single
+# step can't launch unbounded step tasks and overwhelm the pipeline cluster.
+# Set to 0 to restore fully-unbounded default behaviour.
+_DEFAULT_FANOUT_MAX_CONCURRENCY = int(
+    os.getenv("PULSE_DEFAULT_FANOUT_MAX_CONCURRENCY", "20")
+)
 
 _F = TypeVar("_F", bound=Callable[..., Any])
 
@@ -1293,6 +1299,10 @@ def _execute_layer(
             # below is a no-op (it's already done). Net effect: sync mode
             # is unaffected by max_concurrency.
             max_c = step_config.get("max_concurrency")
+            # When a step omits an explicit cap, fall back to the global safety
+            # default so unbounded fan-out can't flood the pipeline cluster.
+            if (not max_c or max_c <= 0) and _DEFAULT_FANOUT_MAX_CONCURRENCY > 0:
+                max_c = _DEFAULT_FANOUT_MAX_CONCURRENCY
             n_items = len(items)
             if not max_c or max_c <= 0 or max_c >= n_items:
                 futures = [

{pulse_engine-0.2.14 → pulse_engine-0.2.15}/src/pulse_engine/services/bootstrap.py

@@ -154,12 +154,17 @@ async def bootstrap_services(
             PrefectTranslator,
         )
 
-        pipeline_translators["prefect"] = PrefectTranslator(
+        prefect_translator = PrefectTranslator(
             prefect_api_url=settings.prefect_api_url,
             prefect_api_key=settings.prefect_api_key,
             work_pool_name=settings.prefect_ecs_work_pool_name,
             engine_image=settings.prefect_engine_image,
         )
+        # Apply the fleet-wide queue throttle (idempotent, best-effort).
+        await prefect_translator.ensure_work_pool_concurrency_limit(
+            settings.prefect_work_pool_concurrency_limit
+        )
+        pipeline_translators["prefect"] = prefect_translator
         pipeline_status_providers["prefect"] = PrefectStatusProvider(
             prefect_api_url=settings.prefect_api_url,
             prefect_api_key=settings.prefect_api_key,