pulse-engine 0.2.0.dev20260407065251__py3-none-any.whl

pulse_engine/api/v1/auth.py

@@ -0,0 +1,91 @@
+"""Authentication endpoint — exchanges email/password for Cognito tokens."""
+
+import base64
+import hashlib
+import hmac
+
+import boto3
+import structlog
+from botocore.exceptions import ClientError
+from fastapi import APIRouter, HTTPException, Request
+from jose import jwt as jose_jwt
+from pydantic import BaseModel, EmailStr, Field
+
+from pulse_engine.config import get_settings
+from pulse_engine.middleware.rate_limit import check_rate_limit
+
+logger = structlog.get_logger()
+
+router = APIRouter(prefix="/auth", tags=["Auth"])
+
+
+class LoginRequest(BaseModel):
+    email: EmailStr
+    password: str = Field(min_length=8, max_length=128)
+
+
+class LoginResponse(BaseModel):
+    id_token: str
+    access_token: str
+    refresh_token: str
+    expires_in: int
+    token_type: str = "Bearer"
+    tenant_id: str
+    email: str
+
+
+def _compute_secret_hash(email: str, client_id: str, client_secret: str) -> str:
+    message = email + client_id
+    digest = hmac.new(
+        client_secret.encode("utf-8"),
+        message.encode("utf-8"),
+        hashlib.sha256,
+    ).digest()
+    return base64.b64encode(digest).decode("utf-8")
+
+
+@router.post("/login", response_model=LoginResponse)
+async def login(request: Request, body: LoginRequest) -> LoginResponse:
+    """Authenticate with email and password, returns JWT tokens."""
+    # Strict per-IP rate limit: 5 attempts per 60 seconds
+    check_rate_limit(request, scope="login", max_requests=5, window_seconds=60)
+    settings = get_settings()
+
+    client_id = settings.cognito_app_client_id
+    client_secret = settings.cognito_app_client_secret
+
+    auth_params: dict[str, str] = {
+        "USERNAME": body.email,
+        "PASSWORD": body.password,
+    }
+
+    if client_secret:
+        auth_params["SECRET_HASH"] = _compute_secret_hash(
+            body.email, client_id, client_secret
+        )
+
+    try:
+        cognito = boto3.client("cognito-idp", region_name=settings.aws_region)
+        result = cognito.initiate_auth(
+            ClientId=client_id,
+            AuthFlow="USER_PASSWORD_AUTH",
+            AuthParameters=auth_params,
+        )
+    except ClientError as e:
+        code = e.response["Error"]["Code"]
+        if code in ("NotAuthorizedException", "UserNotFoundException"):
+            raise HTTPException(status_code=401, detail="Invalid email or password")
+        logger.error("cognito_auth_error", error=str(e), code=code)
+        raise HTTPException(status_code=500, detail="Authentication service error")
+
+    auth = result["AuthenticationResult"]
+    # Decode id_token (without verification) to extract tenant_id
+    claims = jose_jwt.get_unverified_claims(auth["IdToken"])
+    return LoginResponse(
+        id_token=auth["IdToken"],
+        access_token=auth["AccessToken"],
+        refresh_token=auth["RefreshToken"],
+        expires_in=auth["ExpiresIn"],
+        tenant_id=claims.get("custom:tenant_id", ""),
+        email=claims.get("email", body.email),
+    )

pulse_engine/api/v1/health.py

@@ -0,0 +1,62 @@
+import httpx
+import structlog
+from fastapi import APIRouter, Depends, Request
+from pydantic import BaseModel
+
+from pulse_engine.config import Settings, get_settings
+from pulse_engine.services.opensearch import OpenSearchService
+
+router = APIRouter()
+logger = structlog.get_logger()
+
+
+class ServiceHealth(BaseModel):
+    opensearch: str
+    prefect: str
+
+
+class HealthResponse(BaseModel):
+    status: str
+    version: str
+    environment: str
+    services: ServiceHealth
+
+
+async def _check_opensearch(opensearch: OpenSearchService) -> str:
+    if await opensearch.ping():
+        return "up"
+    return "down"
+
+
+async def _check_prefect(url: str) -> str:
+    try:
+        async with httpx.AsyncClient(timeout=5.0) as client:
+            resp = await client.get(f"{url}/health")
+            resp.raise_for_status()
+            return "up"
+    except Exception:
+        logger.warning("prefect_health_check_failed", url=url)
+        return "down"
+
+
+@router.get("/health", response_model=HealthResponse)
+async def health_check(
+    request: Request,
+    settings: Settings = Depends(get_settings),
+) -> HealthResponse:
+    opensearch: OpenSearchService = request.app.state.opensearch
+    opensearch_status = await _check_opensearch(opensearch)
+    prefect_status = await _check_prefect(settings.prefect_api_url)
+
+    all_up = opensearch_status == "up" and prefect_status == "up"
+    overall = "ok" if all_up else "degraded"
+
+    return HealthResponse(
+        status=overall,
+        version=settings.app_version,
+        environment=settings.app_env,
+        services=ServiceHealth(
+            opensearch=opensearch_status,
+            prefect=prefect_status,
+        ),
+    )

pulse_engine/api/v1/router.py

@@ -0,0 +1,16 @@
+from fastapi import APIRouter
+
+from pulse_engine.api.v1.auth import router as auth_router
+from pulse_engine.api.v1.health import router as health_router
+from pulse_engine.deployment.router import router as deployments_router
+from pulse_engine.extractor.router import router as jobs_router
+from pulse_engine.processor.router import router as process_router
+from pulse_engine.storage.router import router as kb_router
+
+v1_router = APIRouter(prefix="/api/v1")
+v1_router.include_router(auth_router)
+v1_router.include_router(health_router)
+v1_router.include_router(kb_router)
+v1_router.include_router(jobs_router)
+v1_router.include_router(process_router)
+v1_router.include_router(deployments_router)

pulse_engine/chain_recovery.py

@@ -0,0 +1,131 @@
+"""Background task that monitors Prefect for stalled or failed chained flows."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING
+
+import structlog
+
+if TYPE_CHECKING:
+    from pulse_engine.config import Settings
+    from pulse_engine.deployment.job_launcher import JobLauncher
+    from pulse_engine.extractor.orchestrator.base import BaseOrchestratorAdapter
+    from pulse_engine.extractor.repository import JobRepository
+    from pulse_engine.extractor.stage_repository import StageRepository
+
+logger = structlog.get_logger(__name__)
+
+_NEXT_STAGE = {
+    "extraction": ("processor", "processing"),
+    "processing": ("storage", "storage"),
+}
+
+
+class ChainRecoveryTask:
+    """Polls Prefect for stalled chained flows and auto-recovers."""
+
+    def __init__(
+        self,
+        stage_repo: StageRepository,
+        job_repo: JobRepository,
+        job_launcher: JobLauncher,
+        orchestrator: BaseOrchestratorAdapter,
+        settings: Settings,
+    ) -> None:
+        self._stage_repo = stage_repo
+        self._job_repo = job_repo
+        self._job_launcher = job_launcher
+        self._orchestrator = orchestrator
+        self._grace_seconds = settings.pulse_chain_grace_period_seconds
+
+    async def check_once(self) -> None:
+        """Check running stages against Prefect for failures."""
+        stages = await self._stage_repo.get_running_stages()
+        for stage in stages:
+            if not stage.prefect_flow_run_id:
+                continue
+            try:
+                run_status = await self._orchestrator.get_run_status(
+                    stage.prefect_flow_run_id
+                )
+                if run_status.status in ("failed", "cancelled"):
+                    await self._stage_repo.update_status(
+                        stage.id,
+                        run_status.status,
+                    )
+                    job = await self._job_repo.get_by_id(stage.job_id)
+                    if job:
+                        await self._job_repo.update_status(
+                            stage.job_id, job.tenant_id, "failed"
+                        )
+                    logger.warning(
+                        "chain_recovery_stage_failed",
+                        job_id=stage.job_id,
+                        stage=stage.stage,
+                        flow_run_status=run_status.status,
+                    )
+            except Exception:
+                logger.warning(
+                    "chain_recovery_check_error",
+                    stage_id=stage.id,
+                    exc_info=True,
+                )
+
+    async def check_stalled_chains(self) -> None:
+        """Auto-trigger next stage for completed stages where chain stalled."""
+        cutoff = datetime.now(UTC) - timedelta(
+            seconds=self._grace_seconds,
+        )
+        stages = await self._stage_repo.get_completed_unchained_stages()
+
+        for stage in stages:
+            if stage.completed_at and stage.completed_at < cutoff:
+                next_info = _NEXT_STAGE.get(stage.stage)
+                if not next_info:
+                    continue
+
+                next_stage_key, next_stage_name = next_info
+                job = await self._job_repo.get_by_id(stage.job_id)
+                if not job:
+                    continue
+
+                params = getattr(job, "parameters", {}) or {}
+                if not params.get("chain", False):
+                    continue
+
+                orchestrator = params.get("orchestrator", "prefect")
+                compute = params.get("compute", "ecs")
+
+                try:
+                    result = await self._job_launcher.launch(
+                        job_id=job.job_id,
+                        tenant_id=job.tenant_id,
+                        product=job.product,
+                        stage=next_stage_key,
+                        orchestrator=orchestrator,
+                        compute=compute,
+                        chain=True,
+                        config=params.get("config", {}),
+                    )
+
+                    await self._stage_repo.create(
+                        job_id=job.job_id,
+                        stage=next_stage_name,
+                        prefect_flow_run_id=result.flow_run_id,
+                    )
+
+                    logger.info(
+                        "chain_recovery_triggered_next_stage",
+                        job_id=job.job_id,
+                        from_stage=stage.stage,
+                        to_stage=next_stage_name,
+                        flow_run_id=result.flow_run_id,
+                    )
+                except Exception:
+                    logger.warning(
+                        "chain_recovery_trigger_error",
+                        job_id=stage.job_id,
+                        stage=stage.stage,
+                        exc_info=True,
+                    )

pulse_engine/cli/main.py

@@ -0,0 +1,169 @@
+"""Pulse Engine CLI — scaffold, validate, and run products."""
+
+from __future__ import annotations
+
+import importlib
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+app = typer.Typer(name="pulse", help="Pulse Engine CLI", no_args_is_help=True)
+
+
+# ---------------------------------------------------------------------------
+# pulse init
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def init(
+    product_name: Annotated[
+        str, typer.Argument(help="Name of the new product (e.g. 'political')")
+    ],
+    output_dir: Annotated[
+        Path, typer.Option("--output-dir", "-o", help="Where to scaffold")
+    ] = Path("."),
+) -> None:
+    """Scaffold a new Pulse product from the built-in template."""
+    from cookiecutter.main import cookiecutter
+
+    template_dir = str(Path(__file__).parent / "templates")
+
+    # Normalise name: lowercase, underscores for Python
+    slug = product_name.lower().replace("-", "_").replace(" ", "_")
+    display = slug.replace("_", "-")
+
+    result = cookiecutter(
+        template_dir,
+        output_dir=str(output_dir),
+        no_input=True,
+        extra_context={
+            "product_name": display,
+            "product_slug": slug,
+        },
+    )
+    typer.echo(f"Scaffolded product at {result}")
+
+
+# ---------------------------------------------------------------------------
+# pulse validate
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def validate(
+    module: Annotated[
+        str,
+        typer.Argument(
+            help="Dotted import path to the manifest (e.g. 'pulse_political.manifest')"
+        ),
+    ] = "",
+) -> None:
+    """Import a product manifest and run validation checks."""
+    from pulse_engine.registry import discover_products, validate_manifest
+
+    manifests = []
+
+    if module:
+        mod = importlib.import_module(module)
+        manifest = getattr(mod, "manifest", None)
+        if manifest is None:
+            typer.echo(f"No 'manifest' attribute found in {module}", err=True)
+            raise typer.Exit(code=1)
+        manifests = [manifest]
+    else:
+        manifests = discover_products()
+        if not manifests:
+            typer.echo("No products discovered via entry points.", err=True)
+            raise typer.Exit(code=1)
+
+    all_ok = True
+    for m in manifests:
+        errors = validate_manifest(m)
+        if errors:
+            all_ok = False
+            typer.echo(f"[FAIL] {m.name} v{m.version}:")
+            for e in errors:
+                typer.echo(f"  - {e}")
+        else:
+            typer.echo(f"[OK] {m.name} v{m.version}")
+
+    if not all_ok:
+        raise typer.Exit(code=1)
+
+
+# ---------------------------------------------------------------------------
+# pulse run
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def run(
+    host: Annotated[str, typer.Option(help="Bind host")] = "0.0.0.0",
+    port: Annotated[int, typer.Option(help="Bind port")] = 8000,
+    reload: Annotated[bool, typer.Option(help="Enable auto-reload")] = False,
+) -> None:
+    """Discover the product manifest and start the FastAPI server."""
+    import uvicorn
+
+    from pulse_engine.registry import discover_products
+
+    products = discover_products()
+    manifest = products[0] if products else None
+
+    if manifest:
+        typer.echo(f"Starting {manifest.name} v{manifest.version}")
+
+    # We use a factory string so uvicorn can import and call create_app
+    uvicorn.run(
+        "pulse_engine.main:create_app",
+        factory=True,
+        host=host,
+        port=port,
+        reload=reload,
+    )
+
+
+# ---------------------------------------------------------------------------
+# pulse run-worker
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def run_worker() -> None:
+    """Discover the product manifest and start the Celery worker."""
+    from pulse_engine.config import get_settings
+    from pulse_engine.registry import discover_products
+    from pulse_engine.worker import create_celery_app
+
+    products = discover_products()
+    manifest = products[0] if products else None
+
+    settings = get_settings()
+    celery_app = create_celery_app(settings, manifest)
+
+    celery_app.worker_main(["worker", "--loglevel=info"])
+
+
+# ---------------------------------------------------------------------------
+# pulse run-mcp
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def run_mcp() -> None:
+    """Discover the product manifest and start the MCP server."""
+    import asyncio
+
+    from pulse_engine.mcp.server import run_mcp_server
+    from pulse_engine.registry import discover_products
+
+    products = discover_products()
+    manifest = products[0] if products else None
+
+    asyncio.run(run_mcp_server(manifest))
+
+
+if __name__ == "__main__":
+    app()

pulse_engine/cli/templates/cookiecutter.json

@@ -0,0 +1,4 @@
+{
+  "product_name": "my-product",
+  "product_slug": "my_product"
+}

pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/.gitignore

@@ -0,0 +1,13 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+.env
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+*.egg

pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.12-slim AS builder
+
+WORKDIR /build
+
+RUN pip install --no-cache-dir poetry==1.8.5
+
+COPY pyproject.toml ./
+RUN poetry config virtualenvs.in-project true && \
+    poetry install --only main --no-interaction --no-ansi
+
+COPY src/ src/
+
+FROM python:3.12-slim AS runtime
+
+RUN groupadd -g 1001 appuser && \
+    useradd -u 1001 -g appuser -s /bin/false -m appuser
+
+WORKDIR /app
+
+COPY --from=builder /build/.venv .venv
+COPY --from=builder /build/src src
+
+ENV PATH="/app/.venv/bin:$PATH"
+
+USER appuser
+
+EXPOSE 8000
+
+HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
+    CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/v1/health')"
+
+CMD ["pulse", "run", "--host", "0.0.0.0", "--port", "8000"]

pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/pipeline.yaml

@@ -0,0 +1,17 @@
+name: {{cookiecutter.product_name}}
+version: "1.0"
+modules:
+  - name: extractor
+    compute: ecs
+    retries: 2
+    timeout: 900
+  - name: storage
+    compute: ecs
+
+dag:
+  - step: extract
+    module: extractor
+  - step: store
+    module: storage
+    depends_on:
+      - step: extract

pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "pulse-{{cookiecutter.product_name}}"
+version = "0.1.0"
+description = "Pulse product: {{cookiecutter.product_name}}"
+authors = ["Pulse Team"]
+packages = [{include = "pulse_{{cookiecutter.product_slug}}", from = "src"}]
+
+[tool.poetry.dependencies]
+python = ">=3.11,<3.13"
+pulse-engine = "*"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.0"
+pytest-asyncio = "^0.25.0"
+
+[tool.poetry.plugins."pulse_engine.products"]
+{{cookiecutter.product_slug}} = "pulse_{{cookiecutter.product_slug}}:manifest"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]

pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/src/pulse_{{cookiecutter.product_slug}}/__init__.py

@@ -0,0 +1,8 @@
+"""Pulse product: {{cookiecutter.product_name}}."""
+
+from pulse_engine.registry import ProductManifest
+
+manifest = ProductManifest(
+    name="{{cookiecutter.product_name}}",
+    version="0.1.0",
+)

pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/tests/unit/test_manifest.py

@@ -0,0 +1,15 @@
+"""Tests for the product manifest."""
+
+from pulse_engine.registry import validate_manifest
+
+from pulse_{{cookiecutter.product_slug}} import manifest
+
+
+def test_manifest_is_valid() -> None:
+    errors = validate_manifest(manifest)
+    assert errors == [], f"Manifest validation failed: {errors}"
+
+
+def test_manifest_metadata() -> None:
+    assert manifest.name == "{{cookiecutter.product_name}}"
+    assert manifest.version == "0.1.0"