pulse-engine 0.2.0__py3-none-any.whl

pulse_engine/cli/main.py

@@ -0,0 +1,169 @@
+"""Pulse Engine CLI — scaffold, validate, and run products."""
+
+from __future__ import annotations
+
+import importlib
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+app = typer.Typer(name="pulse", help="Pulse Engine CLI", no_args_is_help=True)
+
+
+# ---------------------------------------------------------------------------
+# pulse init
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def init(
+    product_name: Annotated[
+        str, typer.Argument(help="Name of the new product (e.g. 'political')")
+    ],
+    output_dir: Annotated[
+        Path, typer.Option("--output-dir", "-o", help="Where to scaffold")
+    ] = Path("."),
+) -> None:
+    """Scaffold a new Pulse product from the built-in template."""
+    from cookiecutter.main import cookiecutter
+
+    template_dir = str(Path(__file__).parent / "templates")
+
+    # Normalise name: lowercase, underscores for Python
+    slug = product_name.lower().replace("-", "_").replace(" ", "_")
+    display = slug.replace("_", "-")
+
+    result = cookiecutter(
+        template_dir,
+        output_dir=str(output_dir),
+        no_input=True,
+        extra_context={
+            "product_name": display,
+            "product_slug": slug,
+        },
+    )
+    typer.echo(f"Scaffolded product at {result}")
+
+
+# ---------------------------------------------------------------------------
+# pulse validate
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def validate(
+    module: Annotated[
+        str,
+        typer.Argument(
+            help="Dotted import path to the manifest (e.g. 'pulse_political.manifest')"
+        ),
+    ] = "",
+) -> None:
+    """Import a product manifest and run validation checks."""
+    from pulse_engine.registry import discover_products, validate_manifest
+
+    manifests = []
+
+    if module:
+        mod = importlib.import_module(module)
+        manifest = getattr(mod, "manifest", None)
+        if manifest is None:
+            typer.echo(f"No 'manifest' attribute found in {module}", err=True)
+            raise typer.Exit(code=1)
+        manifests = [manifest]
+    else:
+        manifests = discover_products()
+        if not manifests:
+            typer.echo("No products discovered via entry points.", err=True)
+            raise typer.Exit(code=1)
+
+    all_ok = True
+    for m in manifests:
+        errors = validate_manifest(m)
+        if errors:
+            all_ok = False
+            typer.echo(f"[FAIL] {m.name} v{m.version}:")
+            for e in errors:
+                typer.echo(f"  - {e}")
+        else:
+            typer.echo(f"[OK] {m.name} v{m.version}")
+
+    if not all_ok:
+        raise typer.Exit(code=1)
+
+
+# ---------------------------------------------------------------------------
+# pulse run
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def run(
+    host: Annotated[str, typer.Option(help="Bind host")] = "0.0.0.0",
+    port: Annotated[int, typer.Option(help="Bind port")] = 8000,
+    reload: Annotated[bool, typer.Option(help="Enable auto-reload")] = False,
+) -> None:
+    """Discover the product manifest and start the FastAPI server."""
+    import uvicorn
+
+    from pulse_engine.registry import discover_products
+
+    products = discover_products()
+    manifest = products[0] if products else None
+
+    if manifest:
+        typer.echo(f"Starting {manifest.name} v{manifest.version}")
+
+    # We use a factory string so uvicorn can import and call create_app
+    uvicorn.run(
+        "pulse_engine.main:create_app",
+        factory=True,
+        host=host,
+        port=port,
+        reload=reload,
+    )
+
+
+# ---------------------------------------------------------------------------
+# pulse run-worker
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def run_worker() -> None:
+    """Discover the product manifest and start the Celery worker."""
+    from pulse_engine.config import get_settings
+    from pulse_engine.registry import discover_products
+    from pulse_engine.worker import create_celery_app
+
+    products = discover_products()
+    manifest = products[0] if products else None
+
+    settings = get_settings()
+    celery_app = create_celery_app(settings, manifest)
+
+    celery_app.worker_main(["worker", "--loglevel=info"])
+
+
+# ---------------------------------------------------------------------------
+# pulse run-mcp
+# ---------------------------------------------------------------------------
+
+
+@app.command()
+def run_mcp() -> None:
+    """Discover the product manifest and start the MCP server."""
+    import asyncio
+
+    from pulse_engine.mcp.server import run_mcp_server
+    from pulse_engine.registry import discover_products
+
+    products = discover_products()
+    manifest = products[0] if products else None
+
+    asyncio.run(run_mcp_server(manifest))
+
+
+if __name__ == "__main__":
+    app()

pulse_engine/cli/templates/cookiecutter.json

@@ -0,0 +1,4 @@
+{
+  "product_name": "my-product",
+  "product_slug": "my_product"
+}

pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/.gitignore

@@ -0,0 +1,13 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+.env
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+*.egg

pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.12-slim AS builder
+
+WORKDIR /build
+
+RUN pip install --no-cache-dir poetry==1.8.5
+
+COPY pyproject.toml ./
+RUN poetry config virtualenvs.in-project true && \
+    poetry install --only main --no-interaction --no-ansi
+
+COPY src/ src/
+
+FROM python:3.12-slim AS runtime
+
+RUN groupadd -g 1001 appuser && \
+    useradd -u 1001 -g appuser -s /bin/false -m appuser
+
+WORKDIR /app
+
+COPY --from=builder /build/.venv .venv
+COPY --from=builder /build/src src
+
+ENV PATH="/app/.venv/bin:$PATH"
+
+USER appuser
+
+EXPOSE 8000
+
+HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
+    CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/v1/health')"
+
+CMD ["pulse", "run", "--host", "0.0.0.0", "--port", "8000"]

pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/pipeline.yaml

@@ -0,0 +1,17 @@
+name: {{cookiecutter.product_name}}
+version: "1.0"
+modules:
+  - name: extractor
+    compute: ecs
+    retries: 2
+    timeout: 900
+  - name: storage
+    compute: ecs
+
+dag:
+  - step: extract
+    module: extractor
+  - step: store
+    module: storage
+    depends_on:
+      - step: extract

pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "pulse-{{cookiecutter.product_name}}"
+version = "0.1.0"
+description = "Pulse product: {{cookiecutter.product_name}}"
+authors = ["Pulse Team"]
+packages = [{include = "pulse_{{cookiecutter.product_slug}}", from = "src"}]
+
+[tool.poetry.dependencies]
+python = ">=3.11,<3.13"
+pulse-engine = "*"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.0"
+pytest-asyncio = "^0.25.0"
+
+[tool.poetry.plugins."pulse_engine.products"]
+{{cookiecutter.product_slug}} = "pulse_{{cookiecutter.product_slug}}:manifest"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]

pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/src/pulse_{{cookiecutter.product_slug}}/__init__.py

@@ -0,0 +1,8 @@
+"""Pulse product: {{cookiecutter.product_name}}."""
+
+from pulse_engine.registry import ProductManifest
+
+manifest = ProductManifest(
+    name="{{cookiecutter.product_name}}",
+    version="0.1.0",
+)

pulse_engine/cli/templates/pulse-{{cookiecutter.product_name}}/tests/unit/test_manifest.py

@@ -0,0 +1,15 @@
+"""Tests for the product manifest."""
+
+from pulse_engine.registry import validate_manifest
+
+from pulse_{{cookiecutter.product_slug}} import manifest
+
+
+def test_manifest_is_valid() -> None:
+    errors = validate_manifest(manifest)
+    assert errors == [], f"Manifest validation failed: {errors}"
+
+
+def test_manifest_metadata() -> None:
+    assert manifest.name == "{{cookiecutter.product_name}}"
+    assert manifest.version == "0.1.0"

pulse_engine/client.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+from typing import Any, cast
+
+import httpx
+import structlog
+from jose import jwt as jose_jwt
+
+logger = structlog.get_logger(__name__)
+
+
+class PulseEngineClient:
+    """Client used inside container stages to call Pulse Engine REST API."""
+
+    def __init__(
+        self,
+        base_url: str,
+        token: str,
+        job_id: str,
+        timeout: float = 30.0,
+    ) -> None:
+        self._base_url = base_url.rstrip("/")
+        self._job_id = job_id
+        # Decode token without verification to read backend claims
+        # (signature was already verified by Pulse Engine before injecting the token)
+        try:
+            self._token_claims: dict[str, Any] = jose_jwt.get_unverified_claims(token)
+        except Exception:
+            self._token_claims = {}
+        self._client = httpx.AsyncClient(
+            base_url=self._base_url,
+            headers={
+                "Authorization": f"Bearer {token}",
+                "Content-Type": "application/json",
+            },
+            timeout=timeout,
+        )
+
+    @property
+    def job_id(self) -> str:
+        return self._job_id
+
+    async def report_status(
+        self,
+        stage: str,
+        status: str,
+        error: str | None = None,
+        result_summary: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Report stage status back to Pulse Engine."""
+        payload: dict[str, Any] = {"stage": stage, "status": status}
+        if error is not None:
+            payload["error"] = error
+        if result_summary is not None:
+            payload["result_summary"] = result_summary
+        resp = await self._client.post(
+            f"/api/v1/jobs/{self._job_id}/status",
+            json=payload,
+        )
+        resp.raise_for_status()
+        return cast(dict[str, Any], resp.json())
+
+    async def store_documents(self, documents: list[dict[str, Any]]) -> dict[str, Any]:
+        """Store processed documents in the knowledge base."""
+        resp = await self._client.post(
+            "/api/v1/kb/documents",
+            json={"documents": documents},
+        )
+        resp.raise_for_status()
+        return cast(dict[str, Any], resp.json())
+
+    async def trigger_next_stage(
+        self,
+        product: str,
+        next_stage: str,
+        chain: bool = True,
+        config: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Trigger the next stage, forwarding the same orchestrator+compute backend."""
+        payload: dict[str, Any] = {
+            "product": product,
+            "stage": next_stage,
+            "chain": chain,
+            "job_id": self._job_id,
+            "orchestrator": self._token_claims.get("orchestrator", "prefect"),
+            "compute": self._token_claims.get("compute", "ecs"),
+        }
+        if config:
+            payload["config"] = config
+        resp = await self._client.post("/api/v1/jobs/", json=payload)
+        resp.raise_for_status()
+        return cast(dict[str, Any], resp.json())
+
+    async def close(self) -> None:
+        await self._client.aclose()

pulse_engine/config.py

@@ -0,0 +1,157 @@
+from functools import lru_cache
+
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+    app_env: str = "development"
+    app_version: str = "0.1.0"
+    debug: bool = False
+    log_level: str = "INFO"
+
+    # AWS core credentials
+    aws_access_key_id: str = ""
+    aws_secret_access_key: str = ""
+    aws_region: str = "ap-south-1"
+    cognito_user_pool_id: str = ""
+    cognito_app_client_id: str = ""
+    cognito_app_client_secret: str = ""
+
+    # OpenSearch (hosted AWS)
+    opensearch_url: str = ""
+    opensearch_username: str = ""
+    opensearch_password: str = ""
+    opensearch_use_ssl: bool = True
+    opensearch_verify_certs: bool = True
+    opensearch_index_prefix: str = "pulse_kb"
+
+    # Prefect
+    prefect_api_url: str = ""
+    prefect_database_url: str = ""
+
+    # Athena (separate credentials — product specifies which database)
+    athena_aws_access_key_id: str = ""
+    athena_aws_secret_access_key: str = ""
+    athena_output_location: str = ""
+    athena_workgroup: str = "primary"
+    athena_query_timeout_seconds: int = 60
+    embedding_dimension: int = 1536
+
+    # Database (app)
+    database_url: str = ""
+
+    # Redis / Celery
+    redis_url: str = "redis://localhost:6379/0"
+    celery_broker_url: str = ""  # defaults to redis_url if empty
+    celery_result_backend: str = ""  # defaults to redis_url if empty
+
+    # Orchestrator
+    pulse_orchestrator_backend: str = "none"
+    prefect_api_key: str = ""
+
+    # Prefect ECS backend
+    prefect_ecs_work_pool_name: str = "pipeline-ecs"
+    # ECR image used to run pipeline_flow in ECS work pool
+    prefect_engine_image: str = ""
+    # Override compute backend for all pipeline steps ("ecs", "local", etc.)
+    pipeline_default_compute: str = ""
+
+    # Prefect Lambda backend
+    prefect_lambda_work_pool_name: str = "lambda-worker-pool"
+    prefect_lambda_function_name_template: str = "{product}-{stage}"
+
+    # Prefect Kubernetes backend
+    prefect_k8s_work_pool_name: str = "k8s-worker-pool"
+    prefect_k8s_namespace: str = "pulse-jobs"
+    prefect_k8s_default_cpu: str = "500m"
+    prefect_k8s_default_memory: str = "1Gi"
+    pulse_max_concurrent_jobs_per_tenant: int = 10
+    pulse_embedding_model: str = "text-embedding-3-small"
+    pulse_embedding_provider: str = "openai"
+    pulse_openai_embedding_model: str = "text-embedding-3-small"
+    pulse_openai_api_key: str = ""  # falls back to pulse_llm_api_key
+
+    # LLM
+    pulse_llm_provider: str = "openai"
+    pulse_llm_model: str = "gpt-4o-mini"
+    pulse_llm_api_key: str = ""
+    pulse_llm_temperature: float = 0.0
+    pulse_default_chunk_size: int = 512
+    pulse_default_chunk_strategy: str = "token_count"
+    pulse_job_callback_timeout: int = 10
+    pulse_dedup_similarity_threshold: float = 0.95
+
+    # Pipeline separation
+    pulse_engine_url: str = ""  # public URL containers use to call back
+    pulse_job_token_secret: str = ""  # HMAC secret for job-scoped JWTs
+    pulse_s3_bucket: str = ""  # S3 bucket for inter-stage data
+    pulse_chain_grace_period_seconds: int = (
+        300  # seconds before auto-triggering next stage
+    )
+
+    # Pipeline infrastructure — populated from Terraform outputs
+    pipeline_task_definition: str = "pulse-pipeline-step"  # ECS task def family
+    pipeline_cluster_name: str = ""  # ECS cluster for pipeline steps
+    pipeline_execution_role_arn: str = ""  # ECS task execution role
+    pipeline_task_role_arn: str = ""  # ECS task role (S3, Lambda, ECS perms)
+    pipeline_log_group: str = ""  # CloudWatch log group for ECS steps
+    pipeline_subnets: str = ""  # comma-separated private subnet IDs
+    pipeline_security_groups: str = ""  # comma-separated security group IDs
+    lambda_execution_role_arn: str = ""  # Lambda execution role
+    lambda_subnets: str = ""  # comma-separated subnet IDs for Lambda VPC
+    lambda_security_groups: str = ""  # comma-separated SG IDs for Lambda
+    lambda_log_group: str = ""  # CloudWatch log group for Lambda steps
+
+    # Local dev auth bypass — set this to use MockTokenVerifier instead of Cognito
+    mock_jwt_secret: str = ""
+
+    mcp_transport: str = "sse"
+    mcp_sse_host: str = "127.0.0.1"
+    mcp_sse_port: int = 8001
+
+    @property
+    def effective_celery_broker_url(self) -> str:
+        return self.celery_broker_url or self.redis_url
+
+    @property
+    def effective_celery_result_backend(self) -> str:
+        return self.celery_result_backend or self.redis_url
+
+    @property
+    def cognito_jwks_url(self) -> str:
+        return (
+            f"https://cognito-idp.{self.aws_region}.amazonaws.com/"
+            f"{self.cognito_user_pool_id}/.well-known/jwks.json"
+        )
+
+    @property
+    def cognito_issuer(self) -> str:
+        return (
+            f"https://cognito-idp.{self.aws_region}.amazonaws.com/"
+            f"{self.cognito_user_pool_id}"
+        )
+
+    @property
+    def pipeline_subnet_list(self) -> list[str]:
+        return [s.strip() for s in self.pipeline_subnets.split(",") if s.strip()]
+
+    @property
+    def pipeline_sg_list(self) -> list[str]:
+        return [
+            s.strip() for s in self.pipeline_security_groups.split(",") if s.strip()
+        ]
+
+    @property
+    def lambda_subnet_list(self) -> list[str]:
+        return [s.strip() for s in self.lambda_subnets.split(",") if s.strip()]
+
+    @property
+    def lambda_sg_list(self) -> list[str]:
+        return [s.strip() for s in self.lambda_security_groups.split(",") if s.strip()]
+
+    model_config = {"env_file": ".env", "extra": "ignore"}
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

pulse_engine/core/error_handlers.py

@@ -0,0 +1,64 @@
+import structlog
+from fastapi import FastAPI, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+
+from pulse_engine.core.exceptions import AppError
+
+logger = structlog.get_logger()
+
+
+def _get_request_id(request: Request) -> str:
+    return getattr(request.state, "request_id", "unknown")
+
+
+async def app_error_handler(request: Request, exc: AppError) -> JSONResponse:
+    request_id = _get_request_id(request)
+    logger.warning(
+        "app_error",
+        error=exc.message,
+        status_code=exc.status_code,
+        request_id=request_id,
+        **exc.context,
+    )
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "success": False,
+            "error": exc.message,
+            "request_id": request_id,
+        },
+    )
+
+
+async def validation_error_handler(
+    request: Request, exc: RequestValidationError
+) -> JSONResponse:
+    request_id = _get_request_id(request)
+    return JSONResponse(
+        status_code=422,
+        content={
+            "success": False,
+            "error": "Validation error",
+            "request_id": request_id,
+        },
+    )
+
+
+async def catch_all_handler(request: Request, exc: Exception) -> JSONResponse:
+    request_id = _get_request_id(request)
+    logger.exception("unhandled_exception", request_id=request_id)
+    return JSONResponse(
+        status_code=500,
+        content={
+            "success": False,
+            "error": "Internal server error",
+            "request_id": request_id,
+        },
+    )
+
+
+def register_exception_handlers(app: FastAPI) -> None:
+    app.add_exception_handler(AppError, app_error_handler)  # type: ignore[arg-type]
+    app.add_exception_handler(RequestValidationError, validation_error_handler)  # type: ignore[arg-type]
+    app.add_exception_handler(Exception, catch_all_handler)

pulse_engine/core/exceptions.py

@@ -0,0 +1,67 @@
+from typing import Any
+
+
+class AppError(Exception):
+    status_code: int = 500
+    is_operational: bool = True
+
+    def __init__(self, message: str = "Internal server error", **context: Any) -> None:
+        self.message = message
+        self.context = context
+        super().__init__(message)
+
+
+class BadRequestError(AppError):
+    status_code = 400
+
+    def __init__(self, message: str = "Bad request", **context: Any) -> None:
+        super().__init__(message, **context)
+
+
+class UnauthorizedError(AppError):
+    status_code = 401
+
+    def __init__(self, message: str = "Unauthorized", **context: Any) -> None:
+        super().__init__(message, **context)
+
+
+class ForbiddenError(AppError):
+    status_code = 403
+
+    def __init__(self, message: str = "Forbidden", **context: Any) -> None:
+        super().__init__(message, **context)
+
+
+class NotFoundError(AppError):
+    status_code = 404
+
+    def __init__(self, message: str = "Not found", **context: Any) -> None:
+        super().__init__(message, **context)
+
+
+class ConflictError(AppError):
+    status_code = 409
+
+    def __init__(self, message: str = "Conflict", **context: Any) -> None:
+        super().__init__(message, **context)
+
+
+class UnprocessableEntityError(AppError):
+    status_code = 422
+
+    def __init__(self, message: str = "Unprocessable entity", **context: Any) -> None:
+        super().__init__(message, **context)
+
+
+class TooManyRequestsError(AppError):
+    status_code = 429
+
+    def __init__(self, message: str = "Too many requests", **context: Any) -> None:
+        super().__init__(message, **context)
+
+
+class ServiceUnavailableError(AppError):
+    status_code = 503
+
+    def __init__(self, message: str = "Service unavailable", **context: Any) -> None:
+        super().__init__(message, **context)