pulse-engine 0.2.0__py3-none-any.whl

pulse_engine/mcp/tools_pipelines.py

@@ -0,0 +1,215 @@
+"""MCP tools for Pipeline operations."""
+
+from typing import Any
+
+from pulse_engine.core.exceptions import NotFoundError, ServiceUnavailableError
+from pulse_engine.mcp.server import get_container, handle_mcp_errors, mcp
+from pulse_engine.pipeline.config_parser import (
+    PipelineConfigError,
+    parse_pipeline_config,
+)
+from pulse_engine.pipeline.repositories import (
+    ModuleRegistryRepository,
+    PipelineRunRepository,
+)
+from pulse_engine.pipeline.schemas import PipelineRunStatus, PipelineStatusResponse
+from pulse_engine.pipeline.service import PipelineService
+
+
+def _build_pipeline_service(
+    session: Any,
+    container: Any,
+) -> PipelineService:
+    return PipelineService(
+        module_repo=ModuleRegistryRepository(session),
+        run_repo=PipelineRunRepository(session),
+        translators=container.pipeline_translators,
+        status_providers=container.pipeline_status_providers,
+        settings=container.settings,
+        token_issuer=container.token_issuer,
+    )
+
+
+@mcp.tool()
+@handle_mcp_errors
+async def pipelines_trigger(
+    tenant_id: str,
+    product: str,
+    orchestrator: str,
+    pipeline_yaml: str,
+    global_config: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Trigger a pipeline run.
+
+    Args:
+        tenant_id: Tenant identifier.
+        product: Product name (must have modules registered).
+        orchestrator: Orchestrator backend ("prefect").
+        pipeline_yaml: Pipeline definition as a YAML string.
+        global_config: Config passed to all steps (overrides yaml config block).
+
+    Returns:
+        {"run_id": "<uuid>"} on success.
+    """
+    container = get_container()
+    if container.db_session_factory is None:
+        raise ServiceUnavailableError("Database not configured")
+
+    try:
+        config = parse_pipeline_config(pipeline_yaml)
+    except PipelineConfigError as e:
+        return {"error": True, "error_type": "PipelineConfigError", "errors": e.errors}
+
+    async with container.db_session_factory() as session:
+        service = _build_pipeline_service(session, container)
+        run_id = await service.trigger(
+            tenant_id=tenant_id,
+            product=product,
+            orchestrator=orchestrator,
+            config=config,
+            global_config=global_config or config.config,
+        )
+        return {"run_id": run_id}
+
+
+@mcp.tool()
+@handle_mcp_errors
+async def pipelines_status(
+    tenant_id: str,
+    run_id: str,
+) -> dict[str, Any]:
+    """Get the status and step details of a pipeline run.
+
+    Args:
+        tenant_id: Tenant identifier.
+        run_id: Pipeline run ID (from pipelines_trigger).
+
+    Returns:
+        Pipeline status with run_id, product, orchestrator, status, and steps.
+    """
+    container = get_container()
+    if container.db_session_factory is None:
+        raise ServiceUnavailableError("Database not configured")
+
+    async with container.db_session_factory() as session:
+        service = _build_pipeline_service(session, container)
+        run = await service.get_run(tenant_id, run_id)
+        if run is None:
+            raise NotFoundError("Pipeline run not found", run_id=run_id)
+
+        pipeline_status = await service.get_status(tenant_id, run_id)
+
+        resp = PipelineStatusResponse(
+            run_id=run.id,
+            product=run.product,
+            orchestrator=run.orchestrator,
+            status=PipelineRunStatus(
+                pipeline_status.status if pipeline_status else run.status
+            ),
+            started_at=pipeline_status.started_at if pipeline_status else None,  # type: ignore[arg-type]
+            steps=pipeline_status.steps if pipeline_status else [],
+        )
+        return resp.model_dump(mode="json")
+
+
+@mcp.tool()
+@handle_mcp_errors
+async def pipelines_list(
+    tenant_id: str,
+    product: str | None = None,
+    status: str | None = None,
+    limit: int = 20,
+    offset: int = 0,
+) -> dict[str, Any]:
+    """List pipeline runs for a tenant.
+
+    Args:
+        tenant_id: Tenant identifier.
+        product: Filter by product name.
+        status: Filter by status (pending, running, completed, failed).
+        limit: Max results to return.
+        offset: Pagination offset.
+
+    Returns:
+        {"items": [...], "total": N, "limit": N, "offset": N}
+    """
+    container = get_container()
+    if container.db_session_factory is None:
+        raise ServiceUnavailableError("Database not configured")
+
+    async with container.db_session_factory() as session:
+        service = _build_pipeline_service(session, container)
+        runs, total = await service.list_runs(
+            tenant_id, product=product, status=status, limit=limit, offset=offset
+        )
+        items = [
+            {
+                "run_id": r.id,
+                "product": r.product,
+                "orchestrator": r.orchestrator,
+                "status": r.status,
+                "created_at": r.created_at.isoformat() if r.created_at else None,
+            }
+            for r in runs
+        ]
+        return {"items": items, "total": total, "limit": limit, "offset": offset}
+
+
+@mcp.tool()
+@handle_mcp_errors
+async def pipelines_cancel(
+    tenant_id: str,
+    run_id: str,
+) -> dict[str, Any]:
+    """Cancel a running pipeline.
+
+    Args:
+        tenant_id: Tenant identifier.
+        run_id: Pipeline run ID to cancel.
+
+    Returns:
+        {"run_id": "...", "status": "cancelled"} on success.
+    """
+    container = get_container()
+    if container.db_session_factory is None:
+        raise ServiceUnavailableError("Database not configured")
+
+    async with container.db_session_factory() as session:
+        service = _build_pipeline_service(session, container)
+        result = await service.cancel(tenant_id, run_id)
+        if not result:
+            raise NotFoundError(
+                "Pipeline run not found or cannot be cancelled", run_id=run_id
+            )
+        return {"run_id": run_id, "status": "cancelled"}
+
+
+@mcp.tool()
+@handle_mcp_errors
+async def pipelines_steps(
+    tenant_id: str,
+    run_id: str,
+) -> dict[str, Any]:
+    """Get detailed step execution info for a pipeline run.
+
+    Args:
+        tenant_id: Tenant identifier.
+        run_id: Pipeline run ID.
+
+    Returns:
+        {"run_id": "...", "steps": [{step, module, status, started_at}, ...]}
+    """
+    container = get_container()
+    if container.db_session_factory is None:
+        raise ServiceUnavailableError("Database not configured")
+
+    async with container.db_session_factory() as session:
+        service = _build_pipeline_service(session, container)
+        pipeline_status = await service.get_status(tenant_id, run_id)
+        if pipeline_status is None:
+            raise NotFoundError("Pipeline run not found", run_id=run_id)
+
+        return {
+            "run_id": run_id,
+            "steps": [s.model_dump(mode="json") for s in pipeline_status.steps],
+        }

pulse_engine/mcp/tools_processor.py

@@ -0,0 +1,208 @@
+"""MCP tools for Processor operations."""
+
+import time
+from typing import Any
+
+from pulse_engine.mcp.server import get_container, handle_mcp_errors, mcp
+from pulse_engine.processor.core.analysis import (
+    classify_sentiment,
+    extract_entities,
+    extract_topics,
+    summarize,
+)
+from pulse_engine.processor.core.chunking import chunk_content
+from pulse_engine.processor.postprocessor.tasks import run_postprocessing
+from pulse_engine.processor.preprocessor.tasks import (
+    clean_html,
+    detect_language,
+    normalize_text,
+    validate_content,
+)
+from pulse_engine.processor.schemas import (
+    ContentChunk,
+    ProcessingContext,
+    ProcessingOptions,
+)
+
+
+@mcp.tool()
+@handle_mcp_errors
+async def process_pipeline(
+    tenant_id: str,
+    content: str,
+    source_id: str,
+    source_type: str = "text",
+    product: str = "",
+    job_id: str | None = None,
+    options: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Run the full processing pipeline (preprocess → analyze → postprocess)."""
+    from pulse_engine.processor.pipeline import ProcessingPipeline
+
+    container = get_container()
+    pipeline = ProcessingPipeline(kb_service=container.kb_service)
+
+    opts = ProcessingOptions(**(options or {}))
+    ctx = ProcessingContext(
+        source_id=source_id,
+        source_type=source_type,
+        product=product,
+        tenant_id=tenant_id,
+        job_id=job_id,
+        raw_content=content,
+        options=opts,
+    )
+
+    start = time.monotonic()
+    ctx = await pipeline.run(ctx)
+    elapsed_ms = int((time.monotonic() - start) * 1000)
+
+    doc_count = len(ctx.documents)
+    chunk_count = len(ctx.chunks)
+    total_chunks = chunk_count + (
+        doc_count - chunk_count if doc_count != chunk_count else 0
+    )
+    stored = len(ctx.documents) if ctx.options.store_results else 0
+    deduped = max(0, total_chunks - len(ctx.chunks))
+    languages = [ctx.language] if ctx.language and ctx.language != "unknown" else []
+
+    return {
+        "source_id": ctx.source_id,
+        "chunks_produced": len(ctx.chunks),
+        "chunks_stored": stored,
+        "chunks_deduplicated": deduped,
+        "processing_time_ms": elapsed_ms,
+        "summary": ctx.summary,
+        "languages_detected": languages,
+        "entities_found": len(ctx.entities),
+        "stages_completed": ctx.stages_completed,
+    }
+
+
+@mcp.tool()
+@handle_mcp_errors
+async def process_preprocess(
+    tenant_id: str,
+    content: str,
+    source_type: str = "text",
+) -> dict[str, Any]:
+    """Clean, normalize, and validate content."""
+    cleaned = clean_html(content)
+    cleaned = normalize_text(cleaned)
+    lang = detect_language(cleaned) if validate_content(cleaned) else None
+    is_valid = validate_content(cleaned)
+
+    return {
+        "cleaned_content": cleaned,
+        "language": lang,
+        "is_valid": is_valid,
+    }
+
+
+@mcp.tool()
+@handle_mcp_errors
+async def process_analyze(
+    tenant_id: str,
+    content: str,
+    options: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Chunk content and run NER, sentiment, topic extraction, and summarization."""
+    opts = ProcessingOptions(**(options or {}))
+
+    chunks = chunk_content(
+        text=content,
+        source_id="analyze-temp",
+        strategy=opts.chunk_strategy,
+        chunk_size=opts.chunk_size,
+        overlap=opts.chunk_overlap,
+    )
+
+    for c in chunks:
+        if opts.enable_ner:
+            c.entities = extract_entities(c.content)
+        if opts.enable_sentiment:
+            c.sentiment = classify_sentiment(c.content)
+        if opts.enable_topics:
+            c.topics = extract_topics(c.content)
+
+    entities = extract_entities(content) if opts.enable_ner else []
+    sentiment = classify_sentiment(content) if opts.enable_sentiment else None
+    topics = extract_topics(content) if opts.enable_topics else []
+    summary_text = summarize(content)
+
+    return {
+        "chunks": [c.model_dump(mode="json") for c in chunks],
+        "entities": [e.model_dump(mode="json") for e in entities],
+        "sentiment": sentiment,
+        "topics": topics,
+        "summary": summary_text,
+    }
+
+
+@mcp.tool()
+@handle_mcp_errors
+async def process_postprocess(
+    tenant_id: str,
+    content: str,
+    chunks: list[dict[str, Any]],
+    source_id: str,
+    source_type: str = "text",
+    product: str = "",
+    options: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Run postprocessing: embeddings, quality scoring, dedup, document formatting."""
+    opts = ProcessingOptions(**(options or {}))
+    chunk_objs = [ContentChunk(**c) for c in chunks]
+
+    ctx = ProcessingContext(
+        source_id=source_id,
+        source_type=source_type,
+        product=product,
+        tenant_id=tenant_id,
+        raw_content=content,
+        cleaned_content=content,
+        chunks=chunk_objs,
+        options=opts,
+    )
+
+    ctx = run_postprocessing(ctx)
+
+    return {
+        "documents_created": len(ctx.documents),
+        "embeddings_generated": len(ctx.embeddings),
+        "quality_scores": ctx.quality_scores,
+    }
+
+
+@mcp.tool()
+@handle_mcp_errors
+async def process_chunk(
+    tenant_id: str,
+    content: str,
+    strategy: str = "token_count",
+    chunk_size: int = 512,
+    overlap: int = 50,
+) -> dict[str, Any]:
+    """Split content into chunks using the specified strategy."""
+    chunks = chunk_content(
+        text=content,
+        source_id="chunk-temp",
+        strategy=strategy,
+        chunk_size=chunk_size,
+        overlap=overlap,
+    )
+
+    items = [
+        {
+            "index": c.chunk_index,
+            "content": c.content,
+            "token_count": c.token_count,
+        }
+        for c in chunks
+    ]
+
+    return {
+        "chunks": items,
+        "total_chunks": len(items),
+        "strategy": strategy,
+    }

pulse_engine/middleware/rate_limit.py

@@ -0,0 +1,144 @@
+"""In-memory sliding-window rate limiter.
+
+Provides per-key rate limiting without external dependencies.  Suitable for
+single-instance deployments; for multi-instance, swap the storage backend
+to Redis.
+"""
+
+from __future__ import annotations
+
+import time
+from collections import defaultdict
+from threading import Lock
+from typing import Any
+
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response
+
+
+class _SlidingWindowCounter:
+    """Thread-safe sliding-window counter."""
+
+    def __init__(self) -> None:
+        self._windows: dict[str, list[float]] = defaultdict(list)
+        self._lock = Lock()
+
+    def hit(self, key: str, max_requests: int, window_seconds: int) -> bool:
+        """Record a hit and return True if under the limit, False if exceeded."""
+        now = time.monotonic()
+        cutoff = now - window_seconds
+
+        with self._lock:
+            timestamps = self._windows[key]
+            # Prune expired entries
+            self._windows[key] = timestamps = [t for t in timestamps if t > cutoff]
+            if len(timestamps) >= max_requests:
+                return False
+            timestamps.append(now)
+            return True
+
+    def remaining(self, key: str, max_requests: int, window_seconds: int) -> int:
+        """Return remaining requests in the current window."""
+        now = time.monotonic()
+        cutoff = now - window_seconds
+
+        with self._lock:
+            timestamps = self._windows[key]
+            active = [t for t in timestamps if t > cutoff]
+            return max(0, max_requests - len(active))
+
+    def reset(self) -> None:
+        """Clear all counters — intended for use in tests only."""
+        with self._lock:
+            self._windows.clear()
+
+
+# Global singleton — shared across middleware and endpoint decorators
+_counter = _SlidingWindowCounter()
+
+
+def _client_ip(request: Request) -> str:
+    """Extract the client IP, respecting X-Forwarded-For behind a proxy."""
+    forwarded = request.headers.get("x-forwarded-for")
+    if forwarded:
+        return forwarded.split(",")[0].strip()
+    client = request.client
+    return client.host if client else "unknown"
+
+
+class RateLimitMiddleware(BaseHTTPMiddleware):
+    """Global per-IP rate limiting middleware.
+
+    Args:
+        app: ASGI application.
+        max_requests: Maximum requests per window (default 100).
+        window_seconds: Window duration in seconds (default 60).
+        counter: Optional counter instance; defaults to the shared module-level
+            singleton.  Pass a fresh ``_SlidingWindowCounter()`` in tests to
+            keep state isolated between test cases.
+    """
+
+    def __init__(
+        self,
+        app: Any,
+        max_requests: int = 100,
+        window_seconds: int = 60,
+        counter: _SlidingWindowCounter | None = None,
+    ) -> None:
+        super().__init__(app)
+        self._max = max_requests
+        self._window = window_seconds
+        self._counter = counter if counter is not None else _counter
+
+    async def dispatch(
+        self, request: Request, call_next: RequestResponseEndpoint
+    ) -> Response:
+        key = f"global:{_client_ip(request)}"
+        allowed = self._counter.hit(key, self._max, self._window)
+        remaining = self._counter.remaining(key, self._max, self._window)
+
+        if not allowed:
+            return JSONResponse(
+                status_code=429,
+                content={
+                    "success": False,
+                    "error": "Rate limit exceeded. Try again later.",
+                },
+                headers={
+                    "Retry-After": str(self._window),
+                    "X-RateLimit-Limit": str(self._max),
+                    "X-RateLimit-Remaining": "0",
+                },
+            )
+
+        response = await call_next(request)
+        response.headers["X-RateLimit-Limit"] = str(self._max)
+        response.headers["X-RateLimit-Remaining"] = str(remaining)
+        return response
+
+
+def check_rate_limit(
+    request: Request,
+    *,
+    scope: str,
+    max_requests: int,
+    window_seconds: int,
+) -> None:
+    """Check rate limit for a specific scope.  Raises 429 via JSONResponse.
+
+    Usage in endpoints::
+
+        @router.post("/login")
+        async def login(request: Request, body: LoginRequest):
+            check_rate_limit(request, scope="login", max_requests=5, window_seconds=60)
+            ...
+    """
+    key = f"{scope}:{_client_ip(request)}"
+    allowed = _counter.hit(key, max_requests, window_seconds)
+    if not allowed:
+        from pulse_engine.core.exceptions import TooManyRequestsError
+
+        raise TooManyRequestsError(
+            f"Too many requests. Try again in {window_seconds} seconds."
+        )

pulse_engine/middleware/request_id.py

@@ -0,0 +1,16 @@
+import uuid
+
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.requests import Request
+from starlette.responses import Response
+
+
+class RequestIDMiddleware(BaseHTTPMiddleware):
+    async def dispatch(
+        self, request: Request, call_next: RequestResponseEndpoint
+    ) -> Response:
+        request_id = request.headers.get("x-request-id") or str(uuid.uuid4())
+        request.state.request_id = request_id
+        response = await call_next(request)
+        response.headers["x-request-id"] = request_id
+        return response

pulse_engine/middleware/security_headers.py

@@ -0,0 +1,25 @@
+"""Middleware that adds defensive HTTP security headers to every response."""
+
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.requests import Request
+from starlette.responses import Response
+
+SECURITY_HEADERS = {
+    "X-Content-Type-Options": "nosniff",
+    "X-Frame-Options": "DENY",
+    "X-XSS-Protection": "1; mode=block",
+    "Strict-Transport-Security": "max-age=31536000; includeSubDomains",
+    "Referrer-Policy": "strict-origin-when-cross-origin",
+    "Permissions-Policy": "geolocation=(), microphone=(), camera=()",
+    "Content-Security-Policy": "default-src 'self'",
+}
+
+
+class SecurityHeadersMiddleware(BaseHTTPMiddleware):
+    async def dispatch(
+        self, request: Request, call_next: RequestResponseEndpoint
+    ) -> Response:
+        response = await call_next(request)
+        for header, value in SECURITY_HEADERS.items():
+            response.headers[header] = value
+        return response

pulse_engine/middleware/tenant.py

@@ -0,0 +1,90 @@
+import structlog
+from fastapi import Request
+from jose import jwt as jose_jwt
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.responses import JSONResponse, Response
+from starlette.types import ASGIApp
+
+from pulse_engine.core.exceptions import UnauthorizedError
+from pulse_engine.core.job_token import JobClaims
+from pulse_engine.core.security import (
+    CognitoClaims,
+    TokenVerifier,
+    extract_bearer_token,
+)
+
+logger = structlog.get_logger()
+
+PUBLIC_PATHS = {
+    "/api/v1/health",
+    "/api/v1/auth/login",
+    # OpenAPI/Swagger docs — only reachable in non-production environments
+    # because create_app() sets docs_url/redoc_url/openapi_url to None in production.
+    "/docs",
+    "/docs/oauth2-redirect",
+    "/openapi.json",
+    "/redoc",
+}
+
+
+class TenantMiddleware(BaseHTTPMiddleware):
+    def __init__(
+        self,
+        app: ASGIApp,
+        verifier: TokenVerifier,
+        job_token_secret: str = "",
+    ) -> None:
+        super().__init__(app)
+        self._verifier = verifier
+        self._job_token_secret = job_token_secret
+
+    async def dispatch(
+        self, request: Request, call_next: RequestResponseEndpoint
+    ) -> Response:
+        if request.url.path in PUBLIC_PATHS:
+            return await call_next(request)
+
+        try:
+            token = extract_bearer_token(request.headers.get("authorization"))
+            claims = await self._resolve_token(token)
+            request.state.tenant_id = claims.tenant_id
+            request.state.user_claims = claims
+        except UnauthorizedError as e:
+            request_id = getattr(request.state, "request_id", "unknown")
+            return JSONResponse(
+                status_code=401,
+                content={
+                    "success": False,
+                    "error": e.message,
+                    "request_id": request_id,
+                },
+            )
+
+        return await call_next(request)
+
+    async def _resolve_token(self, token: str) -> CognitoClaims | JobClaims:
+        """Dispatch to job-token or Cognito verifier based on sub prefix."""
+        if self._job_token_secret:
+            try:
+                unverified = jose_jwt.get_unverified_claims(token)
+                sub = unverified.get("sub", "")
+                if sub.startswith("job:"):
+                    from pulse_engine.core.job_token import JobTokenVerifier
+
+                    verifier = JobTokenVerifier(secret=self._job_token_secret)
+                    return await verifier.verify(token)
+            except UnauthorizedError:
+                raise
+            except Exception as exc:
+                logger.debug(
+                    "job_token_parse_failed_falling_back_to_cognito",
+                    error=str(exc),
+                )
+        return await self._verifier.verify(token)
+
+
+def get_tenant_id(request: Request) -> str:
+    tenant_id: str | None = getattr(request.state, "tenant_id", None)
+    if not tenant_id:
+        raise UnauthorizedError("Tenant context not available")
+    return tenant_id