pulse-engine 0.2.0__py3-none-any.whl

pulse_engine/processor/router.py

@@ -0,0 +1,192 @@
+import time
+
+from fastapi import APIRouter, Depends
+
+from pulse_engine.dependencies import get_processing_pipeline
+from pulse_engine.middleware.tenant import get_tenant_id
+from pulse_engine.processor.core.analysis import (
+    classify_sentiment,
+    extract_entities,
+    extract_topics,
+    summarize,
+)
+from pulse_engine.processor.core.chunking import chunk_content
+from pulse_engine.processor.pipeline import ProcessingPipeline
+from pulse_engine.processor.postprocessor.tasks import (
+    run_postprocessing,
+)
+from pulse_engine.processor.preprocessor.tasks import (
+    clean_html,
+    detect_language,
+    normalize_text,
+    validate_content,
+)
+from pulse_engine.processor.schemas import (
+    AnalyzeRequest,
+    AnalyzeResponse,
+    ChunkItem,
+    ChunkRequest,
+    ChunkResponse,
+    PipelineRequest,
+    PipelineResponse,
+    PostprocessRequest,
+    PostprocessResponse,
+    PreprocessRequest,
+    PreprocessResponse,
+    ProcessingContext,
+)
+
+router = APIRouter(prefix="/process", tags=["Processor"])
+
+
+@router.post("/pipeline", response_model=PipelineResponse)
+async def run_pipeline(
+    body: PipelineRequest,
+    tenant_id: str = Depends(get_tenant_id),
+    pipeline: ProcessingPipeline = Depends(get_processing_pipeline),
+) -> PipelineResponse:
+    start = time.monotonic()
+
+    ctx = ProcessingContext(
+        source_id=body.source_id,
+        source_type=body.source_type,
+        product=body.product,
+        tenant_id=tenant_id,
+        job_id=body.job_id,
+        raw_content=body.content,
+        options=body.options,
+    )
+
+    ctx = await pipeline.run(ctx)
+
+    elapsed_ms = int((time.monotonic() - start) * 1000)
+
+    doc_count = len(ctx.documents)
+    chunk_count = len(ctx.chunks)
+    total_chunks = chunk_count + (
+        doc_count - chunk_count if doc_count != chunk_count else 0
+    )
+    stored = len(ctx.documents) if ctx.options.store_results else 0
+    deduped = max(0, total_chunks - len(ctx.chunks))
+
+    languages = [ctx.language] if ctx.language and ctx.language != "unknown" else []
+
+    return PipelineResponse(
+        source_id=ctx.source_id,
+        chunks_produced=len(ctx.chunks),
+        chunks_stored=stored,
+        chunks_deduplicated=deduped,
+        processing_time_ms=elapsed_ms,
+        summary=ctx.summary,
+        languages_detected=languages,
+        entities_found=len(ctx.entities),
+        stages_completed=ctx.stages_completed,
+    )
+
+
+@router.post("/preprocess", response_model=PreprocessResponse)
+async def preprocess(
+    body: PreprocessRequest,
+    tenant_id: str = Depends(get_tenant_id),
+) -> PreprocessResponse:
+    cleaned = clean_html(body.content)
+    cleaned = normalize_text(cleaned)
+    lang = detect_language(cleaned) if validate_content(cleaned) else None
+    is_valid = validate_content(cleaned)
+
+    return PreprocessResponse(
+        cleaned_content=cleaned,
+        language=lang,
+        is_valid=is_valid,
+    )
+
+
+@router.post("/analyze", response_model=AnalyzeResponse)
+async def analyze(
+    body: AnalyzeRequest,
+    tenant_id: str = Depends(get_tenant_id),
+) -> AnalyzeResponse:
+    chunks = chunk_content(
+        text=body.content,
+        source_id="analyze-temp",
+        strategy=body.options.chunk_strategy,
+        chunk_size=body.options.chunk_size,
+        overlap=body.options.chunk_overlap,
+    )
+
+    for c in chunks:
+        if body.options.enable_ner:
+            c.entities = extract_entities(c.content)
+        if body.options.enable_sentiment:
+            c.sentiment = classify_sentiment(c.content)
+        if body.options.enable_topics:
+            c.topics = extract_topics(c.content)
+
+    entities = extract_entities(body.content) if body.options.enable_ner else []
+    sentiment = (
+        classify_sentiment(body.content) if body.options.enable_sentiment else None
+    )
+    topics = extract_topics(body.content) if body.options.enable_topics else []
+    summary_text = summarize(body.content)
+
+    return AnalyzeResponse(
+        chunks=chunks,
+        entities=entities,
+        sentiment=sentiment,
+        topics=topics,
+        summary=summary_text,
+    )
+
+
+@router.post("/postprocess", response_model=PostprocessResponse)
+async def postprocess(
+    body: PostprocessRequest,
+    tenant_id: str = Depends(get_tenant_id),
+) -> PostprocessResponse:
+    ctx = ProcessingContext(
+        source_id=body.source_id,
+        source_type=body.source_type,
+        product=body.product,
+        tenant_id=body.tenant_id or tenant_id,
+        raw_content=body.content,
+        cleaned_content=body.content,
+        chunks=body.chunks,
+        options=body.options,
+    )
+
+    ctx = run_postprocessing(ctx)
+
+    return PostprocessResponse(
+        documents_created=len(ctx.documents),
+        embeddings_generated=len(ctx.embeddings),
+        quality_scores=ctx.quality_scores,
+    )
+
+
+@router.post("/chunk", response_model=ChunkResponse)
+async def chunk(
+    body: ChunkRequest,
+    tenant_id: str = Depends(get_tenant_id),
+) -> ChunkResponse:
+    chunks = chunk_content(
+        text=body.content,
+        source_id="chunk-temp",
+        strategy=body.strategy,
+        chunk_size=body.chunk_size,
+        overlap=body.overlap,
+    )
+
+    items = [
+        ChunkItem(
+            index=c.chunk_index,
+            content=c.content,
+            token_count=c.token_count,
+        )
+        for c in chunks
+    ]
+
+    return ChunkResponse(
+        chunks=items,
+        total_chunks=len(items),
+        strategy=body.strategy,
+    )

pulse_engine/processor/schemas.py

@@ -0,0 +1,167 @@
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class ProcessingOptions(BaseModel):
+    chunk_strategy: str = "token_count"
+    chunk_size: int = 512
+    chunk_overlap: int = 50
+    enable_ner: bool = True
+    enable_sentiment: bool = True
+    enable_topics: bool = True
+    enable_embedding: bool = True
+    enable_dedup: bool = True
+    store_results: bool = True
+    embedding_model: str = "text-embedding-3-small"
+    embedding_provider: str | None = None  # None = use config default
+
+
+class Entity(BaseModel):
+    name: str
+    type: str  # PERSON, ORGANIZATION, LOCATION, DATE, etc.
+    start: int = 0
+    end: int = 0
+
+
+class ContentChunk(BaseModel):
+    chunk_index: int
+    content: str
+    token_count: int
+    parent_source_id: str
+    entities: list[Entity] = Field(default_factory=list)
+    sentiment: str | None = None
+    topics: list[str] = Field(default_factory=list)
+
+
+class ProcessingError(BaseModel):
+    stage: str
+    task: str
+    message: str
+
+
+class ProcessingContext(BaseModel):
+    source_id: str
+    source_type: str
+    product: str
+    tenant_id: str
+    job_id: str | None = None
+    raw_content: str
+    cleaned_content: str | None = None
+    language: str | None = None
+    chunks: list[ContentChunk] = Field(default_factory=list)
+    entities: list[Entity] = Field(default_factory=list)
+    sentiment: str | None = None
+    topics: list[str] = Field(default_factory=list)
+    summary: str | None = None
+    embeddings: list[list[float]] = Field(default_factory=list)
+    quality_scores: list[float] = Field(default_factory=list)
+    documents: list[Any] = Field(default_factory=list)  # Final Document objects
+    options: ProcessingOptions = Field(default_factory=ProcessingOptions)
+    stages_completed: list[str] = Field(default_factory=list)
+    errors: list[ProcessingError] = Field(default_factory=list)
+
+
+# API Request/Response schemas
+
+
+class PipelineRequest(BaseModel):
+    content: str
+    source_id: str
+    source_type: str = "text"
+    product: str = ""
+    job_id: str | None = None
+    options: ProcessingOptions = Field(default_factory=ProcessingOptions)
+
+
+class PipelineResponse(BaseModel):
+    source_id: str
+    chunks_produced: int
+    chunks_stored: int
+    chunks_deduplicated: int
+    processing_time_ms: int
+    summary: str | None = None
+    languages_detected: list[str] = Field(default_factory=list)
+    entities_found: int
+    stages_completed: list[str] = Field(default_factory=list)
+
+
+class ChunkRequest(BaseModel):
+    content: str
+    strategy: str = "token_count"
+    chunk_size: int = 512
+    overlap: int = 50
+
+
+class ChunkItem(BaseModel):
+    index: int
+    content: str
+    token_count: int
+
+
+class ChunkResponse(BaseModel):
+    chunks: list[ChunkItem]
+    total_chunks: int
+    strategy: str
+
+
+class PreprocessRequest(BaseModel):
+    content: str
+    source_type: str = "text"
+
+
+class PreprocessResponse(BaseModel):
+    cleaned_content: str
+    language: str | None = None
+    is_valid: bool
+
+
+class AnalyzeRequest(BaseModel):
+    content: str
+    options: ProcessingOptions = Field(default_factory=ProcessingOptions)
+
+
+class AnalyzeResponse(BaseModel):
+    chunks: list[ContentChunk]
+    entities: list[Entity]
+    sentiment: str | None = None
+    topics: list[str]
+    summary: str | None = None
+
+
+class PostprocessRequest(BaseModel):
+    content: str
+    chunks: list[ContentChunk]
+    source_id: str
+    source_type: str = "text"
+    product: str = ""
+    tenant_id: str = ""
+    options: ProcessingOptions = Field(default_factory=ProcessingOptions)
+
+
+class PostprocessResponse(BaseModel):
+    documents_created: int
+    embeddings_generated: int
+    quality_scores: list[float]
+
+
+class TopicShift(BaseModel):
+    index: int
+    reason: str  # "Topic A to Topic B"
+
+
+class TopicSplitResult(BaseModel):
+    topic_shifts: list[TopicShift] = Field(default_factory=list)
+
+
+class OCRInput(BaseModel):
+    """Input for an OCR extraction request."""
+
+    file_path: str | None = None
+    file_bytes: bytes | None = None
+    mime_type: str = "application/pdf"
+    prompt: str = ""
+    temperature: float = 0.1
+    model: str = "gemini-2.5-flash"
+    api_key: str = ""
+    max_output_tokens: int = 65536

pulse_engine/registry.py

@@ -0,0 +1,117 @@
+"""Product manifest declaration, validation, and discovery."""
+
+from __future__ import annotations
+
+import importlib.metadata
+from dataclasses import dataclass, field
+from types import EllipsisType
+from typing import Any
+
+from pulse_engine.extractor.base import BaseExtractor
+from pulse_engine.processor.base import (
+    BaseCoreProcessor,
+    BasePostprocessor,
+    BasePreprocessor,
+)
+
+
+@dataclass
+class ProductManifest:
+    """Declarative manifest that a product uses to register with the engine.
+
+    For processor stages, the convention is:
+      - ``...`` (Ellipsis) → use the engine's default implementation
+      - ``None`` → skip the stage entirely
+      - an instance → use the provided custom implementation
+    """
+
+    name: str
+    version: str
+    extractors: list[type[BaseExtractor]] = field(default_factory=list)
+    preprocessor: BasePreprocessor | None | EllipsisType = ...
+    core_processor: BaseCoreProcessor | None | EllipsisType = ...
+    postprocessor: BasePostprocessor | None | EllipsisType = ...
+    settings_class: type[Any] | None = None
+    routers: list[Any] = field(default_factory=list)
+    mcp_tool_modules: list[str] = field(default_factory=list)
+    celery_task_modules: list[str] = field(default_factory=list)
+    athena_database: str = ""
+
+
+class ManifestValidationError(Exception):
+    """Raised when a product manifest fails validation."""
+
+    def __init__(self, errors: list[str]) -> None:
+        self.errors = errors
+        super().__init__(f"Manifest validation failed: {'; '.join(errors)}")
+
+
+def validate_manifest(manifest: ProductManifest) -> list[str]:
+    """Validate a ProductManifest and return a list of error strings (empty = valid)."""
+    errors: list[str] = []
+
+    if not manifest.name:
+        errors.append("manifest.name is required")
+    if not manifest.version:
+        errors.append("manifest.version is required")
+
+    # Validate extractors implement BaseExtractor
+    seen_job_types: set[str] = set()
+    for i, ext_cls in enumerate(manifest.extractors):
+        if not (isinstance(ext_cls, type) and issubclass(ext_cls, BaseExtractor)):
+            errors.append(
+                f"extractors[{i}] ({ext_cls!r}) must be a subclass of BaseExtractor"
+            )
+            continue
+
+        # Check for duplicate job_types
+        try:
+            config = ext_cls().get_config()
+            if config.job_type in seen_job_types:
+                errors.append(
+                    f"Duplicate job_type '{config.job_type}' "
+                    f"from extractor '{ext_cls.__name__}'"
+                )
+            seen_job_types.add(config.job_type)
+        except Exception as exc:
+            errors.append(
+                f"extractors[{i}] ({ext_cls.__name__}).get_config() raised: {exc}"
+            )
+
+    # Validate processor stages (must be correct ABC or None or Ellipsis)
+    _validate_stage(manifest.preprocessor, BasePreprocessor, "preprocessor", errors)
+    _validate_stage(
+        manifest.core_processor, BaseCoreProcessor, "core_processor", errors
+    )
+    _validate_stage(manifest.postprocessor, BasePostprocessor, "postprocessor", errors)
+
+    return errors
+
+
+def _validate_stage(
+    value: Any,
+    abc: type[Any],
+    name: str,
+    errors: list[str],
+) -> None:
+    if value is ... or value is None:
+        return
+    if not isinstance(value, abc):
+        errors.append(
+            f"{name} must be an instance of {abc.__name__}, None, or ... (Ellipsis); "
+            f"got {type(value).__name__}"
+        )
+
+
+def discover_products() -> list[ProductManifest]:
+    """Discover installed products via ``pulse_engine.products`` entry-point group."""
+    manifests: list[ProductManifest] = []
+    eps = importlib.metadata.entry_points()
+    group = eps.select(group="pulse_engine.products")
+    for ep in group:
+        obj = ep.load()
+        if isinstance(obj, ProductManifest):
+            manifests.append(obj)
+        elif callable(obj):
+            manifests.append(obj())
+    return manifests

pulse_engine/runners/lambda_runner.py

@@ -0,0 +1,26 @@
+"""Lambda handler wrapper for product containers.
+
+This is the Lambda handler pointed to by Terraform for native Lambda compute.
+Product containers must NOT add any orchestration logic — they expose a plain
+`entrypoint:run` function and this runner unpacks the event and calls it.
+
+Terraform Lambda resource handler:
+    pulse_engine.runners.lambda_runner.handler
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def handler(event: dict[str, Any], context: object) -> None:
+    """Engine-owned Lambda handler. Delegates to the product's plain entrypoint."""
+    from entrypoint import run  # imported at call time from the product container
+
+    run(
+        job_id=event["job_id"],
+        chain=event["chain"],
+        config=event["config"],
+        pulse_api_token=event["pulse_api_token"],
+        pulse_engine_url=event["pulse_engine_url"],
+    )

pulse_engine/runners/pipeline_runner.py

@@ -0,0 +1,43 @@
+"""Generic runner for pipeline module containers.
+
+This module provides the entrypoint shim that the engine's runner package
+(pulse-engine-runner) uses to call the product's entrypoint.run() function
+with the new pipeline contract.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+
+class RunnerError(Exception):
+    pass
+
+
+def execute_step(
+    entrypoint_fn: Callable[..., dict[str, Any]],
+    job_id: str,
+    args: dict[str, Any],
+    inputs: Any,
+    pipeline_run_id: str,
+    pulse_engine_url: str,
+    pulse_api_token: str,
+) -> dict[str, Any]:
+    """Call the product's entrypoint with the pipeline contract and validate output."""
+    result = entrypoint_fn(
+        job_id=job_id,
+        args=args,
+        inputs=inputs,
+        pipeline_run_id=pipeline_run_id,
+        pulse_engine_url=pulse_engine_url,
+        pulse_api_token=pulse_api_token,
+    )
+
+    if not isinstance(result, dict):
+        raise RunnerError(
+            f"Module entrypoint must return a dict (output reference), "
+            f"got {type(result).__name__}"
+        )
+
+    return result