longparser 0.1.0__py3-none-any.whl

longparser/server/embeddings.py

@@ -0,0 +1,181 @@
+"""Embedding engine for LongParser — wraps LangChain providers.
+
+Supported: HuggingFace (local), OpenAI, Gemini.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+import threading
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# Global lock for single-flight dimension discovery within a process
+_dim_lock = threading.Lock()
+
+
+class EmbeddingEngine:
+    """Multi-provider embedding engine.
+
+    Parameters
+    ----------
+    provider:
+        "huggingface", "openai", or "gemini".
+    model_name:
+        Model ID (e.g., "BAAI/bge-base-en-v1.5", "text-embedding-3-small").
+    dimensions:
+        Optional override for embedding dimensions (useful for OpenAI/Gemini
+        to avoid API calls or specifically configure text-embedding-3).
+    """
+
+    def __init__(
+        self,
+        provider: str = "huggingface",
+        model_name: str = "BAAI/bge-base-en-v1.5",
+        dimensions: Optional[int] = None,
+    ) -> None:
+        self.provider = provider.lower()
+        self.model_name = model_name
+        self.configured_dimensions = dimensions
+        self._dim: Optional[int] = dimensions
+
+        # Provider-specific configurations
+        self._gemini_doc_task = "RETRIEVAL_DOCUMENT"
+        self._gemini_query_task = "RETRIEVAL_QUERY"
+        self._hf_normalize = True
+
+        if self.provider == "openai":
+            from langchain_openai import OpenAIEmbeddings
+            # Pass dimensions kwargs if provided (supported for text-embedding-3*)
+            kwargs = {}
+            if self.configured_dimensions:
+                kwargs["dimensions"] = self.configured_dimensions
+            self.model = OpenAIEmbeddings(model=self.model_name, **kwargs)
+            logger.info(f"EmbeddingEngine: Initialized OpenAI {model_name}")
+
+        elif self.provider == "gemini":
+            from langchain_google_genai import GoogleGenerativeAIEmbeddings
+            kwargs = {}
+            if self.configured_dimensions:
+                kwargs["output_dimensionality"] = self.configured_dimensions
+            self.model = GoogleGenerativeAIEmbeddings(model=self.model_name, **kwargs)
+            logger.info(f"EmbeddingEngine: Initialized Gemini {model_name}")
+
+        elif self.provider == "huggingface":
+            from langchain_huggingface import HuggingFaceEmbeddings
+            self.model = HuggingFaceEmbeddings(
+                model_name=self.model_name,
+                encode_kwargs={"normalize_embeddings": self._hf_normalize}
+            )
+            logger.info(f"EmbeddingEngine: Initialized HuggingFace {model_name}")
+
+        else:
+            raise ValueError(f"Unknown embedding provider: {provider}")
+
+    def get_fingerprint(self) -> str:
+        """Return a stable 10-char SHA1 hash of the full embedding configuration.
+        This is used to isolate vector spaces when models/configs change.
+        """
+        config = {
+            "p": self.provider,
+            "m": self.model_name,
+            "d": self.configured_dimensions,
+        }
+        if self.provider == "gemini":
+            config["t_doc"] = self._gemini_doc_task
+            config["t_qry"] = self._gemini_query_task
+        elif self.provider == "huggingface":
+            config["n"] = self._hf_normalize
+
+        # Stable json dump
+        cfg_str = json.dumps(config, sort_keys=True)
+        return hashlib.sha1(cfg_str.encode("utf-8")).hexdigest()[:10]
+
+    @property
+    def dim(self) -> int:
+        """Lazy-evaluated, thread-safe, cross-process cached embedding dimension."""
+        if self._dim is not None:
+            return self._dim
+
+        # Global dimension lock
+        with _dim_lock:
+            # Check if another thread resolved it while waiting
+            if self._dim is not None:
+                return self._dim
+
+            fp = self.get_fingerprint()
+            cache_key = f"cleanrag:embed_dim:{fp}"
+
+            # 1) Try Redis cross-process cache if available
+            try:
+                import redis
+                redis_url = os.getenv("LONGPARSER_REDIS_URL", "redis://localhost:6379/0")
+                r = redis.from_url(redis_url, socket_connect_timeout=1)
+                r.ping()  # Fail fast if unavailable
+                cached = r.get(cache_key)
+                if cached:
+                    self._dim = int(cached)
+                    logger.debug(f"Resolved dim from Redis cache: {self._dim}")
+                    return self._dim
+            except Exception as e:
+                logger.debug(f"Redis cache check failed (likely no redis): {e}")
+
+            # 2) Fallback: perform an API call to discover it.
+            # We strictly use the Document retrieval task setting to ensure dimension matches index.
+            logger.info(f"Discovering embedding dimension for {self.provider}/{self.model_name}...")
+            test_doc = ["test"]
+            try:
+                if self.provider == "gemini":
+                    # Explicitly pass the document task type
+                    test_vec = self.model.embed_documents(test_doc, task_type=self._gemini_doc_task)[0]
+                else:
+                    test_vec = self.model.embed_documents(test_doc)[0]
+            except Exception as e:
+                logger.error(f"Failed to discover embedding dimension: {e}")
+                raise  # Fail loudly! Missing API keys should crash here.
+
+            self._dim = len(test_vec)
+            logger.info(f"Discovered dimension: {self._dim}")
+
+            # 3) Cache it in Redis
+            try:
+                if 'r' in locals():
+                    r.set(cache_key, self._dim)
+            except Exception:
+                pass
+
+            return self._dim
+
+    def embed_chunks(self, texts: list[str], batch_size: int = 64) -> list[list[float]]:
+        """Embed a list of text chunks.
+
+        Returns list of float vectors (one per chunk).
+        """
+        if not texts:
+            return []
+
+        if self.provider == "gemini":
+            # Gemini strictly enforces maximum 100 texts per batch request
+            effective_batch = min(batch_size, 100)
+            all_embeddings = []
+            for i in range(0, len(texts), effective_batch):
+                batch = texts[i:i + effective_batch]
+                # Pass explicit task_type parameter per call
+                all_embeddings.extend(
+                    self.model.embed_documents(batch, task_type=self._gemini_doc_task)
+                )
+            return all_embeddings
+        else:
+            return self.model.embed_documents(texts)
+
+    def embed_query(self, query: str) -> list[float]:
+        """Embed a single search query."""
+        if self.provider == "gemini":
+            # For queries, Gemini optimizes with a different task_type
+            return self.model.embed_query(query, task_type=self._gemini_query_task)
+        else:
+            return self.model.embed_query(query)

longparser/server/queue.py

@@ -0,0 +1,97 @@
+"""Abstracted job queue backend for LongParser.
+
+ARQ (async Redis queue) is the default implementation.
+The QueueBackend ABC allows swapping to Celery/Dramatiq
+without rewriting business logic.
+"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class QueueBackend(ABC):
+    """Abstract queue interface — swap implementations without changing routes."""
+
+    @abstractmethod
+    async def enqueue(self, task_name: str, payload: dict) -> str:
+        """Enqueue a task. Returns task/job reference ID."""
+        ...
+
+    @abstractmethod
+    async def cancel(self, task_id: str) -> bool:
+        """Cancel a queued/running task. Returns True if cancelled."""
+        ...
+
+    @abstractmethod
+    async def status(self, task_id: str) -> dict:
+        """Get task status and progress."""
+        ...
+
+
+class ARQBackend(QueueBackend):
+    """ARQ (async Redis queue) implementation."""
+
+    def __init__(self, redis_url: str = "redis://localhost:6379"):
+        self.redis_url = redis_url
+        self._pool = None
+
+    async def _get_pool(self):
+        if self._pool is None:
+            from arq import create_pool
+            from arq.connections import RedisSettings
+
+            url = self.redis_url.replace("redis://", "")
+            # Strip database number (e.g., /0) if present
+            url = url.split("/")[0]
+            host, _, port_str = url.partition(":")
+            port = int(port_str) if port_str else 6379
+            self._pool = await create_pool(RedisSettings(host=host, port=port))
+        return self._pool
+
+    async def enqueue(self, task_name: str, payload: dict) -> str:
+        """Enqueue a task via ARQ."""
+        pool = await self._get_pool()
+        job = await pool.enqueue_job(task_name, **payload)
+        job_id = job.job_id if job else "unknown"
+        logger.info(f"Enqueued {task_name} → ARQ job {job_id}")
+        return job_id
+
+    async def cancel(self, task_id: str) -> bool:
+        """Cancel via ARQ job abort (best-effort)."""
+        try:
+            from arq.jobs import Job
+            pool = await self._get_pool()
+            job = Job(task_id, pool)
+            await job.abort()
+            return True
+        except Exception as e:
+            logger.warning(f"Failed to abort ARQ job {task_id}: {e}")
+            return False
+
+    async def status(self, task_id: str) -> dict:
+        """Get ARQ job status."""
+        try:
+            from arq.jobs import Job
+            pool = await self._get_pool()
+            job = Job(task_id, pool)
+            info = await job.info()
+            if info is None:
+                return {"status": "unknown"}
+            return {
+                "status": info.status,
+                "result": info.result,
+                "enqueue_time": str(info.enqueue_time) if info.enqueue_time else None,
+            }
+        except Exception:
+            return {"status": "unknown"}
+
+    async def close(self) -> None:
+        """Close the Redis pool."""
+        if self._pool:
+            await self._pool.close()
+            self._pool = None

longparser/server/schemas.py

@@ -0,0 +1,204 @@
+"""API request/response models for LongParser HITL review system."""
+
+from __future__ import annotations
+
+import os
+import uuid
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+from ..schemas import BlockType
+
+
+# ---------------------------------------------------------------------------
+# Enums
+# ---------------------------------------------------------------------------
+
+class JobStatus(str, Enum):
+    """Job lifecycle states."""
+    QUEUED = "queued"
+    EXTRACTING = "extracting"
+    READY_FOR_REVIEW = "ready_for_review"
+    FINALIZING = "finalizing"
+    FINALIZED = "finalized"
+    EMBEDDING = "embedding"
+    INDEXED = "indexed"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+
+
+class ReviewStatus(str, Enum):
+    """Block/chunk review states."""
+    PENDING = "pending"
+    APPROVED = "approved"
+    EDITED = "edited"
+    REJECTED = "rejected"
+
+
+class FinalizePolicy(str, Enum):
+    """What to do with pending items on finalize."""
+    REJECT_PENDING = "reject_pending"
+    APPROVE_PENDING = "approve_pending"
+    REQUIRE_ALL_APPROVED = "require_all_approved"
+
+
+class UserRole(str, Enum):
+    """RBAC roles."""
+    ADMIN = "admin"
+    REVIEWER = "reviewer"
+    VIEWER = "viewer"
+
+
+# ---------------------------------------------------------------------------
+# Revision (append-only audit trail)
+# ---------------------------------------------------------------------------
+
+class Revision(BaseModel):
+    """Immutable record of a block/chunk edit."""
+    revision_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    entity_type: str  # "block" | "chunk"
+    entity_id: str
+    previous_revision_id: Optional[str] = None
+    action: ReviewStatus
+    original_text: str
+    edited_text: Optional[str] = None
+    edited_type: Optional[BlockType] = None
+    reviewer_id: str = ""
+    reviewer_note: str = ""
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+
+
+# ---------------------------------------------------------------------------
+# API Request Models
+# ---------------------------------------------------------------------------
+
+class BlockReviewUpdate(BaseModel):
+    """Request body for PATCH /jobs/{id}/blocks/{bid}."""
+    status: ReviewStatus
+    edited_text: Optional[str] = None
+    edited_type: Optional[BlockType] = None
+    reviewer_note: str = ""
+    version: int  # optimistic locking
+
+
+class ChunkReviewUpdate(BaseModel):
+    """Request body for PATCH /jobs/{id}/chunks/{cid}."""
+    status: ReviewStatus
+    edited_text: Optional[str] = None
+    reviewer_note: str = ""
+    version: int  # optimistic locking
+
+
+class FinalizeRequest(BaseModel):
+    """Request body for POST /jobs/{id}/finalize."""
+    finalize_policy: FinalizePolicy = FinalizePolicy.REJECT_PENDING
+
+
+class EmbedRequest(BaseModel):
+    """Request body for POST /jobs/{id}/embed."""
+    provider: str = Field(
+        default_factory=lambda: os.getenv("LONGPARSER_EMBED_PROVIDER", "huggingface")
+    )
+    model: str = Field(
+        default_factory=lambda: os.getenv("LONGPARSER_EMBED_MODEL", "BAAI/bge-base-en-v1.5")
+    )
+    vector_db: str = Field(
+        default_factory=lambda: os.getenv("LONGPARSER_VECTOR_DB", "chroma")
+    )  # "chroma" | "faiss" | "qdrant"
+    collection_name: Optional[str] = None
+
+
+class SearchRequest(BaseModel):
+    """Request body for POST /search."""
+    query: str
+    job_id: str
+    top_k: int = 5
+    index_version: Optional[str] = None  # defaults to latest
+    filters: dict = Field(default_factory=dict)
+
+
+# ---------------------------------------------------------------------------
+# API Response Models
+# ---------------------------------------------------------------------------
+
+class ReviewProgress(BaseModel):
+    """Review completion stats."""
+    approved: int = 0
+    edited: int = 0
+    rejected: int = 0
+    pending: int = 0
+
+
+class JobResponse(BaseModel):
+    """Response body for GET /jobs/{id}."""
+    job_id: str
+    tenant_id: str
+    status: JobStatus
+    source_file: str
+    file_hash: str = ""
+    total_pages: int = 0
+    total_blocks: int = 0
+    total_chunks: int = 0
+    review_progress: ReviewProgress = Field(default_factory=ReviewProgress)
+    created_at: datetime
+    finalized_at: Optional[datetime] = None
+    error: Optional[str] = None
+
+
+class JobListResponse(BaseModel):
+    """Response body for GET /jobs."""
+    jobs: list[JobResponse]
+    total: int
+
+
+class BlockResponse(BaseModel):
+    """Block data for API responses (confidence excluded)."""
+    block_id: str
+    type: BlockType
+    text: str
+    order_index: int = 0
+    heading_level: Optional[int] = None
+    indent_level: int = 0
+    hierarchy_path: list[str] = Field(default_factory=list)
+    page_number: int = 0
+    review_status: ReviewStatus = ReviewStatus.PENDING
+    current_revision_id: Optional[str] = None
+    version: int = 1
+
+
+class ChunkResponse(BaseModel):
+    """Chunk data for API responses."""
+    chunk_id: str
+    text: str
+    token_count: int = 0
+    chunk_type: str = ""
+    section_path: list[str] = Field(default_factory=list)
+    page_numbers: list[int] = Field(default_factory=list)
+    block_ids: list[str] = Field(default_factory=list)
+    review_status: ReviewStatus = ReviewStatus.PENDING
+    current_revision_id: Optional[str] = None
+    version: int = 1
+
+
+class SearchResult(BaseModel):
+    """Single search result."""
+    chunk_id: str
+    text: str
+    score: float
+    chunk_type: str = ""
+    section_path: list[str] = Field(default_factory=list)
+    page_numbers: list[int] = Field(default_factory=list)
+    block_ids: list[str] = Field(default_factory=list)
+    metadata: dict = Field(default_factory=dict)
+
+
+class SearchResponse(BaseModel):
+    """Response body for POST /search."""
+    results: list[SearchResult]
+    index_version: str
+    model: str
+    query: str
+    total: int