semantixrag 2.0.0__py3-none-any.whl

semantixrag/__init__.py

@@ -0,0 +1,14 @@
+"""RAG Ingestion Pipeline Source Package."""
+from .pipeline import IngestionPipeline
+from .models import Chunk, ExtractionResult, ExtractedElement, ElementType, IndexedDocument
+from .monitoring.logger import setup_logging
+
+__all__ = [
+    "IngestionPipeline",
+    "Chunk",
+    "ExtractionResult",
+    "ExtractedElement",
+    "ElementType",
+    "IndexedDocument",
+    "setup_logging",
+]

semantixrag/__main__.py

@@ -0,0 +1,6 @@
+"""Allow semantixrag to be executed as a module: python -m semantixrag"""
+import sys
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

semantixrag/api/__init__.py

@@ -0,0 +1 @@
+"""FastAPI application for SemantixRAG Platform."""

semantixrag/api/main.py

@@ -0,0 +1,29 @@
+"""FastAPI application entry point for SemantixRAG Platform."""
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from .routes import ingestion, retrieval, admin, observability, compliance
+
+app = FastAPI(
+    title="SemantixRAG Platform",
+    description="AI-native RAG ingestion, retrieval, and governance platform",
+    version="2.0.0",
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.include_router(ingestion.router, prefix="/v1", tags=["ingestion"])
+app.include_router(retrieval.router, prefix="/v1", tags=["retrieval"])
+app.include_router(admin.router, prefix="/v1", tags=["admin"])
+app.include_router(observability.router, prefix="/v1", tags=["observability"])
+app.include_router(compliance.router, prefix="/v1", tags=["compliance"])
+
+
+@app.get("/health")
+async def health_check():
+    return {"status": "ok", "version": "2.0.0"}

semantixrag/api/routes/admin.py

@@ -0,0 +1,68 @@
+"""Admin API routes (AdminCopilot)."""
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from typing import Optional
+from ...pipeline import IngestionPipeline
+from ...observability.metrics import metrics_collector
+
+router = APIRouter()
+pipeline = IngestionPipeline()
+
+
+class AdminQueryRequest(BaseModel):
+    query: str
+    user_id: str = "anonymous"
+    user_role: str = "viewer"
+    tenant_id: str = "default"
+    confirm_destructive: bool = True
+
+
+class AdminQueryResponse(BaseModel):
+    action_taken: str = "query_executed"
+    action_type: str = "read"
+    result: dict = {}
+    requires_confirmation: bool = False
+    summary: str = ""
+
+
+@router.post("/admin/query", response_model=AdminQueryResponse)
+async def admin_query(request: AdminQueryRequest):
+    """Natural-language platform administration."""
+    query = request.query.lower()
+
+    try:
+        if "document" in query and "count" in query:
+            return AdminQueryResponse(
+                summary="Document count query processed",
+                result={"message": "Use GET /metrics for document counts"},
+            )
+
+        elif "pii" in query and ("scan" in query or "find" in query):
+            return AdminQueryResponse(
+                action_type="compliance",
+                summary="PII scan initiated",
+                result={"status": "PII scanning enabled by default on ingestion"},
+            )
+
+        elif "health" in query or "status" in query:
+            return AdminQueryResponse(
+                action_type="read",
+                summary="Platform health check",
+                result=metrics_collector.snapshot(),
+            )
+
+        elif "schema" in query or "ontology" in query:
+            return AdminQueryResponse(
+                action_type="read",
+                summary="Knowledge graph schema",
+                result=pipeline.ontology_manager.to_schema(),
+            )
+
+        else:
+            return AdminQueryResponse(
+                summary=f"Processed: {request.query}",
+                result={"message": f"Query '{request.query}' received. Use specific commands: 'document count', 'PII scan', 'health', 'schema'"},
+            )
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

semantixrag/api/routes/compliance.py

@@ -0,0 +1,108 @@
+"""Compliance API routes (GuardRail)."""
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel
+from typing import Optional
+import uuid
+from datetime import datetime
+from ...compliance.pii_scanner import PIIScanner
+from ...compliance.masking import masking_engine
+from ...compliance.dsar import DSAREngine
+from ...config.settings import settings
+
+router = APIRouter()
+pii_scanner = PIIScanner()
+dsar_engine = DSAREngine()
+
+
+class PIIscanRequest(BaseModel):
+    document_id: Optional[str] = None
+    text: Optional[str] = None
+    tenant_id: str = "default"
+    scan_depth: str = "standard"
+
+
+class DSARRequest(BaseModel):
+    subject_id: str
+    action: str
+    tenant_id: str = "default"
+    requested_by: str
+    reason: Optional[str] = None
+
+
+@router.post("/compliance/pii/scan")
+async def scan_pii(request: PIIscanRequest):
+    """Scan text or document for PII."""
+    if not request.text:
+        raise HTTPException(status_code=400, detail="No text provided for scanning")
+
+    findings = await pii_scanner.scan(request.text)
+    summary = pii_scanner.get_summary(findings)
+
+    return {
+        "scan_id": str(uuid.uuid4()),
+        "pii_findings": [
+            {
+                "pii_type": f.pii_type,
+                "sensitivity": f.sensitivity,
+                "confidence": f.confidence,
+                "location": {"start": f.start, "end": f.end},
+                "context": f.context[:100],
+                "recommended_action": f.recommended_action,
+                "masked_text": f.masked_text,
+            }
+            for f in findings
+        ],
+        "summary": summary,
+    }
+
+
+@router.post("/compliance/dsar")
+async def execute_dsar(request: DSARRequest):
+    """Execute GDPR data subject access/deletion request."""
+    if request.action == "delete" and not request.reason:
+        raise HTTPException(
+            status_code=400,
+            detail="Reason is required for delete actions",
+        )
+
+    result = await dsar_engine.execute_dsar(
+        subject_id=request.subject_id,
+        action=request.action,
+        tenant_id=request.tenant_id,
+        requested_by=request.requested_by,
+        reason=request.reason,
+    )
+
+    return {
+        "dsar_id": result.dsar_id,
+        "status": result.status,
+        "subject_id": result.subject_id,
+        "action": result.action,
+        "affected_records": {
+            "documents": result.affected_documents,
+            "chunks": result.affected_chunks,
+            "embeddings": result.affected_embeddings,
+            "agent_memories": result.affected_memories,
+        },
+        "estimated_completion": result.estimated_completion,
+    }
+
+
+@router.get("/compliance/dsar/{dsar_id}")
+async def get_dsar_status(dsar_id: str):
+    """Get DSAR request status."""
+    result = dsar_engine.get_dsar_status(dsar_id)
+    if not result:
+        raise HTTPException(status_code=404, detail="DSAR request not found")
+    return {
+        "dsar_id": result.dsar_id,
+        "status": result.status,
+        "subject_id": result.subject_id,
+        "action": result.action,
+        "affected_records": {
+            "documents": result.affected_documents,
+            "chunks": result.affected_chunks,
+            "embeddings": result.affected_embeddings,
+            "agent_memories": result.affected_memories,
+        },
+    }

semantixrag/api/routes/ingestion.py

@@ -0,0 +1,76 @@
+"""Ingestion API routes."""
+from fastapi import APIRouter, UploadFile, File, Form, HTTPException
+from pathlib import Path
+import uuid
+from datetime import datetime
+from typing import Optional
+from ...pipeline import IngestionPipeline
+from ...config.settings import settings
+
+router = APIRouter()
+pipeline = IngestionPipeline()
+
+
+@router.post("/ingest")
+async def ingest_document(
+    file: UploadFile = File(...),
+    tenant_id: str = Form("default"),
+    enrich: bool = Form(True),
+    enable_entity_extraction: bool = Form(True),
+    enable_pii_scan: bool = Form(True),
+    enable_summarization: bool = Form(True),
+):
+    """Upload and process a single document."""
+    if not file.filename:
+        raise HTTPException(status_code=400, detail="No file provided")
+
+    # Save uploaded file temporarily
+    temp_dir = Path(settings.watch_directory) / "uploads"
+    temp_dir.mkdir(parents=True, exist_ok=True)
+    temp_path = temp_dir / f"{uuid.uuid4()}_{file.filename}"
+
+    try:
+        content = await file.read()
+        temp_path.write_bytes(content)
+
+        document_id = str(uuid.uuid4())
+        result = pipeline.process_document(
+            temp_path,
+            document_id=document_id,
+            tenant_id=tenant_id,
+        )
+
+        if not result.get("success"):
+            raise HTTPException(
+                status_code=500,
+                detail=result.get("error", "Processing failed"),
+            )
+
+        return {
+            "document_id": result["document_id"],
+            "filename": result["filename"],
+            "status": "completed",
+            "chunks_created": result["chunks_count"],
+            "entities_extracted": result.get("entities_extracted", 0),
+            "pii_findings": result.get("pii_findings", 0),
+            "indexed_at": datetime.utcnow().isoformat(),
+            "trace_id": result.get("trace_id", ""),
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        if temp_path.exists():
+            temp_path.unlink()
+
+
+@router.get("/ingest/{document_id}/status")
+async def get_ingest_status(document_id: str):
+    """Check ingestion status for a document."""
+    return {
+        "document_id": document_id,
+        "status": "completed",
+        "message": "Status tracking requires OpenSearch connection",
+    }

semantixrag/api/routes/observability.py

@@ -0,0 +1,39 @@
+"""Observability API routes (Obsidian)."""
+from fastapi import APIRouter, HTTPException, Query
+from typing import Optional, List
+from datetime import datetime
+from ...observability.metrics import metrics_collector
+
+router = APIRouter()
+
+
+@router.post("/observability/traces")
+async def ingest_traces(traces: list[dict]):
+    """Ingest telemetry traces."""
+    accepted = len(traces)
+    metrics_collector.increment("traces.ingested", accepted)
+    return {"accepted": accepted, "failed": 0}
+
+
+@router.get("/observability/metrics")
+async def get_metrics(
+    start_time: Optional[str] = Query(None),
+    end_time: Optional[str] = Query(None),
+    granularity: str = Query("5m"),
+):
+    """Query pipeline metrics."""
+    return metrics_collector.snapshot()
+
+
+@router.get("/observability/evaluation")
+async def get_evaluation(
+    start_time: Optional[str] = Query(None),
+    end_time: Optional[str] = Query(None),
+    evaluation_id: Optional[str] = Query(None),
+    query_id: Optional[str] = Query(None),
+):
+    """Query RAG quality evaluation metrics."""
+    return {
+        "evaluations": [],
+        "message": "Evaluation requires LLM client for full functionality",
+    }

semantixrag/api/routes/retrieval.py

@@ -0,0 +1,75 @@
+"""Retrieval API routes."""
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from typing import Optional, List
+import uuid
+from ...pipeline import IngestionPipeline
+from ...indexing.hybrid_search import HybridSearch
+from ...indexing.graph_writer import GraphWriter
+from ...config.settings import settings
+
+router = APIRouter()
+pipeline = IngestionPipeline()
+hybrid_search = HybridSearch()
+
+
+class RetrievalRequest(BaseModel):
+    query: str
+    tenant_id: str = "default"
+    strategy: str = "hybrid"
+    vector_top_k: int = 50
+    keyword_top_k: int = 50
+    graph_hops: int = 2
+    rerank: bool = True
+    rerank_top_k: int = 5
+    filters: Optional[dict] = None
+    generation: Optional[dict] = None
+
+
+class RetrievalResponse(BaseModel):
+    query_id: str
+    query: str
+    results: list[dict] = []
+    retrieval_metrics: dict = {}
+    generation: Optional[dict] = None
+    trace_id: str = ""
+
+
+@router.post("/query", response_model=RetrievalResponse)
+async def query(request: RetrievalRequest):
+    """Execute semantic search with hybrid retrieval."""
+    query_id = str(uuid.uuid4())
+    trace_id = str(uuid.uuid4())
+
+    try:
+        search_results = hybrid_search.search(
+            query_text=request.query,
+            top_k=request.vector_top_k,
+            tenant_id=request.tenant_id,
+        )
+
+        results = []
+        for i, result in enumerate(search_results[:request.rerank_top_k]):
+            results.append({
+                "chunk_id": result.get("chunk_id", ""),
+                "document_id": result.get("document_id", ""),
+                "document_title": result.get("document_title", ""),
+                "header_path": result.get("header_path", ""),
+                "chunk_text": result.get("chunk_text", "")[:500],
+                "score": result.get("score", 0.0),
+                "entities": result.get("entities", []),
+            })
+
+        return RetrievalResponse(
+            query_id=query_id,
+            query=request.query,
+            results=results,
+            retrieval_metrics={
+                "total_candidates": len(search_results),
+                "returned": len(results),
+            },
+            trace_id=trace_id,
+        )
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

semantixrag/cdc/__init__.py

@@ -0,0 +1,5 @@
+"""CDC and file-watching package."""
+from .watcher import DirectoryWatcher
+from .incremental import IncrementalUpdater
+
+__all__ = ["DirectoryWatcher", "IncrementalUpdater"]

semantixrag/cdc/incremental.py

@@ -0,0 +1,57 @@
+"""Incremental update logic for keeping the vector store synchronized."""
+import logging
+from pathlib import Path
+from typing import Optional
+
+from ..config.settings import settings
+from ..indexing.bulk_indexer import BulkIndexer
+from ..monitoring.logger import get_logger
+
+logger = logging.getLogger(__name__)
+
+
+class IncrementalUpdater:
+    """Handles incremental updates to the OpenSearch index.
+
+    When a document is updated: deletes all existing chunks for that
+    document_id, then re-triggers the full extraction/chunking/embedding pipeline.
+    """
+
+    def __init__(self, bulk_indexer: Optional[BulkIndexer] = None):
+        self.bulk_indexer = bulk_indexer or BulkIndexer()
+
+    def before_reindex(self, file_path: Path, document_id: str) -> dict:
+        """Prepare for re-indexing by cleaning up existing entries.
+
+        Args:
+            file_path: Path to the document.
+            document_id: Unique document identifier.
+
+        Returns:
+            Deletion result from OpenSearch.
+        """
+        logger.info(
+            f"Preparing to re-index '{file_path.name}' "
+            f"(document_id={document_id})"
+        )
+        return self.bulk_indexer.delete_document_chunks(document_id)
+
+    def process_deletion(self, file_path: Path, document_id: str) -> dict:
+        """Handle file deletion: remove all chunks from index.
+
+        Args:
+            file_path: Path to the deleted document.
+            document_id: Unique document identifier.
+
+        Returns:
+            Deletion result from OpenSearch.
+        """
+        logger.info(
+            f"Processing deletion of '{file_path.name}' "
+            f"(document_id={document_id})"
+        )
+        result = self.bulk_indexer.delete_document_chunks(document_id)
+        logger.info(
+            f"Deleted {result.get('deleted', 0)} chunks for '{document_id}'"
+        )
+        return result

semantixrag/cdc/watcher.py

@@ -0,0 +1,139 @@
+"""Directory watcher using watchdog for Change Data Capture on local files."""
+import os
+import logging
+import time
+from pathlib import Path
+from typing import Callable, Optional
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler, FileModifiedEvent, FileCreatedEvent
+
+from ..config.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+class DocumentEventHandler(FileSystemEventHandler):
+    """Handles file system events for the watched directory."""
+
+    def __init__(
+        self,
+        on_created: Callable[[Path], None],
+        on_modified: Callable[[Path], None],
+        on_deleted: Callable[[Path], None],
+        supported_extensions: set[str] = None,
+    ):
+        self.on_created = on_created
+        self.on_modified = on_modified
+        self.on_deleted = on_deleted
+        self.supported_extensions = supported_extensions or {
+            ".pdf", ".txt", ".md", ".docx", ".html", ".xml", ".csv"
+        }
+        self._last_events: dict[str, float] = {}
+        self._debounce_seconds = 2.0
+
+    def on_created(self, event):
+        """Handle file creation events."""
+        if event.is_directory:
+            return
+        path = Path(event.src_path)
+        if path.suffix.lower() not in self.supported_extensions:
+            return
+        if self._is_debounced(str(path)):
+            return
+        logger.info(f"File created: {path.name}")
+        self.on_created(path)
+
+    def on_modified(self, event):
+        """Handle file modification events."""
+        if event.is_directory:
+            return
+        path = Path(event.src_path)
+        if path.suffix.lower() not in self.supported_extensions:
+            return
+        if self._is_debounced(str(path)):
+            return
+        logger.info(f"File modified: {path.name}")
+        self.on_modified(path)
+
+    def on_deleted(self, event):
+        """Handle file deletion events."""
+        if event.is_directory:
+            return
+        path = Path(event.src_path)
+        if path.suffix.lower() not in self.supported_extensions:
+            return
+        logger.info(f"File deleted: {path.name}")
+        self.on_deleted(path)
+
+    def _is_debounced(self, path_str: str) -> bool:
+        """Debounce rapid-fire file system events."""
+        now = time.time()
+        last = self._last_events.get(path_str, 0)
+        if now - last < self._debounce_seconds:
+            return True
+        self._last_events[path_str] = now
+        return False
+
+
+class DirectoryWatcher:
+    """Watches a directory for file changes and triggers the ingestion pipeline.
+
+    Uses watchdog to monitor file creation, modification, and deletion events.
+    """
+
+    def __init__(
+        self,
+        watch_dir: Optional[Path] = None,
+        on_created: Optional[Callable] = None,
+        on_modified: Optional[Callable] = None,
+        on_deleted: Optional[Callable] = None,
+    ):
+        self.watch_dir = Path(watch_dir or settings.watch_directory)
+        self.recursive = settings.watch_recursive
+        self._observer: Optional[Observer] = None
+        self._running = False
+
+        # Default no-op callbacks
+        self._on_created = on_created or (lambda p: None)
+        self._on_modified = on_modified or (lambda p: None)
+        self._on_deleted = on_deleted or (lambda p: None)
+
+    def start(self) -> None:
+        """Start watching the directory for changes."""
+        if self._running:
+            logger.warning("Watcher is already running")
+            return
+
+        self.watch_dir.mkdir(parents=True, exist_ok=True)
+        logger.info(
+            f"Starting directory watcher on '{self.watch_dir}' "
+            f"(recursive={self.recursive})"
+        )
+
+        event_handler = DocumentEventHandler(
+            on_created=self._on_created,
+            on_modified=self._on_modified,
+            on_deleted=self._on_deleted,
+        )
+
+        self._observer = Observer()
+        self._observer.schedule(
+            event_handler,
+            str(self.watch_dir),
+            recursive=self.recursive,
+        )
+        self._observer.start()
+        self._running = True
+        logger.info(f"Directory watcher started on '{self.watch_dir}'")
+
+    def stop(self) -> None:
+        """Stop watching the directory."""
+        if self._observer is not None and self._running:
+            self._observer.stop()
+            self._observer.join()
+            self._running = False
+            logger.info("Directory watcher stopped")
+
+    @property
+    def is_running(self) -> bool:
+        return self._running

semantixrag/chunking/__init__.py

@@ -0,0 +1,5 @@
+"""Chunking and enrichment package."""
+from .header_splitter import HeaderAwareSplitter
+from .enricher import ContextualEnricher
+
+__all__ = ["HeaderAwareSplitter", "ContextualEnricher"]