maktaba 0.1.0__py3-none-any.whl

maktaba/__init__.py

@@ -0,0 +1,42 @@
+"""
+Maktaba (EC*()) - The library for building libraries.
+
+Production-ready RAG infrastructure for Arabic & multilingual applications.
+By NuhaTech.
+"""
+
+from .exceptions import (
+    ChunkingError,
+    ConfigurationError,
+    EmbeddingError,
+    MaktabaError,
+    PartitionAPIError,
+    StorageError,
+)
+from .models import (
+    EmbeddingConfig,
+    PartitionConfig,
+    SearchResult,
+    VectorChunk,
+    VectorStoreConfig,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Exceptions
+    "MaktabaError",
+    "EmbeddingError",
+    "StorageError",
+    "ChunkingError",
+    "ConfigurationError",
+    "PartitionAPIError",
+    # Models
+    "VectorChunk",
+    "SearchResult",
+    "EmbeddingConfig",
+    "VectorStoreConfig",
+    "PartitionConfig",
+    # Version
+    "__version__",
+]

maktaba/chunking/__init__.py

@@ -0,0 +1,12 @@
+"""Document chunking module."""
+
+from .base import BaseChunker
+from .models import ChunkMetadata, ChunkResult
+from .unstructured import UnstructuredChunker
+
+__all__ = [
+    "BaseChunker",
+    "ChunkMetadata",
+    "ChunkResult",
+    "UnstructuredChunker",
+]

maktaba/chunking/base.py

@@ -0,0 +1,83 @@
+"""Base interface for document chunkers."""
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from .models import ChunkResult
+
+
+class BaseChunker(ABC):
+    """
+    Abstract base class for document chunking implementations.
+
+    All chunkers return LlamaIndex Document objects for compatibility
+    with the broader ecosystem.
+    """
+
+    @abstractmethod
+    async def chunk_text(
+        self,
+        text: str,
+        filename: str = "document.txt",
+        extra_metadata: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> ChunkResult:
+        """
+        Chunk raw text input.
+
+        Args:
+            text: Raw text to chunk
+            filename: Filename to use for metadata
+            extra_metadata: Additional metadata to attach to chunks
+            **kwargs: Implementation-specific options
+
+        Returns:
+            ChunkResult with documents and metadata
+        """
+        pass
+
+    @abstractmethod
+    async def chunk_file(
+        self,
+        file_path: Path | str,
+        extra_metadata: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> ChunkResult:
+        """
+        Chunk a local file.
+
+        Args:
+            file_path: Path to file to chunk
+            extra_metadata: Additional metadata to attach to chunks
+            **kwargs: Implementation-specific options
+
+        Returns:
+            ChunkResult with documents and metadata
+        """
+        pass
+
+    @abstractmethod
+    async def chunk_url(
+        self,
+        url: str,
+        filename: str,
+        extra_metadata: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> ChunkResult:
+        """
+        Download and chunk a file from URL.
+
+        Args:
+            url: URL to download file from
+            filename: Filename to use for metadata and type detection
+            extra_metadata: Additional metadata to attach to chunks
+            **kwargs: Implementation-specific options
+
+        Returns:
+            ChunkResult with documents and metadata
+
+        Raises:
+            ChunkingError: If download or chunking fails
+        """
+        pass

maktaba/chunking/models.py

@@ -0,0 +1,72 @@
+"""Data models for document chunking."""
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class ChunkMetadata:
+    """
+    Metadata about the chunked document.
+
+    Matches partition-api output format.
+    """
+
+    filename: str
+    """Original filename"""
+
+    filetype: str
+    """MIME type (e.g., 'application/pdf')"""
+
+    size_in_bytes: int
+    """File size in bytes"""
+
+
+@dataclass
+class ChunkResult:
+    """
+    Result of document chunking operation.
+
+    Contains chunks as LlamaIndex Document objects plus metadata.
+    Matches partition-api response structure.
+    """
+
+    documents: List[Any]  # List[llama_index.core.schema.Document]
+    """Chunked documents in LlamaIndex format"""
+
+    metadata: ChunkMetadata
+    """File metadata"""
+
+    total_chunks: int
+    """Total number of chunks created"""
+
+    total_characters: int
+    """Total character count across all chunks"""
+
+    total_pages: Optional[int] = None
+    """Total pages (if applicable to document type)"""
+
+    extra_metadata: Dict[str, Any] = field(default_factory=dict)
+    """Additional metadata provided by user"""
+
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary format (matches partition-api response)."""
+        result = {
+            "metadata": {
+                "filename": self.metadata.filename,
+                "filetype": self.metadata.filetype,
+                "sizeInBytes": self.metadata.size_in_bytes,
+            },
+            "total_chunks": self.total_chunks,
+            "total_characters": self.total_characters,
+            "documents": [doc.to_dict() for doc in self.documents],
+        }
+
+        if self.total_pages is not None:
+            result["total_pages"] = self.total_pages
+
+        if self.extra_metadata:
+            result["extra_metadata"] = self.extra_metadata
+
+        return result

maktaba/chunking/unstructured.py

@@ -0,0 +1,343 @@
+"""Unstructured document chunker - extracted from partition-api."""
+
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Dict, List, Literal, Optional, Tuple
+
+import httpx
+
+from ..exceptions import ChunkingError
+from .base import BaseChunker
+from .models import ChunkMetadata, ChunkResult
+
+
+class UnstructuredChunker(BaseChunker):
+    """
+    Document chunker using Unstructured.io library via LlamaIndex.
+
+    This implementation extracts the core logic from partition-api (main.py)
+    and integrates it directly into Maktaba for better performance and simplicity.
+
+    Supports:
+    - Multiple file formats (PDF, DOCX, TXT, HTML, etc.)
+    - Different parsing strategies (auto, fast, hi_res, ocr_only)
+    - Different chunking strategies (basic, by_title)
+    - Automatic file type detection
+    - Page number extraction (for PDFs)
+
+    """
+
+    def __init__(
+        self,
+        strategy: Literal["auto", "fast", "hi_res", "ocr_only"] = "auto",
+        chunking_strategy: Literal["basic", "by_title"] = "basic",
+        allowed_metadata_types: Tuple[type, ...] = (str, int, float, list, dict, type(None)),
+    ):
+        """
+        Initialize UnstructuredChunker.
+
+        Args:
+            strategy: Parsing strategy (default: "auto")
+                - "auto": Automatically choose best strategy
+                - "fast": Fast parsing (no OCR)
+                - "hi_res": High-resolution parsing (with OCR)
+                - "ocr_only": OCR-only parsing
+            chunking_strategy: How to chunk documents (default: "basic")
+                - "basic": Simple text splitting
+                - "by_title": Chunk by document structure (headings)
+            allowed_metadata_types: Allowed metadata value types
+        """
+        self.strategy = strategy
+        self.chunking_strategy = chunking_strategy
+        self.allowed_metadata_types = allowed_metadata_types
+
+    async def chunk_text(
+        self,
+        text: str,
+        filename: str = "document.txt",
+        extra_metadata: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> ChunkResult:
+        """
+        Chunk raw text input.
+
+        Args:
+            text: Raw text to chunk
+            filename: Filename for metadata (default: "document.txt")
+            extra_metadata: Additional metadata to attach to chunks
+            **kwargs: Override strategy/chunking_strategy
+
+        Returns:
+            ChunkResult with documents and metadata
+        """
+        try:
+            # Import here to avoid requiring unstructured for other operations
+            from llama_index.readers.file import UnstructuredReader
+            from unstructured.file_utils.filetype import detect_filetype
+
+            # Convert text to bytes
+            text_bytes = text.encode("utf-8")
+            size_in_bytes = len(text_bytes)
+            file_stream = BytesIO(text_bytes)
+
+            # Detect file type
+            content_type = detect_filetype(
+                file=file_stream,
+                metadata_file_path=filename,
+            ).mime_type
+
+            # Reset stream position after detection
+            file_stream.seek(0)
+
+            # Build unstructured arguments
+            unstructured_args = {
+                "strategy": kwargs.get("strategy", self.strategy),
+                "chunking_strategy": kwargs.get(
+                    "chunking_strategy", self.chunking_strategy
+                ),
+            }
+
+            # Load and chunk document
+            reader = UnstructuredReader(
+                allowed_metadata_types=self.allowed_metadata_types,
+            )
+            documents = reader.load_data(
+                unstructured_kwargs={
+                    "file": file_stream,
+                    "metadata_filename": filename,
+                    "content_type": content_type,
+                    **unstructured_args,
+                },
+                split_documents=True,
+                extra_info=extra_metadata or {},
+            )
+
+            if not documents:
+                raise ChunkingError("No chunks created from text")
+
+            # Calculate statistics
+            total_characters = sum(len(doc.text or "") for doc in documents)
+            total_chunks = len(documents)
+            total_pages = self._extract_total_pages(documents)
+
+            return ChunkResult(
+                documents=documents,
+                metadata=ChunkMetadata(
+                    filename=filename,
+                    filetype=content_type,
+                    size_in_bytes=size_in_bytes,
+                ),
+                total_chunks=total_chunks,
+                total_characters=total_characters,
+                total_pages=total_pages,
+                extra_metadata=extra_metadata or {},
+            )
+
+        except Exception as e:
+            raise ChunkingError(f"Failed to chunk text: {str(e)}") from e
+
+    async def chunk_file(
+        self,
+        file_path: Path | str,
+        extra_metadata: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> ChunkResult:
+        """
+        Chunk a local file.
+
+        Args:
+            file_path: Path to file to chunk
+            extra_metadata: Additional metadata to attach to chunks
+            **kwargs: Override strategy/chunking_strategy
+
+        Returns:
+            ChunkResult with documents and metadata
+        """
+        try:
+            # Import here to avoid requiring unstructured for other operations
+            from llama_index.readers.file import UnstructuredReader
+            from unstructured.file_utils.filetype import detect_filetype
+
+            # Convert to Path object
+            path = Path(file_path) if isinstance(file_path, str) else file_path
+
+            if not path.exists():
+                raise ChunkingError(f"File not found: {file_path}")
+
+            # Read file
+            file_bytes = path.read_bytes()
+            size_in_bytes = len(file_bytes)
+            file_stream = BytesIO(file_bytes)
+
+            # Detect file type
+            content_type = detect_filetype(
+                file=file_stream,
+                metadata_file_path=str(path),
+            ).mime_type
+
+            # Reset stream position after detection
+            file_stream.seek(0)
+
+            # Build unstructured arguments
+            unstructured_args = {
+                "strategy": kwargs.get("strategy", self.strategy),
+                "chunking_strategy": kwargs.get(
+                    "chunking_strategy", self.chunking_strategy
+                ),
+            }
+
+            # Load and chunk document
+            reader = UnstructuredReader(
+                allowed_metadata_types=self.allowed_metadata_types,
+            )
+            documents = reader.load_data(
+                unstructured_kwargs={
+                    "file": file_stream,
+                    "metadata_filename": path.name,
+                    "content_type": content_type,
+                    **unstructured_args,
+                },
+                split_documents=True,
+                extra_info=extra_metadata or {},
+            )
+
+            if not documents:
+                raise ChunkingError(f"No chunks created from file: {file_path}")
+
+            # Calculate statistics
+            total_characters = sum(len(doc.text or "") for doc in documents)
+            total_chunks = len(documents)
+            total_pages = self._extract_total_pages(documents)
+
+            return ChunkResult(
+                documents=documents,
+                metadata=ChunkMetadata(
+                    filename=path.name,
+                    filetype=content_type,
+                    size_in_bytes=size_in_bytes,
+                ),
+                total_chunks=total_chunks,
+                total_characters=total_characters,
+                total_pages=total_pages,
+                extra_metadata=extra_metadata or {},
+            )
+
+        except ChunkingError:
+            raise
+        except Exception as e:
+            raise ChunkingError(f"Failed to chunk file: {str(e)}") from e
+
+    async def chunk_url(
+        self,
+        url: str,
+        filename: str,
+        extra_metadata: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> ChunkResult:
+        """
+        Download and chunk a file from URL.
+
+        Args:
+            url: URL to download file from
+            filename: Filename for metadata and type detection
+            extra_metadata: Additional metadata to attach to chunks
+            **kwargs: Override strategy/chunking_strategy
+
+        Returns:
+            ChunkResult with documents and metadata
+
+        Raises:
+            ChunkingError: If download or chunking fails
+        """
+        try:
+            # Import here to avoid requiring unstructured for other operations
+            from llama_index.readers.file import UnstructuredReader
+            from unstructured.file_utils.filetype import detect_filetype
+
+            # Download file
+            async with httpx.AsyncClient() as client:
+                response = await client.get(url, follow_redirects=True)
+                response.raise_for_status()
+                file_bytes = response.content
+
+            size_in_bytes = len(file_bytes)
+            file_stream = BytesIO(file_bytes)
+
+            # Detect file type
+            content_type = detect_filetype(
+                file=file_stream,
+                metadata_file_path=filename,
+            ).mime_type
+
+            # Reset stream position after detection
+            file_stream.seek(0)
+
+            # Build unstructured arguments
+            unstructured_args = {
+                "strategy": kwargs.get("strategy", self.strategy),
+                "chunking_strategy": kwargs.get(
+                    "chunking_strategy", self.chunking_strategy
+                ),
+            }
+
+            # Load and chunk document
+            reader = UnstructuredReader(
+                allowed_metadata_types=self.allowed_metadata_types,
+            )
+            documents = reader.load_data(
+                unstructured_kwargs={
+                    "file": file_stream,
+                    "metadata_filename": filename,
+                    "content_type": content_type,
+                    **unstructured_args,
+                },
+                split_documents=True,
+                extra_info=extra_metadata or {},
+            )
+
+            if not documents:
+                raise ChunkingError(f"No chunks created from URL: {url}")
+
+            # Calculate statistics
+            total_characters = sum(len(doc.text or "") for doc in documents)
+            total_chunks = len(documents)
+            total_pages = self._extract_total_pages(documents)
+
+            return ChunkResult(
+                documents=documents,
+                metadata=ChunkMetadata(
+                    filename=filename,
+                    filetype=content_type,
+                    size_in_bytes=size_in_bytes,
+                ),
+                total_chunks=total_chunks,
+                total_characters=total_characters,
+                total_pages=total_pages,
+                extra_metadata=extra_metadata or {},
+            )
+
+        except httpx.HTTPError as e:
+            raise ChunkingError(f"Failed to download file from URL: {str(e)}") from e
+        except ChunkingError:
+            raise
+        except Exception as e:
+            raise ChunkingError(f"Failed to chunk URL: {str(e)}") from e
+
+    def _extract_total_pages(self, documents: List[Any]) -> Optional[int]:
+        """
+        Extract total page count from documents.
+
+        Matches partition-api logic: finds max page_number in metadata.
+        """
+        total_pages = None
+
+        for doc in documents:
+            if hasattr(doc, "metadata") and isinstance(doc.metadata, dict):
+                page_number = doc.metadata.get("page_number")
+                if page_number is not None:
+                    total_pages = max(
+                        total_pages if total_pages is not None else 0,
+                        page_number,
+                    )
+
+        return total_pages

maktaba/citation/formatter.py

@@ -0,0 +1,37 @@
+"""Citation formatter for search results."""
+
+from typing import Dict, List
+
+from ..models import SearchResult
+
+
+def format_with_citations(
+    results: List[SearchResult], top_k: int = 10
+) -> Dict[str, object]:
+    """
+    Produce formatted context and citation entries from search results.
+
+    Returns a dict with:
+      - formatted_context: str
+      - citations: List[Dict]
+    """
+    limited = results[:top_k]
+
+    blocks: List[str] = []
+    citations: List[Dict[str, object]] = []
+
+    for idx, res in enumerate(limited, start=1):
+        text = res.text or ""
+        blocks.append(f"[{idx}]: {text}")
+        citations.append(
+            {
+                "index": idx,
+                "id": res.id,
+                "document_id": res.document_id,
+                "chunk_id": res.chunk_id,
+                "metadata": res.metadata,
+            }
+        )
+
+    formatted_context = "\n\n".join(blocks)
+    return {"formatted_context": formatted_context, "citations": citations}

maktaba/embedding/__init__.py

@@ -0,0 +1,9 @@
+"""Embedding providers for Maktaba."""
+
+from .base import BaseEmbedder
+from .openai import OpenAIEmbedder
+
+__all__ = [
+    "BaseEmbedder",
+    "OpenAIEmbedder",
+]

maktaba/embedding/base.py

@@ -0,0 +1,72 @@
+"""Base interface for embedding providers."""
+
+from abc import ABC, abstractmethod
+from typing import List, Literal
+
+from ..models import EmbeddingVector
+
+
+class BaseEmbedder(ABC):
+    """
+    Abstract base class for text embedding providers.
+
+    Design principles:
+    1. Batch-first: embed_batch is the primary method
+    2. Input types: Support 'document' vs 'query' for Voyage AI
+    3. Async-first: All methods are async
+    4. Dimension aware: Each embedder knows its output dimension
+    """
+
+    @abstractmethod
+    async def embed_batch(
+        self,
+        texts: List[str],
+        input_type: Literal["document", "query"] = "document",
+    ) -> List[EmbeddingVector]:
+        """
+        Embed multiple texts in a single batch (primary method).
+
+        Args:
+            texts: List of texts to embed
+            input_type: Type of input - 'document' for indexing, 'query' for search
+                       (required by Voyage AI, ignored by OpenAI/others)
+
+        Returns:
+            List of embedding vectors, one per input text
+
+        Raises:
+            EmbeddingError: If embedding fails
+        """
+        pass
+
+    async def embed_text(
+        self,
+        text: str,
+        input_type: Literal["document", "query"] = "document",
+    ) -> EmbeddingVector:
+        """
+        Embed a single text (convenience wrapper).
+
+        This is implemented as a wrapper around embed_batch for consistency.
+        """
+        results = await self.embed_batch([text], input_type=input_type)
+        return results[0]
+
+    @property
+    @abstractmethod
+    def dimension(self) -> int:
+        """
+        Get the dimension of embeddings produced by this embedder.
+
+        Examples:
+            - text-embedding-3-large: 3072
+            - text-embedding-3-small: 1536
+            - voyage-3-large: 1024
+        """
+        pass
+
+    @property
+    @abstractmethod
+    def model(self) -> str:
+        """Get the model name/identifier."""
+        pass