rag-forge-core 0.1.0__tar.gz

rag_forge_core-0.1.0/.gitignore

@@ -0,0 +1,62 @@
+# Dependencies
+node_modules/
+.pnpm-store/
+
+# Build outputs
+dist/
+build/
+*.tsbuildinfo
+
+# Turborepo
+.turbo/
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+*.egg
+.venv/
+.python-version-local
+
+# Python tools
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+htmlcov/
+.coverage
+.coverage.*
+
+# Environment variables
+.env
+.env.local
+.env.*.local
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+desktop.ini
+
+# Test & coverage
+coverage/
+*.lcov
+
+# Logs
+*.log
+npm-debug.log*
+pnpm-debug.log*
+
+.claude/
+
+# Next.js
+apps/*/.next
+apps/*/out
+apps/*/next-env.d.ts
+.vercel

rag_forge_core-0.1.0/PKG-INFO

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: rag-forge-core
+Version: 0.1.0
+Summary: RAG pipeline primitives: ingestion, retrieval, context management, and security
+Project-URL: Homepage, https://github.com/hallengray/rag-forge
+Project-URL: Repository, https://github.com/hallengray/rag-forge
+Project-URL: Issues, https://github.com/hallengray/rag-forge/issues
+Project-URL: Documentation, https://github.com/hallengray/rag-forge#readme
+Author: Femi Adedayo
+License-Expression: MIT
+Keywords: chunking,embedding,llm,pipeline,rag,retrieval-augmented-generation
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: bm25s>=0.2
+Requires-Dist: lxml>=5.0
+Requires-Dist: openai>=1.30
+Requires-Dist: opentelemetry-api>=1.20
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pymupdf>=1.24
+Requires-Dist: qdrant-client>=1.9
+Requires-Dist: rich>=13.0
+Requires-Dist: tiktoken>=0.7
+Provides-Extra: cohere
+Requires-Dist: cohere>=5.0; extra == 'cohere'
+Provides-Extra: local
+Requires-Dist: sentence-transformers>=3.0; extra == 'local'
+Provides-Extra: presidio
+Requires-Dist: presidio-analyzer>=2.2; extra == 'presidio'
+Provides-Extra: redis
+Requires-Dist: redis>=5.0; extra == 'redis'
+Description-Content-Type: text/markdown
+
+# rag-forge-core
+
+RAG pipeline primitives for the RAG-Forge toolkit: ingestion, chunking, retrieval, context management, and security.
+
+## Installation
+
+```bash
+pip install rag-forge-core
+```
+
+## Usage
+
+This package provides the building blocks used by the `rag-forge` CLI. For end-user usage, see the [main RAG-Forge documentation](https://github.com/hallengray/rag-forge#readme).
+
+```python
+from rag_forge_core.chunking.factory import create_chunker
+from rag_forge_core.chunking.config import ChunkConfig
+
+chunker = create_chunker(ChunkConfig(strategy="recursive", chunk_size=512))
+chunks = chunker.chunk("Some long document text...", source="doc.md")
+```
+
+## Modules
+
+- `rag_forge_core.chunking` — Five chunking strategies (recursive, fixed, semantic, structural, llm-driven)
+- `rag_forge_core.retrieval` — Dense, sparse, and hybrid retrieval with reranking
+- `rag_forge_core.security` — InputGuard, OutputGuard, PII scanning, prompt injection detection
+- `rag_forge_core.context` — Contextual enrichment and semantic caching
+- `rag_forge_core.plugins` — Plugin registry for custom extensions
+
+## License
+
+MIT

rag_forge_core-0.1.0/README.md

@@ -0,0 +1,33 @@
+# rag-forge-core
+
+RAG pipeline primitives for the RAG-Forge toolkit: ingestion, chunking, retrieval, context management, and security.
+
+## Installation
+
+```bash
+pip install rag-forge-core
+```
+
+## Usage
+
+This package provides the building blocks used by the `rag-forge` CLI. For end-user usage, see the [main RAG-Forge documentation](https://github.com/hallengray/rag-forge#readme).
+
+```python
+from rag_forge_core.chunking.factory import create_chunker
+from rag_forge_core.chunking.config import ChunkConfig
+
+chunker = create_chunker(ChunkConfig(strategy="recursive", chunk_size=512))
+chunks = chunker.chunk("Some long document text...", source="doc.md")
+```
+
+## Modules
+
+- `rag_forge_core.chunking` — Five chunking strategies (recursive, fixed, semantic, structural, llm-driven)
+- `rag_forge_core.retrieval` — Dense, sparse, and hybrid retrieval with reranking
+- `rag_forge_core.security` — InputGuard, OutputGuard, PII scanning, prompt injection detection
+- `rag_forge_core.context` — Contextual enrichment and semantic caching
+- `rag_forge_core.plugins` — Plugin registry for custom extensions
+
+## License
+
+MIT

rag_forge_core-0.1.0/pyproject.toml

@@ -0,0 +1,49 @@
+[project]
+name = "rag-forge-core"
+version = "0.1.0"
+description = "RAG pipeline primitives: ingestion, retrieval, context management, and security"
+requires-python = ">=3.11"
+license = "MIT"
+authors = [{ name = "Femi Adedayo" }]
+keywords = ["rag", "retrieval-augmented-generation", "llm", "pipeline", "chunking", "embedding"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+readme = "README.md"
+dependencies = [
+    "pydantic>=2.0",
+    "rich>=13.0",
+    "tiktoken>=0.7",
+    "pymupdf>=1.24",
+    "beautifulsoup4>=4.12",
+    "lxml>=5.0",
+    "openai>=1.30",
+    "qdrant-client>=1.9",
+    "bm25s>=0.2",
+    "opentelemetry-api>=1.20",
+]
+
+[project.urls]
+Homepage = "https://github.com/hallengray/rag-forge"
+Repository = "https://github.com/hallengray/rag-forge"
+Issues = "https://github.com/hallengray/rag-forge/issues"
+Documentation = "https://github.com/hallengray/rag-forge#readme"
+
+[project.optional-dependencies]
+local = ["sentence-transformers>=3.0"]
+cohere = ["cohere>=5.0"]
+presidio = ["presidio-analyzer>=2.2"]
+redis = ["redis>=5.0"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/rag_forge_core"]

rag_forge_core-0.1.0/src/rag_forge_core/__init__.py

@@ -0,0 +1,3 @@
+"""RAG-Forge Core: RAG pipeline primitives for ingestion, retrieval, context, and security."""
+
+__version__ = "0.1.0"

rag_forge_core-0.1.0/src/rag_forge_core/chunking/__init__.py

@@ -0,0 +1,7 @@
+"""Chunking strategies for document splitting."""
+
+from rag_forge_core.chunking.base import ChunkStrategy
+from rag_forge_core.chunking.config import ChunkConfig
+from rag_forge_core.chunking.factory import UnsupportedStrategyError, create_chunker
+
+__all__ = ["ChunkConfig", "ChunkStrategy", "UnsupportedStrategyError", "create_chunker"]

rag_forge_core-0.1.0/src/rag_forge_core/chunking/base.py

@@ -0,0 +1,53 @@
+"""Abstract base class for all chunking strategies."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+from rag_forge_core.chunking.config import ChunkConfig
+
+
+@dataclass
+class Chunk:
+    """A single chunk of text with metadata."""
+
+    text: str
+    chunk_index: int
+    source_document: str
+    strategy_used: str
+    parent_section: str | None = None
+    overlap_tokens: int = 0
+    metadata: dict[str, str | int | float] | None = None
+
+
+@dataclass
+class ChunkStats:
+    """Statistics about a chunking operation."""
+
+    total_chunks: int
+    avg_chunk_size: int
+    min_chunk_size: int
+    max_chunk_size: int
+    total_tokens: int
+
+
+class ChunkStrategy(ABC):
+    """Abstract base class that all chunking strategies must implement.
+
+    Ensures strategies are interchangeable and the evaluation engine
+    can compare performance across strategies on the same dataset.
+    """
+
+    def __init__(self, config: ChunkConfig) -> None:
+        self.config = config
+
+    @abstractmethod
+    def chunk(self, text: str, source: str) -> list[Chunk]:
+        """Split text into chunks according to the strategy."""
+
+    @abstractmethod
+    def preview(self, text: str, source: str) -> list[Chunk]:
+        """Dry-run: show chunk boundaries without committing to storage."""
+
+    @abstractmethod
+    def stats(self, chunks: list[Chunk]) -> ChunkStats:
+        """Compute statistics about the chunking result."""

rag_forge_core-0.1.0/src/rag_forge_core/chunking/config.py

@@ -0,0 +1,50 @@
+"""Chunk configuration with fail-fast validation."""
+
+from typing import Self
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class ChunkConfig(BaseModel):
+    """Configuration for chunking strategies. Validated at init time (fail-fast)."""
+
+    strategy: str = Field(
+        default="recursive",
+        pattern=r"^(fixed|recursive|semantic|structural|llm-driven)$",
+        description="Chunking strategy: fixed, recursive, semantic, structural, llm-driven",
+    )
+    chunk_size: int = Field(
+        default=512,
+        ge=64,
+        le=8192,
+        description="Target chunk size in tokens",
+    )
+    overlap_ratio: float = Field(
+        default=0.1,
+        ge=0.0,
+        le=0.5,
+        description="Overlap ratio between consecutive chunks (0.0 to 0.5)",
+    )
+    separators: list[str] = Field(
+        default_factory=lambda: ["\n\n", "\n", ". ", " "],
+        description="Separator hierarchy for recursive splitting",
+    )
+    cosine_threshold: float = Field(
+        default=0.75,
+        ge=0.0,
+        le=1.0,
+        description="Cosine similarity threshold for semantic chunking",
+    )
+
+    @model_validator(mode="after")
+    def validate_overlap(self) -> Self:
+        overlap_tokens = int(self.chunk_size * self.overlap_ratio)
+        if overlap_tokens >= self.chunk_size:
+            msg = f"Overlap ({overlap_tokens} tokens) must be less than chunk_size ({self.chunk_size})"
+            raise ValueError(msg)
+        return self
+
+    @property
+    def overlap_tokens(self) -> int:
+        """Calculate the overlap in tokens."""
+        return int(self.chunk_size * self.overlap_ratio)

rag_forge_core-0.1.0/src/rag_forge_core/chunking/factory.py

@@ -0,0 +1,71 @@
+"""Factory function for creating chunker instances by strategy name."""
+
+from rag_forge_core.chunking.base import ChunkStrategy
+from rag_forge_core.chunking.config import ChunkConfig
+from rag_forge_core.embedding.base import EmbeddingProvider
+from rag_forge_core.generation.base import GenerationProvider
+
+
+class UnsupportedStrategyError(ValueError):
+    """Raised when an unknown chunking strategy is requested."""
+
+
+def create_chunker(
+    config: ChunkConfig,
+    embedder: EmbeddingProvider | None = None,
+    generator: GenerationProvider | None = None,
+) -> ChunkStrategy:
+    """Create a chunker instance for the given strategy.
+
+    Args:
+        config: Chunk configuration with strategy name.
+        embedder: Required for "semantic" strategy.
+        generator: Required for "llm-driven" strategy.
+
+    Returns:
+        A ChunkStrategy instance ready to use.
+
+    Raises:
+        ValueError: If a required dependency is missing.
+        UnsupportedStrategyError: If the strategy name is unknown.
+    """
+    strategy = config.strategy
+
+    if strategy == "recursive":
+        from rag_forge_core.chunking.recursive import RecursiveChunker
+
+        return RecursiveChunker(config)
+
+    if strategy == "fixed":
+        from rag_forge_core.chunking.fixed import FixedSizeChunker
+
+        return FixedSizeChunker(config)
+
+    if strategy == "structural":
+        from rag_forge_core.chunking.structural import StructuralChunker
+
+        return StructuralChunker(config)
+
+    if strategy == "semantic":
+        if embedder is None:
+            msg = "Semantic chunking requires an embedder. Pass embedder= to create_chunker()."
+            raise ValueError(msg)
+        from rag_forge_core.chunking.semantic import SemanticChunker
+
+        return SemanticChunker(config=config, embedder=embedder)
+
+    if strategy == "llm-driven":
+        if generator is None:
+            msg = (
+                "LLM-driven chunking requires a generator. "
+                "Pass generator= to create_chunker()."
+            )
+            raise ValueError(msg)
+        from rag_forge_core.chunking.llm_driven import LLMDrivenChunker
+
+        return LLMDrivenChunker(config=config, generator=generator)
+
+    raise UnsupportedStrategyError(
+        f"Unknown chunking strategy: {strategy!r}. "
+        "Supported: 'recursive', 'fixed', 'structural', 'semantic', 'llm-driven'."
+    )

rag_forge_core-0.1.0/src/rag_forge_core/chunking/fixed.py

@@ -0,0 +1,87 @@
+"""Fixed-size chunking strategy.
+
+Splits text by token count with configurable overlap.
+Best for structured data and baseline comparisons.
+PRD default: 512 tokens, 10-20% overlap.
+"""
+
+import tiktoken
+
+from rag_forge_core.chunking.base import Chunk, ChunkStats, ChunkStrategy
+from rag_forge_core.chunking.config import ChunkConfig
+
+_ENCODING = tiktoken.get_encoding("cl100k_base")
+
+
+def _token_count(text: str) -> int:
+    """Count tokens using tiktoken cl100k_base encoding."""
+    return len(_ENCODING.encode(text))
+
+
+class FixedSizeChunker(ChunkStrategy):
+    """Split text into fixed-size token windows with overlap.
+
+    Each window is exactly `chunk_size` tokens (or fewer for the last window).
+    Consecutive windows share `overlap_tokens` tokens from the end of the
+    previous window, giving the model context across chunk boundaries.
+    """
+
+    def __init__(self, config: ChunkConfig | None = None) -> None:
+        super().__init__(config or ChunkConfig(strategy="fixed"))
+
+    def chunk(self, text: str, source: str) -> list[Chunk]:
+        """Split text into fixed-size token windows with overlap."""
+        if not text.strip():
+            return []
+
+        tokens = _ENCODING.encode(text)
+        chunk_size = self.config.chunk_size
+        overlap = self.config.overlap_tokens
+        step = max(1, chunk_size - overlap)
+
+        chunks: list[Chunk] = []
+        idx = 0
+        start = 0
+
+        while start < len(tokens):
+            end = min(start + chunk_size, len(tokens))
+            chunk_text = _ENCODING.decode(tokens[start:end])
+            chunks.append(
+                Chunk(
+                    text=chunk_text,
+                    chunk_index=idx,
+                    source_document=source,
+                    strategy_used="fixed",
+                    overlap_tokens=overlap if idx > 0 else 0,
+                )
+            )
+            idx += 1
+            start += step
+            if end == len(tokens):
+                break
+
+        return chunks
+
+    def preview(self, text: str, source: str) -> list[Chunk]:
+        """Dry-run: show chunk boundaries without committing to storage."""
+        return self.chunk(text, source)
+
+    def stats(self, chunks: list[Chunk]) -> ChunkStats:
+        """Compute statistics using tiktoken token counts."""
+        if not chunks:
+            return ChunkStats(
+                total_chunks=0,
+                avg_chunk_size=0,
+                min_chunk_size=0,
+                max_chunk_size=0,
+                total_tokens=0,
+            )
+
+        sizes = [_token_count(c.text) for c in chunks]
+        return ChunkStats(
+            total_chunks=len(chunks),
+            avg_chunk_size=sum(sizes) // len(sizes),
+            min_chunk_size=min(sizes),
+            max_chunk_size=max(sizes),
+            total_tokens=sum(sizes),
+        )

rag_forge_core-0.1.0/src/rag_forge_core/chunking/llm_driven.py

@@ -0,0 +1,163 @@
+"""LLM-driven chunking strategy.
+
+Uses a small LLM to identify meaningful boundary points in text.
+The LLM receives numbered sentences and returns boundary indices as JSON.
+Falls back to size-based splitting when the LLM response is unparseable.
+PRD recommendation: Claude Haiku / GPT-4o-mini for cost efficiency.
+"""
+
+import json
+import logging
+
+import tiktoken
+
+from rag_forge_core.chunking.base import Chunk, ChunkStats, ChunkStrategy
+from rag_forge_core.chunking.config import ChunkConfig
+from rag_forge_core.generation.base import GenerationProvider
+
+_ENCODING = tiktoken.get_encoding("cl100k_base")
+_LOG = logging.getLogger(__name__)
+
+_BOUNDARY_PROMPT = """You are a document chunking assistant. Given the following numbered sentences, identify the indices where topic boundaries occur. A boundary means the content shifts to a different topic or subtopic.
+
+Return a JSON array of sentence indices (0-based) where splits should happen. For example: [3, 7, 12] means split BEFORE sentences 3, 7, and 12.
+
+If there are no clear boundaries, return an empty array: []
+
+Sentences:
+{sentences}"""
+
+
+def _token_count(text: str) -> int:
+    return len(_ENCODING.encode(text))
+
+
+def _split_into_sentences(text: str) -> list[str]:
+    """Split text into sentences for LLM analysis."""
+    # Normalise Windows line endings
+    text = text.replace("\r\n", "\n")
+    paragraphs = text.split("\n\n")
+    sentences: list[str] = []
+    for para in paragraphs:
+        para = para.strip()
+        if not para:
+            continue
+        parts = para.replace(". ", ".\n").replace("? ", "?\n").replace("! ", "!\n").split("\n")
+        for part in parts:
+            part = part.strip()
+            if part:
+                sentences.append(part)
+    return sentences
+
+
+class LLMDrivenChunker(ChunkStrategy):
+    """Use an LLM to identify semantic boundaries in text.
+
+    Sends numbered sentences to the LLM and asks for boundary indices.
+    Falls back to size-based splitting on LLM failure or invalid response.
+    """
+
+    def __init__(self, config: ChunkConfig, generator: GenerationProvider) -> None:
+        super().__init__(config)
+        self._generator = generator
+
+    def chunk(self, text: str, source: str) -> list[Chunk]:
+        if not text.strip():
+            return []
+
+        sentences = _split_into_sentences(text)
+        if not sentences:
+            return []
+
+        if len(sentences) == 1:
+            return [
+                Chunk(
+                    text=sentences[0],
+                    chunk_index=0,
+                    source_document=source,
+                    strategy_used="llm-driven",
+                )
+            ]
+
+        boundaries = self._get_boundaries(sentences)
+        groups = self._apply_boundaries(sentences, boundaries)
+
+        return [
+            Chunk(
+                text=" ".join(group),
+                chunk_index=idx,
+                source_document=source,
+                strategy_used="llm-driven",
+            )
+            for idx, group in enumerate(groups)
+        ]
+
+    def preview(self, text: str, source: str) -> list[Chunk]:
+        return self.chunk(text, source)
+
+    def stats(self, chunks: list[Chunk]) -> ChunkStats:
+        if not chunks:
+            return ChunkStats(
+                total_chunks=0,
+                avg_chunk_size=0,
+                min_chunk_size=0,
+                max_chunk_size=0,
+                total_tokens=0,
+            )
+        sizes = [_token_count(c.text) for c in chunks]
+        return ChunkStats(
+            total_chunks=len(chunks),
+            avg_chunk_size=sum(sizes) // len(sizes),
+            min_chunk_size=min(sizes),
+            max_chunk_size=max(sizes),
+            total_tokens=sum(sizes),
+        )
+
+    def _get_boundaries(self, sentences: list[str]) -> list[int]:
+        """Ask the LLM for boundary indices. Returns sorted list of split points."""
+        numbered = "\n".join(f"[{i}] {s}" for i, s in enumerate(sentences))
+        prompt = _BOUNDARY_PROMPT.format(sentences=numbered)
+
+        try:
+            response = self._generator.generate(
+                system_prompt="You are a document analysis assistant. Respond only with valid JSON.",
+                user_prompt=prompt,
+            )
+            boundaries = json.loads(response)
+            if not isinstance(boundaries, list):
+                _LOG.warning(
+                    "LLM returned non-list: %s, falling back to size-based splitting",
+                    type(boundaries),
+                )
+                return self._fallback_boundaries(sentences)
+            # Empty list from LLM means "no boundaries" — return as-is (single group)
+            if len(boundaries) == 0:
+                return []
+            valid = sorted({int(b) for b in boundaries if 0 < int(b) < len(sentences)})
+            return valid if valid else self._fallback_boundaries(sentences)
+        except (json.JSONDecodeError, ValueError, TypeError) as e:
+            _LOG.warning("LLM boundary parsing failed: %s, falling back", e)
+            return self._fallback_boundaries(sentences)
+
+    def _fallback_boundaries(self, sentences: list[str]) -> list[int]:
+        """Size-based fallback: split every chunk_size tokens."""
+        boundaries: list[int] = []
+        current_tokens = 0
+        for i, sentence in enumerate(sentences):
+            current_tokens += _token_count(sentence)
+            if current_tokens >= self.config.chunk_size and i > 0:
+                boundaries.append(i)
+                current_tokens = _token_count(sentence)
+        return boundaries
+
+    def _apply_boundaries(self, sentences: list[str], boundaries: list[int]) -> list[list[str]]:
+        """Split sentences into groups at the given boundary indices."""
+        if not boundaries:
+            return [sentences]
+        groups: list[list[str]] = []
+        prev = 0
+        for boundary in boundaries:
+            groups.append(sentences[prev:boundary])
+            prev = boundary
+        groups.append(sentences[prev:])
+        return [g for g in groups if g]