ragcheck-cli 0.2.1__tar.gz → 0.2.3__tar.gz

{ragcheck_cli-0.2.1 → ragcheck_cli-0.2.3}/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.2.2] - 2026-06-06
+
+### Fixed
+- **Prompt size reduction** — Shrunk auto-QA prompt from ~1500 to ~800 chars + compact instructions. Stays well under Groq 6000 TPM limit.
+- **Gemini 3+ compatibility** — Skip deprecated `temperature`/`top_p`/`top_k` params for `gemini/gemini-3.*` models to suppress deprecation warnings.
+- **Better Q&A parsing** — Accept both `Q:/A:` and `Question:/Answer:` formats from LLM responses.
+
 ## [0.2.0] - 2026-06-04
 
 ### Added
@@ -33,4 +40,4 @@ All notable changes to this project will be documented in this file.
 - Recommendation engine with decision tree
 - Beautiful HTML reports (single file, no server)
 - CI/CD mode with GitHub Actions
-- PDF/PNG export via Playwright
+- PDF/PNG export via Playwright

{ragcheck_cli-0.2.1/ragcheck_cli.egg-info → ragcheck_cli-0.2.3}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: ragcheck-cli
-Version: 0.2.1
-Summary: Lighthouse for RAG systems — diagnose and fix your retrieval pipeline
+Version: 0.2.3
+Summary: Lighthouse for RAG systems - diagnose and fix your retrieval pipeline
 Author-email: Pranay Mane <pranaymane78@gmail.com>
 License: MIT
 Project-URL: Homepage, https://github.com/pranay7863/ragcheck
@@ -30,6 +30,7 @@ Requires-Dist: sentence-transformers>=2.2.0
 Requires-Dist: chromadb>=0.4.0
 Requires-Dist: PyYAML>=6.0
 Requires-Dist: transformers>=4.30.0
+Requires-Dist: pdfplumber>=0.10.0
 Provides-Extra: pdf
 Requires-Dist: PyPDF2>=3.0.0; extra == "pdf"
 Provides-Extra: export
@@ -180,6 +181,7 @@ MIT — see [LICENSE](LICENSE)
 ## Roadmap
 
 - [x] v0.2.0 — Offline reports, NLI faithfulness, scaled auto-QA, chunk viz
+- [x] v0.2.2 — Prompt size fix, Gemini 3+ support, Groq TPM compliance
 - [ ] v0.3.0 — More vector DBs (Pinecone, Weaviate)
 - [ ] v0.3.0 — SaaS API for teams
 - [ ] v0.4.0 — Enterprise features (SSO, audit logs)

{ragcheck_cli-0.2.1 → ragcheck_cli-0.2.3}/README.md

@@ -140,6 +140,7 @@ MIT — see [LICENSE](LICENSE)
 ## Roadmap
 
 - [x] v0.2.0 — Offline reports, NLI faithfulness, scaled auto-QA, chunk viz
+- [x] v0.2.2 — Prompt size fix, Gemini 3+ support, Groq TPM compliance
 - [ ] v0.3.0 — More vector DBs (Pinecone, Weaviate)
 - [ ] v0.3.0 — SaaS API for teams
 - [ ] v0.4.0 — Enterprise features (SSO, audit logs)

{ragcheck_cli-0.2.1 → ragcheck_cli-0.2.3}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "ragcheck-cli"
-version = "0.2.1"
-description = "Lighthouse for RAG systems — diagnose and fix your retrieval pipeline"
+version = "0.2.3"
+description = "Lighthouse for RAG systems - diagnose and fix your retrieval pipeline"
 readme = "README.md"
 license = {text = "MIT"}
 requires-python = ">=3.10"
@@ -31,6 +31,7 @@ dependencies = [
     "chromadb>=0.4.0",
     "PyYAML>=6.0",
     "transformers>=4.30.0",
+	"pdfplumber>=0.10.0",
 ]
 
 [project.optional-dependencies]

{ragcheck_cli-0.2.1 → ragcheck_cli-0.2.3}/ragcheck/__init__.py

@@ -1,3 +1,3 @@
 """ragcheck — Lighthouse for RAG systems."""
 
-__version__ = "0.2.1"
+__version__ = "0.2.3"

ragcheck_cli-0.2.3/ragcheck/analyzers/chunkers.py

@@ -0,0 +1,193 @@
+"""Document chunking strategies."""
+
+from typing import List, Dict, Any
+import re
+
+from ragcheck.core.config import Config
+
+
+class Chunk:
+    """A text chunk with metadata."""
+
+    def __init__(self, text: str, metadata: Dict[str, Any] = None):
+        self.text = text
+        self.metadata = metadata or {}
+        self.start = metadata.get("start", 0)
+        self.end = metadata.get("end", len(text))
+
+
+class Chunker:
+    """Chunk documents using configurable strategies."""
+
+    def __init__(self, config: Config):
+        self.config = config
+
+    def chunk(self, document: Dict[str, Any]) -> List[Chunk]:
+        """Chunk a document based on configured strategy."""
+        text = document.get("text", "")
+        if not text:
+            return []
+
+        if self.config.chunk_strategy == "recursive":
+            return self._recursive_chunk(text, document)
+        elif self.config.chunk_strategy == "semantic":
+            return self._semantic_chunk(text, document)
+        else:
+            return self._fixed_chunk(text, document)
+
+    def _recursive_chunk(self, text: str, document: Dict[str, Any]) -> List[Chunk]:
+        """Recursively split by separators, preferring larger chunks.
+
+        For legal documents, we use section/paragraph boundaries first,
+        then fall back to sentence boundaries, then fixed size.
+        """
+        chunks = []
+
+        # Legal document separators (in order of preference)
+        separators = [
+            r"
+
+SECTION\s+\d+",  # SECTION headers
+            r"
+
+CHAPTER\s+[IVX]+",  # CHAPTER headers
+            r"
+
+\d+\s+\.",  # Numbered paragraphs
+            r"
+
+",  # Double newline (paragraphs)
+            r"
+",  # Single newline
+            r"\.\s+",  # Sentence end
+        ]
+
+        def split_recursive(text: str, sep_index: int) -> List[str]:
+            if sep_index >= len(separators):
+                # Final fallback: fixed size
+                return self._split_fixed(text)
+
+            sep = separators[sep_index]
+            parts = re.split(f"(?={sep})", text)
+
+            result = []
+            current = ""
+            for part in parts:
+                if not part.strip():
+                    continue
+
+                if len(current) + len(part) <= self.config.chunk_size:
+                    current += part
+                else:
+                    if current:
+                        result.append(current)
+                    # If single part is too big, recurse with next separator
+                    if len(part) > self.config.chunk_size:
+                        result.extend(split_recursive(part, sep_index + 1))
+                    else:
+                        current = part
+
+            if current:
+                result.append(current)
+
+            return result
+
+        parts = split_recursive(text, 0)
+
+        # Apply overlap
+        for i, part in enumerate(parts):
+            start = max(0, i * self.config.chunk_size - i * self.config.chunk_overlap)
+            end = start + len(part)
+            chunks.append(Chunk(
+                text=part.strip(),
+                metadata={
+                    **document,
+                    "chunk_index": i,
+                    "start": start,
+                    "end": end,
+                }
+            ))
+
+        return chunks
+
+    def _semantic_chunk(self, text: str, document: Dict[str, Any]) -> List[Chunk]:
+        """Semantic chunking using sentence boundaries."""
+        # Simple implementation: split by sentences, group semantically
+        sentences = re.split(r"(?<=[.!?])\s+", text)
+
+        chunks = []
+        current = []
+        current_len = 0
+
+        for sent in sentences:
+            sent_len = len(sent)
+            if current_len + sent_len > self.config.chunk_size and current:
+                chunk_text = " ".join(current)
+                chunks.append(Chunk(
+                    text=chunk_text,
+                    metadata={**document, "chunk_index": len(chunks)}
+                ))
+                # Keep overlap
+                overlap_sents = []
+                overlap_len = 0
+                for s in reversed(current):
+                    if overlap_len + len(s) > self.config.chunk_overlap:
+                        break
+                    overlap_sents.insert(0, s)
+                    overlap_len += len(s)
+                current = overlap_sents
+                current_len = overlap_len
+
+            current.append(sent)
+            current_len += sent_len
+
+        if current:
+            chunks.append(Chunk(
+                text=" ".join(current),
+                metadata={**document, "chunk_index": len(chunks)}
+            ))
+
+        return chunks
+
+    def _fixed_chunk(self, text: str, document: Dict[str, Any]) -> List[Chunk]:
+        """Fixed-size chunking with overlap."""
+        chunks = []
+        step = self.config.chunk_size - self.config.chunk_overlap
+
+        for i in range(0, len(text), step):
+            chunk_text = text[i:i + self.config.chunk_size]
+            if len(chunk_text) < 100:  # Skip tiny trailing chunks
+                break
+            chunks.append(Chunk(
+                text=chunk_text.strip(),
+                metadata={
+                    **document,
+                    "chunk_index": i // step,
+                    "start": i,
+                    "end": i + len(chunk_text),
+                }
+            ))
+
+        return chunks
+
+    def _split_fixed(self, text: str) -> List[str]:
+        """Split text into fixed-size pieces."""
+        return [
+            text[i:i + self.config.chunk_size]
+            for i in range(0, len(text), self.config.chunk_size)
+        ]
+
+
+def get_chunk_stats(chunks: List[Chunk]) -> Dict[str, Any]:
+    """Calculate chunk statistics."""
+    if not chunks:
+        return {}
+
+    lengths = [len(c.text) for c in chunks]
+    return {
+        "total_chunks": len(chunks),
+        "avg_length": sum(lengths) / len(lengths),
+        "min_length": min(lengths),
+        "max_length": max(lengths),
+        "median_length": sorted(lengths)[len(lengths) // 2],
+    }

ragcheck_cli-0.2.3/ragcheck/cli.py

@@ -0,0 +1,231 @@
+"""CLI for ragcheck."""
+
+import os
+import sys
+import warnings
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.panel import Panel
+from rich.text import Text
+
+from ragcheck.core.config import Config
+from ragcheck.core.config_loader import load_config
+from ragcheck.core.document_loader import DocumentLoader
+from ragcheck.core.embeddings import Embedder
+from ragcheck.core.vector_store import ChromaVectorStore, MemoryVectorStore
+from ragcheck.core.progress import Progress
+from ragcheck.analyzers.chunkers import Chunker
+from ragcheck.analyzers.failure_classifier import FailureClassifier
+from ragcheck.analyzers.recommender import Recommender
+from ragcheck.testers.auto_qa import AutoQA, generate_dummy_questions
+from ragcheck.testers.retrieval_tester import RetrievalTester
+from ragcheck.reports.generator import ReportGenerator
+from ragcheck.reports.html_report import HTMLReport
+
+app = typer.Typer(help="Lighthouse for RAG systems — diagnose and fix your retrieval pipeline")
+console = Console()
+
+
+def get_version() -> str:
+    try:
+        from ragcheck import __version__
+        return __version__
+    except ImportError:
+        return "0.2.2"
+
+
+@app.command()
+def init(
+    path: Optional[str] = typer.Argument(None, help="Project path"),
+    force: bool = typer.Option(False, "--force", "-f", help="Overwrite existing config"),
+):
+    """Initialize a new ragcheck project."""
+    target = Path(path) if path else Path(".")
+    target.mkdir(parents=True, exist_ok=True)
+
+    config_path = target / "ragcheck.yaml"
+    if config_path.exists() and not force:
+        console.print(f"[yellow]Config already exists at {config_path}[/yellow]")
+        raise typer.Exit(1)
+
+    config = Config()
+    # Write default config
+    config_path.write_text(f"""# ragcheck configuration
+embedding_model: {config.embedding_model}
+chunk_size: {config.chunk_size}
+chunk_overlap: {config.chunk_overlap}
+chunk_strategy: {config.chunk_strategy}
+vector_store: {config.vector_store}
+collection_name: {config.collection_name}
+top_k: {config.top_k}
+similarity_threshold: {config.similarity_threshold}
+answer_model: {config.answer_model}
+qa_model: {config.qa_model}
+max_qa_questions: {config.max_qa_questions}
+""")
+
+    console.print(f"[green]OK[/green] Created {config_path}")
+
+
+@app.command()
+def run(
+    docs: str = typer.Option(..., "--docs", "-d", help="Path to documents directory"),
+    query: Optional[str] = typer.Option(None, "--query", "-q", help="Single test query"),
+    config_path: Optional[str] = typer.Option(None, "--config", "-c", help="Config file path"),
+    output: str = typer.Option("ragcheck_report.html", "--output", "-o", help="Output file"),
+    generate_answers: bool = typer.Option(False, "--generate-answers", "-a", help="Generate answers with LLM"),
+    answer_model: Optional[str] = typer.Option(None, "--answer-model", help="Override answer model"),
+    top_k: Optional[int] = typer.Option(None, "--top-k", "-k", help="Override top_k"),
+    verbose: bool = typer.Option(False, "--verbose", "-v", help="Verbose output"),
+):
+    """Run RAG diagnostics on documents."""
+
+    # Load config
+    config = load_config(config_path) if config_path else Config()
+    if verbose:
+        config.verbose = True
+    if answer_model:
+        config.answer_model = answer_model
+    if top_k:
+        config.top_k = top_k
+
+    progress = Progress(verbose=verbose)
+
+    # Load documents
+    doc_path = Path(docs)
+    if not doc_path.exists():
+        console.print(f"[red]Error:[/red] Path not found: {doc_path}")
+        raise typer.Exit(1)
+
+    progress.start("Loading documents...")
+    loader = DocumentLoader()
+    documents = loader.load(doc_path)
+    progress.complete(f"Loaded {len(documents)} documents")
+
+    if not documents:
+        console.print("[red]No documents found[/red]")
+        raise typer.Exit(1)
+
+    # Chunk documents
+    progress.start("Chunking documents...")
+    chunker = Chunker(config)
+    chunks = []
+    for doc in documents:
+        chunks.extend(chunker.chunk(doc))
+    progress.complete(f"Created {len(chunks)} chunks")
+
+    # Build vector store
+    progress.start("Building vector store...")
+    if config.vector_store == "chroma":
+        store = ChromaVectorStore(config, progress)
+    else:
+        store = MemoryVectorStore(config, progress)
+    store.clear()
+    store.add([c.text for c in chunks])
+    progress.complete("Vector store ready")
+
+    # Generate or use test questions
+    questions = []
+    if query:
+        # Single query mode — create question from user query
+        questions.append(RetrievalTester.TestQuestion(
+            question=query,
+            expected_answer="",  # Will be filled by retrieval
+            source_chunks=[query],  # Use query as proxy for source
+            difficulty="user",
+        ))
+    else:
+        # Auto-generate questions from chunks
+        progress.start("Generating test questions...")
+        qa = AutoQA(config, progress)
+        questions = qa.generate([c.text for c in chunks])
+
+        if not questions:
+            # FALLBACK: Generate meaningful questions from chunk content
+            warnings.warn(
+                "LLM question generation failed. Using content-based fallback questions. "
+                "To use a real LLM:
+"
+                "  1. Get a free Groq key: https://console.groq.com/keys
+"
+                "  2. Run: set GROQ_API_KEY=your_key (Windows)
+"
+                "  3. Or ensure Ollama is running: ollama run phi3:mini",
+                UserWarning,
+            )
+            questions = generate_dummy_questions([c.text for c in chunks])
+
+        progress.complete(f"Generated {len(questions)} test questions")
+
+    # Test retrieval
+    progress.start("Testing retrieval...")
+    tester = RetrievalTester(config, store, progress)
+    retrieval_results = tester.test(questions)
+    progress.complete(
+        f"Retrieval: {retrieval_results['passed']}/{retrieval_results['total']} passed "
+        f"({retrieval_results['score']}%)"
+    )
+
+    # Generate answers if requested
+    answer_results = None
+    if generate_answers:
+        progress.start("Generating answers...")
+        # Answer generation logic here
+        progress.complete("Answers generated")
+
+    # Analyze failures
+    progress.start("Analyzing failures...")
+    classifier = FailureClassifier(config)
+    failures = classifier.classify(retrieval_results["details"])
+
+    recommender = Recommender(config)
+    recommendations = recommender.recommend(failures)
+    progress.complete(f"Found {len(failures)} failures, {len(recommendations)} recommendations")
+
+    # Generate report
+    progress.start("Generating report...")
+    report_data = {
+        "project_name": doc_path.name,
+        "config": config,
+        "retrieval_results": retrieval_results,
+        "answer_results": answer_results,
+        "failures": failures,
+        "recommendations": recommendations,
+        "chunks": chunks,
+        "documents": documents,
+    }
+
+    if config.export_format == "html" or output.endswith(".html"):
+        reporter = HTMLReport(config)
+        html = reporter.generate(report_data)
+        Path(output).write_text(html, encoding="utf-8")
+    else:
+        generator = ReportGenerator(config)
+        generator.generate(report_data, output)
+
+    progress.complete(f"Report saved to {output}")
+
+    # Summary
+    score = retrieval_results["score"]
+    color = "green" if score >= 80 else "yellow" if score >= 60 else "red"
+    console.print(f"
+[{color}]Tests: {retrieval_results['passed']}/{retrieval_results['total']} passed | Score: {score}%[/[{color}]]")
+
+    if score < 100:
+        console.print("
+[bold]Top Recommendations:[/bold]")
+        for rec in recommendations[:3]:
+            console.print(f"  • {rec['title']}: {rec['description'][:60]}...")
+
+
+@app.command()
+def version():
+    """Show version."""
+    console.print(f"ragcheck {get_version()}")
+
+
+if __name__ == "__main__":
+    app()

ragcheck_cli-0.2.3/ragcheck/core/config.py

@@ -0,0 +1,47 @@
+"""Configuration for ragcheck."""
+
+from pydantic import BaseModel, Field
+from typing import Literal
+
+
+class Config(BaseModel):
+    """RAGCheck configuration."""
+
+    # Embedding model
+    embedding_model: str = Field(default="all-MiniLM-L6-v2")
+
+    # Chunking
+    chunk_size: int = Field(default=2048, ge=256, le=8192)
+    chunk_overlap: int = Field(default=256, ge=0, le=2048)
+    chunk_strategy: Literal["recursive", "semantic", "fixed"] = Field(default="recursive")
+
+    # Vector store
+    vector_store: Literal["chroma", "faiss", "memory"] = Field(default="chroma")
+    collection_name: str = Field(default="ragcheck-default")
+
+    # Retrieval
+    top_k: int = Field(default=5, ge=1, le=50)
+    similarity_threshold: float = Field(default=0.3, ge=0.0, le=1.0)
+
+    # Answer generation
+    answer_model: str = Field(default="ollama/phi3:mini")
+    max_answer_tokens: int = Field(default=512, ge=64, le=4096)
+    temperature: float = Field(default=0.3, ge=0.0, le=2.0)
+
+    # QA generation
+    qa_model: str = Field(default="ollama/phi3:mini")
+    max_qa_questions: int = Field(default=50, ge=1, le=200)
+    qa_temperature: float = Field(default=0.7, ge=0.0, le=2.0)
+
+    # Evaluation
+    faithfulness_model: str = Field(default="microsoft/deberta-v2-xlarge-mnli")
+    nli_batch_size: int = Field(default=8, ge=1, le=64)
+
+    # Export
+    export_format: Literal["html", "json", "markdown"] = Field(default="html")
+    include_chunk_visualizer: bool = Field(default=True)
+    include_recommendations: bool = Field(default=True)
+
+    # Display
+    show_progress: bool = Field(default=True)
+    verbose: bool = Field(default=False)

ragcheck_cli-0.2.3/ragcheck/core/config_loader.py

@@ -0,0 +1,24 @@
+"""Configuration loader with encoding fixes."""
+
+from pathlib import Path
+from typing import Optional
+import yaml
+
+from ragcheck.core.config import Config
+
+
+def load_config(path: Optional[str] = None) -> Config:
+    """Load configuration from YAML file."""
+    if path:
+        config_path = Path(path)
+    else:
+        config_path = Path("ragcheck.yaml")
+
+    if not config_path.exists():
+        return Config()
+
+    # CRITICAL FIX: Explicit UTF-8 encoding for Windows
+    with open(config_path, "r", encoding="utf-8", errors="replace") as f:
+        data = yaml.safe_load(f) or {}
+
+    return Config(**data)