tech-hub-skills 1.2.0 → 1.5.2

package/.claude/roles/ai-engineer/skills/02-rag-pipeline/document_chunker.py

@@ -0,0 +1,336 @@
+"""
+Advanced Document Chunking for RAG Systems
+Supports semantic, recursive, and fixed-size chunking strategies.
+"""
+
+from typing import List, Dict, Any, Optional
+from dataclasses import dataclass
+from enum import Enum
+import re
+from langchain.text_splitter import (
+    RecursiveCharacterTextSplitter,
+    CharacterTextSplitter,
+    TokenTextSplitter
+)
+
+
+class ChunkStrategy(Enum):
+    """Available chunking strategies."""
+    FIXED = "fixed"  # Fixed character/token size
+    SEMANTIC = "semantic"  # Semantic boundaries (paragraphs, sentences)
+    RECURSIVE = "recursive"  # Recursive splitting with multiple separators
+    SLIDING_WINDOW = "sliding_window"  # Overlapping windows
+
+
+@dataclass
+class Chunk:
+    """A document chunk with metadata."""
+    content: str
+    chunk_id: str
+    document_id: str
+    chunk_index: int
+    metadata: Dict[str, Any]
+    char_count: int
+    token_count: Optional[int] = None
+
+    def __post_init__(self):
+        if self.char_count == 0:
+            self.char_count = len(self.content)
+
+
+class DocumentChunker:
+    """Advanced document chunker with multiple strategies."""
+
+    def __init__(
+        self,
+        strategy: ChunkStrategy = ChunkStrategy.RECURSIVE,
+        chunk_size: int = 1000,
+        chunk_overlap: int = 200,
+        separators: Optional[List[str]] = None
+    ):
+        """
+        Initialize document chunker.
+
+        Args:
+            strategy: Chunking strategy to use
+            chunk_size: Target chunk size (characters or tokens)
+            chunk_overlap: Overlap between chunks
+            separators: Custom separators for recursive splitting
+        """
+        self.strategy = strategy
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.separators = separators or ["\n\n", "\n", ". ", " ", ""]
+
+        self._init_splitter()
+
+    def _init_splitter(self):
+        """Initialize the appropriate text splitter."""
+        if self.strategy == ChunkStrategy.RECURSIVE:
+            self.splitter = RecursiveCharacterTextSplitter(
+                chunk_size=self.chunk_size,
+                chunk_overlap=self.chunk_overlap,
+                separators=self.separators,
+                length_function=len
+            )
+        elif self.strategy == ChunkStrategy.FIXED:
+            self.splitter = CharacterTextSplitter(
+                chunk_size=self.chunk_size,
+                chunk_overlap=self.chunk_overlap,
+                separator="\n"
+            )
+        elif self.strategy == ChunkStrategy.SEMANTIC:
+            # For semantic chunking, we'll use custom logic
+            self.splitter = None
+        else:
+            self.splitter = RecursiveCharacterTextSplitter(
+                chunk_size=self.chunk_size,
+                chunk_overlap=self.chunk_overlap
+            )
+
+    def chunk_document(
+        self,
+        text: str,
+        document_id: str,
+        metadata: Optional[Dict[str, Any]] = None
+    ) -> List[Chunk]:
+        """
+        Chunk a document into smaller pieces.
+
+        Args:
+            text: Document text
+            document_id: Unique document identifier
+            metadata: Additional metadata
+
+        Returns:
+            List of Chunk objects
+        """
+        metadata = metadata or {}
+
+        if self.strategy == ChunkStrategy.SEMANTIC:
+            text_chunks = self._semantic_chunking(text)
+        elif self.strategy == ChunkStrategy.SLIDING_WINDOW:
+            text_chunks = self._sliding_window_chunking(text)
+        else:
+            text_chunks = self.splitter.split_text(text)
+
+        chunks = []
+        for idx, chunk_text in enumerate(text_chunks):
+            chunk = Chunk(
+                content=chunk_text,
+                chunk_id=f"{document_id}_chunk_{idx}",
+                document_id=document_id,
+                chunk_index=idx,
+                metadata={**metadata, "strategy": self.strategy.value},
+                char_count=len(chunk_text)
+            )
+            chunks.append(chunk)
+
+        return chunks
+
+    def _semantic_chunking(self, text: str) -> List[str]:
+        """
+        Chunk by semantic boundaries (paragraphs with context).
+
+        This strategy:
+        1. Splits on paragraph boundaries
+        2. Combines small paragraphs
+        3. Ensures chunks don't exceed max size
+        """
+        # Split into paragraphs
+        paragraphs = re.split(r'\n\s*\n', text)
+
+        chunks = []
+        current_chunk = []
+        current_length = 0
+
+        for para in paragraphs:
+            para = para.strip()
+            if not para:
+                continue
+
+            para_length = len(para)
+
+            # If paragraph alone exceeds chunk size, split it
+            if para_length > self.chunk_size:
+                # Save current chunk if exists
+                if current_chunk:
+                    chunks.append("\n\n".join(current_chunk))
+                    current_chunk = []
+                    current_length = 0
+
+                # Split large paragraph
+                sentences = re.split(r'(?<=[.!?])\s+', para)
+                temp_chunk = []
+                temp_length = 0
+
+                for sentence in sentences:
+                    sent_length = len(sentence)
+                    if temp_length + sent_length > self.chunk_size:
+                        if temp_chunk:
+                            chunks.append(" ".join(temp_chunk))
+                        temp_chunk = [sentence]
+                        temp_length = sent_length
+                    else:
+                        temp_chunk.append(sentence)
+                        temp_length += sent_length + 1
+
+                if temp_chunk:
+                    chunks.append(" ".join(temp_chunk))
+
+            # If adding paragraph exceeds chunk size, save current chunk
+            elif current_length + para_length > self.chunk_size:
+                if current_chunk:
+                    chunks.append("\n\n".join(current_chunk))
+                current_chunk = [para]
+                current_length = para_length
+
+            # Otherwise, add to current chunk
+            else:
+                current_chunk.append(para)
+                current_length += para_length + 2  # +2 for \n\n
+
+        # Add remaining chunk
+        if current_chunk:
+            chunks.append("\n\n".join(current_chunk))
+
+        return chunks
+
+    def _sliding_window_chunking(self, text: str) -> List[str]:
+        """
+        Create overlapping chunks with sliding window.
+
+        Useful for ensuring important content at chunk boundaries isn't lost.
+        """
+        chunks = []
+        start = 0
+
+        while start < len(text):
+            end = start + self.chunk_size
+            chunk = text[start:end]
+
+            # Try to end at sentence boundary
+            if end < len(text):
+                last_period = chunk.rfind('. ')
+                if last_period > self.chunk_size // 2:
+                    chunk = chunk[:last_period + 1]
+                    end = start + last_period + 1
+
+            chunks.append(chunk.strip())
+
+            # Move start forward (with overlap)
+            start = end - self.chunk_overlap
+
+        return chunks
+
+    def chunk_multiple_documents(
+        self,
+        documents: List[Dict[str, Any]]
+    ) -> List[Chunk]:
+        """
+        Chunk multiple documents.
+
+        Args:
+            documents: List of dicts with 'id', 'text', and optional 'metadata'
+
+        Returns:
+            List of all chunks
+        """
+        all_chunks = []
+
+        for doc in documents:
+            chunks = self.chunk_document(
+                text=doc['text'],
+                document_id=doc['id'],
+                metadata=doc.get('metadata', {})
+            )
+            all_chunks.extend(chunks)
+
+        return all_chunks
+
+    def get_chunk_statistics(self, chunks: List[Chunk]) -> Dict[str, Any]:
+        """Get statistics about chunks."""
+        if not chunks:
+            return {}
+
+        char_counts = [c.char_count for c in chunks]
+
+        return {
+            "total_chunks": len(chunks),
+            "total_characters": sum(char_counts),
+            "avg_chunk_size": sum(char_counts) / len(chunks),
+            "min_chunk_size": min(char_counts),
+            "max_chunk_size": max(char_counts),
+            "unique_documents": len(set(c.document_id for c in chunks)),
+            "strategy": self.strategy.value
+        }
+
+
+# Example usage
+if __name__ == "__main__":
+    # Sample document
+    sample_doc = """
+    Marketing Campaign Analysis Best Practices
+
+    Effective marketing campaign analysis requires a systematic approach to data collection and interpretation.
+
+    Data Collection
+    First, ensure you're tracking the right metrics. Common KPIs include impression count, click-through rates (CTR), conversion rates, and return on ad spend (ROAS). Use tracking pixels and UTM parameters to accurately attribute conversions.
+
+    Campaign Segmentation
+    Break down your analysis by campaign type, channel, audience segment, and time period. This granular view helps identify what's working and what isn't. For example, email campaigns might perform better with certain demographics, while social media ads resonate with others.
+
+    Performance Benchmarking
+    Compare your results against industry benchmarks and historical data. A 2% CTR might seem low in isolation, but could be excellent for your industry. Track performance over time to identify trends and seasonality.
+
+    Attribution Modeling
+    Understand the customer journey. Did they convert after the first touchpoint or after multiple interactions? Multi-touch attribution helps allocate credit appropriately across channels.
+
+    A/B Testing
+    Never stop testing. Test subject lines, ad copy, images, calls-to-action, and landing pages. Use statistical significance testing to ensure your results are valid.
+
+    Reporting and Insights
+    Create actionable reports that tell a story. Don't just show numbers—explain what they mean and what actions should be taken. Use visualizations to make data accessible.
+
+    Continuous Optimization
+    Marketing is iterative. Use insights from each campaign to improve the next one. Build a knowledge base of what works for your audience.
+    """
+
+    print("=" * 80)
+    print("Document Chunking Demonstrations")
+    print("=" * 80)
+
+    # Test different chunking strategies
+    strategies = [
+        (ChunkStrategy.RECURSIVE, "Recursive (smart boundaries)"),
+        (ChunkStrategy.SEMANTIC, "Semantic (paragraph-based)"),
+        (ChunkStrategy.SLIDING_WINDOW, "Sliding Window (overlapping)"),
+        (ChunkStrategy.FIXED, "Fixed Size")
+    ]
+
+    for strategy, description in strategies:
+        print(f"\n📄 Strategy: {description}")
+        print("-" * 80)
+
+        chunker = DocumentChunker(
+            strategy=strategy,
+            chunk_size=300,
+            chunk_overlap=50
+        )
+
+        chunks = chunker.chunk_document(
+            text=sample_doc,
+            document_id="campaign_analysis_guide",
+            metadata={"category": "marketing", "author": "Tech Hub"}
+        )
+
+        stats = chunker.get_chunk_statistics(chunks)
+
+        print(f"Total chunks: {stats['total_chunks']}")
+        print(f"Avg chunk size: {stats['avg_chunk_size']:.0f} chars")
+        print(f"Size range: {stats['min_chunk_size']}-{stats['max_chunk_size']} chars")
+
+        print(f"\nFirst chunk preview:")
+        print(f"{chunks[0].content[:200]}...")
+
+        print(f"\nChunk IDs: {[c.chunk_id for c in chunks]}")

package/.claude/roles/ai-engineer/skills/02-rag-pipeline/rag_pipeline.sql

@@ -0,0 +1,213 @@
+-- RAG Pipeline Analytics Queries
+-- Track knowledge base usage, query patterns, and performance
+
+-- ================================================================
+-- 1. KNOWLEDGE BASE INVENTORY
+-- ================================================================
+
+-- Count documents by source
+SELECT
+    source_type,
+    COUNT(*) as document_count,
+    SUM(chunk_count) as total_chunks,
+    AVG(chunk_count) as avg_chunks_per_doc,
+    MAX(last_updated) as latest_update
+FROM knowledge_base_documents
+GROUP BY source_type
+ORDER BY document_count DESC;
+
+-- ================================================================
+-- 2. QUERY ANALYTICS
+-- ================================================================
+
+-- Top queries by frequency (last 30 days)
+SELECT
+    query_text,
+    COUNT(*) as query_count,
+    AVG(latency_ms) as avg_latency_ms,
+    AVG(relevance_score) as avg_relevance,
+    COUNT(DISTINCT user_id) as unique_users
+FROM rag_query_log
+WHERE query_timestamp >= CURRENT_DATE - INTERVAL '30 days'
+GROUP BY query_text
+HAVING COUNT(*) > 5
+ORDER BY query_count DESC
+LIMIT 20;
+
+-- ================================================================
+-- 3. RETRIEVAL PERFORMANCE
+-- ================================================================
+
+-- Retrieval performance by top_k setting
+SELECT
+    top_k,
+    COUNT(*) as query_count,
+    AVG(latency_ms) as avg_latency_ms,
+    PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY latency_ms) as p50_latency,
+    PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY latency_ms) as p95_latency,
+    PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY latency_ms) as p99_latency,
+    AVG(relevance_score) as avg_relevance_score
+FROM rag_query_log
+WHERE query_timestamp >= CURRENT_DATE - INTERVAL '7 days'
+GROUP BY top_k
+ORDER BY top_k;
+
+-- ================================================================
+-- 4. SOURCE ATTRIBUTION
+-- ================================================================
+
+-- Which documents are most frequently retrieved?
+SELECT
+    d.document_id,
+    d.title,
+    d.source_type,
+    COUNT(*) as retrieval_count,
+    AVG(r.relevance_score) as avg_relevance,
+    MAX(r.query_timestamp) as last_retrieved
+FROM rag_retrievals r
+JOIN knowledge_base_documents d ON r.document_id = d.document_id
+WHERE r.query_timestamp >= CURRENT_DATE - INTERVAL '30 days'
+GROUP BY d.document_id, d.title, d.source_type
+ORDER BY retrieval_count DESC
+LIMIT 50;
+
+-- ================================================================
+-- 5. USER ENGAGEMENT
+-- ================================================================
+
+-- User engagement with RAG system
+SELECT
+    DATE_TRUNC('day', query_timestamp) as query_date,
+    COUNT(DISTINCT user_id) as unique_users,
+    COUNT(*) as total_queries,
+    COUNT(*) / COUNT(DISTINCT user_id) as queries_per_user,
+    AVG(relevance_score) as avg_relevance
+FROM rag_query_log
+WHERE query_timestamp >= CURRENT_DATE - INTERVAL '90 days'
+GROUP BY DATE_TRUNC('day', query_timestamp)
+ORDER BY query_date DESC;
+
+-- ================================================================
+-- 6. CHUNK PERFORMANCE
+-- ================================================================
+
+-- Which chunk size performs best?
+SELECT
+    c.chunk_size_range,
+    COUNT(DISTINCT r.query_id) as query_count,
+    AVG(r.relevance_score) as avg_relevance,
+    AVG(r.rank_position) as avg_rank
+FROM rag_retrievals r
+JOIN knowledge_base_chunks c ON r.chunk_id = c.chunk_id
+WHERE r.query_timestamp >= CURRENT_DATE - INTERVAL '30 days'
+GROUP BY c.chunk_size_range
+ORDER BY avg_relevance DESC;
+
+-- ================================================================
+-- 7. FAILED QUERIES
+-- ================================================================
+
+-- Queries with low relevance (need improvement)
+SELECT
+    query_text,
+    COUNT(*) as failure_count,
+    AVG(relevance_score) as avg_relevance,
+    MIN(relevance_score) as min_relevance,
+    MAX(query_timestamp) as last_failed
+FROM rag_query_log
+WHERE relevance_score < 0.5
+  AND query_timestamp >= CURRENT_DATE - INTERVAL '7 days'
+GROUP BY query_text
+HAVING COUNT(*) > 2
+ORDER BY failure_count DESC
+LIMIT 30;
+
+-- ================================================================
+-- 8. EMBEDDING MODEL PERFORMANCE
+-- ================================================================
+
+-- Compare performance across embedding models
+SELECT
+    embedding_model,
+    COUNT(*) as query_count,
+    AVG(embedding_latency_ms) as avg_embedding_latency,
+    AVG(retrieval_latency_ms) as avg_retrieval_latency,
+    AVG(relevance_score) as avg_relevance
+FROM rag_query_log
+WHERE query_timestamp >= CURRENT_DATE - INTERVAL '30 days'
+GROUP BY embedding_model
+ORDER BY avg_relevance DESC;
+
+-- ================================================================
+-- 9. KNOWLEDGE GAPS
+-- ================================================================
+
+-- Identify topics with no good answers
+WITH poor_coverage AS (
+    SELECT
+        query_text,
+        COUNT(*) as frequency,
+        AVG(relevance_score) as avg_relevance
+    FROM rag_query_log
+    WHERE query_timestamp >= CURRENT_DATE - INTERVAL '30 days'
+    GROUP BY query_text
+    HAVING AVG(relevance_score) < 0.6 AND COUNT(*) > 3
+)
+SELECT
+    query_text,
+    frequency,
+    avg_relevance,
+    'Add documentation' as recommendation
+FROM poor_coverage
+ORDER BY frequency DESC;
+
+-- ================================================================
+-- 10. RAG PIPELINE HEALTH
+-- ================================================================
+
+-- Daily RAG pipeline health metrics
+SELECT
+    DATE(query_timestamp) as date,
+    COUNT(*) as total_queries,
+    AVG(total_latency_ms) as avg_latency_ms,
+    AVG(relevance_score) as avg_relevance,
+    PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY total_latency_ms) as p95_latency,
+    COUNT(CASE WHEN total_latency_ms > 1000 THEN 1 END) as slow_queries,
+    COUNT(CASE WHEN relevance_score < 0.5 THEN 1 END) as low_relevance_queries
+FROM rag_query_log
+WHERE query_timestamp >= CURRENT_DATE - INTERVAL '14 days'
+GROUP BY DATE(query_timestamp)
+ORDER BY date DESC;
+
+-- ================================================================
+-- 11. VECTOR DATABASE STATISTICS
+-- ================================================================
+
+-- Vector database usage statistics
+SELECT
+    collection_name,
+    COUNT(DISTINCT vector_id) as total_vectors,
+    AVG(vector_dimension) as avg_dimension,
+    MAX(last_updated) as last_updated,
+    SUM(storage_bytes) / (1024*1024) as storage_mb
+FROM vector_database_collections
+GROUP BY collection_name
+ORDER BY total_vectors DESC;
+
+-- ================================================================
+-- 12. COST TRACKING
+-- ================================================================
+
+-- Estimated costs by provider
+SELECT
+    DATE(query_timestamp) as date,
+    llm_provider,
+    COUNT(*) as query_count,
+    SUM(input_tokens) as total_input_tokens,
+    SUM(output_tokens) as total_output_tokens,
+    SUM(estimated_cost) as total_cost,
+    AVG(estimated_cost) as avg_cost_per_query
+FROM rag_query_log
+WHERE query_timestamp >= CURRENT_DATE - INTERVAL '30 days'
+GROUP BY DATE(query_timestamp), llm_provider
+ORDER BY date DESC, total_cost DESC;