sqlshell 0.4.4__py3-none-any.whl

sqlshell/utils/profile_ohe_comparison.py

@@ -0,0 +1,237 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+import warnings
+warnings.filterwarnings('ignore')
+
+# Import OHE functions
+try:
+    from .profile_ohe import get_ohe  # Basic OHE
+    from .profile_ohe_advanced import get_advanced_ohe, analyze_concept_correlations  # Advanced OHE
+except ImportError:
+    # Try without relative imports
+    import sys
+    import os
+    sys.path.insert(0, os.path.dirname(__file__))
+    from profile_ohe import get_ohe
+    from profile_ohe_advanced import get_advanced_ohe, analyze_concept_correlations
+
+# Optional: Word embeddings support
+try:
+    from sentence_transformers import SentenceTransformer
+    from sklearn.metrics.pairwise import cosine_similarity
+    from sklearn.cluster import AgglomerativeClustering
+    EMBEDDINGS_AVAILABLE = True
+except ImportError:
+    EMBEDDINGS_AVAILABLE = False
+    print("Note: sentence-transformers not available. Install with: pip install sentence-transformers")
+
+class EmbeddingAnalyzer:
+    """
+    Advanced semantic analysis using transformer-based embeddings.
+    """
+    
+    def __init__(self, model_name='all-MiniLM-L6-v2'):
+        """Initialize embedding analyzer with pre-trained model."""
+        if not EMBEDDINGS_AVAILABLE:
+            self.model = None
+            return
+            
+        try:
+            self.model = SentenceTransformer(model_name)
+        except Exception as e:
+            print(f"Failed to load embedding model: {e}")
+            self.model = None
+    
+    def extract_semantic_clusters_embeddings(self, texts, n_clusters=8):
+        """
+        Extract semantic clusters using sentence embeddings.
+        More sophisticated than TF-IDF clustering.
+        """
+        if self.model is None or not texts:
+            return []
+        
+        try:
+            # Generate embeddings for all texts
+            embeddings = self.model.encode(texts)
+            
+            # Perform hierarchical clustering
+            clustering = AgglomerativeClustering(
+                n_clusters=min(n_clusters, len(texts)),
+                linkage='ward'
+            )
+            cluster_labels = clustering.fit_predict(embeddings)
+            
+            # Group texts by cluster
+            clusters = {}
+            for i, label in enumerate(cluster_labels):
+                if label not in clusters:
+                    clusters[label] = []
+                clusters[label].append(texts[i])
+            
+            # Extract representative terms from each cluster
+            cluster_concepts = []
+            for cluster_id, cluster_texts in clusters.items():
+                if len(cluster_texts) >= 2:  # Only clusters with multiple texts
+                    # Get most common words across cluster texts
+                    all_words = []
+                    for text in cluster_texts:
+                        words = text.lower().split()
+                        all_words.extend([w for w in words if len(w) > 3])
+                    
+                    from collections import Counter
+                    common_words = Counter(all_words).most_common(5)
+                    
+                    cluster_concepts.append({
+                        'name': f"embedding_cluster_{cluster_id}",
+                        'words': [word for word, _ in common_words],
+                        'texts': cluster_texts,
+                        'size': len(cluster_texts)
+                    })
+            
+            return cluster_concepts
+            
+        except Exception as e:
+            print(f"Embedding clustering failed: {e}")
+            return []
+
+
+def demo_advanced_algorithms():
+    """Demonstrate the power of advanced algorithms with AI-related text."""
+    print("\n" + "="*80)
+    print("DEMONSTRATION: Advanced OHE Algorithms vs Basic Approach")
+    print("="*80)
+    
+    # Create AI/ML focused dataset (your use case!)
+    ai_data = {
+        'description': [
+            "Machine learning engineer developing neural networks for computer vision applications",
+            "AI researcher working on natural language processing and large language models", 
+            "Data scientist implementing deep learning algorithms for predictive analytics",
+            "Software engineer building recommendation systems using collaborative filtering",
+            "ML ops engineer deploying artificial intelligence models to cloud infrastructure",
+            "Computer vision specialist creating object detection systems for autonomous vehicles",
+            "NLP engineer developing chatbots and conversational AI systems",
+            "Deep learning researcher working on transformer architectures and attention mechanisms",
+            "AI product manager overseeing machine learning product development lifecycle",
+            "Data engineer building pipelines for real-time AI model inference and training"
+        ]
+    }
+    
+    df = pd.DataFrame(ai_data)
+    
+    print("\nOriginal Data:")
+    for i, desc in enumerate(ai_data['description'][:3]):
+        print(f"{i+1}. {desc}")
+    print("... (and 7 more)")
+    
+    # Test Basic OHE
+    print("\n" + "-"*50)
+    print("BASIC OHE RESULTS:")
+    print("-"*50)
+    basic_result = get_ohe(df.copy(), 'description', binary_format="numeric")
+    basic_features = [col for col in basic_result.columns if col.startswith('has_')]
+    print(f"Features created: {len(basic_features)}")
+    for feature in basic_features:
+        coverage = (basic_result[feature] == 1).sum() / len(basic_result) * 100
+        print(f"  • {feature}: {coverage:.1f}% coverage")
+    
+    # Test Advanced OHE
+    print("\n" + "-"*50)
+    print("ADVANCED OHE RESULTS:")
+    print("-"*50)
+    advanced_result = get_advanced_ohe(
+        df.copy(), 
+        'description', 
+        binary_format="numeric",
+        analysis_type="comprehensive",
+        max_features=15
+    )
+    advanced_features = [col for col in advanced_result.columns if col.startswith('has_')]
+    print(f"Features created: {len(advanced_features)}")
+    
+    # Group features by type
+    feature_types = {}
+    for feature in advanced_features:
+        if 'topic_lda' in feature:
+            feature_types.setdefault('LDA Topics', []).append(feature)
+        elif 'topic_nmf' in feature:
+            feature_types.setdefault('NMF Topics', []).append(feature)
+        elif 'semantic_cluster' in feature:
+            feature_types.setdefault('Semantic Clusters', []).append(feature)
+        elif 'domain_' in feature:
+            feature_types.setdefault('Domain Concepts', []).append(feature)
+        elif 'ngram_' in feature:
+            feature_types.setdefault('Key N-grams', []).append(feature)
+        else:
+            feature_types.setdefault('Other', []).append(feature)
+    
+    for ftype, features in feature_types.items():
+        print(f"\n{ftype}:")
+        for feature in features[:3]:  # Show first 3 of each type
+            coverage = (advanced_result[feature] == 1).sum() / len(advanced_result) * 100
+            print(f"  • {feature}: {coverage:.1f}% coverage")
+        if len(features) > 3:
+            print(f"  ... and {len(features) - 3} more")
+    
+    # Analyze correlations
+    print("\n" + "-"*50)
+    print("CORRELATION ANALYSIS:")
+    print("-"*50)
+    correlation_analysis = analyze_concept_correlations(advanced_result, advanced_features)
+    
+    if correlation_analysis and correlation_analysis.get('strong_correlations'):
+        print("Strong correlations found (shows semantic relationships):")
+        for corr in correlation_analysis['strong_correlations'][:5]:
+            print(f"  • {corr['feature1']} ↔ {corr['feature2']}: {corr['correlation']:.3f}")
+        
+        print("\nThis shows the advanced algorithm captured semantic relationships!")
+        print("For example, 'AI' and 'machine learning' concepts are properly linked.")
+    else:
+        print("No strong correlations found - features are orthogonal")
+    
+    # Show specific example of AI correlation capture
+    print("\n" + "-"*50)
+    print("AI CORRELATION ANALYSIS (Your Original Problem):")
+    print("-"*50)
+    
+    # Check which features capture AI-related concepts
+    ai_related_features = []
+    for feature in advanced_features:
+        feature_name = feature.lower()
+        if any(term in feature_name for term in ['ai', 'artificial', 'intelligence', 'machine', 'learning', 'neural', 'deep']):
+            ai_related_features.append(feature)
+    
+    if ai_related_features:
+        print(f"Found {len(ai_related_features)} AI-related features:")
+        for feature in ai_related_features:
+            coverage = (advanced_result[feature] == 1).sum() / len(advanced_result) * 100
+            print(f"  • {feature}: {coverage:.1f}% coverage")
+        
+        # Show which descriptions match these features
+        print("\nDescriptions matching AI-related features:")
+        for i, desc in enumerate(ai_data['description']):
+            matches = []
+            for feature in ai_related_features:
+                if advanced_result.iloc[i][feature] == 1:
+                    matches.append(feature.replace('has_', ''))
+            if matches:
+                print(f"  {i+1}. '{desc[:50]}...' → {', '.join(matches[:2])}")
+    else:
+        print("No explicit AI-related features found in feature names")
+        print("However, topic modeling may have captured these concepts in broader themes")
+    
+    print("\n" + "="*80)
+    print("CONCLUSION: Advanced algorithms provide much richer semantic understanding!")
+    print("• Basic OHE: Only captures individual word frequency")
+    print("• Advanced OHE: Captures topics, themes, semantic clusters, and domain concepts")
+    print("• This solves your AI correlation problem by grouping related concepts!")
+    print("="*80)
+    
+    return basic_result, advanced_result
+
+
+if __name__ == "__main__":
+    # Run the demonstration
+    demo_advanced_algorithms()