jseye 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

jseye/modules/tiered_analysis.py

@@ -0,0 +1,304 @@
+"""
+Tiered Analysis Engine - Smart analysis based on JS file importance
+"""
+
+import asyncio
+from pathlib import Path
+from typing import List, Dict, Any
+from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
+import time
+
+from ..utils.logger import log_progress
+from ..utils.fs import save_json
+from .analyze_regex import RegexAnalyzer
+from .analyze_ast import ASTAnalyzer
+from .linkfinder import LinkFinderIntegration
+from .secrets import SecretsDetector
+
+class TieredAnalysisEngine:
+    """
+    🔥 TIERED ANALYSIS MODEL
+    
+    Tier 1: Regex + AST + LinkFinder + Secrets (HEAVY) - Top 20%
+    Tier 2: Regex + LinkFinder (MEDIUM) - Next 30% 
+    Tier 3: Regex only (FAST) - Remaining 50%
+    """
+    
+    def __init__(self, output_dir: Path):
+        self.output_dir = output_dir
+        self.regex_analyzer = RegexAnalyzer(output_dir)
+        self.ast_analyzer = ASTAnalyzer(output_dir)
+        self.linkfinder = LinkFinderIntegration(output_dir)
+        self.secrets_detector = SecretsDetector(output_dir)
+        
+        # Performance tracking
+        self.analysis_stats = {
+            'tier1_time': 0,
+            'tier2_time': 0,
+            'tier3_time': 0,
+            'files_analyzed': 0,
+            'time_saved': 0
+        }
+    
+    async def analyze_tier1_file(self, js_file: Dict) -> Dict[str, Any]:
+        """
+        TIER 1: Full analysis (Heavy)
+        - Regex analysis
+        - AST analysis  
+        - LinkFinder
+        - Secrets detection
+        """
+        start_time = time.time()
+        
+        try:
+            results = {
+                'file': js_file['filepath'],
+                'tier': 1,
+                'analysis': {
+                    'regex': [],
+                    'ast': [],
+                    'linkfinder': [],
+                    'secrets': []
+                }
+            }
+            
+            # Run all analysis types
+            if js_file.get('status') == 'success' and js_file.get('filepath'):
+                # Regex analysis
+                regex_results = self.regex_analyzer.analyze_files([js_file])
+                results['analysis']['regex'] = regex_results.get('endpoints', [])
+                
+                # AST analysis
+                ast_results = self.ast_analyzer.analyze_files([js_file])
+                results['analysis']['ast'] = ast_results.get('endpoints', [])
+                
+                # LinkFinder
+                lf_results = self.linkfinder.analyze_js_file(js_file['filepath'])
+                results['analysis']['linkfinder'] = lf_results
+                
+                # Secrets
+                secret_results = self.secrets_detector.run_mantra([js_file])
+                results['analysis']['secrets'] = secret_results
+            
+            analysis_time = time.time() - start_time
+            results['analysis_time'] = analysis_time
+            self.analysis_stats['tier1_time'] += analysis_time
+            
+            return results
+            
+        except Exception as e:
+            return {
+                'file': js_file.get('filepath', 'unknown'),
+                'tier': 1,
+                'error': str(e),
+                'analysis_time': time.time() - start_time
+            }
+    
+    async def analyze_tier2_file(self, js_file: Dict) -> Dict[str, Any]:
+        """
+        TIER 2: Medium analysis
+        - Regex analysis
+        - LinkFinder
+        """
+        start_time = time.time()
+        
+        try:
+            results = {
+                'file': js_file['filepath'],
+                'tier': 2,
+                'analysis': {
+                    'regex': [],
+                    'linkfinder': []
+                }
+            }
+            
+            if js_file.get('status') == 'success' and js_file.get('filepath'):
+                # Regex analysis
+                regex_results = self.regex_analyzer.analyze_files([js_file])
+                results['analysis']['regex'] = regex_results.get('endpoints', [])
+                
+                # Skip AST if regex finds nothing (SMART OPTIMIZATION)
+                if len(results['analysis']['regex']) > 0:
+                    # LinkFinder
+                    lf_results = self.linkfinder.analyze_js_file(js_file['filepath'])
+                    results['analysis']['linkfinder'] = lf_results
+                else:
+                    log_progress(f"Tier 2: Skipping LinkFinder for {js_file['filepath']} (no regex results)")
+            
+            analysis_time = time.time() - start_time
+            results['analysis_time'] = analysis_time
+            self.analysis_stats['tier2_time'] += analysis_time
+            
+            return results
+            
+        except Exception as e:
+            return {
+                'file': js_file.get('filepath', 'unknown'),
+                'tier': 2,
+                'error': str(e),
+                'analysis_time': time.time() - start_time
+            }
+    
+    async def analyze_tier3_file(self, js_file: Dict) -> Dict[str, Any]:
+        """
+        TIER 3: Light analysis (Fast)
+        - Regex analysis only
+        """
+        start_time = time.time()
+        
+        try:
+            results = {
+                'file': js_file['filepath'],
+                'tier': 3,
+                'analysis': {
+                    'regex': []
+                }
+            }
+            
+            if js_file.get('status') == 'success' and js_file.get('filepath'):
+                # Regex analysis only
+                regex_results = self.regex_analyzer.analyze_files([js_file])
+                results['analysis']['regex'] = regex_results.get('endpoints', [])
+            
+            analysis_time = time.time() - start_time
+            results['analysis_time'] = analysis_time
+            self.analysis_stats['tier3_time'] += analysis_time
+            
+            return results
+            
+        except Exception as e:
+            return {
+                'file': js_file.get('filepath', 'unknown'),
+                'tier': 3,
+                'error': str(e),
+                'analysis_time': time.time() - start_time
+            }
+    
+    async def run_tiered_analysis(self, tiered_js_files: Dict[str, List[str]], 
+                                 downloaded_files: List[Dict]) -> Dict[str, Any]:
+        """
+        >> Run tiered analysis with parallel processing
+        
+        Args:
+            tiered_js_files: Dictionary with tier1_full, tier2_medium, tier3_light
+            downloaded_files: List of downloaded JS file info
+            
+        Returns:
+            Combined analysis results
+        """
+        log_progress("🔥 Starting TIERED ANALYSIS ENGINE")
+        
+        # Map URLs to file info
+        url_to_file = {f['url']: f for f in downloaded_files if f.get('status') == 'success'}
+        
+        # Prepare files for each tier
+        tier1_files = [url_to_file[url] for url in tiered_js_files.get('tier1_full', []) if url in url_to_file]
+        tier2_files = [url_to_file[url] for url in tiered_js_files.get('tier2_medium', []) if url in url_to_file]
+        tier3_files = [url_to_file[url] for url in tiered_js_files.get('tier3_light', []) if url in url_to_file]
+        
+        log_progress(f"Tier 1 (HEAVY): {len(tier1_files)} files")
+        log_progress(f"Tier 2 (MEDIUM): {len(tier2_files)} files") 
+        log_progress(f"Tier 3 (LIGHT): {len(tier3_files)} files")
+        
+        all_results = []
+        
+        # Process each tier with appropriate concurrency
+        if tier1_files:
+            log_progress(">> Processing Tier 1 (Full Analysis)...")
+            # Limit concurrency for heavy analysis
+            semaphore1 = asyncio.Semaphore(2)
+            
+            async def analyze_tier1_with_semaphore(js_file):
+                async with semaphore1:
+                    return await self.analyze_tier1_file(js_file)
+            
+            tier1_tasks = [analyze_tier1_with_semaphore(f) for f in tier1_files]
+            tier1_results = await asyncio.gather(*tier1_tasks, return_exceptions=True)
+            all_results.extend([r for r in tier1_results if not isinstance(r, Exception)])
+        
+        if tier2_files:
+            log_progress(">> Processing Tier 2 (Medium Analysis)...")
+            # More concurrency for medium analysis
+            semaphore2 = asyncio.Semaphore(4)
+            
+            async def analyze_tier2_with_semaphore(js_file):
+                async with semaphore2:
+                    return await self.analyze_tier2_file(js_file)
+            
+            tier2_tasks = [analyze_tier2_with_semaphore(f) for f in tier2_files]
+            tier2_results = await asyncio.gather(*tier2_tasks, return_exceptions=True)
+            all_results.extend([r for r in tier2_results if not isinstance(r, Exception)])
+        
+        if tier3_files:
+            log_progress(">> Processing Tier 3 (Light Analysis)...")
+            # High concurrency for light analysis
+            semaphore3 = asyncio.Semaphore(8)
+            
+            async def analyze_tier3_with_semaphore(js_file):
+                async with semaphore3:
+                    return await self.analyze_tier3_file(js_file)
+            
+            tier3_tasks = [analyze_tier3_with_semaphore(f) for f in tier3_files]
+            tier3_results = await asyncio.gather(*tier3_tasks, return_exceptions=True)
+            all_results.extend([r for r in tier3_results if not isinstance(r, Exception)])
+        
+        # Aggregate results
+        aggregated = self.aggregate_results(all_results)
+        
+        # Save detailed results
+        save_json({
+            'summary': aggregated['summary'],
+            'performance_stats': self.analysis_stats,
+            'detailed_results': all_results
+        }, self.output_dir / "tiered_analysis_results.json")
+        
+        log_progress(f">> Tiered analysis complete: {aggregated['summary']['total_findings']} findings")
+        log_progress(f"[T] Time breakdown: T1({self.analysis_stats['tier1_time']:.1f}s) T2({self.analysis_stats['tier2_time']:.1f}s) T3({self.analysis_stats['tier3_time']:.1f}s)")
+        
+        return aggregated
+    
+    def aggregate_results(self, all_results: List[Dict]) -> Dict[str, Any]:
+        """Aggregate results from all tiers"""
+        endpoints = []
+        secrets = []
+        sinks = []
+        functions = []
+        
+        tier_stats = {'tier1': 0, 'tier2': 0, 'tier3': 0}
+        
+        for result in all_results:
+            if 'error' in result:
+                continue
+                
+            tier_stats[f"tier{result['tier']}"] += 1
+            analysis = result.get('analysis', {})
+            
+            # Collect endpoints
+            if 'regex' in analysis:
+                endpoints.extend(analysis['regex'])
+            if 'ast' in analysis:
+                endpoints.extend(analysis['ast'])
+            if 'linkfinder' in analysis:
+                endpoints.extend(analysis['linkfinder'])
+            
+            # Collect secrets
+            if 'secrets' in analysis:
+                secrets.extend(analysis['secrets'])
+        
+        return {
+            'summary': {
+                'total_findings': len(endpoints) + len(secrets),
+                'endpoints_found': len(endpoints),
+                'secrets_found': len(secrets),
+                'files_by_tier': tier_stats,
+                'total_analysis_time': sum([
+                    self.analysis_stats['tier1_time'],
+                    self.analysis_stats['tier2_time'], 
+                    self.analysis_stats['tier3_time']
+                ])
+            },
+            'endpoints': endpoints,
+            'secrets': secrets,
+            'sinks': sinks,
+            'functions': functions
+        }