src2purl 1.3.2__py3-none-any.whl

src2id/search/hash_search.py

@@ -0,0 +1,206 @@
+"""Optimized hash-based search functionality."""
+
+import hashlib
+from pathlib import Path
+from typing import Dict, List, Optional
+import asyncio
+
+from rich.console import Console
+
+from .providers import SearchProviderRegistry
+
+console = Console()
+
+
+class HashSearcher:
+    """Optimized search for files by their content hashes."""
+    
+    def __init__(
+        self,
+        search_registry: Optional[SearchProviderRegistry] = None,
+        verbose: bool = False
+    ):
+        """Initialize the hash searcher.
+        
+        Args:
+            search_registry: Registry of search providers to use
+            verbose: Whether to show verbose output
+        """
+        self.verbose = verbose
+        self.search_registry = search_registry
+    
+    def compute_file_hashes(self, file_path: Path) -> Dict[str, str]:
+        """Compute various hashes for a file.
+        
+        Args:
+            file_path: Path to the file
+            
+        Returns:
+            Dictionary of hash types to hash values
+        """
+        hashes = {}
+        
+        try:
+            content = file_path.read_bytes()
+            
+            # SHA1 (used by Git and SWH)
+            sha1 = hashlib.sha1(content).hexdigest()
+            hashes['sha1'] = sha1
+            
+            # Git blob hash (includes header)
+            git_header = f"blob {len(content)}\0".encode()
+            git_content = git_header + content
+            git_sha1 = hashlib.sha1(git_content).hexdigest()
+            hashes['sha1_git'] = git_sha1
+            
+            # SHA256 (modern standard)
+            sha256 = hashlib.sha256(content).hexdigest()
+            hashes['sha256'] = sha256
+            
+            # MD5 (legacy, but still used)
+            md5 = hashlib.md5(content).hexdigest()
+            hashes['md5'] = md5
+            
+        except Exception as e:
+            if self.verbose:
+                console.print(f"[yellow]Error computing hashes for {file_path}: {e}[/yellow]")
+        
+        return hashes
+    
+    def compute_directory_hash(self, directory: Path) -> str:
+        """Compute directory hash similar to SWH directory identifier.
+        
+        Args:
+            directory: Path to directory
+            
+        Returns:
+            Directory hash
+        """
+        entries = []
+        
+        try:
+            for item in sorted(directory.iterdir()):
+                if item.name.startswith('.'):
+                    continue
+                    
+                if item.is_file():
+                    try:
+                        content = item.read_bytes()
+                        sha1 = hashlib.sha1(content).hexdigest()
+                        entries.append(f"100644 {item.name}\0{bytes.fromhex(sha1)}")
+                    except:
+                        pass
+                elif item.is_dir() and not item.is_symlink():
+                    # Recursive directory hash
+                    dir_hash = self.compute_directory_hash(item)
+                    if dir_hash:
+                        entries.append(f"40000 {item.name}\0{bytes.fromhex(dir_hash)}")
+            
+            if entries:
+                tree_content = b''.join(e.encode() if isinstance(e, str) else e for e in entries)
+                return hashlib.sha1(tree_content).hexdigest()
+                
+        except Exception as e:
+            if self.verbose:
+                console.print(f"[yellow]Error computing directory hash: {e}[/yellow]")
+        
+        return ""
+    
+    async def search_hash(
+        self,
+        hash_value: str,
+        hash_type: str = "auto"
+    ) -> List[str]:
+        """Search for a hash value across all providers.
+        
+        Args:
+            hash_value: The hash value to search for
+            hash_type: Type of hash (sha1, sha256, md5, auto)
+            
+        Returns:
+            List of unique repository URLs found
+        """
+        if not self.search_registry:
+            return []
+        
+        all_urls = set()
+        
+        # Determine hash type if auto
+        if hash_type == "auto":
+            if len(hash_value) == 32:
+                hash_type = "md5"
+            elif len(hash_value) == 40:
+                hash_type = "sha1"
+            elif len(hash_value) == 64:
+                hash_type = "sha256"
+        
+        # Build optimized search queries
+        queries = [
+            f'"{hash_value}" site:github.com',
+            f'"{hash_value}" site:gitlab.com',
+            f'"sha1_git:{hash_value}" site:archive.softwareheritage.org'
+        ]
+        
+        # Execute searches in parallel
+        tasks = []
+        for query in queries:
+            if self.verbose:
+                console.print(f"[dim]Searching: {query}[/dim]")
+            tasks.append(self.search_registry.search_all(query))
+        
+        try:
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+            
+            for result in results:
+                if isinstance(result, dict):
+                    for provider, urls in result.items():
+                        all_urls.update(urls)
+                        
+        except Exception as e:
+            if self.verbose:
+                console.print(f"[yellow]Search error: {e}[/yellow]")
+        
+        return list(all_urls)
+    
+    async def search_file(
+        self,
+        file_path: Path
+    ) -> Dict[str, List[str]]:
+        """Search for a file by computing and searching its hashes.
+        
+        Args:
+            file_path: Path to the file
+            
+        Returns:
+            Dictionary mapping hash types to lists of repository URLs
+        """
+        if not file_path.exists():
+            if self.verbose:
+                console.print(f"[red]File not found: {file_path}[/red]")
+            return {}
+        
+        hashes = self.compute_file_hashes(file_path)
+        
+        if self.verbose:
+            console.print(f"\n[bold]Computed hashes for {file_path.name}:[/bold]")
+            for hash_type, hash_value in hashes.items():
+                console.print(f"  {hash_type}: {hash_value}")
+        
+        results = {}
+        
+        # Search for most relevant hashes in parallel
+        search_tasks = []
+        for hash_type in ['sha1_git', 'sha1']:  # Prioritize git hashes
+            if hash_type in hashes:
+                search_tasks.append((hash_type, self.search_hash(hashes[hash_type], hash_type)))
+        
+        if search_tasks:
+            task_results = await asyncio.gather(*[task for _, task in search_tasks])
+            
+            for (hash_type, _), urls in zip(search_tasks, task_results):
+                if urls:
+                    results[hash_type] = urls
+                    if self.verbose:
+                        console.print(f"[green]Found {len(urls)} results for {hash_type}[/green]")
+        
+        return results

src2id/search/providers.py

@@ -0,0 +1,310 @@
+"""Search providers for source identification."""
+
+import os
+import json
+import hashlib
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Dict, List, Optional, Any
+import asyncio
+
+import aiohttp
+from rich.console import Console
+
+console = Console()
+
+
+class SearchProvider(ABC):
+    """Abstract base class for search providers."""
+    
+    def __init__(self, api_key: Optional[str] = None, verbose: bool = False):
+        """Initialize the search provider."""
+        self.api_key = api_key
+        self.verbose = verbose
+        self.session = None
+    
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Return the name of the search provider."""
+        pass
+    
+    @property
+    @abstractmethod
+    def requires_api_key(self) -> bool:
+        """Return whether this provider requires an API key."""
+        pass
+    
+    @abstractmethod
+    async def search(self, query: str, **kwargs) -> List[str]:
+        """Perform a search and return repository URLs."""
+        pass
+    
+    async def ensure_session(self):
+        """Ensure we have an active session."""
+        if not self.session:
+            self.session = aiohttp.ClientSession()
+    
+    async def close(self):
+        """Close the session."""
+        if self.session:
+            await self.session.close()
+            self.session = None
+    
+    def extract_repo_urls(self, urls: List[str]) -> List[str]:
+        """Extract and filter repository URLs from a list of URLs."""
+        from urllib.parse import urlparse
+
+        repo_urls = []
+        for url in urls:
+            try:
+                parsed = urlparse(url)
+                hostname = parsed.hostname.lower() if parsed.hostname else ''
+
+                if hostname in ('github.com', 'gitlab.com', 'bitbucket.org'):
+                    # Clean up the URL - remove file-specific paths
+                    if "/blob/" in url or "/tree/" in url:
+                        if hostname in ('github.com', 'gitlab.com'):
+                            path_parts = parsed.path.strip('/').split('/')
+                            if len(path_parts) >= 2:
+                                # Keep only owner/repo part
+                                clean_path = f"/{path_parts[0]}/{path_parts[1]}"
+                                clean_url = f"{parsed.scheme}://{hostname}{clean_path}"
+                                repo_urls.append(clean_url)
+                    else:
+                        repo_urls.append(url)
+            except Exception:
+                # If URL parsing fails, skip this URL
+                continue
+        return list(set(repo_urls))
+
+
+class SerpAPIProvider(SearchProvider):
+    """Search provider using SerpAPI (Google Search)."""
+    
+    @property
+    def name(self) -> str:
+        return "SerpAPI"
+    
+    @property
+    def requires_api_key(self) -> bool:
+        return True
+    
+    async def search(self, query: str, **kwargs) -> List[str]:
+        """Search using SerpAPI."""
+        if not self.api_key:
+            return []
+        
+        await self.ensure_session()
+        
+        params = {
+            "q": query,
+            "api_key": self.api_key,
+            "engine": kwargs.get("engine", "google"),
+            "num": kwargs.get("num", 10)
+        }
+        
+        try:
+            async with self.session.get("https://serpapi.com/search", params=params) as response:
+                if response.status == 200:
+                    data = await response.json()
+                    urls = [
+                        result.get("link", "")
+                        for result in data.get("organic_results", [])
+                        if result.get("link")
+                    ]
+                    return self.extract_repo_urls(urls)
+                return []
+        except Exception as e:
+            if self.verbose:
+                console.print(f"[red]{self.name} error: {e}[/red]")
+            return []
+
+
+class GitHubSearchProvider(SearchProvider):
+    """Search provider using GitHub's search API."""
+    
+    @property
+    def name(self) -> str:
+        return "GitHub"
+    
+    @property
+    def requires_api_key(self) -> bool:
+        return False  # Optional
+    
+    async def search(self, query: str, **kwargs) -> List[str]:
+        """Search using GitHub API."""
+        await self.ensure_session()
+        
+        headers = {"Accept": "application/vnd.github.v3+json"}
+        if self.api_key:
+            headers["Authorization"] = f"token {self.api_key}"
+        
+        # Extract search terms
+        import re
+        search_terms = re.findall(r'"([^"]+)"', query)
+        if not search_terms:
+            search_terms = [query]
+        
+        urls = []
+        for term in search_terms[:1]:  # Limit to avoid rate limits
+            try:
+                params = {"q": term[:100], "per_page": 5, "sort": "stars"}
+                async with self.session.get(
+                    "https://api.github.com/search/repositories",
+                    params=params,
+                    headers=headers
+                ) as response:
+                    if response.status == 200:
+                        data = await response.json()
+                        urls.extend([
+                            item.get("html_url", "")
+                            for item in data.get("items", [])
+                            if item.get("html_url")
+                        ])
+            except Exception:
+                pass
+        
+        return self.extract_repo_urls(urls)
+
+
+class SourcegraphProvider(SearchProvider):
+    """Search provider using Sourcegraph code search."""
+    
+    @property
+    def name(self) -> str:
+        return "Sourcegraph"
+    
+    @property
+    def requires_api_key(self) -> bool:
+        return False
+    
+    async def search(self, query: str, **kwargs) -> List[str]:
+        """Search using Sourcegraph (simplified implementation)."""
+        # Note: Full implementation would use Sourcegraph API
+        # This is a placeholder that returns empty results
+        return []
+
+
+class SCANOSSProvider(SearchProvider):
+    """Provider for SCANOSS code identification service."""
+    
+    DEFAULT_URL = 'https://api.osskb.org/scan/direct'
+    PREMIUM_URL = 'https://api.scanoss.com/scan/direct'
+    
+    def __init__(self, api_key: Optional[str] = None, verbose: bool = False):
+        """Initialize SCANOSS provider."""
+        super().__init__(api_key, verbose)
+        self.url = self.PREMIUM_URL if api_key else self.DEFAULT_URL
+    
+    @property
+    def name(self) -> str:
+        return "SCANOSS"
+    
+    @property
+    def requires_api_key(self) -> bool:
+        return False  # Works without key
+    
+    async def search(self, query: str, **kwargs) -> List[str]:
+        """SCANOSS doesn't do text search - use scan_file instead."""
+        return []
+    
+    async def scan_file(self, file_path: Path) -> Dict:
+        """Scan a file using SCANOSS winnowing algorithm."""
+        await self.ensure_session()
+        
+        try:
+            content = file_path.read_bytes()
+            wfp = self._create_wfp(file_path, content)
+            
+            headers = {'User-Agent': 'swhpi-scanner/1.0'}
+            if self.api_key:
+                headers['X-Session'] = self.api_key
+            
+            form_data = aiohttp.FormData()
+            form_data.add_field('file', wfp, filename='scan.wfp')
+            form_data.add_field('format', 'plain')
+            
+            async with self.session.post(
+                self.url,
+                data=form_data,
+                headers=headers,
+                timeout=aiohttp.ClientTimeout(total=30)
+            ) as response:
+                if response.status == 200:
+                    result_text = await response.text()
+                    return json.loads(result_text)
+                return {}
+        except Exception as e:
+            if self.verbose:
+                console.print(f"[red]SCANOSS error: {e}[/red]")
+            return {}
+    
+    def _create_wfp(self, file_path: Path, content: bytes) -> str:
+        """Create simplified WFP for SCANOSS."""
+        md5_hash = hashlib.md5(content).hexdigest()
+        return f"file={md5_hash},{len(content)},{file_path.name}\n1=00000000"
+
+
+class SearchProviderRegistry:
+    """Registry for managing search providers."""
+    
+    def __init__(self, verbose: bool = False):
+        """Initialize the registry."""
+        self.providers: Dict[str, SearchProvider] = {}
+        self.verbose = verbose
+    
+    def register_provider(self, name: str, provider: SearchProvider):
+        """Register a search provider."""
+        self.providers[name] = provider
+    
+    def get_provider(self, name: str) -> Optional[SearchProvider]:
+        """Get a provider by name."""
+        return self.providers.get(name)
+    
+    async def search_all(self, query: str, **kwargs) -> Dict[str, List[str]]:
+        """Search using all registered providers."""
+        results = {}
+        for name, provider in self.providers.items():
+            if provider.requires_api_key and not provider.api_key:
+                continue
+            try:
+                urls = await provider.search(query, **kwargs)
+                if urls:
+                    results[name] = urls
+            except Exception:
+                pass
+        return results
+    
+    async def close_all(self):
+        """Close all provider sessions."""
+        for provider in self.providers.values():
+            await provider.close()
+
+
+def create_default_registry(verbose: bool = False) -> SearchProviderRegistry:
+    """Create a registry with default providers configured from environment."""
+    registry = SearchProviderRegistry(verbose=verbose)
+    
+    # SerpAPI
+    if serpapi_key := os.environ.get("SERPAPI_KEY"):
+        registry.register_provider(
+            "serpapi",
+            SerpAPIProvider(api_key=serpapi_key, verbose=verbose)
+        )
+    
+    # GitHub
+    github_token = os.environ.get("GITHUB_TOKEN")
+    registry.register_provider(
+        "github",
+        GitHubSearchProvider(api_key=github_token, verbose=verbose)
+    )
+    
+    # SCANOSS
+    scanoss_key = os.environ.get("SCANOSS_API_KEY")
+    registry.register_provider(
+        "scanoss",
+        SCANOSSProvider(api_key=scanoss_key, verbose=verbose)
+    )
+    
+    return registry