src2purl 1.3.2__py3-none-any.whl

src2id/integrations/oslili.py

@@ -0,0 +1,228 @@
+"""Integration with oslili for enhanced license detection.
+
+This module provides integration with the oslili (Open Source License 
+Identification Library) tool for more accurate license detection in packages.
+"""
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+try:
+    from semantic_copycat_oslili import (
+        LicenseCopyrightDetector,
+        DetectionResult,
+        DetectedLicense,
+        CopyrightInfo,
+        Config
+    )
+    HAS_OSLILI = True
+except ImportError:
+    HAS_OSLILI = False
+    Config = None  # Define Config as None for type hints when oslili not available
+
+
+class OsliliIntegration:
+    """Integration with oslili for license detection."""
+    
+    def __init__(self, config: Optional[Any] = None):
+        """Initialize oslili integration.
+        
+        Args:
+            config: Optional Config object for oslili configuration
+        """
+        self.available = HAS_OSLILI
+        if self.available:
+            try:
+                # Create config if not provided
+                if config is None and HAS_OSLILI:
+                    config = Config(
+                        verbose=False,
+                        debug=False,
+                        thread_count=4,
+                        similarity_threshold=0.97,
+                        max_recursion_depth=5
+                    )
+                self.detector = LicenseCopyrightDetector(config)
+            except Exception:
+                self.detector = None
+                self.available = False
+        else:
+            self.detector = None
+    
+    def detect_licenses(self, path: Path) -> Dict[str, Any]:
+        """
+        Detect licenses in a directory or file.
+        
+        Args:
+            path: Path to analyze
+            
+        Returns:
+            Dictionary with license information:
+            - licenses: List of detected license identifiers
+            - confidence: Confidence score (0-1)
+            - files: Dict mapping files to their licenses
+            - summary: Human-readable summary
+        """
+        if not self.available or not self.detector:
+            return {
+                "licenses": [],
+                "confidence": 0.0,
+                "files": {},
+                "summary": "oslili not available",
+                "error": "oslili integration not available"
+            }
+        
+        try:
+            # Process the path using oslili v1.3.2
+            result: DetectionResult = self.detector.process_local_path(str(path))
+            
+            if result and result.licenses:
+                # Extract license information
+                licenses = []
+                confidence_scores = []
+                files = {}
+                
+                # Group licenses by category for better understanding
+                declared = [l for l in result.licenses if l.category == "declared"]
+                detected = [l for l in result.licenses if l.category == "detected"]
+                referenced = [l for l in result.licenses if l.category == "referenced"]
+                
+                # Prioritize declared licenses, then detected, then referenced
+                all_licenses = declared + detected + referenced
+                
+                for license_info in all_licenses:
+                    licenses.append(license_info.spdx_id)
+                    confidence_scores.append(license_info.confidence)
+                    
+                    # Map files to licenses
+                    if license_info.source_file:
+                        if license_info.source_file not in files:
+                            files[license_info.source_file] = []
+                        files[license_info.source_file].append({
+                            "spdx_id": license_info.spdx_id,
+                            "confidence": license_info.confidence,
+                            "category": license_info.category,
+                            "method": license_info.detection_method
+                        })
+                
+                # Calculate average confidence
+                avg_confidence = sum(confidence_scores) / len(confidence_scores) if confidence_scores else 0.8
+                
+                # Generate summary
+                if licenses:
+                    # Remove duplicates while preserving order
+                    unique_licenses = self._deduplicate_licenses(licenses)
+                    summary = f"Found {len(unique_licenses)} unique license(s): {', '.join(unique_licenses[:3])}"
+                    if len(unique_licenses) > 3:
+                        summary += f" and {len(unique_licenses) - 3} more"
+                    
+                    # Add category info to summary
+                    if declared:
+                        summary += f" (Declared: {len(declared)})"
+                    if detected:
+                        summary += f" (Detected: {len(detected)})"
+                else:
+                    summary = "No licenses detected"
+                
+                return {
+                    "licenses": licenses,
+                    "confidence": avg_confidence,
+                    "files": files,
+                    "summary": summary,
+                    "copyrights": [c.to_dict() for c in result.copyrights] if result.copyrights else []
+                }
+            else:
+                return {
+                    "licenses": [],
+                    "confidence": 0.0,
+                    "files": {},
+                    "summary": "No licenses detected",
+                    "copyrights": []
+                }
+                
+        except Exception as e:
+            return {
+                "licenses": [],
+                "confidence": 0.0,
+                "files": {},
+                "summary": f"Error during detection: {e}",
+                "error": str(e)
+            }
+    
+    def _deduplicate_licenses(self, licenses: List[str]) -> List[str]:
+        """Remove duplicates while preserving order."""
+        return list(dict.fromkeys(licenses))
+    
+    def enhance_package_match(self, match: "PackageMatch", path: Path) -> "PackageMatch":
+        """
+        Enhance a package match with oslili license detection.
+        
+        Args:
+            match: Package match to enhance
+            path: Path to the package directory
+            
+        Returns:
+            Enhanced package match with better license information
+        """
+        if not self.available:
+            return match
+        
+        # Detect licenses
+        license_info = self.detect_licenses(path)
+        
+        # Update match if we found licenses with good confidence
+        if license_info["licenses"] and license_info["confidence"] > 0.7:
+            # Remove duplicates while preserving order
+            unique_licenses = self._deduplicate_licenses(license_info["licenses"])
+            
+            # Use the most confident license as primary
+            primary_license = unique_licenses[0]
+            
+            # Update match license if not already set or if ours is more confident
+            if not match.license or license_info["confidence"] > 0.85:
+                match.license = primary_license
+            
+            # Add metadata about additional licenses and copyrights
+            if not hasattr(match, "metadata"):
+                match.metadata = {}
+            
+            if len(unique_licenses) > 1:
+                match.metadata["additional_licenses"] = unique_licenses[1:]
+            
+            match.metadata["license_confidence"] = license_info["confidence"]
+            
+            # Add copyright information if available
+            if license_info.get("copyrights"):
+                match.metadata["copyrights"] = license_info["copyrights"]
+            
+            # Add file mapping if available
+            if license_info.get("files"):
+                match.metadata["license_files"] = license_info["files"]
+        
+        return match
+    
+
+
+def enhance_with_oslili(package_matches: List["PackageMatch"], base_path: Path) -> List["PackageMatch"]:
+    """
+    Enhance package matches with oslili license detection.
+    
+    Args:
+        package_matches: List of package matches to enhance
+        base_path: Base path where the code is located
+        
+    Returns:
+        Enhanced package matches
+    """
+    integration = OsliliIntegration()
+    
+    if not integration.available:
+        print("oslili not available - skipping license enhancement")
+        return package_matches
+    
+    enhanced_matches = []
+    for match in package_matches:
+        enhanced = integration.enhance_package_match(match, base_path)
+        enhanced_matches.append(enhanced)
+    
+    return enhanced_matches

src2id/integrations/upmex.py

@@ -0,0 +1,305 @@
+"""Integration with UPMEX for package metadata extraction."""
+
+import json
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Set
+
+from src2id.core.models import PackageMatch, MatchType
+from src2id.integrations.manifest_parser import DirectManifestParser
+
+logger = logging.getLogger(__name__)
+
+class UpmexIntegration:
+    """Integration with UPMEX Universal Package Metadata Extractor."""
+
+    # Package file patterns that UPMEX can handle
+    UPMEX_SUPPORTED_FILES = {
+        # Python
+        'setup.py', 'pyproject.toml', 'setup.cfg',
+        # Node.js/NPM
+        'package.json', 'package-lock.json',
+        # Java/Maven
+        'pom.xml',
+        # Gradle
+        'build.gradle', 'build.gradle.kts',
+        # Go
+        'go.mod', 'go.sum',
+        # Rust
+        'Cargo.toml', 'Cargo.lock',
+        # Ruby
+        '*.gemspec', 'Gemfile', 'Gemfile.lock',
+        # CocoaPods
+        '*.podspec', '*.podspec.json',
+        # Conda
+        'meta.yaml', 'conda.yaml',
+        # Perl
+        'META.json', 'META.yml', 'Makefile.PL',
+        # Conan
+        'conanfile.py', 'conanfile.txt',
+        # NuGet
+        '*.csproj', '*.nuspec', 'packages.config',
+        # Debian
+        'control', 'debian/control',
+    }
+
+    def __init__(self, enabled: bool = True):
+        """
+        Initialize UPMEX integration.
+
+        Args:
+            enabled: Whether UPMEX integration is enabled
+        """
+        self.enabled = enabled
+        self._manifest_parser = DirectManifestParser()
+
+        if enabled:
+            from upmex import PackageExtractor
+            self._extractor = PackageExtractor()
+            logger.info("UPMEX integration initialized successfully")
+
+    def scan_directory_for_packages(self, directory: Path) -> List[Path]:
+        """
+        Scan directory for package files using manifest parser.
+
+        Args:
+            directory: Directory to scan
+
+        Returns:
+            List of package file paths found
+        """
+        if not self.enabled:
+            return []
+
+        return self._manifest_parser.scan_directory_for_manifests(directory)
+
+    # Note: Directory scanning is now handled by DirectManifestParser
+
+    def extract_metadata_from_file(self, file_path: Path) -> Optional[PackageMatch]:
+        """
+        Extract package metadata from a single package file.
+
+        Args:
+            file_path: Path to package file
+
+        Returns:
+            PackageMatch if extraction successful, None otherwise
+        """
+        if not self.enabled:
+            return None
+
+        try:
+            # Try UPMEX extractor first
+            upmex_metadata = self._extractor.extract_from_file(str(file_path))
+            if upmex_metadata:
+                return self._convert_upmex_metadata(upmex_metadata, file_path)
+        except Exception as e:
+            logger.debug(f"UPMEX extraction failed for {file_path}: {e}")
+
+        # Fallback to direct manifest parser
+        return self._manifest_parser.extract_metadata_from_file(file_path)
+
+    def extract_metadata_from_directory(self, directory: Path) -> List[PackageMatch]:
+        """
+        Extract metadata from all package files found in a directory.
+
+        Args:
+            directory: Directory to scan and extract from
+
+        Returns:
+            List of PackageMatch objects (deduplicated by package name)
+        """
+        if not self.enabled:
+            return []
+
+        matches = []
+
+        try:
+            # Try UPMEX extractor for directory
+            upmex_results = self._extractor.extract_from_directory(str(directory))
+            for upmex_metadata in upmex_results:
+                if upmex_metadata:
+                    match = self._convert_upmex_metadata(upmex_metadata, directory)
+                    if match:
+                        matches.append(match)
+        except Exception as e:
+            logger.debug(f"UPMEX directory extraction failed for {directory}: {e}")
+
+        # Also use direct manifest parser for additional coverage
+        manifest_matches = self._manifest_parser.extract_metadata_from_directory(directory)
+        matches.extend(manifest_matches)
+
+        # Deduplicate by package name
+        seen_packages = set()
+        unique_matches = []
+        for match in matches:
+            if match.name and match.name not in seen_packages:
+                seen_packages.add(match.name)
+                unique_matches.append(match)
+
+        return unique_matches
+
+    def _convert_upmex_metadata(self, upmex_metadata, source_file: Path) -> PackageMatch:
+        """
+        Convert UPMEX metadata to PackageMatch format.
+
+        Args:
+            upmex_metadata: UPMEX PackageMetadata object
+            source_file: Source file that was processed
+
+        Returns:
+            PackageMatch object
+        """
+        # Extract basic package information
+        name = getattr(upmex_metadata, 'name', None)
+        version = getattr(upmex_metadata, 'version', None)
+
+        # Debug: Print the metadata structure
+        if logger.isEnabledFor(logging.DEBUG):
+            logger.debug(f"UPMEX metadata for {source_file.name}: {vars(upmex_metadata)}")
+
+        # Get download URL from repository information
+        download_url = self._extract_download_url(upmex_metadata)
+
+        # Extract license information
+        license_info = self._extract_license_info(upmex_metadata)
+
+        # Generate PURL if available
+        purl = getattr(upmex_metadata, 'purl', None)
+
+        # Determine confidence based on data completeness
+        confidence = self._calculate_confidence(upmex_metadata)
+
+        # Create PackageMatch
+        package_match = PackageMatch(
+            download_url=download_url or f"file://{source_file.parent}",
+            match_type=MatchType.EXACT,  # Direct metadata extraction is exact
+            confidence_score=confidence,
+            name=name,
+            version=version,
+            license=license_info,
+            sh_url=None,  # No Software Heritage URL for direct extraction
+            frequency_count=0,
+            is_official_org=self._is_official_organization(download_url),
+            purl=purl,
+        )
+
+        return package_match
+
+    def _extract_download_url(self, metadata) -> Optional[str]:
+        """Extract download URL from UPMEX metadata."""
+        # Try to get repository URL
+        if hasattr(metadata, 'repository') and metadata.repository:
+            if isinstance(metadata.repository, dict):
+                return metadata.repository.get('url')
+            elif isinstance(metadata.repository, str):
+                return metadata.repository
+
+        # Try homepage
+        if hasattr(metadata, 'homepage') and metadata.homepage:
+            return metadata.homepage
+
+        # Try to construct from package type and name
+        if hasattr(metadata, 'package_type') and hasattr(metadata, 'name'):
+            package_type = getattr(metadata.package_type, 'value', str(metadata.package_type))
+            name = metadata.name
+
+            # Generate conventional URLs for major ecosystems
+            url_patterns = {
+                'python': f"https://pypi.org/project/{name}/",
+                'npm': f"https://www.npmjs.com/package/{name}",
+                'maven': f"https://mvnrepository.com/artifact/{name}",
+                'gem': f"https://rubygems.org/gems/{name}",
+                'cargo': f"https://crates.io/crates/{name}",
+                'go': f"https://pkg.go.dev/{name}",
+            }
+
+            return url_patterns.get(package_type.lower())
+
+        return None
+
+    def _extract_license_info(self, metadata) -> Optional[str]:
+        """Extract license information from UPMEX metadata."""
+        # Try licenses list first
+        if hasattr(metadata, 'licenses') and metadata.licenses:
+            license_ids = []
+            for license_obj in metadata.licenses:
+                if hasattr(license_obj, 'spdx_id') and license_obj.spdx_id:
+                    license_ids.append(license_obj.spdx_id)
+                elif hasattr(license_obj, 'name') and license_obj.name:
+                    license_ids.append(license_obj.name)
+
+            if license_ids:
+                return ', '.join(license_ids)
+
+        # Try direct license field
+        if hasattr(metadata, 'license') and metadata.license:
+            return metadata.license
+
+        return None
+
+    def _calculate_confidence(self, metadata) -> float:
+        """Calculate confidence score based on metadata completeness."""
+        confidence = 0.7  # Base confidence for direct metadata extraction
+
+        # Boost confidence for complete metadata
+        if hasattr(metadata, 'name') and metadata.name:
+            confidence += 0.1
+
+        if hasattr(metadata, 'version') and metadata.version:
+            confidence += 0.1
+
+        if hasattr(metadata, 'licenses') and metadata.licenses:
+            confidence += 0.05
+
+        if (hasattr(metadata, 'repository') and metadata.repository) or \
+           (hasattr(metadata, 'homepage') and metadata.homepage):
+            confidence += 0.05
+
+        return min(1.0, confidence)
+
+    def _is_official_organization(self, url: Optional[str]) -> bool:
+        """Check if URL belongs to an official organization."""
+        if not url:
+            return False
+
+        # Official organization patterns
+        official_patterns = [
+            'github.com/python/',
+            'github.com/nodejs/',
+            'github.com/golang/',
+            'github.com/rust-lang/',
+            'github.com/microsoft/',
+            'github.com/google/',
+            'github.com/apache/',
+            'github.com/eclipse/',
+            'gitlab.com/gitlab-org/',
+        ]
+
+        return any(pattern in url.lower() for pattern in official_patterns)
+
+    def _deduplicate_matches(self, matches: List[PackageMatch]) -> List[PackageMatch]:
+        """
+        Deduplicate matches by package name, keeping the best quality match.
+
+        Args:
+            matches: List of package matches
+
+        Returns:
+            Deduplicated list of matches
+        """
+        if not matches:
+            return matches
+
+        # Group by package name
+        grouped = {}
+        for match in matches:
+            name = match.name or "unknown"
+            if name not in grouped or match.confidence_score > grouped[name].confidence_score:
+                grouped[name] = match
+
+        return list(grouped.values())
+
+    def get_supported_file_types(self) -> Set[str]:
+        """Get set of file types supported by UPMEX integration."""
+        return self.UPMEX_SUPPORTED_FILES.copy()

src2id/search/__init__.py

@@ -0,0 +1,34 @@
+"""Unified search module for source identification."""
+
+from .providers import (
+    SearchProvider,
+    SerpAPIProvider,
+    GitHubSearchProvider,
+    SourcegraphProvider,
+    SCANOSSProvider,
+    SearchProviderRegistry,
+    create_default_registry
+)
+
+from .strategies import (
+    SourceIdentifier,
+    identify_source
+)
+
+from .hash_search import HashSearcher
+
+__all__ = [
+    # Providers
+    'SearchProvider',
+    'SerpAPIProvider', 
+    'GitHubSearchProvider',
+    'SourcegraphProvider',
+    'SCANOSSProvider',
+    'SearchProviderRegistry',
+    'create_default_registry',
+    # Strategies
+    'SourceIdentifier',
+    'identify_source',
+    # Hash search
+    'HashSearcher',
+]