src2purl 1.3.2__py3-none-any.whl

src2id/core/subcomponent_detector.py

@@ -0,0 +1,353 @@
+"""Detect and identify multiple subcomponents in a project."""
+
+import asyncio
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+from dataclasses import dataclass
+
+from rich.console import Console
+from rich.table import Table
+
+console = Console()
+
+
+@dataclass
+class Subcomponent:
+    """Represents a detected subcomponent in a project."""
+    path: Path
+    type: str  # 'npm', 'python', 'rust', 'go', 'java', 'mono', 'multi'
+    name: Optional[str] = None
+    markers: List[str] = None
+    
+    def __post_init__(self):
+        if self.markers is None:
+            self.markers = []
+
+
+class SubcomponentDetector:
+    """Detects multiple subcomponents in a project structure."""
+    
+    # Package markers that indicate a component boundary
+    PACKAGE_MARKERS = {
+        'package.json': 'npm',
+        'pyproject.toml': 'python',
+        'setup.py': 'python',
+        'requirements.txt': 'python',
+        'Cargo.toml': 'rust',
+        'go.mod': 'go',
+        'pom.xml': 'java',
+        'build.gradle': 'java',
+        'build.gradle.kts': 'java',
+        'Gemfile': 'ruby',
+        'composer.json': 'php',
+        'CMakeLists.txt': 'cmake',
+        'Makefile': 'make',
+        '.csproj': 'dotnet',
+        'mix.exs': 'elixir',
+    }
+    
+    # Directories that typically indicate subcomponents
+    SUBCOMPONENT_PATTERNS = {
+        'packages/*': 'monorepo',  # Lerna/Yarn workspaces
+        'apps/*': 'monorepo',      # Nx/Turborepo
+        'services/*': 'microservices',
+        'libs/*': 'libraries',
+        'modules/*': 'modules',
+        'components/*': 'components',
+        'plugins/*': 'plugins',
+        'extensions/*': 'extensions',
+    }
+    
+    def __init__(self, verbose: bool = False):
+        """Initialize the detector."""
+        self.verbose = verbose
+    
+    def detect_subcomponents(
+        self,
+        root_path: Path,
+        max_depth: int = 3
+    ) -> List[Subcomponent]:
+        """
+        Detect all subcomponents in a project.
+        
+        Args:
+            root_path: Root directory to scan
+            max_depth: Maximum depth to search for subcomponents
+            
+        Returns:
+            List of detected subcomponents
+        """
+        subcomponents = []
+        visited = set()
+        
+        # Check if root itself is a component
+        root_markers = self._check_markers(root_path)
+        if root_markers:
+            root_component = Subcomponent(
+                path=root_path,
+                type=self._determine_type(root_markers),
+                name=root_path.name,
+                markers=root_markers
+            )
+            subcomponents.append(root_component)
+            visited.add(root_path)
+        
+        # Search for nested subcomponents
+        self._scan_for_subcomponents(
+            root_path,
+            subcomponents,
+            visited,
+            current_depth=0,
+            max_depth=max_depth
+        )
+        
+        # Deduplicate and organize
+        subcomponents = self._organize_subcomponents(subcomponents)
+        
+        if self.verbose:
+            self._print_detection_results(subcomponents)
+        
+        return subcomponents
+    
+    def _scan_for_subcomponents(
+        self,
+        path: Path,
+        subcomponents: List[Subcomponent],
+        visited: set,
+        current_depth: int,
+        max_depth: int
+    ):
+        """Recursively scan for subcomponents."""
+        if current_depth >= max_depth:
+            return
+        
+        try:
+            for item in path.iterdir():
+                if item.is_dir() and item not in visited:
+                    # Skip common non-component directories
+                    if item.name in {'.git', '__pycache__', 'node_modules', 
+                                   'venv', 'build', 'dist', '.idea', '.vscode'}:
+                        continue
+                    
+                    # Check for package markers
+                    markers = self._check_markers(item)
+                    
+                    if markers:
+                        # Found a subcomponent
+                        component = Subcomponent(
+                            path=item,
+                            type=self._determine_type(markers),
+                            name=item.name,
+                            markers=markers
+                        )
+                        subcomponents.append(component)
+                        visited.add(item)
+                        
+                        # Don't scan inside detected components by default
+                        # unless it's a monorepo pattern
+                        if self._is_monorepo_pattern(item):
+                            self._scan_for_subcomponents(
+                                item, subcomponents, visited,
+                                current_depth + 1, max_depth
+                            )
+                    else:
+                        # Continue scanning subdirectories
+                        self._scan_for_subcomponents(
+                            item, subcomponents, visited,
+                            current_depth + 1, max_depth
+                        )
+        except PermissionError:
+            pass
+    
+    def _check_markers(self, path: Path) -> List[str]:
+        """Check for package markers in a directory."""
+        markers = []
+        
+        for marker_file, _ in self.PACKAGE_MARKERS.items():
+            marker_path = path / marker_file
+            if marker_path.exists():
+                markers.append(marker_file)
+        
+        return markers
+    
+    def _determine_type(self, markers: List[str]) -> str:
+        """Determine component type from markers."""
+        types = set()
+        for marker in markers:
+            if marker in self.PACKAGE_MARKERS:
+                types.add(self.PACKAGE_MARKERS[marker])
+        
+        if len(types) == 1:
+            return list(types)[0]
+        elif len(types) > 1:
+            return 'multi'
+        else:
+            return 'unknown'
+    
+    def _is_monorepo_pattern(self, path: Path) -> bool:
+        """Check if this looks like a monorepo."""
+        # Check for workspace configuration files
+        workspace_files = [
+            'lerna.json',
+            'nx.json',
+            'pnpm-workspace.yaml',
+            'rush.json',
+            'turbo.json'
+        ]
+        
+        for ws_file in workspace_files:
+            if (path / ws_file).exists():
+                return True
+        
+        # Check for packages/apps directories
+        monorepo_dirs = {'packages', 'apps', 'services', 'libs'}
+        subdirs = {item.name for item in path.iterdir() if item.is_dir()}
+        
+        return bool(monorepo_dirs & subdirs)
+    
+    def _organize_subcomponents(
+        self,
+        subcomponents: List[Subcomponent]
+    ) -> List[Subcomponent]:
+        """Organize and deduplicate subcomponents."""
+        # Remove duplicates based on path
+        seen_paths = set()
+        unique_components = []
+        
+        for comp in subcomponents:
+            if comp.path not in seen_paths:
+                unique_components.append(comp)
+                seen_paths.add(comp.path)
+        
+        # Sort by path depth (root first, then nested)
+        unique_components.sort(key=lambda c: len(c.path.parts))
+        
+        return unique_components
+    
+    def _print_detection_results(self, subcomponents: List[Subcomponent]):
+        """Print detected subcomponents in a nice table."""
+        if not subcomponents:
+            console.print("[yellow]No subcomponents detected[/yellow]")
+            return
+        
+        table = Table(title=f"Detected {len(subcomponents)} Subcomponents")
+        table.add_column("Path", style="cyan")
+        table.add_column("Type", style="green")
+        table.add_column("Markers", style="yellow")
+        
+        for comp in subcomponents:
+            # Try to get relative path, fallback to absolute
+            try:
+                rel_path = str(comp.path.relative_to(Path.cwd()))
+            except ValueError:
+                # If not relative to cwd, just use the path as-is
+                rel_path = str(comp.path)
+            
+            markers_str = ", ".join(comp.markers[:3])
+            if len(comp.markers) > 3:
+                markers_str += f" +{len(comp.markers)-3}"
+            
+            table.add_row(rel_path, comp.type, markers_str)
+        
+        console.print(table)
+
+
+async def identify_subcomponents(
+    root_path: Path,
+    max_depth: int = 3,
+    confidence_threshold: float = 0.5,
+    verbose: bool = False,
+    use_swh: bool = False
+) -> Dict[str, Any]:
+    """
+    Identify all subcomponents in a project.
+    
+    Args:
+        root_path: Root directory to analyze
+        max_depth: Maximum depth for scanning
+        confidence_threshold: Minimum confidence for identification
+        verbose: Enable verbose output
+        use_swh: Include Software Heritage checking
+        
+    Returns:
+        Dictionary with identification results for all subcomponents
+    """
+    from src2id.search import identify_source
+    
+    # Detect subcomponents
+    detector = SubcomponentDetector(verbose=verbose)
+    subcomponents = detector.detect_subcomponents(root_path, max_depth)
+    
+    if not subcomponents:
+        # No subcomponents detected, identify as single project
+        if verbose:
+            console.print("[dim]No subcomponents detected, analyzing as single project[/dim]")
+        
+        result = await identify_source(
+            path=root_path,
+            max_depth=max_depth,
+            confidence_threshold=confidence_threshold,
+            verbose=verbose,
+            use_swh=use_swh
+        )
+        
+        return {
+            "root": root_path,
+            "subcomponents": [],
+            "single_result": result
+        }
+    
+    # Identify each subcomponent
+    results = {
+        "root": root_path,
+        "subcomponents": [],
+        "total_identified": 0,
+        "total_components": len(subcomponents)
+    }
+    
+    if verbose:
+        console.print(f"\n[bold]Identifying {len(subcomponents)} subcomponents...[/bold]\n")
+    
+    for i, comp in enumerate(subcomponents, 1):
+        if verbose:
+            console.print(f"[cyan]Component {i}/{len(subcomponents)}: {comp.path.name}[/cyan]")
+        
+        # Identify this component
+        comp_result = await identify_source(
+            path=comp.path,
+            max_depth=1,  # Don't go too deep for subcomponents
+            confidence_threshold=confidence_threshold,
+            verbose=False,  # Less verbose for individual components
+            use_swh=use_swh
+        )
+        
+        # Add component info to result
+        comp_info = {
+            "path": str(comp.path),
+            "type": comp.type,
+            "markers": comp.markers,
+            "identified": comp_result["identified"],
+            "confidence": comp_result["confidence"],
+            "repository": comp_result.get("final_origin"),
+            "strategies_used": comp_result.get("strategies_used", [])
+        }
+        
+        results["subcomponents"].append(comp_info)
+        
+        if comp_result["identified"]:
+            results["total_identified"] += 1
+            
+        if verbose:
+            if comp_result["identified"]:
+                console.print(f"  ✓ Identified: {comp_result['final_origin']}")
+            else:
+                console.print(f"  ✗ Not identified")
+    
+    # Print summary
+    if verbose:
+        console.print(f"\n[bold green]Summary:[/bold green]")
+        console.print(f"Total components: {results['total_components']}")
+        console.print(f"Identified: {results['total_identified']}")
+        console.print(f"Success rate: {results['total_identified']/results['total_components']:.1%}")
+    
+    return results

src2id/core/swhid.py

@@ -0,0 +1,324 @@
+"""SWHID generation using Software Heritage tools."""
+
+import hashlib
+import os
+from pathlib import Path
+from typing import Optional
+
+# Try different SWHID generation methods in order of preference
+HAS_SWH_MODEL = False
+HAS_MINISWHID = False
+
+try:
+    from swh.model.cli import model_of_dir
+    from swh.model.from_disk import Content
+    HAS_SWH_MODEL = True
+except ImportError:
+    try:
+        import miniswhid
+        HAS_MINISWHID = True
+    except ImportError:
+        import warnings
+        warnings.warn(
+            "No accurate SWHID generation available. "
+            "Install swh.model or miniswhid for accurate SWHID generation: "
+            "pip install swh.model"
+        )
+
+
+class SWHIDGenerator:
+    """
+    Generates Software Heritage Identifiers using miniswhid or custom implementation.
+    """
+    
+    def __init__(self, use_swh_model: bool = True):
+        """
+        Initialize the SWHID generator.
+        
+        Args:
+            use_swh_model: Whether to use swh.model if available
+        """
+        self.use_swh_model = use_swh_model and HAS_SWH_MODEL
+        self.use_miniswhid = (not self.use_swh_model) and HAS_MINISWHID
+    
+    def generate_directory_swhid(self, path: Path) -> str:
+        """
+        Generate SWHID for directory content.
+        
+        Args:
+            path: Directory path
+            
+        Returns:
+            SWHID string in format swh:1:dir:HASH
+        """
+        if not path.is_dir():
+            raise ValueError(f"Path {path} is not a directory")
+        
+        if self.use_swh_model:
+            return self._generate_with_swh_model(path)
+        elif self.use_miniswhid:
+            return self._generate_with_miniswhid(path)
+        else:
+            return self._generate_fallback(path)
+    
+    def generate_content_swhid(self, file_path: Path) -> str:
+        """
+        Generate SWHID for individual file content.
+        
+        Args:
+            file_path: File path
+            
+        Returns:
+            SWHID string in format swh:1:cnt:HASH
+        """
+        if not file_path.is_file():
+            raise ValueError(f"Path {file_path} is not a file")
+        
+        if self.use_swh_model:
+            # Use swh.model for file content
+            try:
+                from swh.model.from_disk import Content
+                content = Content.from_file(path=bytes(file_path))
+                return f"swh:1:cnt:{content.hash}"
+            except Exception as e:
+                print(f"Warning: swh.model failed for {file_path}: {e}")
+                return self._hash_file_content(file_path)
+        elif self.use_miniswhid:
+            # Use miniswhid for file content
+            try:
+                result = miniswhid.compute_swhid(str(file_path))
+                if isinstance(result, dict) and 'swhid' in result:
+                    return result['swhid']
+                return str(result)
+            except Exception as e:
+                if hasattr(miniswhid, 'hash_file'):
+                    # Alternative API
+                    return f"swh:1:cnt:{miniswhid.hash_file(str(file_path))}"
+                raise e
+        else:
+            # Fallback implementation
+            return self._hash_file_content(file_path)
+    
+    def _generate_with_swh_model(self, path: Path) -> str:
+        """
+        Generate SWHID using swh.model library.
+        
+        Args:
+            path: Directory path
+            
+        Returns:
+            SWHID string
+        """
+        try:
+            # Use official exclusion patterns like SWH scanner
+            exclusion_patterns = [
+                b'.git', b'.hg', b'.svn', b'__pycache__', 
+                b'.mypy_cache', b'.tox', b'*.egg-info',
+                b'.bzr', b'.coverage', b'.eggs'
+            ]
+            
+            # Use official model_of_dir method (same as SWH scanner)
+            source_tree = model_of_dir(
+                str(path).encode(),
+                exclusion_patterns
+            )
+            
+            # Generate SWHID using official method
+            swhid = source_tree.swhid()
+            return str(swhid)
+            
+        except Exception as e:
+            print(f"Warning: swh.model failed for {path}: {e}")
+            print("Falling back to custom implementation")
+            return self._generate_fallback(path)
+    
+    def _generate_with_miniswhid(self, path: Path) -> str:
+        """
+        Generate SWHID using miniswhid library.
+        
+        Args:
+            path: Directory path
+            
+        Returns:
+            SWHID string
+        """
+        try:
+            # Try the main API
+            result = miniswhid.compute_swhid(str(path))
+            
+            # Handle different return types from miniswhid
+            if isinstance(result, dict):
+                if 'swhid' in result:
+                    return result['swhid']
+                elif 'directory' in result:
+                    return f"swh:1:dir:{result['directory']}"
+            elif isinstance(result, str):
+                if result.startswith('swh:'):
+                    return result
+                else:
+                    return f"swh:1:dir:{result}"
+            
+            # If we get here, try alternative API if available
+            if hasattr(miniswhid, 'hash_directory'):
+                dir_hash = miniswhid.hash_directory(str(path))
+                return f"swh:1:dir:{dir_hash}"
+            
+            # Last resort: use the result as-is
+            return str(result)
+            
+        except Exception as e:
+            print(f"Warning: miniswhid failed for {path}: {e}")
+            print("Falling back to custom implementation")
+            return self._generate_fallback(path)
+    
+    def _generate_fallback(self, path: Path) -> str:
+        """
+        Custom fallback implementation for SWHID generation.
+        
+        This is a simplified version that may not match SH exactly
+        but provides a consistent hash for the directory content.
+        
+        Args:
+            path: Directory path
+            
+        Returns:
+            SWHID string
+        """
+        # Create a hash of the directory structure and content
+        hasher = hashlib.sha1()
+        
+        # Get all files in sorted order for consistency
+        all_files = []
+        for root, dirs, files in os.walk(path):
+            # Skip hidden and build directories
+            dirs[:] = [d for d in dirs if not d.startswith('.') and d not in {
+                '__pycache__', 'node_modules', 'build', 'dist', 'target'
+            }]
+            dirs.sort()  # Ensure consistent ordering
+            
+            for file in sorted(files):
+                if not file.startswith('.'):
+                    file_path = Path(root) / file
+                    relative_path = file_path.relative_to(path)
+                    all_files.append(relative_path)
+        
+        # Sort all files for consistent ordering
+        all_files.sort()
+        
+        # Hash the directory structure
+        for relative_path in all_files:
+            file_path = path / relative_path
+            
+            # Add file path to hash
+            hasher.update(str(relative_path).encode('utf-8'))
+            
+            # Add file content hash
+            try:
+                with open(file_path, 'rb') as f:
+                    file_hasher = hashlib.sha1()
+                    while chunk := f.read(8192):
+                        file_hasher.update(chunk)
+                    hasher.update(file_hasher.digest())
+            except (PermissionError, OSError):
+                # Skip files we can't read
+                continue
+            
+            # Add file mode (simplified)
+            try:
+                mode = oct(file_path.stat().st_mode)[-3:]
+                hasher.update(mode.encode('utf-8'))
+            except (PermissionError, OSError):
+                hasher.update(b'644')  # Default mode
+        
+        # Generate the final hash
+        dir_hash = hasher.hexdigest()
+        
+        # Return in SWHID format
+        # Note: This is a simplified format and may not match SH exactly
+        return f"swh:1:dir:{dir_hash}"
+    
+    def generate_content_swhid(self, file_path: Path) -> str:
+        """
+        Generate SWHID for file content.
+        
+        Args:
+            file_path: File path
+            
+        Returns:
+            SWHID string for the file content
+        """
+        if not file_path.is_file():
+            raise ValueError(f"Not a file: {file_path}")
+        
+        if self.use_swh_model:
+            try:
+                # Use official SWH model for accurate content hashing
+                content = Content.from_file(path=file_path, max_content_length=10_000_000)
+                return str(content.swhid())
+            except Exception:
+                # Fall back to basic implementation
+                pass
+        
+        # Fallback: Generate git-compatible SHA1
+        return self._hash_file_content(file_path)
+    
+    def _hash_file_content(self, file_path: Path) -> str:
+        """
+        Generate content hash for a single file.
+        
+        Args:
+            file_path: File path
+            
+        Returns:
+            SWHID string for content
+        """
+        hasher = hashlib.sha1()
+        
+        try:
+            with open(file_path, 'rb') as f:
+                # Add git-style header
+                content = f.read()
+                header = f"blob {len(content)}\0".encode('utf-8')
+                hasher.update(header)
+                hasher.update(content)
+        except (PermissionError, OSError) as e:
+            raise ValueError(f"Cannot read file {file_path}: {e}")
+        
+        content_hash = hasher.hexdigest()
+        return f"swh:1:cnt:{content_hash}"
+    
+    def validate_swhid(self, swhid: str) -> bool:
+        """
+        Validate SWHID format.
+        
+        Args:
+            swhid: SWHID string to validate
+            
+        Returns:
+            True if valid SWHID format
+        """
+        if not isinstance(swhid, str):
+            return False
+        
+        parts = swhid.split(':')
+        if len(parts) != 4:
+            return False
+        
+        if parts[0] != 'swh':
+            return False
+        
+        if parts[1] != '1':  # Version
+            return False
+        
+        if parts[2] not in {'cnt', 'dir', 'rev', 'rel', 'snp', 'ori'}:
+            return False
+        
+        # Check if hash is valid hex
+        try:
+            int(parts[3], 16)
+            if len(parts[3]) != 40:  # SHA1 length
+                return False
+        except ValueError:
+            return False
+        
+        return True

src2id/integrations/__init__.py

@@ -0,0 +1 @@
+"""External integrations for SWHPI."""