chatsbom 0.2.1__py3-none-any.whl

chatsbom/commands/convert.py

@@ -0,0 +1,263 @@
+import concurrent.futures
+import subprocess
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+import structlog
+import typer
+from rich.console import Console
+from rich.progress import BarColumn
+from rich.progress import Progress
+from rich.progress import SpinnerColumn
+from rich.progress import TextColumn
+from rich.progress import TimeElapsedColumn
+from rich.table import Table
+
+from chatsbom.models.language import Language
+
+logger = structlog.get_logger('converter')
+console = Console()
+
+
+@dataclass
+class ConvertResult:
+    project_path: str
+    status_msg: str
+    converted: int = 0
+    skipped: int = 0
+    failed: int = 0
+
+
+def find_project_dirs(base_dir: Path, language: Language | None = None) -> list[Path]:
+    """
+    Finds leaf project directories.
+    Structure: base_dir / language / owner / repo / branch / [files]
+    We assume the 'branch' directory is the project root.
+    """
+    projects = []
+    # base_dir / lang / owner / repo / branch
+    if not base_dir.exists():
+        return []
+
+    # Iterate languages
+    for lang_dir in base_dir.iterdir():
+        if not lang_dir.is_dir():
+            continue
+
+        # Filter by language if specified
+        if language and lang_dir.name != language.value:
+            continue
+
+        # Iterate owners
+        for owner_dir in lang_dir.iterdir():
+            if not owner_dir.is_dir():
+                continue
+
+            # Iterate repos
+            for repo_dir in owner_dir.iterdir():
+                if not repo_dir.is_dir():
+                    continue
+
+                # Iterate branches (project roots)
+                for branch_dir in repo_dir.iterdir():
+                    if branch_dir.is_dir():
+                        projects.append(branch_dir)
+    return projects
+
+
+def convert_project(project_dir: Path, output_format: str, overwrite: bool) -> ConvertResult:
+    """Runs syft on a project directory."""
+    output_file = project_dir / 'sbom.json'
+    stats = ConvertResult(project_path=str(project_dir), status_msg='')
+
+    if output_file.exists() and not overwrite:
+        stats.skipped += 1
+        stats.status_msg = '[dim]Skip[/dim]'
+
+        # Log skipped/cached
+        timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        log_msg = (
+            f"{timestamp} \\[info     ] SBOM Generated                 "
+            f"elapsed=0.00s output={output_file} "
+            f"project={project_dir} size={output_file.stat().st_size} "
+            f"[green](Cached)[/green]"
+        )
+        console.print(f"[dim]{log_msg}[/dim]")
+
+        return stats
+
+    # syft dir:. -o json
+    cmd = [
+        'syft',
+        f"dir:{project_dir.absolute()}",
+        '-o', output_format,
+    ]
+
+    try:
+        # Capture output to avoid polluting CLI
+        start_time = time.time()
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        elapsed = time.time() - start_time
+
+        # Write output to file
+        with open(output_file, 'w', encoding='utf-8') as f:
+            f.write(result.stdout)
+
+        logger.info(
+            'SBOM Generated',
+            project=str(project_dir),
+            output=str(output_file),
+            size=len(result.stdout),
+            elapsed=f"{elapsed:.2f}s",
+        )
+        stats.converted += 1
+        stats.status_msg = '[green]Done[/green]'
+        return stats
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Syft failed for {project_dir}: {e.stderr}")
+        stats.failed += 1
+        stats.status_msg = '[red]Fail[/red]'
+        return stats
+    except Exception as e:
+        logger.error(f"Error {project_dir}: {e}")
+        stats.failed += 1
+        stats.status_msg = '[red]Err[/red]'
+        return stats
+
+
+def main(
+    input_dir: str = typer.Option(
+        'data', help='Root data directory',
+    ),
+    concurrency: int = typer.Option(
+        4, help='Number of concurrent syft processes',
+    ),
+    output_format: str = typer.Option(
+        'json', '--format', help='Syft output format (json, spdx-json, cyclonedx-json)',
+    ),
+    overwrite: bool = typer.Option(
+        False, help='Overwrite existing SBOM files',
+    ),
+    limit: int | None = typer.Option(
+        None, help='Limit number of projects to convert (for testing)',
+    ),
+    language: Language | None = typer.Option(
+        None, help='Filter by Language (default: all)',
+    ),
+):
+    """
+    Convert downloaded project manifests to SBOMs using Syft.
+    """
+    # Check if syft is installed
+    try:
+        subprocess.run(
+            ['syft', 'version'],
+            capture_output=True,
+            check=True,
+        )
+    except FileNotFoundError:
+        console.print(
+            '[bold red]Error:[/] syft is not installed.\n\n'
+            'The convert-sbom command requires Syft to generate SBOMs. '
+            'Please install Syft:\n\n'
+            '  [cyan]# macOS / Linux (Homebrew)[/]\n'
+            '  [cyan]brew tap anchore/syft[/]\n'
+            '  [cyan]brew install syft[/]\n\n'
+            '  [cyan]# Or via install script[/]\n'
+            '  [cyan]curl -sSfL https://get.anchore.io/syft | sudo sh -s -- -b /usr/local/bin[/]\n\n'
+            'For more options, visit: '
+            '[link=https://github.com/anchore/syft?tab=readme-ov-file#installation]'
+            'https://github.com/anchore/syft?tab=readme-ov-file#installation[/link]',
+        )
+        raise typer.Exit(1)
+    except subprocess.CalledProcessError:
+        console.print(
+            '[bold yellow]Warning:[/] Could not verify syft version, proceeding anyway.',
+        )
+
+    base_path = Path(input_dir)
+    if not base_path.exists():
+        logger.error(f"Input directory not found: {base_path}")
+        raise typer.Exit(1)
+
+    if limit:
+        logger.warning(f"Test Mode: Limiting to top {limit} projects")
+
+    projects = find_project_dirs(base_path, language)
+    if limit:
+        projects = projects[:limit]
+
+    logger.info(
+        'Starting SBOM Conversion',
+        input_dir=input_dir,
+        concurrency=concurrency,
+        format=output_format,
+        overwrite=overwrite,
+        found_projects=len(projects),
+    )
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn('[bold blue]{task.description}'),
+        BarColumn(),
+        TextColumn('[progress.percentage]{task.percentage:>3.0f}%'),
+        TextColumn('•'),
+        TextColumn('[green]{task.completed}/{task.total}'),
+        TextColumn('•'),
+        TextColumn('[dim]{task.fields[status]}'),
+        TextColumn('•'),
+        TimeElapsedColumn(),
+        console=console,
+    ) as progress:
+
+        overall_stats = {
+            'converted': 0,
+            'skipped': 0,
+            'failed': 0,
+            'total': len(projects),
+        }
+        start_time = time.time()
+
+        task = progress.add_task(
+            'Converting...',
+            total=len(projects),
+            status='Starting',
+        )
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
+            future_to_project = {
+                executor.submit(convert_project, p, output_format, overwrite): p
+                for p in projects
+            }
+
+            for future in concurrent.futures.as_completed(future_to_project):
+                result = future.result()
+                overall_stats['converted'] += result.converted
+                overall_stats['skipped'] += result.skipped
+                overall_stats['failed'] += result.failed
+                progress.update(task, advance=1, status=result.status_msg)
+
+    # Print Summary Table
+    total_time = time.time() - start_time
+    table = Table(title='Conversion Summary')
+    table.add_column('Metric', style='cyan')
+    table.add_column('Value', style='magenta')
+
+    table.add_row('Total Projects', str(overall_stats['total']))
+    table.add_row('Converted (Success)', str(overall_stats['converted']))
+    table.add_row('Skipped (Exists)', str(overall_stats['skipped']))
+    table.add_row('Failed', str(overall_stats['failed']))
+    table.add_row('Total Duration', f"{total_time:.2f}s")
+
+    console.print(table)
+
+
+if __name__ == '__main__':
+    typer.run(main)

chatsbom/commands/download.py

@@ -0,0 +1,293 @@
+import concurrent.futures
+import json
+import os
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+import dotenv
+import requests
+import structlog
+import typer
+from rich.console import Console
+from rich.progress import BarColumn
+from rich.progress import Progress
+from rich.progress import SpinnerColumn
+from rich.progress import TextColumn
+from rich.table import Table
+
+from chatsbom.core.client import get_http_client
+from chatsbom.models.language import Language
+from chatsbom.models.language import LanguageFactory
+
+dotenv.load_dotenv()
+console = Console()
+logger = structlog.get_logger('downloader')
+
+
+@dataclass
+class DownloadResult:
+    repo: str
+    status_msg: str
+    downloaded_files: int = 0
+    missing_files: int = 0
+    failed_files: int = 0
+    skipped_files: int = 0
+    cache_hits: int = 0
+
+
+class SBOMDownloader:
+    """Handles concurrent downloading of SBOM files from GitHub."""
+
+    def __init__(self, token: str | None, base_dir: str, timeout: int = 10, pool_size: int = 50):
+        self.session = get_http_client(pool_size=pool_size)
+
+        if token:
+            self.session.headers.update({'Authorization': f"Bearer {token}"})
+
+        self.base_dir = Path(base_dir)
+        self.timeout = timeout
+
+    def download_repo(self, repo: dict, lang: Language) -> DownloadResult:
+        """Downloads SBOM files for a single repository."""
+        full_name = repo['full_name']
+        owner, name = full_name.split('/')
+        branch = repo.get('default_branch', 'master')
+
+        target_dir = self.base_dir / lang / owner / name / branch
+        target_dir.mkdir(parents=True, exist_ok=True)
+
+        base_url = f"https://raw.githubusercontent.com/{owner}/{name}/{branch}"
+        language_handler = LanguageFactory.get_handler(lang)
+        targets: list[str] = language_handler.get_sbom_paths()
+
+        result_msgs = []
+        stats = DownloadResult(repo=full_name, status_msg='')
+
+        for filename in targets:
+            file_path = target_dir / filename
+
+            try:
+                start_time = time.time()
+                url = f"{base_url}/{filename}"
+                resp = self.session.get(url, timeout=self.timeout)
+                elapsed = time.time() - start_time
+
+                # Check for cache hit (requests-cache adds 'from_cache' attribute)
+                if getattr(resp, 'from_cache', False):
+                    stats.cache_hits += 1
+
+                if resp.status_code == 200:
+                    file_path.parent.mkdir(parents=True, exist_ok=True)
+                    with open(file_path, 'wb') as f:
+                        f.write(resp.content)
+
+                    # Visual Caching Indicator
+                    is_cached = getattr(resp, 'from_cache', False)
+                    if is_cached:
+                        timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                        log_msg = (
+                            f"{timestamp} \\[info     ] Downloaded                     "
+                            f"elapsed={elapsed:.2f}s file={filename} "
+                            f"repo={full_name} size={len(resp.content)} "
+                            f"url={resp.url} [green](Cached)[/green]"
+                        )
+                        console.print(f"[dim]{log_msg}[/dim]")
+                    else:
+                        logger.info(
+                            'Downloaded',
+                            repo=full_name,
+                            file=filename,
+                            size=len(resp.content),
+                            elapsed=f"{elapsed:.2f}s",
+                            url=resp.url,
+                        )
+
+                    stats.downloaded_files += 1
+                    result_msgs.append(f"[green]{filename}[/green]")
+                elif resp.status_code == 404:
+                    stats.missing_files += 1
+                    result_msgs.append(
+                        f"[dim yellow]no {filename}[/dim yellow]",
+                    )
+                else:
+                    stats.failed_files += 1
+                    logger.warning(
+                        'Download Failed',
+                        repo=full_name,
+                        file=filename,
+                        status=resp.status_code,
+                        elapsed=f"{elapsed:.2f}s",
+                        url=url,
+                    )
+
+                    result_msgs.append(
+                        f"[red]{filename} {resp.status_code}[/red]",
+                    )
+
+            except requests.RequestException as e:
+                logger.error(
+                    'Download Error',
+                    repo=full_name,
+                    file=filename,
+                    error=str(e),
+                )
+                stats.failed_files += 1
+                result_msgs.append(f"[red]{filename} Err[/red]")
+
+        if not result_msgs:
+            stats.status_msg = f"[dim]{full_name} skip[/dim]"
+        else:
+            stats.status_msg = f"{full_name}: {', '.join(result_msgs)}"
+
+        return stats
+
+
+def load_targets(jsonl_path: str) -> list[dict]:
+    """Loads repository targets from a JSONL file."""
+    targets: list[dict] = []
+    path = Path(jsonl_path)
+    if not path.exists():
+        return targets
+
+    with path.open(encoding='utf-8') as f:
+        for line in f:
+            if line.strip():
+                try:
+                    targets.append(json.loads(line))
+                except json.JSONDecodeError:
+                    pass
+    return targets
+
+
+def main(
+    input_file: str | None = typer.Option(
+        None, help='Input JSONL file path (default: {language}.jsonl)',
+    ),
+    output_dir: str = typer.Option(
+        'data', help='Download destination directory',
+    ),
+    language: Language | None = typer.Option(
+        None, help='Target Language (default: all)',
+    ),
+    token: str = typer.Option(
+        None, envvar='GITHUB_TOKEN', help='GitHub Token',
+    ),
+    concurrency: int = typer.Option(32, help='Number of concurrent threads'),
+    limit: int | None = typer.Option(
+        None, help='Limit number of processed repos (for testing)',
+    ),
+):
+    """
+    Download SBOM files from repositories.
+    """
+    if language is None:
+        if input_file:
+            logger.error(
+                'Cannot specify input_file when targeting ALL languages.',
+            )
+            raise typer.Exit(1)
+        logger.warning('No language specified. Downloading ALL languages...')
+        target_languages = list(Language)
+    else:
+        target_languages = [language]
+
+    downloader = SBOMDownloader(token, output_dir, pool_size=concurrency)
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn('[bold blue]{task.description}'),
+        BarColumn(),
+        TextColumn('[progress.percentage]{task.percentage:>3.0f}%'),
+        TextColumn('•'),
+        TextColumn('[green]{task.completed}/{task.total}'),
+        TextColumn('•'),
+        TextColumn('[dim]{task.fields[status]}', justify='left'),
+        console=console,
+    ) as progress:
+
+        overall_stats = {
+            'repos': 0,
+            'downloaded': 0,
+            'missing': 0,
+            'failed': 0,
+            'cache_hits': 0,
+        }
+        start_time_all = time.time()
+
+        for lang in target_languages:
+            if input_file:
+                target_file = input_file
+            else:
+                target_file = f"{lang}.jsonl"
+
+            # Check if file exists, if not, skip efficiently
+            if not os.path.exists(target_file):
+                logger.debug(
+                    f"Target file {target_file} not found. Skipping {lang}.",
+                )
+                continue
+
+            tasks = load_targets(target_file)
+            if not tasks:
+                logger.warning(f"Input file empty: {target_file}. Skipping.")
+                continue
+
+            if limit:
+                tasks = tasks[:limit]
+
+            logger.info(
+                'Starting Processing',
+                language=str(lang),
+                target_file=target_file,
+                total_tasks=len(tasks),
+            )
+
+            main_task = progress.add_task(
+                f'Downloading {lang}...', total=len(tasks), status='Starting...',
+            )
+
+            with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
+                future_to_repo = {
+                    executor.submit(downloader.download_repo, repo, lang): repo
+                    for repo in tasks
+                }
+
+                for future in concurrent.futures.as_completed(future_to_repo):
+                    try:
+                        result = future.result()
+                        overall_stats['repos'] += 1
+                        overall_stats['downloaded'] += result.downloaded_files
+                        overall_stats['missing'] += result.missing_files
+                        overall_stats['failed'] += result.failed_files
+                        overall_stats['cache_hits'] += result.cache_hits
+                        progress.update(
+                            main_task, advance=1,
+                            status=result.status_msg,
+                        )
+                    except Exception as e:
+                        logger.error(f"Error processing repo: {e}")
+                        progress.update(
+                            main_task, advance=1,
+                            status='[red]Error[/red]',
+                        )
+
+    # Print Summary Table
+    total_time = time.time() - start_time_all
+    table = Table(title='Download Summary')
+    table.add_column('Metric', style='cyan')
+    table.add_column('Value', style='magenta')
+
+    table.add_row('Total Repos Processed', str(overall_stats['repos']))
+    table.add_row('Files Downloaded', str(overall_stats['downloaded']))
+    table.add_row('Files Missing (404)', str(overall_stats['missing']))
+    table.add_row('Failed Downloads', str(overall_stats['failed']))
+    table.add_row('Cache Hits', str(overall_stats['cache_hits']))
+    table.add_row('Total Duration', f"{total_time:.2f}s")
+
+    console.print(table)
+
+
+if __name__ == '__main__':
+    typer.run(main)