ossa-scanner 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

ossa_scanner/__init__.py

@@ -1 +1 @@
-__version__ = "0.1.1"
+__version__ = "0.1.3"

ossa_scanner/cli.py

@@ -1,35 +1,50 @@
 import argparse
+import os
+import shutil
 from .scanner import Scanner
 from .uploader import GitHubUploader
 
 def main():
     parser = argparse.ArgumentParser(description="OSSA Scanner CLI Tool")
-    parser.add_argument('--output-dir', type=str, required=True, help="Directory to save downloaded source")
-    parser.add_argument('--results-file', type=str, required=True, help="Path to save the JSON results")
     parser.add_argument('--threads', type=int, default=4, help="Number of threads for parallel processing")
     parser.add_argument('--upload', action='store_true', help="Upload results to GitHub")
-    parser.add_argument('--repo-owner', type=str, help="GitHub repository owner")
-    parser.add_argument('--repo-name', type=str, help="GitHub repository name")
-    parser.add_argument('--token', type=str, help="GitHub token")
-    parser.add_argument('--repo-dir', type=str, help="Target directory in GitHub repo for results")
+    parser.add_argument('--repo-owner', type=str, help="GitHub repository owner (required for upload)")
+    parser.add_argument('--repo-name', type=str, help="GitHub repository name (required for upload)")
+    parser.add_argument('--token', type=str, help="GitHub token (required for upload)")
+    parser.add_argument('--repo-dir', type=str, help="Target directory in GitHub repo for results (required for upload)")
+    parser.add_argument('--retain-temp', action='store_true', help="Retain the temporary directory for downloaded and extracted packages")
     args = parser.parse_args()
 
-    # Initialize the scanner
-    scanner = Scanner(output_dir=args.output_dir, threads=args.threads)
-    
-    # Perform scanning
-    results = scanner.scan_packages()
+    # Define directories
+    reports_dir = os.path.join(os.getcwd(), "ossa_reports")
+    temp_dir = "/tmp/ossa_temp"
 
-    # Save results locally
-    scanner.save_results(results, args.results_file)
+    os.makedirs(reports_dir, exist_ok=True)
+    os.makedirs(temp_dir, exist_ok=True)
 
-    # Upload results to GitHub if specified
-    if args.upload:
-        if not (args.repo_owner and args.repo_name and args.token and args.repo_dir):
-            raise ValueError("GitHub upload requires --repo-owner, --repo-name, --token, and --repo-dir")
+    try:
+        # Initialize the scanner
+        scanner = Scanner(threads=args.threads, output_dir=reports_dir, temp_dir=temp_dir)
 
-        uploader = GitHubUploader(args.token, args.repo_owner, args.repo_name)
-        scanner.upload_results(args.results_file, uploader, args.repo_dir)
+        # Perform scanning
+        results = scanner.scan_packages()
+
+        # Handle GitHub upload if specified
+        if args.upload:
+            if not (args.repo_owner and args.repo_name and args.token and args.repo_dir):
+                raise ValueError("GitHub upload requires --repo-owner, --repo-name, --token, and --repo-dir")
+
+            uploader = GitHubUploader(args.token, args.repo_owner, args.repo_name)
+            for report_file in os.listdir(reports_dir):
+                report_path = os.path.join(reports_dir, report_file)
+                if os.path.isfile(report_path):
+                    uploader.upload_file(report_path, os.path.join(args.repo_dir, report_file), "Add OSSA report")
+
+    finally:
+        # Clean up the temporary directory unless the user opts to retain it
+        if not args.retain_temp:
+            print(f"Cleaning up temporary directory: {temp_dir}")
+            shutil.rmtree(temp_dir, ignore_errors=True)
 
 if __name__ == "__main__":
     main()

ossa_scanner/scanner.py

@@ -1,18 +1,21 @@
 import os
 import json
+import hashlib
+from datetime import datetime
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from .utils.os_detection import detect_os
 from .utils.package_manager import list_packages, get_package_info
 from .utils.downloader import download_source
 from .utils.hash_calculator import calculate_file_hash
 from .utils.swhid_calculator import calculate_swhid
-from .uploader import GitHubUploader
 
 class Scanner:
-    def __init__(self, output_dir, threads=4):
+    def __init__(self, threads=4, output_dir="ossa_reports", temp_dir="/tmp/ossa_temp"):
         self.output_dir = output_dir
+        self.temp_dir = temp_dir
         self.os_type = detect_os()
         self.threads = threads
+        os.makedirs(self.temp_dir, exist_ok=True)
 
     def process_package(self, package):
         """
@@ -29,48 +32,35 @@ class Scanner:
             package_info = get_package_info(self.os_type, package)
             print(f"Fetched metadata for {package}")
 
-            # Download the source code
-            source_file = download_source(self.os_type, package, self.output_dir)
+            # Download the source code to temp_dir
+            source_file = download_source(self.os_type, package, self.temp_dir)
             print(f"Downloaded source file: {source_file}")
 
             # Calculate hash of the source file
             file_hash = calculate_file_hash(source_file)
             print(f"Hash (SHA256) for {package}: {file_hash}")
 
-            # Extract source code directory
-            source_dir = os.path.join(self.output_dir, package)
+            # Extract source code directory in temp_dir
+            source_dir = os.path.join(self.temp_dir, package)
             os.makedirs(source_dir, exist_ok=True)
 
             # Calculate SWHID
             swhid = calculate_swhid(source_dir)
             print(f"SWHID for {package}: {swhid}")
 
-            return {
-                "package": package,
-                "info": package_info,
-                "hash": file_hash,
-                "swhid": swhid,
-            }
+            # Save report
+            self.save_package_report(package, package_info, file_hash, swhid, source_file)
 
         except Exception as e:
             print(f"Error processing package {package}: {e}")
-            return {
-                "package": package,
-                "error": str(e)
-            }
 
     def scan_packages(self):
         """
         Scans all packages in the repository and processes them in parallel.
-
-        Returns:
-            list: List of results for each package.
         """
         print(f"Detected OS: {self.os_type}")
         print("Listing available packages...")
         packages = list_packages(self.os_type)
-
-        results = []
         with ThreadPoolExecutor(max_workers=self.threads) as executor:
             # Submit tasks for parallel processing
             future_to_package = {
@@ -81,34 +71,53 @@ class Scanner:
             for future in as_completed(future_to_package):
                 package = future_to_package[future]
                 try:
-                    result = future.result()
-                    results.append(result)
+                    future.result()
                 except Exception as e:
                     print(f"Exception occurred for package {package}: {e}")
-        return results
 
-    def save_results(self, results, output_file):
+    def save_package_report(self, package, package_info, file_hash, swhid, source_file):
         """
-        Save the scan results to a JSON file.
+        Save the report for a single package.
 
         Args:
-            results (list): List of results for each package.
-            output_file (str): Path to save the JSON file.
+            package (str): Package name.
+            package_info (dict): Information about the package.
+            file_hash (str): SHA256 hash of the downloaded source.
+            swhid (str): Software Heritage ID of the package.
         """
-        with open(output_file, "w") as f:
-            json.dump(results, f, indent=4)
-        print(f"Results saved to {output_file}")
-
-    def upload_results(self, results_file, github_uploader, repo_dir):
-        """
-        Uploads the results file to GitHub.
-
-        Args:
-            results_file (str): Local results file path to upload.
-            github_uploader (GitHubUploader): Instance of the GitHubUploader class.
-            repo_dir (str): Path in the GitHub repository where the results will be uploaded.
-        """
-        print(f"Uploading results to GitHub: {repo_dir}")
-        repo_path = os.path.join(repo_dir, os.path.basename(results_file))
-        github_uploader.upload_file(results_file, repo_path, "Add scanning results")
-
+        # Generate report filename
+        sha1_name = hashlib.sha1(package.encode()).hexdigest()
+        date_str = datetime.now().strftime("%Y%m%d")
+        report_filename = f"ossa-{date_str}-{sha1_name}-{package}.json"
+        report_path = os.path.join(self.output_dir, report_filename)
+
+        # Create the report content
+        report = {
+            "id": f"OSSA-{date_str}-{sha1_name.upper()}",
+            "version": "1.0.0",
+            "severity": "Informational",
+            "title": f"Advisory for {package}",
+            "package_name": package,
+            "publisher": "Generated by OSSA Collector",
+            "last_updated": datetime.now().isoformat(),
+            "approvals": [{"consumption": True, "externalization": True}],
+            "description": f"Automatically generated OSSA for the package {package}.",
+            "purls": [f"pkg:{self.os_type}/{package}"],
+            "regex": [f"^pkg:{self.os_type}/{package}.*"],
+            "affected_versions": ["*.*"],
+            "artifacts": [
+                {
+                    "url": f"file://{source_file}",
+                    "hashes": {"sha256": file_hash},
+                    "swhid": swhid
+                }
+            ],
+            "licenses": package_info.get("licenses", []),
+            "aliases": package_info.get("aliases", []),
+            "references": package_info.get("references", [])
+        }
+
+        # Save the report to the output directory
+        with open(report_path, "w") as f:
+            json.dump(report, f, indent=4)
+        print(f"Report saved: {report_path}")

ossa_scanner/utils/downloader.py

@@ -1,11 +1,47 @@
 import subprocess
+import os
+import shutil
+import glob
 
 def download_source(package_manager, package_name, output_dir):
-    if package_manager == 'apt':
-        cmd = ['apt-get', 'source', package_name, '-d', output_dir]
-    elif package_manager in ['yum', 'dnf']:
-        cmd = ['dnf', 'download', '--source', package_name, '--downloaddir', output_dir]
-    else:
-        raise ValueError("Unsupported package manager")
-
-    subprocess.run(cmd)
+    try:
+        if package_manager == 'apt':
+            cmd = ['apt-get', 'source', package_name, '-d', output_dir]
+            subprocess.run(cmd, check=True)
+        elif package_manager in ['yum', 'dnf']:
+            cmd = ['dnf', 'download', '--source', package_name, '--downloaddir', output_dir]
+            subprocess.run(cmd, check=True)
+        elif package_manager == 'brew':
+            # Fetch the source tarball
+            cmd = ['brew', 'fetch', '--build-from-source', package_name]
+            subprocess.run(cmd, check=True, capture_output=True, text=True)
+            cache_dir = subprocess.run(
+                ['brew', '--cache', package_name],
+                capture_output=True,
+                text=True,
+                check=True
+            ).stdout.strip()
+            prefixes_to_remove = ['aarch64-elf-', 'arm-none-eabi-', 'other-prefix-']
+            stripped_package_name = package_name
+            for prefix in prefixes_to_remove:
+                if package_name.startswith(prefix):
+                    stripped_package_name = package_name[len(prefix):]
+                    break
+            cache_folder = os.path.dirname(cache_dir)
+            tarball_pattern = os.path.join(cache_folder, f"*{stripped_package_name}*")
+            matching_files = glob.glob(tarball_pattern)
+            if not matching_files:
+                raise FileNotFoundError(f"Tarball not found for {package_name} in {cache_folder}")
+            tarball_path = matching_files[0]
+            os.makedirs(output_dir, exist_ok=True)
+            target_path = os.path.join(output_dir, os.path.basename(tarball_path))
+            shutil.move(tarball_path, target_path)
+            return target_path
+        else:
+            raise ValueError("Unsupported package manager")
+    except subprocess.CalledProcessError as e:
+        print(f"Command failed: {e}")
+        return None
+    except Exception as e:
+        print(f"Error: {e}")
+        return None

ossa_scanner/utils/package_manager.py

@@ -1,5 +1,6 @@
 import subprocess
 
+
 def list_packages(package_manager):
     if package_manager == 'apt':
         result = subprocess.run(
@@ -20,18 +21,108 @@ def list_packages(package_manager):
             text=True
         )
     else:
-        raise ValueError("Unsupported package manager")
+        raise ValueError("ER1: Unsupported package manager for search")
 
     packages = result.stdout.splitlines()
-    return [pkg.split()[0] for pkg in packages]
+    extracted_packages = []
+    max_packages = 5
+    k_packages = 0
+    for line in packages:
+        if not line.strip() or line.startswith("==>"):
+            continue
+        extracted_packages.append(line.split()[0])
+        if k_packages >= max_packages:
+            break
+        k_packages += 1
+
+    return extracted_packages
+
 
 def get_package_info(package_manager, package_name):
     if package_manager == 'apt':
         cmd = ['apt-cache', 'show', package_name]
     elif package_manager in ['yum', 'dnf']:
         cmd = ['repoquery', '--info', package_name]
+    elif package_manager == 'brew':
+        cmd = ['brew', 'info', package_name]
     else:
-        raise ValueError("Unsupported package manager")
+        raise ValueError("ER: Unsupported package manager for info")
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+        output = result.stdout
+
+        # Parse the output based on the package manager
+        if package_manager == 'brew':
+            return parse_brew_info(output)
+        elif package_manager in ['yum', 'dnf']:
+            return parse_yum_info(output)
+        elif package_manager == 'apt':
+            return parse_apt_info(output)
+    except subprocess.CalledProcessError as e:
+        print(f"Command failed: {e}")
+        return None
+
+
+def parse_brew_info(output):
+    """Parses brew info output to extract license, website, and description."""
+    info = {}
+    lines = output.splitlines()
+    info["license"] = "Unknown"
+    info["website"] = "Unknown"
+    info["description"] = "Unknown"
+
+    for i, line in enumerate(lines):
+        if i == 1:  # The description is usually on the second line
+            info["description"] = line.strip()
+        elif line.startswith("https://"):  # The website URL
+            info["website"] = line.strip()
+        elif line.startswith("License:"):  # The license information
+            info["license"] = line.split(":", 1)[1].strip()
+
+    # Ensure all keys are present even if some fields are missing
+    return info
+
+
+
+def parse_yum_info(output):
+    """Parses yum repoquery --info output."""
+    info = {}
+    lines = output.splitlines()
+
+    for line in lines:
+        if line.startswith("License"):
+            info["license"] = line.split(":", 1)[1].strip()
+        elif line.startswith("URL"):
+            info["website"] = line.split(":", 1)[1].strip()
+        elif "Copyright" in line:
+            info["copyright"] = line.strip()
+
+    # Ensure all keys are present even if data is missing
+    return {
+        "license": info.get("license", "Unknown"),
+        "copyright": info.get("copyright", "Unknown"),
+        "website": info.get("website", "Unknown"),
+    }
+
+
+def parse_apt_info(output):
+    """Parses apt-cache show output."""
+    info = {}
+    lines = output.splitlines()
+
+    for line in lines:
+        if line.startswith("License:") or "License" in line:
+            info["license"] = line.split(":", 1)[1].strip()
+        elif line.startswith("Homepage:"):
+            info["website"] = line.split(":", 1)[1].strip()
+        elif "Copyright" in line:
+            info["copyright"] = line.strip()
+
+    # Ensure all keys are present even if data is missing
+    return {
+        "license": info.get("license", "Unknown"),
+        "copyright": info.get("copyright", "Unknown"),
+        "website": info.get("website", "Unknown"),
+    }
 
-    result = subprocess.run(cmd, capture_output=True, text=True)
-    return result.stdout

{ossa_scanner-0.1.1.dist-info → ossa_scanner-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ossa_scanner
-Version: 0.1.1
+Version: 0.1.3
 Summary: A Python library for scanning Linux packages, managing metadata, and generating SWHIDs.
 Home-page: https://github.com/oscarvalenzuelab/ossa_scanner
 Author: Oscar Valenzuela

ossa_scanner-0.1.3.dist-info/RECORD

@@ -0,0 +1,16 @@
+ossa_scanner/__init__.py,sha256=XEqb2aiIn8fzGE68Mph4ck1FtQqsR_am0wRWvrYPffQ,22
+ossa_scanner/cli.py,sha256=sgr8NFpf_Ut84KYFQjOKRxv8CfAMaTPhMo7DbR53lT4,2311
+ossa_scanner/scanner.py,sha256=YOYB4-7EwQyZE6KU6_dyRD09tq6ntgmYvyxX02KgB5c,4885
+ossa_scanner/uploader.py,sha256=X8bo7GqfpBjz2NlnvSwDR_rVqNoZDRPF2pnQMaVENbc,2436
+ossa_scanner/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ossa_scanner/utils/downloader.py,sha256=3ccwcde9yJ_SEP0mG9TDr2O0MMdA1p-K6hpzqme-KQ4,2081
+ossa_scanner/utils/hash_calculator.py,sha256=i47KS_HoZNiSbGyd0iP9_TcDwxWS2SrmkIcNF2MWLcA,254
+ossa_scanner/utils/os_detection.py,sha256=QdRKQ4li4SOHgBofe1qWf8OOcw8XvhM-XWUNu0Cy0a4,315
+ossa_scanner/utils/package_manager.py,sha256=tWuQwgkFQjTzeisem0Gz8uFvWw5Cxd-Tft5HM8tIQmk,4028
+ossa_scanner/utils/swhid_calculator.py,sha256=4Z0H2GmECMAJlvH6JBbUmaLXSLRNntyYEdxsS6CTEMQ,63
+ossa_scanner-0.1.3.dist-info/LICENSE,sha256=9slQ_XNiEkio28l90NwihP7a90fCL2GQ6YhcVXTBls4,1064
+ossa_scanner-0.1.3.dist-info/METADATA,sha256=22Fo5X2J06UlI-94hUZLBSGJvdzpHaK-GqKFDIDkF_Q,1043
+ossa_scanner-0.1.3.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ossa_scanner-0.1.3.dist-info/entry_points.txt,sha256=UVoAo-wTPxT82g3cfqTs2CmQnazd57TAwhd9VwEKD1c,55
+ossa_scanner-0.1.3.dist-info/top_level.txt,sha256=uUp5CvhZfJLapXn9DyUXvgH7QK3uzF2ibH943lWN5Bs,13
+ossa_scanner-0.1.3.dist-info/RECORD,,

ossa_scanner-0.1.1.dist-info/RECORD

@@ -1,16 +0,0 @@
-ossa_scanner/__init__.py,sha256=rnObPjuBcEStqSO0S6gsdS_ot8ITOQjVj_-P1LUUYpg,22
-ossa_scanner/cli.py,sha256=hyRUOgp9kcwFtQrIeyth5vTxeK7eOlxfn5R9E7HX5sA,1640
-ossa_scanner/scanner.py,sha256=O1gKFfa1yknTNcQWGJOR3sKnFgcEZ0qZzhf2VqORLNM,4083
-ossa_scanner/uploader.py,sha256=X8bo7GqfpBjz2NlnvSwDR_rVqNoZDRPF2pnQMaVENbc,2436
-ossa_scanner/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ossa_scanner/utils/downloader.py,sha256=5fV531x-oiFTyh6B17Afi4W72zFC2ejopvMpn1qNpw4,408
-ossa_scanner/utils/hash_calculator.py,sha256=i47KS_HoZNiSbGyd0iP9_TcDwxWS2SrmkIcNF2MWLcA,254
-ossa_scanner/utils/os_detection.py,sha256=QdRKQ4li4SOHgBofe1qWf8OOcw8XvhM-XWUNu0Cy0a4,315
-ossa_scanner/utils/package_manager.py,sha256=SC6NMHsH4EX689OL_D4lqMzBkXHYeGlt2wum_uCV4tA,1124
-ossa_scanner/utils/swhid_calculator.py,sha256=4Z0H2GmECMAJlvH6JBbUmaLXSLRNntyYEdxsS6CTEMQ,63
-ossa_scanner-0.1.1.dist-info/LICENSE,sha256=9slQ_XNiEkio28l90NwihP7a90fCL2GQ6YhcVXTBls4,1064
-ossa_scanner-0.1.1.dist-info/METADATA,sha256=sgvxMtorRONY1Mfp22MW6aYltiILEyKXVb5K1x9p-rQ,1043
-ossa_scanner-0.1.1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ossa_scanner-0.1.1.dist-info/entry_points.txt,sha256=UVoAo-wTPxT82g3cfqTs2CmQnazd57TAwhd9VwEKD1c,55
-ossa_scanner-0.1.1.dist-info/top_level.txt,sha256=uUp5CvhZfJLapXn9DyUXvgH7QK3uzF2ibH943lWN5Bs,13
-ossa_scanner-0.1.1.dist-info/RECORD,,