ossa-scanner 0.1.0__py3-none-any.whl

ossa_scanner/scanner.py

@@ -0,0 +1,120 @@
+import os
+import json
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from ossa_scanner.utils.os_detection import detect_os
+from ossa_scanner.utils.package_manager import list_packages, get_package_info, download_source
+from ossa_scanner.utils.hash_calculator import calculate_file_hash
+from ossa_scanner.utils.swhid_calculator import calculate_swhid
+from ossa_scanner.uploader import GitHubUploader
+
+class Scanner:
+    def __init__(self, output_dir, threads=4):
+        """
+        Initialize the scanner with the output directory and thread count.
+
+        Args:
+            output_dir (str): Directory to store downloaded files and extracted sources.
+            threads (int): Number of threads for parallel processing.
+        """
+        self.output_dir = output_dir
+        self.os_type = detect_os()
+        self.threads = threads
+
+    def process_package(self, package):
+        """
+        Processes a single package: downloads source, extracts, calculates hash and SWHID.
+
+        Args:
+            package (str): Package name to process.
+
+        Returns:
+            dict: Result of the processed package including hash and SWHID.
+        """
+        try:
+            print(f"Processing package: {package}")
+            package_info = get_package_info(self.os_type, package)
+            print(f"Fetched metadata for {package}")
+
+            # Download the source code
+            source_file = download_source(self.os_type, package, self.output_dir)
+            print(f"Downloaded source file: {source_file}")
+
+            # Calculate hash of the source file
+            file_hash = calculate_file_hash(source_file)
+            print(f"Hash (SHA256) for {package}: {file_hash}")
+
+            # Extract source code directory
+            source_dir = os.path.join(self.output_dir, package)
+            os.makedirs(source_dir, exist_ok=True)
+
+            # Calculate SWHID
+            swhid = calculate_swhid(source_dir)
+            print(f"SWHID for {package}: {swhid}")
+
+            return {
+                "package": package,
+                "info": package_info,
+                "hash": file_hash,
+                "swhid": swhid,
+            }
+
+        except Exception as e:
+            print(f"Error processing package {package}: {e}")
+            return {
+                "package": package,
+                "error": str(e)
+            }
+
+    def scan_packages(self):
+        """
+        Scans all packages in the repository and processes them in parallel.
+
+        Returns:
+            list: List of results for each package.
+        """
+        print(f"Detected OS: {self.os_type}")
+        print("Listing available packages...")
+        packages = list_packages(self.os_type)
+
+        results = []
+        with ThreadPoolExecutor(max_workers=self.threads) as executor:
+            # Submit tasks for parallel processing
+            future_to_package = {
+                executor.submit(self.process_package, package): package
+                for package in packages
+            }
+
+            for future in as_completed(future_to_package):
+                package = future_to_package[future]
+                try:
+                    result = future.result()
+                    results.append(result)
+                except Exception as e:
+                    print(f"Exception occurred for package {package}: {e}")
+        return results
+
+    def save_results(self, results, output_file):
+        """
+        Save the scan results to a JSON file.
+
+        Args:
+            results (list): List of results for each package.
+            output_file (str): Path to save the JSON file.
+        """
+        with open(output_file, "w") as f:
+            json.dump(results, f, indent=4)
+        print(f"Results saved to {output_file}")
+
+    def upload_results(self, results_file, github_uploader, repo_dir):
+        """
+        Uploads the results file to GitHub.
+
+        Args:
+            results_file (str): Local results file path to upload.
+            github_uploader (GitHubUploader): Instance of the GitHubUploader class.
+            repo_dir (str): Path in the GitHub repository where the results will be uploaded.
+        """
+        print(f"Uploading results to GitHub: {repo_dir}")
+        repo_path = os.path.join(repo_dir, os.path.basename(results_file))
+        github_uploader.upload_file(results_file, repo_path, "Add scanning results")
+

ossa_scanner/uploader.py

@@ -0,0 +1,68 @@
+import http.client
+import json
+import base64
+import os
+
+
+class GitHubUploader:
+    def __init__(self, token, repo_owner, repo_name):
+        self.token = token
+        self.repo_owner = repo_owner
+        self.repo_name = repo_name
+        self.base_url = "api.github.com"
+
+    def upload_file(self, file_path, repo_path, commit_message="Add scanner results"):
+        """
+        Uploads a file to a GitHub repository.
+        
+        Args:
+            file_path (str): Local file path to upload.
+            repo_path (str): Path in the GitHub repository.
+            commit_message (str): Commit message for the upload.
+        """
+        # Read the file and encode it in base64
+        with open(file_path, "rb") as f:
+            content = f.read()
+        encoded_content = base64.b64encode(content).decode("utf-8")
+
+        # Create the payload
+        payload = {
+            "message": commit_message,
+            "content": encoded_content,
+        }
+
+        # GitHub API endpoint
+        endpoint = f"/repos/{self.repo_owner}/{self.repo_name}/contents/{repo_path}"
+
+        # Make the API request
+        conn = http.client.HTTPSConnection(self.base_url)
+        headers = {
+            "Authorization": f"Bearer {self.token}",
+            "User-Agent": "ossa-scanner",
+            "Content-Type": "application/json",
+        }
+
+        conn.request("PUT", endpoint, body=json.dumps(payload), headers=headers)
+        response = conn.getresponse()
+        data = response.read().decode("utf-8")
+        conn.close()
+
+        if response.status == 201:
+            print(f"File '{file_path}' successfully uploaded to {repo_path} in {self.repo_name}")
+        else:
+            print(f"Failed to upload file '{file_path}'. Response: {data}")
+            raise Exception(f"GitHub API Error: {response.status}")
+
+    def upload_results(self, results_dir, repo_dir):
+        """
+        Uploads all files in a directory to a specified path in the GitHub repo.
+        
+        Args:
+            results_dir (str): Local directory containing results to upload.
+            repo_dir (str): Target directory in the GitHub repository.
+        """
+        for root, _, files in os.walk(results_dir):
+            for file_name in files:
+                local_path = os.path.join(root, file_name)
+                repo_path = os.path.join(repo_dir, file_name).replace("\\", "/")
+                self.upload_file(local_path, repo_path)

ossa_scanner/utils/downloader.py

@@ -0,0 +1,11 @@
+import subprocess
+
+def download_source(package_manager, package_name, output_dir):
+    if package_manager == 'apt':
+        cmd = ['apt-get', 'source', package_name, '-d', output_dir]
+    elif package_manager in ['yum', 'dnf']:
+        cmd = ['dnf', 'download', '--source', package_name, '--downloaddir', output_dir]
+    else:
+        raise ValueError("Unsupported package manager")
+
+    subprocess.run(cmd)

ossa_scanner/utils/hash_calculator.py

@@ -0,0 +1,8 @@
+import hashlib
+
+def calculate_file_hash(file_path, algorithm='sha256'):
+    hash_func = hashlib.new(algorithm)
+    with open(file_path, 'rb') as f:
+        while chunk := f.read(8192):
+            hash_func.update(chunk)
+    return hash_func.hexdigest()

ossa_scanner/utils/os_detection.py

@@ -0,0 +1,10 @@
+import platform
+
+def detect_os():
+    dist, _, _ = platform.linux_distribution(full_distribution_name=False)
+    if 'Ubuntu' in dist or 'Debian' in dist:
+        return 'apt'
+    elif 'Red Hat' in dist or 'CentOS' in dist or 'AlmaLinux' in dist:
+        return 'yum'
+    else:
+        raise ValueError("Unsupported OS")

ossa_scanner/utils/package_manager.py

@@ -0,0 +1,31 @@
+import subprocess
+
+def list_packages(package_manager):
+    if package_manager == 'apt':
+        result = subprocess.run(
+            ['apt-cache', 'search', '.'],
+            capture_output=True,
+            text=True
+        )
+    elif package_manager in ['yum', 'dnf']:
+        result = subprocess.run(
+            ['repoquery', '--all'],
+            capture_output=True,
+            text=True
+        )
+    else:
+        raise ValueError("Unsupported package manager")
+
+    packages = result.stdout.splitlines()
+    return [pkg.split()[0] for pkg in packages]
+
+def get_package_info(package_manager, package_name):
+    if package_manager == 'apt':
+        cmd = ['apt-cache', 'show', package_name]
+    elif package_manager in ['yum', 'dnf']:
+        cmd = ['repoquery', '--info', package_name]
+    else:
+        raise ValueError("Unsupported package manager")
+
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    return result.stdout

ossa_scanner/utils/swhid_calculator.py

@@ -0,0 +1,4 @@
+from swh.model.hashutil import hash_directory
+
+def calculate_swhid(directory_path):
+    return hash_directory(directory_path)

ossa_scanner-0.1.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Oscar V
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ossa_scanner-0.1.0.dist-info/METADATA

@@ -0,0 +1,41 @@
+Metadata-Version: 2.1
+Name: ossa-scanner
+Version: 0.1.0
+Summary: A CLI tool to scan Linux packages, manage metadata, and upload results to GitHub.
+Author: Oscar Valenzuela
+Author-email: Oscar Valenzuela <oscar.valenzuela.b@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2024 Oscar V
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Keywords: linux,packages,SWHID,GitHub,open-source
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: POSIX :: Linux
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: swh.model
+
+# ossa_scanner
+Open Source Advisory Scanner (Generator)

ossa_scanner-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+ossa_scanner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ossa_scanner/scanner.py,sha256=nku8pJR1n7Z2Fjs5F3lbxBlv9xFUWrUb3dfucZZcM30,4387
+ossa_scanner/uploader.py,sha256=X8bo7GqfpBjz2NlnvSwDR_rVqNoZDRPF2pnQMaVENbc,2436
+ossa_scanner/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ossa_scanner/utils/downloader.py,sha256=5fV531x-oiFTyh6B17Afi4W72zFC2ejopvMpn1qNpw4,408
+ossa_scanner/utils/hash_calculator.py,sha256=i47KS_HoZNiSbGyd0iP9_TcDwxWS2SrmkIcNF2MWLcA,254
+ossa_scanner/utils/os_detection.py,sha256=h2hlS5QHHVyyGSHI5jDwq7aivGZTEHk1v_Ml9lTTxbQ,320
+ossa_scanner/utils/package_manager.py,sha256=N_noGTVHxxCTfH9q6ftq2GhkACj1ExOt1AXhNdFFgNQ,953
+ossa_scanner/utils/swhid_calculator.py,sha256=mvjx4wlTDCrWhH3Z_6TitKtt1WsPMC7QYwmWSMck7Ro,126
+ossa_scanner-0.1.0.dist-info/LICENSE,sha256=9slQ_XNiEkio28l90NwihP7a90fCL2GQ6YhcVXTBls4,1064
+ossa_scanner-0.1.0.dist-info/METADATA,sha256=VdiOAhrJ6M75qRG6ntG8UGxYFUR8IQoPNvtFX-bB_Nk,1927
+ossa_scanner-0.1.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ossa_scanner-0.1.0.dist-info/top_level.txt,sha256=uUp5CvhZfJLapXn9DyUXvgH7QK3uzF2ibH943lWN5Bs,13
+ossa_scanner-0.1.0.dist-info/RECORD,,

ossa_scanner-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.6.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ossa_scanner-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+ossa_scanner