ossa-scanner 0.1.4__tar.gz → 0.1.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/PKG-INFO +12 -3
- ossa_scanner-0.1.7/README.md +11 -0
- ossa_scanner-0.1.7/ossa_scanner/__init__.py +1 -0
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/ossa_scanner/scanner.py +51 -55
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/ossa_scanner/uploader.py +0 -16
- ossa_scanner-0.1.7/ossa_scanner/utils/downloader.py +119 -0
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/ossa_scanner/utils/hash_calculator.py +0 -1
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/ossa_scanner/utils/os_detection.py +7 -2
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/ossa_scanner/utils/package_manager.py +72 -27
- ossa_scanner-0.1.7/ossa_scanner/utils/swhid_calculator.py +35 -0
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/ossa_scanner.egg-info/PKG-INFO +12 -3
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/setup.py +2 -2
- ossa_scanner-0.1.4/README.md +0 -2
- ossa_scanner-0.1.4/ossa_scanner/__init__.py +0 -1
- ossa_scanner-0.1.4/ossa_scanner/utils/downloader.py +0 -47
- ossa_scanner-0.1.4/ossa_scanner/utils/swhid_calculator.py +0 -3
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/LICENSE +0 -0
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/ossa_scanner/cli.py +0 -0
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/ossa_scanner/utils/__init__.py +0 -0
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/ossa_scanner.egg-info/SOURCES.txt +0 -0
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/ossa_scanner.egg-info/dependency_links.txt +0 -0
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/ossa_scanner.egg-info/entry_points.txt +0 -0
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/ossa_scanner.egg-info/requires.txt +0 -0
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/ossa_scanner.egg-info/top_level.txt +0 -0
- {ossa_scanner-0.1.4 → ossa_scanner-0.1.7}/setup.cfg +0 -0
@@ -1,12 +1,12 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: ossa_scanner
|
3
|
-
Version: 0.1.
|
4
|
-
Summary:
|
3
|
+
Version: 0.1.7
|
4
|
+
Summary: Open Source Software Advisory generator for Core and Base Linux Packages.
|
5
5
|
Home-page: https://github.com/oscarvalenzuelab/ossa_scanner
|
6
6
|
Author: Oscar Valenzuela
|
7
7
|
Author-email: oscar.valenzuela.b@gmail.com
|
8
8
|
License: MIT
|
9
|
-
Keywords: linux packages SWHID open-source compliance
|
9
|
+
Keywords: linux packages SWHID open-source compliance ossa advisory
|
10
10
|
Classifier: Development Status :: 3 - Alpha
|
11
11
|
Classifier: Intended Audience :: Developers
|
12
12
|
Classifier: License :: OSI Approved :: MIT License
|
@@ -27,3 +27,12 @@ Requires-Dist: ssdeep
|
|
27
27
|
|
28
28
|
# ossa_scanner
|
29
29
|
Open Source Advisory Scanner (Generator)
|
30
|
+
|
31
|
+
## Centos/AL/AlmaLinux
|
32
|
+
Install Python PyPI:
|
33
|
+
|
34
|
+
> sudo yum groupinstall "Development Tools"
|
35
|
+
> sudo yum -y install python-pip python3-devel
|
36
|
+
> pip3 install swh-scanner
|
37
|
+
> BUILD_LIB=1 pip install ssdeep
|
38
|
+
> pip3 install ossa-scanner
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# ossa_scanner
|
2
|
+
Open Source Advisory Scanner (Generator)
|
3
|
+
|
4
|
+
## Centos/AL/AlmaLinux
|
5
|
+
Install Python PyPI:
|
6
|
+
|
7
|
+
> sudo yum groupinstall "Development Tools"
|
8
|
+
> sudo yum -y install python-pip python3-devel
|
9
|
+
> pip3 install swh-scanner
|
10
|
+
> BUILD_LIB=1 pip install ssdeep
|
11
|
+
> pip3 install ossa-scanner
|
@@ -0,0 +1 @@
|
|
1
|
+
__version__ = "0.1.7"
|
@@ -1,9 +1,13 @@
|
|
1
1
|
import os
|
2
|
+
import re
|
2
3
|
import json
|
4
|
+
import glob
|
5
|
+
import shutil
|
6
|
+
import subprocess
|
3
7
|
import hashlib
|
4
8
|
from datetime import datetime
|
5
9
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
6
|
-
from .utils.os_detection import detect_os
|
10
|
+
from .utils.os_detection import detect_os, detect_pm
|
7
11
|
from .utils.package_manager import list_packages, get_package_info
|
8
12
|
from .utils.downloader import download_source
|
9
13
|
from .utils.hash_calculator import calculate_file_hash
|
@@ -14,32 +18,16 @@ class Scanner:
|
|
14
18
|
self.output_dir = output_dir
|
15
19
|
self.temp_dir = temp_dir
|
16
20
|
self.os_type = detect_os()
|
21
|
+
self.pm_type = detect_pm()
|
17
22
|
self.threads = threads
|
18
23
|
os.makedirs(self.temp_dir, exist_ok=True)
|
19
24
|
|
20
25
|
def process_package(self, package):
|
21
26
|
try:
|
22
27
|
print(f"Processing package: {package}")
|
23
|
-
package_info = get_package_info(self.
|
24
|
-
|
25
|
-
|
26
|
-
source_file = download_source(self.os_type, package, self.temp_dir)
|
27
|
-
print(f"Downloaded source file: {source_file}")
|
28
|
-
|
29
|
-
file_hash = calculate_file_hash(source_file)
|
30
|
-
print(f"Hash (SHA256) for {package}: {file_hash}")
|
31
|
-
|
32
|
-
# Extract source code directory in temp_dir
|
33
|
-
source_dir = os.path.join(self.temp_dir, package)
|
34
|
-
os.makedirs(source_dir, exist_ok=True)
|
35
|
-
|
36
|
-
# Calculate SWHID
|
37
|
-
swhid = calculate_swhid(source_dir)
|
38
|
-
print(f"SWHID for {package}: {swhid}")
|
39
|
-
|
40
|
-
# Save report
|
41
|
-
self.save_package_report(package, package_info, file_hash, swhid, source_file)
|
42
|
-
|
28
|
+
package_info = get_package_info(self.pm_type, package)
|
29
|
+
source_files = download_source(self.pm_type, package, self.temp_dir)
|
30
|
+
self.save_package_report(package, package_info, source_files)
|
43
31
|
except Exception as e:
|
44
32
|
print(f"Error processing package {package}: {e}")
|
45
33
|
|
@@ -47,9 +35,9 @@ class Scanner:
|
|
47
35
|
"""
|
48
36
|
Scans all packages in the repository and processes them in parallel.
|
49
37
|
"""
|
50
|
-
print(f"Detected
|
38
|
+
print(f"Detected Package Manager: {self.pm_type}")
|
51
39
|
print("Listing available packages...")
|
52
|
-
packages = list_packages(self.
|
40
|
+
packages = list_packages(self.pm_type)
|
53
41
|
with ThreadPoolExecutor(max_workers=self.threads) as executor:
|
54
42
|
# Submit tasks for parallel processing
|
55
43
|
future_to_package = {
|
@@ -64,51 +52,59 @@ class Scanner:
|
|
64
52
|
except Exception as e:
|
65
53
|
print(f"Exception occurred for package {package}: {e}")
|
66
54
|
|
67
|
-
def save_package_report(self, package, package_info,
|
68
|
-
"""
|
69
|
-
Save the report for a single package.
|
70
|
-
|
71
|
-
Args:
|
72
|
-
package (str): Package name.
|
73
|
-
package_info (dict): Information about the package.
|
74
|
-
file_hash (str): SHA256 hash of the downloaded source.
|
75
|
-
swhid (str): Software Heritage ID of the package.
|
76
|
-
"""
|
55
|
+
def save_package_report(self, package, package_info, source_files):
|
77
56
|
# Generate report filename
|
57
|
+
purl_name = package_info.get("name")
|
58
|
+
purl_version = package_info.get("version")
|
59
|
+
pkg_type = "deb" if self.pm_type == "apt" else "rpm" if self.pm_type == "yum" else self.pm_type
|
60
|
+
os_type = self.os_type
|
78
61
|
date_str = datetime.now().strftime("%Y%m%d")
|
79
|
-
report_filename = f"ossa-{date_str}-{hash(package) % 10000}-{
|
62
|
+
report_filename = f"ossa-{date_str}-{hash(package) % 10000}-{purl_name}.json"
|
80
63
|
report_path = os.path.join(self.output_dir, report_filename)
|
81
64
|
|
82
|
-
|
83
|
-
|
84
|
-
|
65
|
+
if package_info.get("version") != "*":
|
66
|
+
affected_versions = ["*.*", package_info.get("version")]
|
67
|
+
else:
|
68
|
+
affected_versions = ["*.*"]
|
69
|
+
|
70
|
+
artifacts = []
|
71
|
+
for source_file in source_files:
|
72
|
+
artifact = {}
|
73
|
+
|
74
|
+
# Clean up the artifact name
|
85
75
|
artifact_name = os.path.basename(source_file)
|
86
|
-
|
87
|
-
|
76
|
+
if "--" in artifact_name:
|
77
|
+
artifact_name = artifact_name.split("--")[-1]
|
78
|
+
artifact['url'] = "file://" + artifact_name
|
79
|
+
|
80
|
+
file_hash = calculate_file_hash(source_file)
|
81
|
+
artifact['hashes'] = file_hash
|
82
|
+
|
83
|
+
# Extract source code directory in temp_dir
|
84
|
+
# Only required if calculating SWHID
|
85
|
+
source_dir = os.path.join(self.temp_dir, package)
|
86
|
+
os.makedirs(source_dir, exist_ok=True)
|
87
|
+
swhid = calculate_swhid(source_dir, source_file)
|
88
|
+
artifact['swhid'] = swhid
|
89
|
+
|
90
|
+
artifacts.append(artifact)
|
88
91
|
|
89
92
|
# Create the report content
|
90
93
|
report = {
|
91
|
-
"id": f"OSSA-{date_str}-{hash(
|
94
|
+
"id": f"OSSA-{date_str}-{hash(purl_name) % 10000}",
|
92
95
|
"version": "1.0.0",
|
93
96
|
"severity": package_info.get("severity", []),
|
94
|
-
"
|
95
|
-
"
|
97
|
+
"description": package_info.get("rason", []),
|
98
|
+
"title": f"Advisory for {purl_name}",
|
99
|
+
"package_name": purl_name,
|
96
100
|
"publisher": "Generated by OSSA Collector",
|
97
101
|
"last_updated": datetime.now().isoformat(),
|
98
102
|
"approvals": [{"consumption": True, "externalization": True}],
|
99
|
-
"description":
|
100
|
-
"purls": [f"pkg:{
|
101
|
-
"regex": [f"^pkg:{
|
102
|
-
"affected_versions":
|
103
|
-
"artifacts":
|
104
|
-
{
|
105
|
-
"url": f"file://{artifact_name}",
|
106
|
-
"hashes": {
|
107
|
-
"sha1": file_hash['sha1'], "sha256": file_hash['sha256'],
|
108
|
-
"ssdeep": file_hash['ssdeep'], "swhid": file_hash['swhid']},
|
109
|
-
"swhid": swhid
|
110
|
-
}
|
111
|
-
],
|
103
|
+
"description": package_info.get("summary", []),
|
104
|
+
"purls": [f"pkg:{pkg_type}/{os_type}/{purl_name}@{purl_version}"],
|
105
|
+
"regex": [f"^pkg:{pkg_type}/{os_type}/{purl_name}.*"],
|
106
|
+
"affected_versions": affected_versions,
|
107
|
+
"artifacts": artifacts,
|
112
108
|
"licenses": package_info.get("licenses", []),
|
113
109
|
"aliases": package_info.get("aliases", []),
|
114
110
|
"references": package_info.get("references", [])
|
@@ -12,15 +12,6 @@ class GitHubUploader:
|
|
12
12
|
self.base_url = "api.github.com"
|
13
13
|
|
14
14
|
def upload_file(self, file_path, repo_path, commit_message="Add scanner results"):
|
15
|
-
"""
|
16
|
-
Uploads a file to a GitHub repository.
|
17
|
-
|
18
|
-
Args:
|
19
|
-
file_path (str): Local file path to upload.
|
20
|
-
repo_path (str): Path in the GitHub repository.
|
21
|
-
commit_message (str): Commit message for the upload.
|
22
|
-
"""
|
23
|
-
# Read the file and encode it in base64
|
24
15
|
with open(file_path, "rb") as f:
|
25
16
|
content = f.read()
|
26
17
|
encoded_content = base64.b64encode(content).decode("utf-8")
|
@@ -54,13 +45,6 @@ class GitHubUploader:
|
|
54
45
|
raise Exception(f"GitHub API Error: {response.status}")
|
55
46
|
|
56
47
|
def upload_results(self, results_dir, repo_dir):
|
57
|
-
"""
|
58
|
-
Uploads all files in a directory to a specified path in the GitHub repo.
|
59
|
-
|
60
|
-
Args:
|
61
|
-
results_dir (str): Local directory containing results to upload.
|
62
|
-
repo_dir (str): Target directory in the GitHub repository.
|
63
|
-
"""
|
64
48
|
for root, _, files in os.walk(results_dir):
|
65
49
|
for file_name in files:
|
66
50
|
local_path = os.path.join(root, file_name)
|
@@ -0,0 +1,119 @@
|
|
1
|
+
import subprocess
|
2
|
+
import os
|
3
|
+
import shutil
|
4
|
+
import glob
|
5
|
+
|
6
|
+
def cleanup_extracted_files(folder_path):
|
7
|
+
"""Recursively clean up files and directories in the specified folder."""
|
8
|
+
try:
|
9
|
+
for file_path in glob.glob(f"{folder_path}/*"):
|
10
|
+
if os.path.isdir(file_path):
|
11
|
+
shutil.rmtree(file_path) # Recursively delete directories
|
12
|
+
print(f"Deleted directory: {file_path}")
|
13
|
+
else:
|
14
|
+
os.remove(file_path) # Delete files
|
15
|
+
print(f"Deleted file: {file_path}")
|
16
|
+
except Exception as e:
|
17
|
+
print(f"Failed to clean up {folder_path}: {e}")
|
18
|
+
|
19
|
+
def download_source(package_manager, package_name, output_dir):
|
20
|
+
try:
|
21
|
+
if package_manager == 'apt':
|
22
|
+
cmd = ['apt-get', 'source', package_name, '-d', output_dir]
|
23
|
+
subprocess.run(cmd, check=True)
|
24
|
+
elif package_manager in ['yum', 'dnf']:
|
25
|
+
p_hash = hash(package_name) % 10000
|
26
|
+
output_dir = os.path.join(output_dir, str(p_hash))
|
27
|
+
os.makedirs(output_dir, exist_ok=True)
|
28
|
+
source_path = get_rpm_source_package(package_name, output_dir)
|
29
|
+
if not source_path:
|
30
|
+
print(f"Source package for {package_name} not found in {package_name}.")
|
31
|
+
return
|
32
|
+
spec_file = extract_rpm_spec_file(source_path, output_dir)
|
33
|
+
project_url, source_url = (None, None)
|
34
|
+
if spec_file:
|
35
|
+
project_url, source_url, license = extract_rpm_info_from_spec(spec_file)
|
36
|
+
tarballs = extract_rpm_tarballs(source_path, output_dir)
|
37
|
+
return tarballs
|
38
|
+
elif package_manager == 'brew':
|
39
|
+
# Fetch the source tarball
|
40
|
+
cmd = ['brew', 'fetch', '--build-from-source', package_name]
|
41
|
+
subprocess.run(cmd, check=True, capture_output=True, text=True)
|
42
|
+
cache_dir = subprocess.run(
|
43
|
+
['brew', '--cache', package_name],
|
44
|
+
capture_output=True,
|
45
|
+
text=True,
|
46
|
+
check=True
|
47
|
+
).stdout.strip()
|
48
|
+
prefixes_to_remove = ['aarch64-elf-', 'arm-none-eabi-', 'other-prefix-']
|
49
|
+
stripped_package_name = package_name
|
50
|
+
for prefix in prefixes_to_remove:
|
51
|
+
if package_name.startswith(prefix):
|
52
|
+
stripped_package_name = package_name[len(prefix):]
|
53
|
+
break
|
54
|
+
cache_folder = os.path.dirname(cache_dir)
|
55
|
+
tarball_pattern = os.path.join(cache_folder, f"*{stripped_package_name}*")
|
56
|
+
matching_files = glob.glob(tarball_pattern)
|
57
|
+
if not matching_files:
|
58
|
+
raise FileNotFoundError(f"Tarball not found for {package_name} in {cache_folder}")
|
59
|
+
tarball_path = matching_files[0]
|
60
|
+
os.makedirs(output_dir, exist_ok=True)
|
61
|
+
target_path = os.path.join(output_dir, os.path.basename(tarball_path))
|
62
|
+
shutil.move(tarball_path, target_path)
|
63
|
+
return [target_path]
|
64
|
+
else:
|
65
|
+
raise ValueError("Unsupported package manager")
|
66
|
+
except subprocess.CalledProcessError as e:
|
67
|
+
print(f"Command failed: {e}")
|
68
|
+
return None
|
69
|
+
except Exception as e:
|
70
|
+
print(f"Error: {e}")
|
71
|
+
return None
|
72
|
+
|
73
|
+
def get_rpm_source_package(package_name, dest_dir="./source_packages"):
|
74
|
+
os.makedirs(dest_dir, exist_ok=True)
|
75
|
+
command = ["yumdownloader", "--source", "--destdir", dest_dir, package_name]
|
76
|
+
result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
77
|
+
if result.returncode == 0:
|
78
|
+
for file in os.listdir(dest_dir):
|
79
|
+
if file.endswith(".src.rpm"):
|
80
|
+
return os.path.join(dest_dir, file)
|
81
|
+
return None
|
82
|
+
|
83
|
+
def extract_rpm_spec_file(srpm_path, dest_dir="./extracted_specs"):
|
84
|
+
os.makedirs(dest_dir, exist_ok=True)
|
85
|
+
try:
|
86
|
+
command = f"rpm2cpio {srpm_path} | cpio -idmv -D {dest_dir} > /tmp/ossa_gen.log"
|
87
|
+
subprocess.run(command, shell=True, check=True)
|
88
|
+
spec_files = [os.path.join(dest_dir, f) for f in os.listdir(dest_dir) if f.endswith(".spec")]
|
89
|
+
if spec_files:
|
90
|
+
return spec_files[0]
|
91
|
+
except subprocess.CalledProcessError as e:
|
92
|
+
print(f"Failed to extract spec file from {srpm_path}: {e}")
|
93
|
+
return None
|
94
|
+
|
95
|
+
def extract_rpm_tarballs(srpm_path, dest_dir="./extracted_sources"):
|
96
|
+
os.makedirs(dest_dir, exist_ok=True)
|
97
|
+
try:
|
98
|
+
tarballs = [os.path.join(dest_dir, f) for f in os.listdir(dest_dir) if f.endswith((".tar.gz", ".tar.bz2", ".tar.xz", ".tgz"))]
|
99
|
+
return tarballs
|
100
|
+
except subprocess.CalledProcessError as e:
|
101
|
+
print(f"Failed to extract tarballs from {srpm_path}: {e}")
|
102
|
+
return []
|
103
|
+
|
104
|
+
def extract_rpm_info_from_spec(spec_file_path):
|
105
|
+
project_url = None
|
106
|
+
source_url = None
|
107
|
+
license = None
|
108
|
+
try:
|
109
|
+
with open(spec_file_path, "r") as spec_file:
|
110
|
+
for line in spec_file:
|
111
|
+
if line.startswith("URL:"):
|
112
|
+
project_url = line.split(":", 1)[1].strip()
|
113
|
+
elif line.startswith("Source0:"):
|
114
|
+
source_url = line.split(":", 1)[1].strip()
|
115
|
+
elif line.startswith("License:"):
|
116
|
+
license = line.split(":", 1)[1].strip()
|
117
|
+
except FileNotFoundError:
|
118
|
+
print(f"Spec file not found: {spec_file_path}")
|
119
|
+
return project_url, source_url, license
|
@@ -1,13 +1,18 @@
|
|
1
|
+
import os
|
1
2
|
import distro
|
3
|
+
import subprocess
|
2
4
|
|
3
5
|
def detect_os():
|
6
|
+
dist = distro.id()
|
7
|
+
return dist
|
8
|
+
|
9
|
+
def detect_pm():
|
4
10
|
dist = distro.id()
|
5
11
|
if 'ubuntu' in dist or 'debian' in dist:
|
6
12
|
return 'apt'
|
7
|
-
elif 'redhat' in dist or 'centos' in dist or 'almalinux' in dist:
|
13
|
+
elif 'redhat' in dist or 'centos' in dist or 'almalinux' in dist or 'amzn' in dist:
|
8
14
|
return 'yum'
|
9
15
|
elif 'darwin' in dist:
|
10
16
|
return 'brew'
|
11
17
|
else:
|
12
18
|
raise ValueError("Unsupported OS")
|
13
|
-
|
@@ -1,4 +1,5 @@
|
|
1
1
|
import subprocess
|
2
|
+
import re
|
2
3
|
|
3
4
|
|
4
5
|
def list_packages(package_manager):
|
@@ -25,7 +26,7 @@ def list_packages(package_manager):
|
|
25
26
|
|
26
27
|
packages = result.stdout.splitlines()
|
27
28
|
extracted_packages = []
|
28
|
-
max_packages =
|
29
|
+
max_packages = 500000
|
29
30
|
k_packages = 0
|
30
31
|
for line in packages:
|
31
32
|
if not line.strip() or line.startswith("==>"):
|
@@ -65,44 +66,60 @@ def get_package_info(package_manager, package_name):
|
|
65
66
|
def parse_brew_info(output):
|
66
67
|
"""Parses brew info output to extract license, website, and description."""
|
67
68
|
info = {}
|
68
|
-
|
69
|
+
info["name"] = "NOASSERTION"
|
70
|
+
info["version"] = "NOASSERTION"
|
69
71
|
info["licenses"] = "NOASSERTION"
|
72
|
+
info["severity"] = "NOASSERTION"
|
70
73
|
info["references"] = "NOASSERTION"
|
71
|
-
info["
|
74
|
+
info["summary"] = "NOASSERTION"
|
75
|
+
lines = output.splitlines()
|
72
76
|
|
73
77
|
for i, line in enumerate(lines):
|
74
|
-
if
|
75
|
-
|
78
|
+
if line.startswith("==>") and ":" in line:
|
79
|
+
new_line = line.lstrip("==>").strip()
|
80
|
+
match1 = re.match(r"([^:]+):.*?([\d\.a-zA-Z]+)\s*\(", new_line)
|
81
|
+
match2 = re.match(r"([^:]+):", new_line)
|
82
|
+
if match1:
|
83
|
+
pname = match1.group(1).strip()
|
84
|
+
version = match1.group(2).strip()
|
85
|
+
elif match2:
|
86
|
+
pname = match2.group(1).strip()
|
87
|
+
version = "*"
|
88
|
+
info["name"] = pname
|
89
|
+
info["version"] = version
|
90
|
+
elif i == 1:
|
91
|
+
info["summary"] = line.strip()
|
76
92
|
elif line.startswith("https://"): # The website URL
|
77
93
|
info["references"] = line.strip()
|
78
94
|
elif line.startswith("License:"): # The license information
|
79
95
|
info["licenses"] = line.split(":", 1)[1].strip()
|
80
|
-
|
81
|
-
|
96
|
+
info["licenses"] = extract_spdx_ids(info["licenses"])
|
97
|
+
info["severity"], info["rason"] = license_classificaton(info["licenses"])
|
82
98
|
return info
|
83
99
|
|
84
100
|
def parse_yum_info(output):
|
85
|
-
"""Parses yum repoquery --info output."""
|
86
101
|
info = {}
|
102
|
+
info["name"] = "NOASSERTION"
|
103
|
+
info["version"] = "NOASSERTION"
|
104
|
+
info["licenses"] = "NOASSERTION"
|
105
|
+
info["severity"] = "NOASSERTION"
|
106
|
+
info["references"] = "NOASSERTION"
|
107
|
+
info["summary"] = "NOASSERTION"
|
87
108
|
lines = output.splitlines()
|
88
|
-
|
89
109
|
for line in lines:
|
90
110
|
if line.startswith("License"):
|
91
111
|
info["licenses"] = line.split(":", 1)[1].strip()
|
112
|
+
info["licenses"] = extract_spdx_ids(info["licenses"])
|
113
|
+
info["severity"], info["rason"] = license_classificaton(info["licenses"])
|
92
114
|
elif line.startswith("URL"):
|
93
115
|
info["references"] = line.split(":", 1)[1].strip()
|
94
|
-
elif "
|
95
|
-
info["
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
"copyright": info.get("copyright", "NOASSERTION"),
|
102
|
-
"references": info.get("references", "NOASSERTION"),
|
103
|
-
"severity": severity,
|
104
|
-
}
|
105
|
-
|
116
|
+
elif line.startswith("Name"):
|
117
|
+
info["name"] = line.split(":", 1)[1].strip()
|
118
|
+
elif line.startswith("Version"):
|
119
|
+
info["version"] = line.split(":", 1)[1].strip()
|
120
|
+
elif line.startswith("Summary"):
|
121
|
+
info["summary"] = line.split(":", 1)[1].strip()
|
122
|
+
return info
|
106
123
|
|
107
124
|
def parse_apt_info(output):
|
108
125
|
"""Parses apt-cache show output."""
|
@@ -116,6 +133,7 @@ def parse_apt_info(output):
|
|
116
133
|
info["website"] = line.split(":", 1)[1].strip()
|
117
134
|
elif "Copyright" in line:
|
118
135
|
info["references"] = line.strip()
|
136
|
+
info["licenses"] = extract_spdx_ids(info["licenses"])
|
119
137
|
severity = license_classificaton(info["licenses"])
|
120
138
|
|
121
139
|
# Ensure all keys are present even if data is missing
|
@@ -126,10 +144,37 @@ def parse_apt_info(output):
|
|
126
144
|
"severity": severity,
|
127
145
|
}
|
128
146
|
|
147
|
+
def extract_spdx_ids(license_string):
|
148
|
+
if not license_string.strip():
|
149
|
+
return "No valid SPDX licenses found"
|
150
|
+
raw_ids = re.split(r'(?i)\sAND\s|\sOR\s|\(|\)', license_string)
|
151
|
+
cleaned_ids = [spdx.strip() for spdx in raw_ids if spdx.strip()]
|
152
|
+
unique_spdx_ids = sorted(set(cleaned_ids))
|
153
|
+
return ", ".join(unique_spdx_ids) if unique_spdx_ids else "No valid SPDX licenses found"
|
154
|
+
|
129
155
|
def license_classificaton(licenses):
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
156
|
+
license_categories = {
|
157
|
+
"copyleft": ["GPL", "AGPL"],
|
158
|
+
"weak_copyleft": ["LGPL", "MPL", "EPL", "CDDL"],
|
159
|
+
"permissive": ["MIT", "BSD", "Apache"]
|
160
|
+
}
|
161
|
+
# Priority levels for each category
|
162
|
+
priority = {"copyleft": 1, "weak_copyleft": 2, "permissive": 3}
|
163
|
+
severity_map = {
|
164
|
+
"copyleft": ("High", "This package contains copyleft licenses, which impose strong obligations."),
|
165
|
+
"weak_copyleft": ("Medium", "This package contains weak copyleft licenses, which impose moderate obligations."),
|
166
|
+
"permissive": ("Informational", "This package contains permissive licenses, which impose minimal obligations."),
|
167
|
+
}
|
168
|
+
# Split multiple licenses and normalize them
|
169
|
+
license_list = [l.strip() for l in licenses.split(",")]
|
170
|
+
current_priority = float("inf")
|
171
|
+
selected_severity = "Informational"
|
172
|
+
selected_reason = "PURL identification for OSSBOMER"
|
173
|
+
for license in license_list:
|
174
|
+
for category, patterns in license_categories.items():
|
175
|
+
if any(license.upper().startswith(pattern.upper()) for pattern in patterns):
|
176
|
+
if priority[category] < current_priority:
|
177
|
+
current_priority = priority[category]
|
178
|
+
selected_severity, selected_reason = severity_map[category]
|
179
|
+
|
180
|
+
return selected_severity, selected_reason
|
@@ -0,0 +1,35 @@
|
|
1
|
+
import os
|
2
|
+
import glob
|
3
|
+
import shutil
|
4
|
+
import subprocess
|
5
|
+
|
6
|
+
def calculate_swhid(directory_path, file_path):
|
7
|
+
os.makedirs(directory_path, exist_ok=True)
|
8
|
+
try:
|
9
|
+
command = f"tar -xf {file_path} -C {directory_path}"
|
10
|
+
subprocess.run(command, shell=True, check=True)
|
11
|
+
command = ["swh.identify", directory_path]
|
12
|
+
result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
13
|
+
if result.returncode == 0:
|
14
|
+
for line in result.stdout.strip().split("\n"):
|
15
|
+
if line.startswith("swh:1:dir:"):
|
16
|
+
swhid = line.split("\t")[0]
|
17
|
+
cleanup_extracted_files(directory_path)
|
18
|
+
return swhid
|
19
|
+
else:
|
20
|
+
print(f"Failed to compute folder SWHID: {result.stderr}")
|
21
|
+
except subprocess.CalledProcessError as e:
|
22
|
+
print(f"Failed to process tarball {file_path}: {e}")
|
23
|
+
finally:
|
24
|
+
cleanup_extracted_files(directory_path)
|
25
|
+
return None
|
26
|
+
|
27
|
+
def cleanup_extracted_files(directory_path):
|
28
|
+
try:
|
29
|
+
for file_path in glob.glob(f"{directory_path}/*"):
|
30
|
+
if os.path.isdir(file_path):
|
31
|
+
shutil.rmtree(file_path)
|
32
|
+
else:
|
33
|
+
os.remove(file_path)
|
34
|
+
except Exception as e:
|
35
|
+
print(f"Failed to clean up {directory_path}: {e}")
|
@@ -1,12 +1,12 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: ossa_scanner
|
3
|
-
Version: 0.1.
|
4
|
-
Summary:
|
3
|
+
Version: 0.1.7
|
4
|
+
Summary: Open Source Software Advisory generator for Core and Base Linux Packages.
|
5
5
|
Home-page: https://github.com/oscarvalenzuelab/ossa_scanner
|
6
6
|
Author: Oscar Valenzuela
|
7
7
|
Author-email: oscar.valenzuela.b@gmail.com
|
8
8
|
License: MIT
|
9
|
-
Keywords: linux packages SWHID open-source compliance
|
9
|
+
Keywords: linux packages SWHID open-source compliance ossa advisory
|
10
10
|
Classifier: Development Status :: 3 - Alpha
|
11
11
|
Classifier: Intended Audience :: Developers
|
12
12
|
Classifier: License :: OSI Approved :: MIT License
|
@@ -27,3 +27,12 @@ Requires-Dist: ssdeep
|
|
27
27
|
|
28
28
|
# ossa_scanner
|
29
29
|
Open Source Advisory Scanner (Generator)
|
30
|
+
|
31
|
+
## Centos/AL/AlmaLinux
|
32
|
+
Install Python PyPI:
|
33
|
+
|
34
|
+
> sudo yum groupinstall "Development Tools"
|
35
|
+
> sudo yum -y install python-pip python3-devel
|
36
|
+
> pip3 install swh-scanner
|
37
|
+
> BUILD_LIB=1 pip install ssdeep
|
38
|
+
> pip3 install ossa-scanner
|
@@ -20,7 +20,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
|
|
20
20
|
setup(
|
21
21
|
name="ossa_scanner",
|
22
22
|
version=get_version(),
|
23
|
-
description="
|
23
|
+
description="Open Source Software Advisory generator for Core and Base Linux Packages.",
|
24
24
|
long_description=long_description,
|
25
25
|
long_description_content_type='text/markdown',
|
26
26
|
author="Oscar Valenzuela",
|
@@ -52,5 +52,5 @@ setup(
|
|
52
52
|
"Programming Language :: Python :: 3.10",
|
53
53
|
"Operating System :: POSIX :: Linux",
|
54
54
|
],
|
55
|
-
keywords="linux packages SWHID open-source compliance",
|
55
|
+
keywords="linux packages SWHID open-source compliance ossa advisory",
|
56
56
|
)
|
ossa_scanner-0.1.4/README.md
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
__version__ = "0.1.4"
|
@@ -1,47 +0,0 @@
|
|
1
|
-
import subprocess
|
2
|
-
import os
|
3
|
-
import shutil
|
4
|
-
import glob
|
5
|
-
|
6
|
-
def download_source(package_manager, package_name, output_dir):
|
7
|
-
try:
|
8
|
-
if package_manager == 'apt':
|
9
|
-
cmd = ['apt-get', 'source', package_name, '-d', output_dir]
|
10
|
-
subprocess.run(cmd, check=True)
|
11
|
-
elif package_manager in ['yum', 'dnf']:
|
12
|
-
cmd = ['dnf', 'download', '--source', package_name, '--downloaddir', output_dir]
|
13
|
-
subprocess.run(cmd, check=True)
|
14
|
-
elif package_manager == 'brew':
|
15
|
-
# Fetch the source tarball
|
16
|
-
cmd = ['brew', 'fetch', '--build-from-source', package_name]
|
17
|
-
subprocess.run(cmd, check=True, capture_output=True, text=True)
|
18
|
-
cache_dir = subprocess.run(
|
19
|
-
['brew', '--cache', package_name],
|
20
|
-
capture_output=True,
|
21
|
-
text=True,
|
22
|
-
check=True
|
23
|
-
).stdout.strip()
|
24
|
-
prefixes_to_remove = ['aarch64-elf-', 'arm-none-eabi-', 'other-prefix-']
|
25
|
-
stripped_package_name = package_name
|
26
|
-
for prefix in prefixes_to_remove:
|
27
|
-
if package_name.startswith(prefix):
|
28
|
-
stripped_package_name = package_name[len(prefix):]
|
29
|
-
break
|
30
|
-
cache_folder = os.path.dirname(cache_dir)
|
31
|
-
tarball_pattern = os.path.join(cache_folder, f"*{stripped_package_name}*")
|
32
|
-
matching_files = glob.glob(tarball_pattern)
|
33
|
-
if not matching_files:
|
34
|
-
raise FileNotFoundError(f"Tarball not found for {package_name} in {cache_folder}")
|
35
|
-
tarball_path = matching_files[0]
|
36
|
-
os.makedirs(output_dir, exist_ok=True)
|
37
|
-
target_path = os.path.join(output_dir, os.path.basename(tarball_path))
|
38
|
-
shutil.move(tarball_path, target_path)
|
39
|
-
return target_path
|
40
|
-
else:
|
41
|
-
raise ValueError("Unsupported package manager")
|
42
|
-
except subprocess.CalledProcessError as e:
|
43
|
-
print(f"Command failed: {e}")
|
44
|
-
return None
|
45
|
-
except Exception as e:
|
46
|
-
print(f"Error: {e}")
|
47
|
-
return None
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|