scanoss 1.14.0__py3-none-any.whl → 1.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scanoss/__init__.py +1 -1
- scanoss/cli.py +219 -48
- scanoss/cyclonedx.py +13 -0
- scanoss/data/build_date.txt +1 -1
- scanoss/results.py +301 -0
- scanoss/scancodedeps.py +29 -3
- scanoss/scanner.py +42 -30
- scanoss/scanoss_settings.py +189 -0
- scanoss/scanossapi.py +10 -20
- scanoss/scanossbase.py +21 -0
- scanoss/scanpostprocessor.py +159 -0
- scanoss/threadeddependencies.py +75 -5
- {scanoss-1.14.0.dist-info → scanoss-1.16.0.dist-info}/METADATA +1 -1
- {scanoss-1.14.0.dist-info → scanoss-1.16.0.dist-info}/RECORD +18 -15
- {scanoss-1.14.0.dist-info → scanoss-1.16.0.dist-info}/WHEEL +1 -1
- {scanoss-1.14.0.dist-info → scanoss-1.16.0.dist-info}/LICENSE +0 -0
- {scanoss-1.14.0.dist-info → scanoss-1.16.0.dist-info}/entry_points.txt +0 -0
- {scanoss-1.14.0.dist-info → scanoss-1.16.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""
|
|
2
|
+
SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
Copyright (c) 2024, SCANOSS
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the 'Software'), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in
|
|
14
|
+
all copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
22
|
+
THE SOFTWARE.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
import json
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Dict, List, TypedDict
|
|
28
|
+
|
|
29
|
+
from .scanossbase import ScanossBase
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class BomEntry(TypedDict, total=False):
|
|
33
|
+
purl: str
|
|
34
|
+
path: str
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ScanossSettings(ScanossBase):
|
|
38
|
+
"""Handles the loading and parsing of the SCANOSS settings file"""
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
debug: bool = False,
|
|
43
|
+
trace: bool = False,
|
|
44
|
+
quiet: bool = False,
|
|
45
|
+
filepath: str = None,
|
|
46
|
+
):
|
|
47
|
+
"""
|
|
48
|
+
Args:
|
|
49
|
+
debug (bool, optional): Debug. Defaults to False.
|
|
50
|
+
trace (bool, optional): Trace. Defaults to False.
|
|
51
|
+
quiet (bool, optional): Quiet. Defaults to False.
|
|
52
|
+
filepath (str, optional): Path to settings file. Defaults to None.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
super().__init__(debug, trace, quiet)
|
|
56
|
+
self.data = {}
|
|
57
|
+
self.settings_file_type = None
|
|
58
|
+
self.scan_type = None
|
|
59
|
+
|
|
60
|
+
if filepath:
|
|
61
|
+
self.load_json_file(filepath)
|
|
62
|
+
|
|
63
|
+
def load_json_file(self, filepath: str):
|
|
64
|
+
"""Load the scan settings file
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
filepath (str): Path to the SCANOSS settings file
|
|
68
|
+
"""
|
|
69
|
+
json_file = Path(filepath).resolve()
|
|
70
|
+
|
|
71
|
+
if not json_file.exists():
|
|
72
|
+
self.print_stderr(f"Scan settings file not found: {filepath}")
|
|
73
|
+
self.data = {}
|
|
74
|
+
|
|
75
|
+
with open(json_file, "r") as jsonfile:
|
|
76
|
+
self.print_debug(f"Loading scan settings from: {filepath}")
|
|
77
|
+
try:
|
|
78
|
+
self.data = json.load(jsonfile)
|
|
79
|
+
except Exception as e:
|
|
80
|
+
self.print_stderr(f"ERROR: Problem parsing input JSON: {e}")
|
|
81
|
+
return self
|
|
82
|
+
|
|
83
|
+
def set_file_type(self, file_type: str):
|
|
84
|
+
"""Set the file type in order to support both legacy SBOM.json and new scanoss.json files
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
file_type (str): 'legacy' or 'new'
|
|
88
|
+
|
|
89
|
+
Raises:
|
|
90
|
+
Exception: Invalid scan settings file, missing "components" or "bom"
|
|
91
|
+
"""
|
|
92
|
+
self.settings_file_type = file_type
|
|
93
|
+
if not self._is_valid_sbom_file:
|
|
94
|
+
raise Exception(
|
|
95
|
+
'Invalid scan settings file, missing "components" or "bom")'
|
|
96
|
+
)
|
|
97
|
+
return self
|
|
98
|
+
|
|
99
|
+
def set_scan_type(self, scan_type: str):
|
|
100
|
+
"""Set the scan type to support legacy SBOM.json files
|
|
101
|
+
|
|
102
|
+
Args:
|
|
103
|
+
scan_type (str): 'identify' or 'exclude'
|
|
104
|
+
"""
|
|
105
|
+
self.scan_type = scan_type
|
|
106
|
+
return self
|
|
107
|
+
|
|
108
|
+
def _is_valid_sbom_file(self):
|
|
109
|
+
"""Check if the scan settings file is valid
|
|
110
|
+
|
|
111
|
+
Returns:
|
|
112
|
+
bool: True if the file is valid, False otherwise
|
|
113
|
+
"""
|
|
114
|
+
if not self.data.get("components") or not self.data.get("bom"):
|
|
115
|
+
return False
|
|
116
|
+
return True
|
|
117
|
+
|
|
118
|
+
def _get_bom(self):
|
|
119
|
+
"""Get the Billing of Materials from the settings file
|
|
120
|
+
|
|
121
|
+
Returns:
|
|
122
|
+
dict: If using scanoss.json
|
|
123
|
+
list: If using SBOM.json
|
|
124
|
+
"""
|
|
125
|
+
if self.settings_file_type == "legacy":
|
|
126
|
+
return self.data.get("components", [])
|
|
127
|
+
return self.data.get("bom", {})
|
|
128
|
+
|
|
129
|
+
def get_bom_include(self) -> List[BomEntry]:
|
|
130
|
+
"""Get the list of components to include in the scan
|
|
131
|
+
|
|
132
|
+
Returns:
|
|
133
|
+
list: List of components to include in the scan
|
|
134
|
+
"""
|
|
135
|
+
if self.settings_file_type == "legacy":
|
|
136
|
+
return self._get_bom()
|
|
137
|
+
return self._get_bom().get("include", [])
|
|
138
|
+
|
|
139
|
+
def get_bom_remove(self) -> List[BomEntry]:
|
|
140
|
+
"""Get the list of components to remove from the scan
|
|
141
|
+
|
|
142
|
+
Returns:
|
|
143
|
+
list: List of components to remove from the scan
|
|
144
|
+
"""
|
|
145
|
+
if self.settings_file_type == "legacy":
|
|
146
|
+
return self._get_bom()
|
|
147
|
+
return self._get_bom().get("remove", [])
|
|
148
|
+
|
|
149
|
+
def get_sbom(self):
|
|
150
|
+
"""Get the SBOM to be sent to the SCANOSS API
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
dict: SBOM
|
|
154
|
+
"""
|
|
155
|
+
if not self.data:
|
|
156
|
+
return None
|
|
157
|
+
return {
|
|
158
|
+
"scan_type": self.scan_type,
|
|
159
|
+
"assets": json.dumps(self._get_sbom_assets()),
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
def _get_sbom_assets(self):
|
|
163
|
+
"""Get the SBOM assets
|
|
164
|
+
|
|
165
|
+
Returns:
|
|
166
|
+
List: List of SBOM assets
|
|
167
|
+
"""
|
|
168
|
+
if self.scan_type == "identify":
|
|
169
|
+
return self.normalize_bom_entries(self.get_bom_include())
|
|
170
|
+
return self.normalize_bom_entries(self.get_bom_remove())
|
|
171
|
+
|
|
172
|
+
@staticmethod
|
|
173
|
+
def normalize_bom_entries(bom_entries) -> List[BomEntry]:
|
|
174
|
+
"""Normalize the BOM entries
|
|
175
|
+
|
|
176
|
+
Args:
|
|
177
|
+
bom_entries (List[Dict]): List of BOM entries
|
|
178
|
+
|
|
179
|
+
Returns:
|
|
180
|
+
List: Normalized BOM entries
|
|
181
|
+
"""
|
|
182
|
+
normalized_bom_entries = []
|
|
183
|
+
for entry in bom_entries:
|
|
184
|
+
normalized_bom_entries.append(
|
|
185
|
+
{
|
|
186
|
+
"purl": entry.get("purl", ""),
|
|
187
|
+
}
|
|
188
|
+
)
|
|
189
|
+
return normalized_bom_entries
|
scanoss/scanossapi.py
CHANGED
|
@@ -34,6 +34,7 @@ import urllib3
|
|
|
34
34
|
from pypac import PACSession
|
|
35
35
|
from pypac.parser import PACFile
|
|
36
36
|
from urllib3.exceptions import InsecureRequestWarning
|
|
37
|
+
|
|
37
38
|
from .scanossbase import ScanossBase
|
|
38
39
|
from . import __version__
|
|
39
40
|
|
|
@@ -50,14 +51,12 @@ class ScanossApi(ScanossBase):
|
|
|
50
51
|
Currently support posting scan requests to the SCANOSS streaming API
|
|
51
52
|
"""
|
|
52
53
|
|
|
53
|
-
def __init__(self,
|
|
54
|
+
def __init__(self, scan_format: str = None, flags: str = None,
|
|
54
55
|
url: str = None, api_key: str = None, debug: bool = False, trace: bool = False, quiet: bool = False,
|
|
55
56
|
timeout: int = 180, ver_details: str = None, ignore_cert_errors: bool = False,
|
|
56
57
|
proxy: str = None, ca_cert: str = None, pac: PACFile = None, retry: int = 5):
|
|
57
58
|
"""
|
|
58
59
|
Initialise the SCANOSS API
|
|
59
|
-
:param scan_type: Scan type (default identify)
|
|
60
|
-
:param sbom_path: Input SBOM file to match scan type (default None)
|
|
61
60
|
:param scan_format: Scan format (default plain)
|
|
62
61
|
:param flags: Scanning flags (default None)
|
|
63
62
|
:param url: API URL (default https://api.osskb.org/scan/direct)
|
|
@@ -77,9 +76,8 @@ class ScanossApi(ScanossBase):
|
|
|
77
76
|
self.api_key = api_key if api_key else SCANOSS_API_KEY
|
|
78
77
|
if self.api_key and not url and not os.environ.get("SCANOSS_SCAN_URL"):
|
|
79
78
|
self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium
|
|
80
|
-
self.
|
|
79
|
+
self.sbom = None
|
|
81
80
|
self.scan_format = scan_format if scan_format else 'plain'
|
|
82
|
-
self.sbom_path = sbom_path
|
|
83
81
|
self.flags = flags
|
|
84
82
|
self.timeout = timeout if timeout > 5 else 180
|
|
85
83
|
self.retry_limit = retry if retry >= 0 else 5
|
|
@@ -92,8 +90,6 @@ class ScanossApi(ScanossBase):
|
|
|
92
90
|
self.headers['x-api-key'] = self.api_key
|
|
93
91
|
self.headers['User-Agent'] = f'scanoss-py/{__version__}'
|
|
94
92
|
self.headers['user-agent'] = f'scanoss-py/{__version__}'
|
|
95
|
-
self.sbom = None
|
|
96
|
-
self.load_sbom() # Load an input SBOM if one is specified
|
|
97
93
|
if self.trace:
|
|
98
94
|
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
|
|
99
95
|
http_client.HTTPConnection.debuglevel = 1
|
|
@@ -115,17 +111,6 @@ class ScanossApi(ScanossBase):
|
|
|
115
111
|
if self. proxies:
|
|
116
112
|
self.session.proxies = self.proxies
|
|
117
113
|
|
|
118
|
-
def load_sbom(self):
|
|
119
|
-
"""
|
|
120
|
-
Load the input SBOM if one exists
|
|
121
|
-
"""
|
|
122
|
-
if self.sbom_path:
|
|
123
|
-
if not self.scan_type:
|
|
124
|
-
self.scan_type = 'identify' # Default to identify SBOM type if it's not set
|
|
125
|
-
self.print_debug(f'Loading {self.scan_type} SBOM {self.sbom_path}...')
|
|
126
|
-
with open(self.sbom_path) as f:
|
|
127
|
-
self.sbom = f.read()
|
|
128
|
-
|
|
129
114
|
def scan(self, wfp: str, context: str = None, scan_id: int = None):
|
|
130
115
|
"""
|
|
131
116
|
Scan the specified WFP and return the JSON object
|
|
@@ -137,14 +122,15 @@ class ScanossApi(ScanossBase):
|
|
|
137
122
|
request_id = str(uuid.uuid4())
|
|
138
123
|
form_data = {}
|
|
139
124
|
if self.sbom:
|
|
140
|
-
form_data['type'] = self.scan_type
|
|
141
|
-
form_data['assets'] = self.sbom
|
|
125
|
+
form_data['type'] = self.sbom.get("scan_type")
|
|
126
|
+
form_data['assets'] = self.sbom.get("assets")
|
|
142
127
|
if self.scan_format:
|
|
143
128
|
form_data['format'] = self.scan_format
|
|
144
129
|
if self.flags:
|
|
145
130
|
form_data['flags'] = self.flags
|
|
146
131
|
if context:
|
|
147
132
|
form_data['context'] = context
|
|
133
|
+
|
|
148
134
|
scan_files = {'file': ("%s.wfp" % request_id, wfp)}
|
|
149
135
|
headers = self.headers
|
|
150
136
|
headers['x-request-id'] = request_id # send a unique request id for each post
|
|
@@ -242,6 +228,10 @@ class ScanossApi(ScanossBase):
|
|
|
242
228
|
except Exception as ee:
|
|
243
229
|
self.print_stderr(f'Warning: Issue writing bad request file - {bad_req_file} ({ee.__class__.__name__}):'
|
|
244
230
|
f' {ee}')
|
|
231
|
+
|
|
232
|
+
def set_sbom(self, sbom):
|
|
233
|
+
self.sbom = sbom
|
|
234
|
+
return self
|
|
245
235
|
|
|
246
236
|
#
|
|
247
237
|
# End of ScanossApi Class
|
scanoss/scanossbase.py
CHANGED
|
@@ -68,3 +68,24 @@ class ScanossBase:
|
|
|
68
68
|
"""
|
|
69
69
|
if self.trace:
|
|
70
70
|
self.print_stderr(*args, **kwargs)
|
|
71
|
+
|
|
72
|
+
@staticmethod
|
|
73
|
+
def print_stdout(*args, **kwargs):
|
|
74
|
+
"""
|
|
75
|
+
Print message to STDOUT
|
|
76
|
+
"""
|
|
77
|
+
print(
|
|
78
|
+
*args,
|
|
79
|
+
file=sys.stdout,
|
|
80
|
+
**kwargs,
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
def print_to_file_or_stdout(self, msg: str, file: str = None):
|
|
84
|
+
"""
|
|
85
|
+
Print message to file if provided or stdout
|
|
86
|
+
"""
|
|
87
|
+
if file:
|
|
88
|
+
with open(file, "w") as f:
|
|
89
|
+
f.write(msg)
|
|
90
|
+
else:
|
|
91
|
+
self.print_stdout(msg)
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
"""
|
|
2
|
+
SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
Copyright (c) 2024, SCANOSS
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in
|
|
14
|
+
all copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
22
|
+
THE SOFTWARE.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from typing import List
|
|
26
|
+
|
|
27
|
+
from .scanoss_settings import BomEntry, ScanossSettings
|
|
28
|
+
from .scanossbase import ScanossBase
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ScanPostProcessor(ScanossBase):
|
|
32
|
+
"""Handles post-processing of the scan results"""
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
scan_settings: ScanossSettings,
|
|
37
|
+
debug: bool = False,
|
|
38
|
+
trace: bool = False,
|
|
39
|
+
quiet: bool = False,
|
|
40
|
+
results: dict = None,
|
|
41
|
+
):
|
|
42
|
+
"""
|
|
43
|
+
Args:
|
|
44
|
+
scan_settings (ScanossSettings): Scan settings object
|
|
45
|
+
debug (bool, optional): Debug mode. Defaults to False.
|
|
46
|
+
trace (bool, optional): Traces. Defaults to False.
|
|
47
|
+
quiet (bool, optional): Quiet mode. Defaults to False.
|
|
48
|
+
results (dict | str, optional): Results to be processed. Defaults to None.
|
|
49
|
+
"""
|
|
50
|
+
super().__init__(debug, trace, quiet)
|
|
51
|
+
self.scan_settings = scan_settings
|
|
52
|
+
self.results = results
|
|
53
|
+
|
|
54
|
+
def load_results(self, raw_results: dict):
|
|
55
|
+
"""Load the raw results
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
raw_results (dict): Raw scan results
|
|
59
|
+
"""
|
|
60
|
+
self.results = raw_results
|
|
61
|
+
return self
|
|
62
|
+
|
|
63
|
+
def post_process(self):
|
|
64
|
+
"""Post-process the scan results
|
|
65
|
+
|
|
66
|
+
Returns:
|
|
67
|
+
dict: Processed results
|
|
68
|
+
"""
|
|
69
|
+
self.remove_dismissed_files()
|
|
70
|
+
return self.results
|
|
71
|
+
|
|
72
|
+
def remove_dismissed_files(self):
|
|
73
|
+
"""Remove entries from the results based on files and/or purls specified in the SCANOSS settings file"""
|
|
74
|
+
|
|
75
|
+
to_remove_entries = self.scan_settings.get_bom_remove()
|
|
76
|
+
|
|
77
|
+
if not to_remove_entries:
|
|
78
|
+
return
|
|
79
|
+
|
|
80
|
+
self.results = {
|
|
81
|
+
result_path: result
|
|
82
|
+
for result_path, result in self.results.items()
|
|
83
|
+
if not self._should_remove_result(result_path, result, to_remove_entries)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
def _should_remove_result(
|
|
87
|
+
self, result_path: str, result: dict, to_remove_entries: List[BomEntry]
|
|
88
|
+
) -> bool:
|
|
89
|
+
"""Check if a result should be removed based on the SCANOSS settings"""
|
|
90
|
+
result = result[0] if isinstance(result, list) else result
|
|
91
|
+
result_purls = result.get("purl", [])
|
|
92
|
+
|
|
93
|
+
for to_remove_entry in to_remove_entries:
|
|
94
|
+
to_remove_path = to_remove_entry.get("path")
|
|
95
|
+
to_remove_purl = to_remove_entry.get("purl")
|
|
96
|
+
|
|
97
|
+
if not to_remove_path and not to_remove_purl:
|
|
98
|
+
continue
|
|
99
|
+
|
|
100
|
+
# Bom entry has both path and purl
|
|
101
|
+
if self._is_full_match(result_path, result_purls, to_remove_entry):
|
|
102
|
+
self._print_removal_message(result_path, result_purls, to_remove_entry)
|
|
103
|
+
return True
|
|
104
|
+
|
|
105
|
+
# Bom entry has only purl
|
|
106
|
+
if not to_remove_path and to_remove_purl in result_purls:
|
|
107
|
+
self._print_removal_message(result_path, result_purls, to_remove_entry)
|
|
108
|
+
return True
|
|
109
|
+
|
|
110
|
+
# Bom entry has only path
|
|
111
|
+
if not to_remove_purl and to_remove_path == result_path:
|
|
112
|
+
self._print_removal_message(result_path, result_purls, to_remove_entry)
|
|
113
|
+
return True
|
|
114
|
+
|
|
115
|
+
return False
|
|
116
|
+
|
|
117
|
+
def _print_removal_message(
|
|
118
|
+
self, result_path: str, result_purls: List[str], to_remove_entry: BomEntry
|
|
119
|
+
) -> None:
|
|
120
|
+
"""Print a message about removing a result"""
|
|
121
|
+
if to_remove_entry.get("path") and to_remove_entry.get("purl"):
|
|
122
|
+
message = f"Removing '{result_path}' from the results. Full match found."
|
|
123
|
+
elif to_remove_entry.get("purl"):
|
|
124
|
+
message = f"Removing '{result_path}' from the results. Found PURL match."
|
|
125
|
+
else:
|
|
126
|
+
message = f"Removing '{result_path}' from the results. Found path match."
|
|
127
|
+
|
|
128
|
+
self.print_msg(
|
|
129
|
+
f"{message}\n"
|
|
130
|
+
f"Details:\n"
|
|
131
|
+
f" - PURLs: {', '.join(result_purls)}\n"
|
|
132
|
+
f" - Path: '{result_path}'\n"
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
def _is_full_match(
|
|
136
|
+
self,
|
|
137
|
+
result_path: str,
|
|
138
|
+
result_purls: List[str],
|
|
139
|
+
bom_entry: BomEntry,
|
|
140
|
+
) -> bool:
|
|
141
|
+
"""Check if path and purl matches fully with the bom entry
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
result_path (str): Scan result path
|
|
145
|
+
result_purls (List[str]): Scan result purls
|
|
146
|
+
bom_entry (BomEntry): BOM entry to compare with
|
|
147
|
+
|
|
148
|
+
Returns:
|
|
149
|
+
bool: True if the path and purl match, False otherwise
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
if not result_purls:
|
|
153
|
+
return False
|
|
154
|
+
|
|
155
|
+
return bool(
|
|
156
|
+
(bom_entry.get("purl") and bom_entry.get("path"))
|
|
157
|
+
and (bom_entry.get("path") == result_path)
|
|
158
|
+
and (bom_entry.get("purl") in result_purls)
|
|
159
|
+
)
|
scanoss/threadeddependencies.py
CHANGED
|
@@ -24,7 +24,9 @@
|
|
|
24
24
|
|
|
25
25
|
import threading
|
|
26
26
|
import queue
|
|
27
|
-
|
|
27
|
+
import json
|
|
28
|
+
from enum import Enum
|
|
29
|
+
from typing import Dict, Optional, Set
|
|
28
30
|
from dataclasses import dataclass
|
|
29
31
|
|
|
30
32
|
from .scancodedeps import ScancodeDeps
|
|
@@ -33,6 +35,14 @@ from .scanossgrpc import ScanossGrpc
|
|
|
33
35
|
|
|
34
36
|
DEP_FILE_PREFIX = "file=" # Default prefix to signify an existing parsed dependency file
|
|
35
37
|
|
|
38
|
+
DEV_DEPENDENCIES = { "dev", "test", "development", "provided", "runtime", "devDependencies", "dev-dependencies", "testImplementation", "testCompile", "Test", "require-dev" }
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
# Define an enum class
|
|
42
|
+
class SCOPE(Enum):
|
|
43
|
+
PRODUCTION = 'prod'
|
|
44
|
+
DEVELOPMENT = 'dev'
|
|
45
|
+
|
|
36
46
|
|
|
37
47
|
@dataclass
|
|
38
48
|
class ThreadedDependencies(ScanossBase):
|
|
@@ -66,9 +76,13 @@ class ThreadedDependencies(ScanossBase):
|
|
|
66
76
|
return resp
|
|
67
77
|
return None
|
|
68
78
|
|
|
69
|
-
def run(self, what_to_scan: str = None, deps_file: str = None, wait: bool = True
|
|
79
|
+
def run(self, what_to_scan: str = None, deps_file: str = None, wait: bool = True, dep_scope: SCOPE = None,
|
|
80
|
+
dep_scope_include: str = None, dep_scope_exclude: str = None) -> bool:
|
|
70
81
|
"""
|
|
71
82
|
Initiate a background scan for the specified file/dir
|
|
83
|
+
:param dep_scope_exclude: comma separated list of dependency scopes to exclude
|
|
84
|
+
:param dep_scope_include: comma separated list of dependency scopes to include
|
|
85
|
+
:param dep_scope: Enum dependency scope to use
|
|
72
86
|
:param what_to_scan: file/folder to scan
|
|
73
87
|
:param deps_file: file to decorate instead of scan (overrides what_to_scan option)
|
|
74
88
|
:param wait: wait for completion
|
|
@@ -82,8 +96,9 @@ class ThreadedDependencies(ScanossBase):
|
|
|
82
96
|
self.inputs.put(f'{DEP_FILE_PREFIX}{deps_file}') # Add to queue and have parent wait on it
|
|
83
97
|
else: # Search for dependencies to decorate
|
|
84
98
|
self.print_msg(f'Searching {what_to_scan} for dependencies...')
|
|
85
|
-
self.inputs.put(what_to_scan)
|
|
86
|
-
|
|
99
|
+
self.inputs.put(what_to_scan)
|
|
100
|
+
# Add to queue and have parent wait on it
|
|
101
|
+
self._thread = threading.Thread(target=self.scan_dependencies(dep_scope, dep_scope_include, dep_scope_exclude), daemon=True)
|
|
87
102
|
self._thread.start()
|
|
88
103
|
except Exception as e:
|
|
89
104
|
self.print_stderr(f'ERROR: Problem running threaded dependencies: {e}')
|
|
@@ -92,7 +107,54 @@ class ThreadedDependencies(ScanossBase):
|
|
|
92
107
|
self.complete()
|
|
93
108
|
return False if self._errors else True
|
|
94
109
|
|
|
95
|
-
def
|
|
110
|
+
def filter_dependencies(self,deps ,filter_dep)-> json:
|
|
111
|
+
files = deps.get('files', [])
|
|
112
|
+
# Iterate over files and their purls
|
|
113
|
+
for file in files:
|
|
114
|
+
if 'purls' in file:
|
|
115
|
+
# Filter purls with scope 'dependencies' and remove the scope field
|
|
116
|
+
file['purls'] = [
|
|
117
|
+
{key: value for key, value in purl.items() if key != 'scope'}
|
|
118
|
+
for purl in file['purls']
|
|
119
|
+
if filter_dep(purl.get('scope'))
|
|
120
|
+
]
|
|
121
|
+
# End of for loop
|
|
122
|
+
|
|
123
|
+
return {
|
|
124
|
+
'files': [
|
|
125
|
+
file for file in deps.get('files', [])
|
|
126
|
+
if file.get('purls')
|
|
127
|
+
]
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
def filter_dependencies_by_scopes(self,deps: json, dep_scope: SCOPE = None, dep_scope_include: str = None,
|
|
131
|
+
dep_scope_exclude: str = None) -> json:
|
|
132
|
+
# Predefined set of scopes to filter
|
|
133
|
+
|
|
134
|
+
# Include all scopes
|
|
135
|
+
include_all = (dep_scope is None or dep_scope == "") and dep_scope_include is None and dep_scope_exclude is None
|
|
136
|
+
## All dependencies, remove scope key
|
|
137
|
+
if include_all:
|
|
138
|
+
return self.filter_dependencies(deps, lambda purl:True)
|
|
139
|
+
|
|
140
|
+
# Use default list of scopes if a custom list is not set
|
|
141
|
+
if (dep_scope is not None and dep_scope != "") and dep_scope_include is None and dep_scope_exclude is None:
|
|
142
|
+
return self.filter_dependencies(deps, lambda purl: (dep_scope == SCOPE.PRODUCTION and purl not in DEV_DEPENDENCIES) or
|
|
143
|
+
dep_scope == SCOPE.DEVELOPMENT and purl in DEV_DEPENDENCIES)
|
|
144
|
+
|
|
145
|
+
if ((dep_scope_include is not None and dep_scope_include != "")
|
|
146
|
+
or dep_scope_exclude is not None and dep_scope_exclude != ""):
|
|
147
|
+
# Create sets from comma-separated strings, if provided
|
|
148
|
+
exclude = set(dep_scope_exclude.split(',')) if dep_scope_exclude else set()
|
|
149
|
+
include = set(dep_scope_include.split(',')) if dep_scope_include else set()
|
|
150
|
+
|
|
151
|
+
# Define a lambda function that checks the inclusion/exclusion logic
|
|
152
|
+
return self.filter_dependencies(
|
|
153
|
+
deps,
|
|
154
|
+
lambda purl: (exclude and purl not in exclude) or (not exclude and purl in include)
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
def scan_dependencies(self, dep_scope: SCOPE = None, dep_scope_include: str = None, dep_scope_exclude: str = None) -> None:
|
|
96
158
|
"""
|
|
97
159
|
Scan for dependencies from the given file/dir or from an input file (from the input queue).
|
|
98
160
|
"""
|
|
@@ -108,6 +170,14 @@ class ThreadedDependencies(ScanossBase):
|
|
|
108
170
|
self._errors = True
|
|
109
171
|
else:
|
|
110
172
|
deps = self.sc_deps.produce_from_file()
|
|
173
|
+
if dep_scope is not None:
|
|
174
|
+
self.print_debug(f'Filtering {dep_scope.name} dependencies')
|
|
175
|
+
if dep_scope_include is not None:
|
|
176
|
+
self.print_debug(f"Including dependencies with '{dep_scope_include.split(',')}' scopes")
|
|
177
|
+
if dep_scope_exclude is not None:
|
|
178
|
+
self.print_debug(f"Excluding dependencies with '{dep_scope_exclude.split(',')}' scopes")
|
|
179
|
+
deps = self.filter_dependencies_by_scopes(deps, dep_scope,dep_scope_include, dep_scope_exclude)
|
|
180
|
+
|
|
111
181
|
if not self._errors:
|
|
112
182
|
if deps is None:
|
|
113
183
|
self.print_stderr(f'Problem searching for dependencies for: {what_to_scan}')
|
|
@@ -4,20 +4,23 @@ protoc_gen_swagger/options/annotations_pb2.py,sha256=b25EDD6gssUWnFby9gxgcpLIROT
|
|
|
4
4
|
protoc_gen_swagger/options/annotations_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
|
|
5
5
|
protoc_gen_swagger/options/openapiv2_pb2.py,sha256=vYElGp8E1vGHszvWqX97zNG9GFJ7u2QcdK9ouq0XdyI,14939
|
|
6
6
|
protoc_gen_swagger/options/openapiv2_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
|
|
7
|
-
scanoss/__init__.py,sha256=
|
|
8
|
-
scanoss/cli.py,sha256=
|
|
7
|
+
scanoss/__init__.py,sha256=0h0EuhMJeA8cpYqOnCxwfM-gtnXaB0BFmm8FwkEwnKI,1163
|
|
8
|
+
scanoss/cli.py,sha256=k7RJVh36CwxUwDqOVHkXdArWp4QpzJVCSQ0haJWenKA,46686
|
|
9
9
|
scanoss/components.py,sha256=ZHZ1KA69shxOASZK7USD9yPTITpAc_RXL5q5zpDK23o,12590
|
|
10
10
|
scanoss/csvoutput.py,sha256=hBwr_Fc6mBdOdXgyQcdFrockYH-PJ0jblowlExJ6OPg,9925
|
|
11
|
-
scanoss/cyclonedx.py,sha256=
|
|
11
|
+
scanoss/cyclonedx.py,sha256=JVBYeR3D-i4yP9cVSyWvm0_7Y8Kr2MC5GxMgRGAf8R0,12585
|
|
12
12
|
scanoss/filecount.py,sha256=o7xb6m387ucnsU4H1OXGzf_AdWsudhAHe49T8uX4Ieo,6660
|
|
13
|
-
scanoss/
|
|
14
|
-
scanoss/
|
|
15
|
-
scanoss/
|
|
16
|
-
scanoss/
|
|
13
|
+
scanoss/results.py,sha256=8AJgXeZRc7Ugf4iM_NvFD8zPD2Y4NLePwNzZKUmmFi4,9885
|
|
14
|
+
scanoss/scancodedeps.py,sha256=_9d7MAV20-FrET7mF7gW-BZiz2eHrtwudgrEcSX0oZQ,11321
|
|
15
|
+
scanoss/scanner.py,sha256=Boxk0A-AuS0DMB4UYArU0PWZ0yJlK4v1YgdeVnKmJck,52023
|
|
16
|
+
scanoss/scanoss_settings.py,sha256=NpNZ2aCpRG2EqfJc9_BK6SnODqkOwVBEq3u-9s0KxPI,5986
|
|
17
|
+
scanoss/scanossapi.py,sha256=TJxPctr-0DTn_26LfM__OAMfntaXzvheFTbdmU-5pnM,11953
|
|
18
|
+
scanoss/scanossbase.py,sha256=ucG85doysZT3KLgApg2CiVm3-YLcdBQ2HmkZ9YnJSxA,2806
|
|
17
19
|
scanoss/scanossgrpc.py,sha256=ythZkr6F0P0hl_KPYoHkos_IL97TxLKeYfAouX_CUnM,20491
|
|
20
|
+
scanoss/scanpostprocessor.py,sha256=tfQk6GBmW1Yd2rqHHp6QKiYVdmTkBAcpoE4HHN__oKo,5899
|
|
18
21
|
scanoss/scantype.py,sha256=R2-ExLGOrYxaJFtIK2AEo2caD0XrN1zpF5q1qT9Zsyc,1326
|
|
19
22
|
scanoss/spdxlite.py,sha256=IsWP9o1D8ryT1_5LeobIEhWJXNFbffoWCy1yaeZY2X0,15638
|
|
20
|
-
scanoss/threadeddependencies.py,sha256=
|
|
23
|
+
scanoss/threadeddependencies.py,sha256=sOIAjiPTmxybKz2yhT4-ixXBeC4K8UQVq6JQj4e8mLc,9906
|
|
21
24
|
scanoss/threadedscanning.py,sha256=T0tL8W1IEX_hLY5ksrAl_iQqtxT_KbyDhTDHo6a7xFE,9387
|
|
22
25
|
scanoss/winnowing.py,sha256=HzMWRYh1XB4so71br-DUPpV6OlmymDfsnU-EOCCObJM,18734
|
|
23
26
|
scanoss/api/__init__.py,sha256=KlDD87JmyZP-10T-fuJo0_v2zt1gxWfTgs70wjky9xg,1139
|
|
@@ -47,12 +50,12 @@ scanoss/api/vulnerabilities/__init__.py,sha256=FLQtiDiv85Q1Chk-sJ9ky9WOV1mulZhEK
|
|
|
47
50
|
scanoss/api/vulnerabilities/v2/__init__.py,sha256=FLQtiDiv85Q1Chk-sJ9ky9WOV1mulZhEKjiBihlwiaM,1139
|
|
48
51
|
scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py,sha256=CFhF80av8tenGvn9AIsGEtRJPuV2dC_syA5JLZb2lDw,5464
|
|
49
52
|
scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py,sha256=HlS4k4Zmx6RIAqaO9I96jD-eyF5yU6Xx04pVm7pdqOg,6864
|
|
50
|
-
scanoss/data/build_date.txt,sha256=
|
|
53
|
+
scanoss/data/build_date.txt,sha256=Y087LKNPYOukJKz1qnrj5bG17xeBavkMK7KMr_uNZZk,40
|
|
51
54
|
scanoss/data/spdx-exceptions.json,sha256=s7UTYxC7jqQXr11YBlIWYCNwN6lRDFTR33Y8rpN_dA4,17953
|
|
52
55
|
scanoss/data/spdx-licenses.json,sha256=A6Z0q82gaTLtnopBfzeIVZjJFxkdRW1g2TuumQc-lII,228794
|
|
53
|
-
scanoss-1.
|
|
54
|
-
scanoss-1.
|
|
55
|
-
scanoss-1.
|
|
56
|
-
scanoss-1.
|
|
57
|
-
scanoss-1.
|
|
58
|
-
scanoss-1.
|
|
56
|
+
scanoss-1.16.0.dist-info/LICENSE,sha256=LLUaXoiyOroIbr5ubAyrxBOwSRLTm35ETO2FmLpy8QQ,1074
|
|
57
|
+
scanoss-1.16.0.dist-info/METADATA,sha256=b9mSpB5aYU5kLfcWSjYwiAwAd1uBGI1Stw30NU0rKsY,5936
|
|
58
|
+
scanoss-1.16.0.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
|
|
59
|
+
scanoss-1.16.0.dist-info/entry_points.txt,sha256=Uy28xnaDL5KQ7V77sZD5VLDXPNxYYzSr5tsqtiXVzAs,48
|
|
60
|
+
scanoss-1.16.0.dist-info/top_level.txt,sha256=V11PrQ6Pnrc-nDF9xnisnJ8e6-i7HqSIKVNqduRWcL8,27
|
|
61
|
+
scanoss-1.16.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|