scanoss 1.20.0__py3-none-any.whl → 1.20.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- protoc_gen_swagger/__init__.py +13 -13
- protoc_gen_swagger/options/__init__.py +13 -13
- protoc_gen_swagger/options/annotations_pb2.py +12 -9
- protoc_gen_swagger/options/annotations_pb2_grpc.py +1 -1
- protoc_gen_swagger/options/openapiv2_pb2.py +98 -96
- protoc_gen_swagger/options/openapiv2_pb2_grpc.py +1 -1
- scanoss/__init__.py +18 -18
- scanoss/api/__init__.py +17 -17
- scanoss/api/common/__init__.py +17 -17
- scanoss/api/common/v2/__init__.py +17 -17
- scanoss/api/common/v2/scanoss_common_pb2.py +18 -18
- scanoss/api/common/v2/scanoss_common_pb2_grpc.py +1 -1
- scanoss/api/components/__init__.py +17 -17
- scanoss/api/components/v2/__init__.py +17 -17
- scanoss/api/components/v2/scanoss_components_pb2.py +48 -38
- scanoss/api/components/v2/scanoss_components_pb2_grpc.py +142 -96
- scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py +22 -16
- scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py +75 -49
- scanoss/api/dependencies/__init__.py +17 -17
- scanoss/api/dependencies/v2/__init__.py +17 -17
- scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +30 -24
- scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py +75 -49
- scanoss/api/scanning/__init__.py +17 -17
- scanoss/api/scanning/v2/__init__.py +17 -17
- scanoss/api/scanning/v2/scanoss_scanning_pb2.py +10 -8
- scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +40 -32
- scanoss/api/semgrep/__init__.py +17 -17
- scanoss/api/semgrep/v2/__init__.py +17 -17
- scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py +22 -18
- scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py +71 -49
- scanoss/api/vulnerabilities/__init__.py +17 -17
- scanoss/api/vulnerabilities/v2/__init__.py +17 -17
- scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +37 -27
- scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py +109 -72
- scanoss/cli.py +576 -293
- scanoss/components.py +67 -45
- scanoss/csvoutput.py +83 -56
- scanoss/cyclonedx.py +48 -46
- scanoss/data/build_date.txt +1 -1
- scanoss/file_filters.py +13 -15
- scanoss/filecount.py +43 -36
- scanoss/inspection/__init__.py +17 -17
- scanoss/inspection/copyleft.py +71 -58
- scanoss/inspection/policy_check.py +76 -53
- scanoss/inspection/undeclared_component.py +98 -75
- scanoss/inspection/utils/license_utils.py +66 -44
- scanoss/results.py +51 -60
- scanoss/scancodedeps.py +61 -38
- scanoss/scanner.py +203 -135
- scanoss/scanoss_settings.py +5 -3
- scanoss/scanossapi.py +98 -69
- scanoss/scanossbase.py +19 -19
- scanoss/scanossgrpc.py +73 -51
- scanoss/scanpostprocessor.py +9 -6
- scanoss/scantype.py +22 -21
- scanoss/spdxlite.py +265 -171
- scanoss/threadeddependencies.py +91 -61
- scanoss/threadedscanning.py +37 -31
- scanoss/utils/file.py +4 -4
- scanoss/winnowing.py +111 -47
- {scanoss-1.20.0.dist-info → scanoss-1.20.1.dist-info}/METADATA +1 -1
- scanoss-1.20.1.dist-info/RECORD +74 -0
- scanoss-1.20.0.dist-info/RECORD +0 -74
- {scanoss-1.20.0.dist-info → scanoss-1.20.1.dist-info}/LICENSE +0 -0
- {scanoss-1.20.0.dist-info → scanoss-1.20.1.dist-info}/WHEEL +0 -0
- {scanoss-1.20.0.dist-info → scanoss-1.20.1.dist-info}/entry_points.txt +0 -0
- {scanoss-1.20.0.dist-info → scanoss-1.20.1.dist-info}/top_level.txt +0 -0
scanoss/scanoss_settings.py
CHANGED
|
@@ -119,7 +119,9 @@ class ScanossSettings(ScanossBase):
|
|
|
119
119
|
result = validate_json_file(json_file)
|
|
120
120
|
if not result.is_valid:
|
|
121
121
|
if result.error_code == JSON_ERROR_FILE_NOT_FOUND or result.error_code == JSON_ERROR_FILE_EMPTY:
|
|
122
|
-
self.print_msg(
|
|
122
|
+
self.print_msg(
|
|
123
|
+
f'WARNING: The supplied settings file "{filepath}" was not found or is empty. Skipping...'
|
|
124
|
+
)
|
|
123
125
|
return self
|
|
124
126
|
else:
|
|
125
127
|
raise ScanossSettingsError(f'Problem with settings file. {result.error}')
|
|
@@ -234,8 +236,8 @@ class ScanossSettings(ScanossBase):
|
|
|
234
236
|
replace_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_replace()))
|
|
235
237
|
self.print_debug(
|
|
236
238
|
f"Scan type set to 'identify'. Adding {len(include_bom_entries) + len(replace_bom_entries)} components as context to the scan. \n"
|
|
237
|
-
f
|
|
238
|
-
f
|
|
239
|
+
f'From Include list: {[entry["purl"] for entry in include_bom_entries]} \n'
|
|
240
|
+
f'From Replace list: {[entry["purl"] for entry in replace_bom_entries]} \n'
|
|
239
241
|
)
|
|
240
242
|
return include_bom_entries + replace_bom_entries
|
|
241
243
|
return self.normalize_bom_entries(self.get_bom_remove())
|
scanoss/scanossapi.py
CHANGED
|
@@ -1,26 +1,27 @@
|
|
|
1
1
|
"""
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
2
|
+
SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
Copyright (c) 2022, SCANOSS
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in
|
|
14
|
+
all copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
22
|
+
THE SOFTWARE.
|
|
23
23
|
"""
|
|
24
|
+
|
|
24
25
|
import logging
|
|
25
26
|
import os
|
|
26
27
|
import sys
|
|
@@ -39,10 +40,10 @@ from .scanossbase import ScanossBase
|
|
|
39
40
|
from . import __version__
|
|
40
41
|
|
|
41
42
|
|
|
42
|
-
DEFAULT_URL =
|
|
43
|
-
DEFAULT_URL2 =
|
|
44
|
-
SCANOSS_SCAN_URL = os.environ.get(
|
|
45
|
-
SCANOSS_API_KEY = os.environ.get(
|
|
43
|
+
DEFAULT_URL = 'https://api.osskb.org/scan/direct' # default free service URL
|
|
44
|
+
DEFAULT_URL2 = 'https://api.scanoss.com/scan/direct' # default premium service URL
|
|
45
|
+
SCANOSS_SCAN_URL = os.environ.get('SCANOSS_SCAN_URL') if os.environ.get('SCANOSS_SCAN_URL') else DEFAULT_URL
|
|
46
|
+
SCANOSS_API_KEY = os.environ.get('SCANOSS_API_KEY') if os.environ.get('SCANOSS_API_KEY') else ''
|
|
46
47
|
|
|
47
48
|
|
|
48
49
|
class ScanossApi(ScanossBase):
|
|
@@ -51,10 +52,23 @@ class ScanossApi(ScanossBase):
|
|
|
51
52
|
Currently support posting scan requests to the SCANOSS streaming API
|
|
52
53
|
"""
|
|
53
54
|
|
|
54
|
-
def __init__(
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
55
|
+
def __init__(
|
|
56
|
+
self,
|
|
57
|
+
scan_format: str = None,
|
|
58
|
+
flags: str = None,
|
|
59
|
+
url: str = None,
|
|
60
|
+
api_key: str = None,
|
|
61
|
+
debug: bool = False,
|
|
62
|
+
trace: bool = False,
|
|
63
|
+
quiet: bool = False,
|
|
64
|
+
timeout: int = 180,
|
|
65
|
+
ver_details: str = None,
|
|
66
|
+
ignore_cert_errors: bool = False,
|
|
67
|
+
proxy: str = None,
|
|
68
|
+
ca_cert: str = None,
|
|
69
|
+
pac: PACFile = None,
|
|
70
|
+
retry: int = 5,
|
|
71
|
+
):
|
|
58
72
|
"""
|
|
59
73
|
Initialise the SCANOSS API
|
|
60
74
|
:param scan_format: Scan format (default plain)
|
|
@@ -74,7 +88,7 @@ class ScanossApi(ScanossBase):
|
|
|
74
88
|
super().__init__(debug, trace, quiet)
|
|
75
89
|
self.url = url if url else SCANOSS_SCAN_URL
|
|
76
90
|
self.api_key = api_key if api_key else SCANOSS_API_KEY
|
|
77
|
-
if self.api_key and not url and not os.environ.get(
|
|
91
|
+
if self.api_key and not url and not os.environ.get('SCANOSS_SCAN_URL'):
|
|
78
92
|
self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium
|
|
79
93
|
self.sbom = None
|
|
80
94
|
self.scan_format = scan_format if scan_format else 'plain'
|
|
@@ -108,7 +122,7 @@ class ScanossApi(ScanossBase):
|
|
|
108
122
|
self.verify = ca_cert
|
|
109
123
|
self.session.verify = ca_cert
|
|
110
124
|
self.proxies = {'https': proxy, 'http': proxy} if proxy else None
|
|
111
|
-
if self.
|
|
125
|
+
if self.proxies:
|
|
112
126
|
self.session.proxies = self.proxies
|
|
113
127
|
|
|
114
128
|
def scan(self, wfp: str, context: str = None, scan_id: int = None):
|
|
@@ -122,16 +136,16 @@ class ScanossApi(ScanossBase):
|
|
|
122
136
|
request_id = str(uuid.uuid4())
|
|
123
137
|
form_data = {}
|
|
124
138
|
if self.sbom:
|
|
125
|
-
form_data['type'] = self.sbom.get(
|
|
126
|
-
form_data['assets'] = self.sbom.get(
|
|
139
|
+
form_data['type'] = self.sbom.get('scan_type')
|
|
140
|
+
form_data['assets'] = self.sbom.get('assets')
|
|
127
141
|
if self.scan_format:
|
|
128
142
|
form_data['format'] = self.scan_format
|
|
129
143
|
if self.flags:
|
|
130
144
|
form_data['flags'] = self.flags
|
|
131
145
|
if context:
|
|
132
146
|
form_data['context'] = context
|
|
133
|
-
|
|
134
|
-
scan_files = {'file': (
|
|
147
|
+
|
|
148
|
+
scan_files = {'file': ('%s.wfp' % request_id, wfp)}
|
|
135
149
|
headers = self.headers
|
|
136
150
|
headers['x-request-id'] = request_id # send a unique request id for each post
|
|
137
151
|
r = None
|
|
@@ -140,75 +154,87 @@ class ScanossApi(ScanossBase):
|
|
|
140
154
|
retry += 1
|
|
141
155
|
try:
|
|
142
156
|
r = None
|
|
143
|
-
r = self.session.post(
|
|
144
|
-
|
|
145
|
-
|
|
157
|
+
r = self.session.post(
|
|
158
|
+
self.url, files=scan_files, data=form_data, headers=self.headers, timeout=self.timeout
|
|
159
|
+
)
|
|
146
160
|
except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as e:
|
|
147
161
|
self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}.')
|
|
148
|
-
raise Exception(f
|
|
162
|
+
raise Exception(f'ERROR: The SCANOSS API request failed for {self.url}') from e
|
|
149
163
|
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
|
|
150
164
|
if retry > self.retry_limit: # Timed out retry_limit or more times, fail
|
|
151
165
|
self.print_stderr(f'ERROR: {e.__class__.__name__} POSTing data ({request_id}) - {e}: {scan_files}')
|
|
152
|
-
raise Exception(
|
|
153
|
-
|
|
166
|
+
raise Exception(
|
|
167
|
+
f'ERROR: The SCANOSS API request timed out ({e.__class__.__name__}) for {self.url}'
|
|
168
|
+
) from e
|
|
154
169
|
else:
|
|
155
170
|
self.print_stderr(f'Warning: {e.__class__.__name__} communicating with {self.url}. Retrying...')
|
|
156
171
|
time.sleep(5)
|
|
157
172
|
except Exception as e:
|
|
158
|
-
self.print_stderr(
|
|
159
|
-
|
|
160
|
-
|
|
173
|
+
self.print_stderr(
|
|
174
|
+
f'ERROR: Exception ({e.__class__.__name__}) POSTing data ({request_id}) - {e}: {scan_files}'
|
|
175
|
+
)
|
|
176
|
+
raise Exception(f'ERROR: The SCANOSS API request failed for {self.url}') from e
|
|
161
177
|
else:
|
|
162
178
|
if r is None:
|
|
163
179
|
if retry > self.retry_limit: # No response retry_limit or more times, fail
|
|
164
180
|
self.save_bad_req_wfp(scan_files, request_id, scan_id)
|
|
165
|
-
raise Exception(
|
|
166
|
-
|
|
181
|
+
raise Exception(
|
|
182
|
+
f'ERROR: The SCANOSS API request ({request_id}) response object is empty for {self.url}'
|
|
183
|
+
)
|
|
167
184
|
else:
|
|
168
185
|
self.print_stderr(f'Warning: No response received from {self.url}. Retrying...')
|
|
169
186
|
time.sleep(5)
|
|
170
187
|
elif r.status_code == 503: # Service limits have most likely been reached
|
|
171
|
-
self.print_stderr(
|
|
172
|
-
|
|
188
|
+
self.print_stderr(
|
|
189
|
+
f'ERROR: SCANOSS API rejected the scan request ({request_id}) due to '
|
|
190
|
+
f'service limits being exceeded'
|
|
191
|
+
)
|
|
173
192
|
self.print_stderr(f'ERROR: Details: {r.text.strip()}')
|
|
174
|
-
raise Exception(
|
|
175
|
-
|
|
193
|
+
raise Exception(
|
|
194
|
+
f'ERROR: {r.status_code} - The SCANOSS API request ({request_id}) rejected '
|
|
195
|
+
f'for {self.url} due to service limits being exceeded.'
|
|
196
|
+
)
|
|
176
197
|
elif r.status_code >= 400:
|
|
177
198
|
if retry > self.retry_limit: # No response retry_limit or more times, fail
|
|
178
199
|
self.save_bad_req_wfp(scan_files, request_id, scan_id)
|
|
179
200
|
raise Exception(
|
|
180
|
-
f
|
|
181
|
-
f
|
|
201
|
+
f'ERROR: The SCANOSS API returned the following error: HTTP {r.status_code}, '
|
|
202
|
+
f'{r.text.strip()}'
|
|
203
|
+
)
|
|
182
204
|
else:
|
|
183
205
|
self.save_bad_req_wfp(scan_files, request_id, scan_id)
|
|
184
|
-
self.print_stderr(
|
|
185
|
-
|
|
206
|
+
self.print_stderr(
|
|
207
|
+
f'Warning: Error response code {r.status_code} ({r.text.strip()}) from '
|
|
208
|
+
f'{self.url}. Retrying...'
|
|
209
|
+
)
|
|
186
210
|
time.sleep(5)
|
|
187
211
|
else:
|
|
188
212
|
break # Valid response, break out of the retry loop
|
|
189
213
|
# End of while loop
|
|
190
214
|
if r is None:
|
|
191
215
|
self.save_bad_req_wfp(scan_files, request_id, scan_id)
|
|
192
|
-
raise Exception(f
|
|
216
|
+
raise Exception(f'ERROR: The SCANOSS API request response object is empty for {self.url}')
|
|
193
217
|
try:
|
|
194
218
|
if 'xml' in self.scan_format: # TODO remove XML parsing option?
|
|
195
219
|
return r.text
|
|
196
220
|
json_resp = r.json()
|
|
197
221
|
return json_resp
|
|
198
222
|
except (JSONDecodeError, Exception) as e:
|
|
199
|
-
self.print_stderr(
|
|
200
|
-
|
|
223
|
+
self.print_stderr(
|
|
224
|
+
f'ERROR: The SCANOSS API returned an invalid JSON ({e.__class__.__name__} - {request_id}): {e}'
|
|
225
|
+
)
|
|
201
226
|
bad_json_file = f'bad_json-{scan_id}-{request_id}.txt' if scan_id else f'bad_json-{request_id}.txt'
|
|
202
227
|
self.print_stderr(f'Ignoring result. Please look in "{bad_json_file}" for more details.')
|
|
203
228
|
try:
|
|
204
229
|
with open(bad_json_file, 'w') as f:
|
|
205
|
-
f.write(f
|
|
206
|
-
f.write(f
|
|
230
|
+
f.write(f'---Request ID Begin---\n{request_id}\n---Request ID End---\n')
|
|
231
|
+
f.write(f'---WFP Begin---\n{scan_files}\n---WFP End---\n---Bad JSON Begin---\n')
|
|
207
232
|
f.write(r.text)
|
|
208
|
-
f.write(
|
|
233
|
+
f.write('---Bad JSON End---\n')
|
|
209
234
|
except Exception as ee:
|
|
210
|
-
self.print_stderr(
|
|
211
|
-
|
|
235
|
+
self.print_stderr(
|
|
236
|
+
f'Warning: Issue writing bad json file - {bad_json_file} ({ee.__class__.__name__}): {ee}'
|
|
237
|
+
)
|
|
212
238
|
return None
|
|
213
239
|
|
|
214
240
|
def save_bad_req_wfp(self, scan_files, request_id, scan_id):
|
|
@@ -220,19 +246,22 @@ class ScanossApi(ScanossBase):
|
|
|
220
246
|
"""
|
|
221
247
|
bad_req_file = f'bad_request-{scan_id}-{request_id}.txt' if scan_id else f'bad_request-{request_id}.txt'
|
|
222
248
|
try:
|
|
223
|
-
self.print_stderr(
|
|
224
|
-
|
|
249
|
+
self.print_stderr(
|
|
250
|
+
f'No response object returned from API. Please look in "{bad_req_file}" for the offending WFP.'
|
|
251
|
+
)
|
|
225
252
|
with open(bad_req_file, 'w') as f:
|
|
226
|
-
f.write(f
|
|
227
|
-
f.write(f
|
|
253
|
+
f.write(f'---Request ID Begin---\n{request_id}\n---Request ID End---\n')
|
|
254
|
+
f.write(f'---WFP Begin---\n{scan_files}\n---WFP End---\n')
|
|
228
255
|
except Exception as ee:
|
|
229
|
-
self.print_stderr(
|
|
230
|
-
|
|
231
|
-
|
|
256
|
+
self.print_stderr(
|
|
257
|
+
f'Warning: Issue writing bad request file - {bad_req_file} ({ee.__class__.__name__}): {ee}'
|
|
258
|
+
)
|
|
259
|
+
|
|
232
260
|
def set_sbom(self, sbom):
|
|
233
261
|
self.sbom = sbom
|
|
234
262
|
return self
|
|
235
263
|
|
|
264
|
+
|
|
236
265
|
#
|
|
237
266
|
# End of ScanossApi Class
|
|
238
267
|
#
|
scanoss/scanossbase.py
CHANGED
|
@@ -1,25 +1,25 @@
|
|
|
1
1
|
"""
|
|
2
|
-
|
|
2
|
+
SPDX-License-Identifier: MIT
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
Copyright (c) 2021, SCANOSS
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
The above copyright notice and this permission notice shall be included in
|
|
14
|
+
all copies or substantial portions of the Software.
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
22
|
+
THE SOFTWARE.
|
|
23
23
|
"""
|
|
24
24
|
|
|
25
25
|
import sys
|
|
@@ -85,7 +85,7 @@ class ScanossBase:
|
|
|
85
85
|
Print message to file if provided or stdout
|
|
86
86
|
"""
|
|
87
87
|
if file:
|
|
88
|
-
with open(file,
|
|
88
|
+
with open(file, 'w') as f:
|
|
89
89
|
f.write(msg)
|
|
90
90
|
else:
|
|
91
91
|
self.print_stdout(msg)
|
|
@@ -95,7 +95,7 @@ class ScanossBase:
|
|
|
95
95
|
Print message to file if provided or stderr
|
|
96
96
|
"""
|
|
97
97
|
if file:
|
|
98
|
-
with open(file,
|
|
98
|
+
with open(file, 'w') as f:
|
|
99
99
|
f.write(msg)
|
|
100
100
|
else:
|
|
101
101
|
self.print_stderr(msg)
|
scanoss/scanossgrpc.py
CHANGED
|
@@ -1,25 +1,25 @@
|
|
|
1
1
|
"""
|
|
2
|
-
|
|
2
|
+
SPDX-License-Identifier: MIT
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
Copyright (c) 2021, SCANOSS
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
The above copyright notice and this permission notice shall be included in
|
|
14
|
+
all copies or substantial portions of the Software.
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
22
|
+
THE SOFTWARE.
|
|
23
23
|
"""
|
|
24
24
|
|
|
25
25
|
import os
|
|
@@ -44,17 +44,20 @@ from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest, Dep
|
|
|
44
44
|
from .api.common.v2.scanoss_common_pb2 import EchoRequest, EchoResponse, StatusResponse, StatusCode, PurlRequest
|
|
45
45
|
from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import VulnerabilityResponse
|
|
46
46
|
from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse
|
|
47
|
-
from .api.components.v2.scanoss_components_pb2 import (
|
|
48
|
-
|
|
47
|
+
from .api.components.v2.scanoss_components_pb2 import (
|
|
48
|
+
CompSearchRequest,
|
|
49
|
+
CompSearchResponse,
|
|
50
|
+
CompVersionRequest,
|
|
51
|
+
CompVersionResponse,
|
|
52
|
+
)
|
|
49
53
|
from .api.provenance.v2.scanoss_provenance_pb2 import ProvenanceResponse
|
|
50
|
-
|
|
51
54
|
from .scanossbase import ScanossBase
|
|
52
55
|
from . import __version__
|
|
53
56
|
|
|
54
|
-
DEFAULT_URL =
|
|
55
|
-
DEFAULT_URL2 =
|
|
56
|
-
SCANOSS_GRPC_URL = os.environ.get(
|
|
57
|
-
SCANOSS_API_KEY = os.environ.get(
|
|
57
|
+
DEFAULT_URL = 'https://api.osskb.org' # default free service URL
|
|
58
|
+
DEFAULT_URL2 = 'https://api.scanoss.com' # default premium service URL
|
|
59
|
+
SCANOSS_GRPC_URL = os.environ.get('SCANOSS_GRPC_URL') if os.environ.get('SCANOSS_GRPC_URL') else DEFAULT_URL
|
|
60
|
+
SCANOSS_API_KEY = os.environ.get('SCANOSS_API_KEY') if os.environ.get('SCANOSS_API_KEY') else ''
|
|
58
61
|
|
|
59
62
|
|
|
60
63
|
class ScanossGrpc(ScanossBase):
|
|
@@ -62,9 +65,20 @@ class ScanossGrpc(ScanossBase):
|
|
|
62
65
|
Client for gRPC functionality
|
|
63
66
|
"""
|
|
64
67
|
|
|
65
|
-
def __init__(
|
|
66
|
-
|
|
67
|
-
|
|
68
|
+
def __init__(
|
|
69
|
+
self,
|
|
70
|
+
url: str = None,
|
|
71
|
+
debug: bool = False,
|
|
72
|
+
trace: bool = False,
|
|
73
|
+
quiet: bool = False,
|
|
74
|
+
ca_cert: str = None,
|
|
75
|
+
api_key: str = None,
|
|
76
|
+
ver_details: str = None,
|
|
77
|
+
timeout: int = 600,
|
|
78
|
+
proxy: str = None,
|
|
79
|
+
grpc_proxy: str = None,
|
|
80
|
+
pac: PACFile = None,
|
|
81
|
+
):
|
|
68
82
|
"""
|
|
69
83
|
|
|
70
84
|
:param url:
|
|
@@ -83,7 +97,7 @@ class ScanossGrpc(ScanossBase):
|
|
|
83
97
|
super().__init__(debug, trace, quiet)
|
|
84
98
|
self.url = url if url else SCANOSS_GRPC_URL
|
|
85
99
|
self.api_key = api_key if api_key else SCANOSS_API_KEY
|
|
86
|
-
if self.api_key and not url and not os.environ.get(
|
|
100
|
+
if self.api_key and not url and not os.environ.get('SCANOSS_GRPC_URL'):
|
|
87
101
|
self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium
|
|
88
102
|
self.url = self.url.lower()
|
|
89
103
|
self.orig_url = self.url # Used for proxy lookup
|
|
@@ -148,8 +162,9 @@ class ScanossGrpc(ScanossBase):
|
|
|
148
162
|
metadata.append(('x-request-id', request_id)) # Set a Request ID
|
|
149
163
|
resp = self.dependencies_stub.Echo(EchoRequest(message=message), metadata=metadata, timeout=3)
|
|
150
164
|
except Exception as e:
|
|
151
|
-
self.print_stderr(
|
|
152
|
-
|
|
165
|
+
self.print_stderr(
|
|
166
|
+
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
167
|
+
)
|
|
153
168
|
else:
|
|
154
169
|
# self.print_stderr(f'resp: {resp} - call: {call}')
|
|
155
170
|
# response_id = ""
|
|
@@ -181,8 +196,9 @@ class ScanossGrpc(ScanossBase):
|
|
|
181
196
|
metadata.append(('x-request-id', request_id)) # Set a Request ID
|
|
182
197
|
resp = self.crypto_stub.Echo(EchoRequest(message=message), metadata=metadata, timeout=3)
|
|
183
198
|
except Exception as e:
|
|
184
|
-
self.print_stderr(
|
|
185
|
-
|
|
199
|
+
self.print_stderr(
|
|
200
|
+
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
201
|
+
)
|
|
186
202
|
else:
|
|
187
203
|
if resp:
|
|
188
204
|
return resp.message
|
|
@@ -211,7 +227,7 @@ class ScanossGrpc(ScanossBase):
|
|
|
211
227
|
request_id = str(uuid.uuid4())
|
|
212
228
|
resp: DependencyResponse
|
|
213
229
|
try:
|
|
214
|
-
files_json = dependencies.get(
|
|
230
|
+
files_json = dependencies.get('files')
|
|
215
231
|
if files_json is None or len(files_json) == 0:
|
|
216
232
|
self.print_stderr(f'ERROR: No dependency data supplied to send to gRPC service.')
|
|
217
233
|
return None
|
|
@@ -222,8 +238,9 @@ class ScanossGrpc(ScanossBase):
|
|
|
222
238
|
self.print_debug(f'Sending dependency data for decoration (rqId: {request_id})...')
|
|
223
239
|
resp = self.dependencies_stub.GetDependencies(request, metadata=metadata, timeout=self.timeout)
|
|
224
240
|
except Exception as e:
|
|
225
|
-
self.print_stderr(
|
|
226
|
-
|
|
241
|
+
self.print_stderr(
|
|
242
|
+
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
243
|
+
)
|
|
227
244
|
else:
|
|
228
245
|
if resp:
|
|
229
246
|
if not self._check_status_response(resp.status, request_id):
|
|
@@ -249,8 +266,9 @@ class ScanossGrpc(ScanossBase):
|
|
|
249
266
|
self.print_debug(f'Sending crypto data for decoration (rqId: {request_id})...')
|
|
250
267
|
resp = self.crypto_stub.GetAlgorithms(request, metadata=metadata, timeout=self.timeout)
|
|
251
268
|
except Exception as e:
|
|
252
|
-
self.print_stderr(
|
|
253
|
-
|
|
269
|
+
self.print_stderr(
|
|
270
|
+
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
271
|
+
)
|
|
254
272
|
else:
|
|
255
273
|
if resp:
|
|
256
274
|
if not self._check_status_response(resp.status, request_id):
|
|
@@ -278,8 +296,9 @@ class ScanossGrpc(ScanossBase):
|
|
|
278
296
|
self.print_debug(f'Sending crypto data for decoration (rqId: {request_id})...')
|
|
279
297
|
resp = self.vuln_stub.GetVulnerabilities(request, metadata=metadata, timeout=self.timeout)
|
|
280
298
|
except Exception as e:
|
|
281
|
-
self.print_stderr(
|
|
282
|
-
|
|
299
|
+
self.print_stderr(
|
|
300
|
+
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
301
|
+
)
|
|
283
302
|
else:
|
|
284
303
|
if resp:
|
|
285
304
|
if not self._check_status_response(resp.status, request_id):
|
|
@@ -307,8 +326,9 @@ class ScanossGrpc(ScanossBase):
|
|
|
307
326
|
self.print_debug(f'Sending semgrep data for decoration (rqId: {request_id})...')
|
|
308
327
|
resp = self.semgrep_stub.GetIssues(request, metadata=metadata, timeout=self.timeout)
|
|
309
328
|
except Exception as e:
|
|
310
|
-
self.print_stderr(
|
|
311
|
-
|
|
329
|
+
self.print_stderr(
|
|
330
|
+
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
331
|
+
)
|
|
312
332
|
else:
|
|
313
333
|
if resp:
|
|
314
334
|
if not self._check_status_response(resp.status, request_id):
|
|
@@ -336,8 +356,9 @@ class ScanossGrpc(ScanossBase):
|
|
|
336
356
|
self.print_debug(f'Sending component search data (rqId: {request_id})...')
|
|
337
357
|
resp = self.comp_search_stub.SearchComponents(request, metadata=metadata, timeout=self.timeout)
|
|
338
358
|
except Exception as e:
|
|
339
|
-
self.print_stderr(
|
|
340
|
-
|
|
359
|
+
self.print_stderr(
|
|
360
|
+
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
361
|
+
)
|
|
341
362
|
else:
|
|
342
363
|
if resp:
|
|
343
364
|
if not self._check_status_response(resp.status, request_id):
|
|
@@ -365,8 +386,9 @@ class ScanossGrpc(ScanossBase):
|
|
|
365
386
|
self.print_debug(f'Sending component version data (rqId: {request_id})...')
|
|
366
387
|
resp = self.comp_search_stub.GetComponentVersions(request, metadata=metadata, timeout=self.timeout)
|
|
367
388
|
except Exception as e:
|
|
368
|
-
self.print_stderr(
|
|
369
|
-
|
|
389
|
+
self.print_stderr(
|
|
390
|
+
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
391
|
+
)
|
|
370
392
|
else:
|
|
371
393
|
if resp:
|
|
372
394
|
if not self._check_status_response(resp.status, request_id):
|
|
@@ -394,7 +416,7 @@ class ScanossGrpc(ScanossBase):
|
|
|
394
416
|
msg = "Succeeded with warnings"
|
|
395
417
|
ret_val = True # No need to fail as it succeeded with warnings
|
|
396
418
|
elif status_code == 3:
|
|
397
|
-
msg =
|
|
419
|
+
msg = 'Failed with warnings'
|
|
398
420
|
self.print_stderr(f'{msg} (rqId: {request_id} - status: {status_code}): {status_response.message}')
|
|
399
421
|
return ret_val
|
|
400
422
|
return True
|
|
@@ -407,19 +429,19 @@ class ScanossGrpc(ScanossBase):
|
|
|
407
429
|
"""
|
|
408
430
|
if self.grpc_proxy:
|
|
409
431
|
self.print_debug(f'Setting GRPC (grpc_proxy) proxy...')
|
|
410
|
-
os.environ[
|
|
432
|
+
os.environ['grpc_proxy'] = self.grpc_proxy
|
|
411
433
|
elif self.proxy:
|
|
412
434
|
self.print_debug(f'Setting GRPC (http_proxy/https_proxy) proxies...')
|
|
413
|
-
os.environ[
|
|
414
|
-
os.environ[
|
|
435
|
+
os.environ['http_proxy'] = self.proxy
|
|
436
|
+
os.environ['https_proxy'] = self.proxy
|
|
415
437
|
elif self.pac:
|
|
416
438
|
self.print_debug(f'Attempting to get GRPC proxy details from PAC for {self.orig_url}...')
|
|
417
439
|
resolver = ProxyResolver(self.pac)
|
|
418
440
|
proxies = resolver.get_proxy_for_requests(self.orig_url)
|
|
419
441
|
if proxies:
|
|
420
442
|
self.print_trace(f'Setting proxies: {proxies}')
|
|
421
|
-
os.environ[
|
|
422
|
-
os.environ[
|
|
443
|
+
os.environ['http_proxy'] = proxies.get('http') or ''
|
|
444
|
+
os.environ['https_proxy'] = proxies.get('https') or ''
|
|
423
445
|
|
|
424
446
|
def get_provenance_json(self, purls: dict) -> dict:
|
|
425
447
|
"""
|
scanoss/scanpostprocessor.py
CHANGED
|
@@ -207,8 +207,9 @@ class ScanPostProcessor(ScanossBase):
|
|
|
207
207
|
|
|
208
208
|
return result
|
|
209
209
|
|
|
210
|
-
def _should_replace_result(
|
|
211
|
-
|
|
210
|
+
def _should_replace_result(
|
|
211
|
+
self, result_path: str, result: dict, to_replace_entries: List[BomEntry]
|
|
212
|
+
) -> Tuple[bool, str]:
|
|
212
213
|
"""
|
|
213
214
|
Check if a result should be replaced based on the SCANOSS settings
|
|
214
215
|
|
|
@@ -278,14 +279,16 @@ class ScanPostProcessor(ScanossBase):
|
|
|
278
279
|
:return:
|
|
279
280
|
"""
|
|
280
281
|
message = (
|
|
281
|
-
f
|
|
282
|
-
f
|
|
283
|
-
f
|
|
282
|
+
f'{_get_match_type_message(result_path, bom_entry, action)} \n'
|
|
283
|
+
f'Details:\n'
|
|
284
|
+
f' - PURLs: {", ".join(result_purls)}\n'
|
|
284
285
|
f" - Path: '{result_path}'\n"
|
|
285
286
|
)
|
|
286
287
|
if action == 'Replacing':
|
|
287
288
|
message += f" - {action} with '{bom_entry.get('replace_with')}'"
|
|
288
289
|
self.print_debug(message)
|
|
290
|
+
|
|
291
|
+
|
|
289
292
|
#
|
|
290
293
|
# End of ScanPostProcessor Class
|
|
291
|
-
#
|
|
294
|
+
#
|