scanoss 1.20.0__py3-none-any.whl → 1.20.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- protoc_gen_swagger/__init__.py +13 -13
- protoc_gen_swagger/options/__init__.py +13 -13
- protoc_gen_swagger/options/annotations_pb2.py +12 -9
- protoc_gen_swagger/options/annotations_pb2_grpc.py +1 -1
- protoc_gen_swagger/options/openapiv2_pb2.py +98 -96
- protoc_gen_swagger/options/openapiv2_pb2_grpc.py +1 -1
- scanoss/__init__.py +18 -18
- scanoss/api/__init__.py +17 -17
- scanoss/api/common/__init__.py +17 -17
- scanoss/api/common/v2/__init__.py +17 -17
- scanoss/api/common/v2/scanoss_common_pb2.py +18 -18
- scanoss/api/common/v2/scanoss_common_pb2_grpc.py +1 -1
- scanoss/api/components/__init__.py +17 -17
- scanoss/api/components/v2/__init__.py +17 -17
- scanoss/api/components/v2/scanoss_components_pb2.py +48 -38
- scanoss/api/components/v2/scanoss_components_pb2_grpc.py +142 -96
- scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py +22 -16
- scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py +75 -49
- scanoss/api/dependencies/__init__.py +17 -17
- scanoss/api/dependencies/v2/__init__.py +17 -17
- scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +30 -24
- scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py +75 -49
- scanoss/api/scanning/__init__.py +17 -17
- scanoss/api/scanning/v2/__init__.py +17 -17
- scanoss/api/scanning/v2/scanoss_scanning_pb2.py +10 -8
- scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +40 -32
- scanoss/api/semgrep/__init__.py +17 -17
- scanoss/api/semgrep/v2/__init__.py +17 -17
- scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py +22 -18
- scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py +71 -49
- scanoss/api/vulnerabilities/__init__.py +17 -17
- scanoss/api/vulnerabilities/v2/__init__.py +17 -17
- scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +37 -27
- scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py +109 -72
- scanoss/cli.py +576 -293
- scanoss/components.py +67 -45
- scanoss/csvoutput.py +83 -56
- scanoss/cyclonedx.py +48 -46
- scanoss/data/build_date.txt +1 -1
- scanoss/file_filters.py +13 -15
- scanoss/filecount.py +43 -36
- scanoss/inspection/__init__.py +17 -17
- scanoss/inspection/copyleft.py +71 -58
- scanoss/inspection/policy_check.py +76 -53
- scanoss/inspection/undeclared_component.py +98 -75
- scanoss/inspection/utils/license_utils.py +66 -44
- scanoss/results.py +51 -60
- scanoss/scancodedeps.py +61 -38
- scanoss/scanner.py +203 -135
- scanoss/scanoss_settings.py +5 -3
- scanoss/scanossapi.py +98 -69
- scanoss/scanossbase.py +19 -19
- scanoss/scanossgrpc.py +73 -51
- scanoss/scanpostprocessor.py +9 -6
- scanoss/scantype.py +22 -21
- scanoss/spdxlite.py +265 -171
- scanoss/threadeddependencies.py +91 -61
- scanoss/threadedscanning.py +37 -31
- scanoss/utils/file.py +4 -4
- scanoss/winnowing.py +111 -47
- {scanoss-1.20.0.dist-info → scanoss-1.20.1.dist-info}/METADATA +1 -1
- scanoss-1.20.1.dist-info/RECORD +74 -0
- scanoss-1.20.0.dist-info/RECORD +0 -74
- {scanoss-1.20.0.dist-info → scanoss-1.20.1.dist-info}/LICENSE +0 -0
- {scanoss-1.20.0.dist-info → scanoss-1.20.1.dist-info}/WHEEL +0 -0
- {scanoss-1.20.0.dist-info → scanoss-1.20.1.dist-info}/entry_points.txt +0 -0
- {scanoss-1.20.0.dist-info → scanoss-1.20.1.dist-info}/top_level.txt +0 -0
scanoss/scanner.py
CHANGED
|
@@ -1,26 +1,27 @@
|
|
|
1
1
|
"""
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
2
|
+
SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
Copyright (c) 2021, SCANOSS
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in
|
|
14
|
+
all copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
22
|
+
THE SOFTWARE.
|
|
23
23
|
"""
|
|
24
|
+
|
|
24
25
|
import json
|
|
25
26
|
import os
|
|
26
27
|
from pathlib import Path
|
|
@@ -52,12 +53,13 @@ from . import __version__
|
|
|
52
53
|
FAST_WINNOWING = False
|
|
53
54
|
try:
|
|
54
55
|
from scanoss_winnowing.winnowing import Winnowing
|
|
56
|
+
|
|
55
57
|
FAST_WINNOWING = True
|
|
56
58
|
except ModuleNotFoundError or ImportError:
|
|
57
59
|
FAST_WINNOWING = False
|
|
58
60
|
from .winnowing import Winnowing
|
|
59
61
|
|
|
60
|
-
WFP_FILE_START =
|
|
62
|
+
WFP_FILE_START = 'file='
|
|
61
63
|
MAX_POST_SIZE = 64 * 1024 # 64k Max post size
|
|
62
64
|
|
|
63
65
|
|
|
@@ -103,7 +105,7 @@ class Scanner(ScanossBase):
|
|
|
103
105
|
strip_hpsm_ids=None,
|
|
104
106
|
strip_snippet_ids=None,
|
|
105
107
|
skip_md5_ids=None,
|
|
106
|
-
scan_settings: 'ScanossSettings | None' = None
|
|
108
|
+
scan_settings: 'ScanossSettings | None' = None,
|
|
107
109
|
):
|
|
108
110
|
"""
|
|
109
111
|
Initialise scanning class, including Winnowing, ScanossApi, ThreadedScanning
|
|
@@ -113,7 +115,7 @@ class Scanner(ScanossBase):
|
|
|
113
115
|
skip_folders = []
|
|
114
116
|
if skip_extensions is None:
|
|
115
117
|
skip_extensions = []
|
|
116
|
-
self.wfp = wfp if wfp else
|
|
118
|
+
self.wfp = wfp if wfp else 'scanner_output.wfp'
|
|
117
119
|
self.scan_output = scan_output
|
|
118
120
|
self.output_format = output_format
|
|
119
121
|
self.no_wfp_file = no_wfp_file
|
|
@@ -129,26 +131,51 @@ class Scanner(ScanossBase):
|
|
|
129
131
|
self.skip_extensions = skip_extensions
|
|
130
132
|
ver_details = Scanner.version_details()
|
|
131
133
|
|
|
132
|
-
self.winnowing = Winnowing(
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
134
|
+
self.winnowing = Winnowing(
|
|
135
|
+
debug=debug,
|
|
136
|
+
quiet=quiet,
|
|
137
|
+
skip_snippets=self._skip_snippets,
|
|
138
|
+
all_extensions=all_extensions,
|
|
139
|
+
obfuscate=obfuscate,
|
|
140
|
+
hpsm=self.hpsm,
|
|
141
|
+
strip_hpsm_ids=strip_hpsm_ids,
|
|
142
|
+
strip_snippet_ids=strip_snippet_ids,
|
|
143
|
+
skip_md5_ids=skip_md5_ids,
|
|
144
|
+
)
|
|
145
|
+
self.scanoss_api = ScanossApi(
|
|
146
|
+
debug=debug,
|
|
147
|
+
trace=trace,
|
|
148
|
+
quiet=quiet,
|
|
149
|
+
api_key=api_key,
|
|
150
|
+
url=url,
|
|
151
|
+
flags=flags,
|
|
152
|
+
timeout=timeout,
|
|
153
|
+
ver_details=ver_details,
|
|
154
|
+
ignore_cert_errors=ignore_cert_errors,
|
|
155
|
+
proxy=proxy,
|
|
156
|
+
ca_cert=ca_cert,
|
|
157
|
+
pac=pac,
|
|
158
|
+
retry=retry,
|
|
159
|
+
)
|
|
142
160
|
sc_deps = ScancodeDeps(debug=debug, quiet=quiet, trace=trace, timeout=sc_timeout, sc_command=sc_command)
|
|
143
|
-
grpc_api = ScanossGrpc(
|
|
144
|
-
|
|
145
|
-
|
|
161
|
+
grpc_api = ScanossGrpc(
|
|
162
|
+
url=grpc_url,
|
|
163
|
+
debug=debug,
|
|
164
|
+
quiet=quiet,
|
|
165
|
+
trace=trace,
|
|
166
|
+
api_key=api_key,
|
|
167
|
+
ver_details=ver_details,
|
|
168
|
+
ca_cert=ca_cert,
|
|
169
|
+
proxy=proxy,
|
|
170
|
+
pac=pac,
|
|
171
|
+
grpc_proxy=grpc_proxy,
|
|
172
|
+
)
|
|
146
173
|
self.threaded_deps = ThreadedDependencies(sc_deps, grpc_api, debug=debug, quiet=quiet, trace=trace)
|
|
147
174
|
self.nb_threads = nb_threads
|
|
148
175
|
if nb_threads and nb_threads > 0:
|
|
149
|
-
self.threaded_scan = ThreadedScanning(
|
|
150
|
-
|
|
151
|
-
|
|
176
|
+
self.threaded_scan = ThreadedScanning(
|
|
177
|
+
self.scanoss_api, debug=debug, trace=trace, quiet=quiet, nb_threads=nb_threads
|
|
178
|
+
)
|
|
152
179
|
else:
|
|
153
180
|
self.threaded_scan = None
|
|
154
181
|
self.max_post_size = post_size * 1024 if post_size > 0 else MAX_POST_SIZE # Set the max post size (default 64k)
|
|
@@ -157,7 +184,9 @@ class Scanner(ScanossBase):
|
|
|
157
184
|
self.max_post_size = 8 * 1024 # 8k Max post size if we're skipping snippets
|
|
158
185
|
|
|
159
186
|
self.scan_settings = scan_settings
|
|
160
|
-
self.post_processor =
|
|
187
|
+
self.post_processor = (
|
|
188
|
+
ScanPostProcessor(scan_settings, debug=debug, trace=trace, quiet=quiet) if scan_settings else None
|
|
189
|
+
)
|
|
161
190
|
self._maybe_set_api_sbom()
|
|
162
191
|
|
|
163
192
|
def _maybe_set_api_sbom(self):
|
|
@@ -183,7 +212,6 @@ class Scanner(ScanossBase):
|
|
|
183
212
|
if WFP_FILE_START in line:
|
|
184
213
|
count += 1
|
|
185
214
|
return count
|
|
186
|
-
|
|
187
215
|
|
|
188
216
|
@staticmethod
|
|
189
217
|
def version_details() -> str:
|
|
@@ -211,7 +239,7 @@ class Scanner(ScanossBase):
|
|
|
211
239
|
if not outfile and self.scan_output:
|
|
212
240
|
outfile = self.scan_output
|
|
213
241
|
if outfile:
|
|
214
|
-
with open(outfile,
|
|
242
|
+
with open(outfile, 'a') as rf:
|
|
215
243
|
rf.write(string + '\n')
|
|
216
244
|
else:
|
|
217
245
|
print(string)
|
|
@@ -252,9 +280,15 @@ class Scanner(ScanossBase):
|
|
|
252
280
|
return True
|
|
253
281
|
return False
|
|
254
282
|
|
|
255
|
-
def scan_folder_with_options(
|
|
256
|
-
|
|
257
|
-
|
|
283
|
+
def scan_folder_with_options(
|
|
284
|
+
self,
|
|
285
|
+
scan_dir: str,
|
|
286
|
+
deps_file: str = None,
|
|
287
|
+
file_map: dict = None,
|
|
288
|
+
dep_scope: SCOPE = None,
|
|
289
|
+
dep_scope_include: str = None,
|
|
290
|
+
dep_scope_exclude: str = None,
|
|
291
|
+
) -> bool:
|
|
258
292
|
"""
|
|
259
293
|
Scan the given folder for whatever scaning options that have been configured
|
|
260
294
|
:param dep_scope_exclude: comma separated list of dependency scopes to exclude
|
|
@@ -268,17 +302,23 @@ class Scanner(ScanossBase):
|
|
|
268
302
|
|
|
269
303
|
success = True
|
|
270
304
|
if not scan_dir:
|
|
271
|
-
raise Exception(f
|
|
305
|
+
raise Exception(f'ERROR: Please specify a folder to scan')
|
|
272
306
|
if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir):
|
|
273
|
-
raise Exception(f
|
|
307
|
+
raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}')
|
|
274
308
|
if not self.is_file_or_snippet_scan() and not self.is_dependency_scan():
|
|
275
|
-
raise Exception(f
|
|
309
|
+
raise Exception(f'ERROR: No scan options defined to scan folder: {scan_dir}')
|
|
276
310
|
|
|
277
311
|
if self.scan_output:
|
|
278
312
|
self.print_msg(f'Writing results to {self.scan_output}...')
|
|
279
313
|
if self.is_dependency_scan():
|
|
280
|
-
if not self.threaded_deps.run(
|
|
281
|
-
|
|
314
|
+
if not self.threaded_deps.run(
|
|
315
|
+
what_to_scan=scan_dir,
|
|
316
|
+
deps_file=deps_file,
|
|
317
|
+
wait=False,
|
|
318
|
+
dep_scope=dep_scope,
|
|
319
|
+
dep_scope_include=dep_scope_include,
|
|
320
|
+
dep_scope_exclude=dep_scope_exclude,
|
|
321
|
+
): # Kick off a background dependency scan
|
|
282
322
|
success = False
|
|
283
323
|
if self.is_file_or_snippet_scan():
|
|
284
324
|
if not self.scan_folder(scan_dir):
|
|
@@ -302,16 +342,19 @@ class Scanner(ScanossBase):
|
|
|
302
342
|
if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir):
|
|
303
343
|
raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}')
|
|
304
344
|
|
|
305
|
-
file_filters = FileFilters(
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
345
|
+
file_filters = FileFilters(
|
|
346
|
+
debug=self.debug,
|
|
347
|
+
trace=self.trace,
|
|
348
|
+
quiet=self.quiet,
|
|
349
|
+
scanoss_settings=self.scan_settings,
|
|
350
|
+
all_extensions=self.all_extensions,
|
|
351
|
+
all_folders=self.all_folders,
|
|
352
|
+
hidden_files_folders=self.hidden_files_folders,
|
|
353
|
+
skip_size=self.skip_size,
|
|
354
|
+
skip_folders=self.skip_folders,
|
|
355
|
+
skip_extensions=self.skip_extensions,
|
|
356
|
+
operation_type='scanning',
|
|
357
|
+
)
|
|
315
358
|
self.print_msg(f'Searching {scan_dir} for files to fingerprint...')
|
|
316
359
|
spinner = None
|
|
317
360
|
if not self.quiet and self.isatty:
|
|
@@ -342,15 +385,15 @@ class Scanner(ScanossBase):
|
|
|
342
385
|
wfp_list.append(wfp)
|
|
343
386
|
file_count += 1
|
|
344
387
|
if self.threaded_scan:
|
|
345
|
-
wfp_size = len(wfp.encode(
|
|
388
|
+
wfp_size = len(wfp.encode('utf-8'))
|
|
346
389
|
# If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue
|
|
347
390
|
if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size:
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
391
|
+
self.threaded_scan.queue_add(scan_block)
|
|
392
|
+
queue_size += 1
|
|
393
|
+
scan_block = ''
|
|
394
|
+
wfp_file_count = 0
|
|
352
395
|
scan_block += wfp
|
|
353
|
-
scan_size = len(scan_block.encode(
|
|
396
|
+
scan_size = len(scan_block.encode('utf-8'))
|
|
354
397
|
wfp_file_count += 1
|
|
355
398
|
# If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue
|
|
356
399
|
if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size:
|
|
@@ -421,9 +464,7 @@ class Scanner(ScanossBase):
|
|
|
421
464
|
if self.is_dependency_scan():
|
|
422
465
|
self.print_msg('Retrieving dependency data...')
|
|
423
466
|
if not self.threaded_deps.complete():
|
|
424
|
-
self.print_stderr(
|
|
425
|
-
f'Warning: Dependency analysis ran into some trouble.'
|
|
426
|
-
)
|
|
467
|
+
self.print_stderr(f'Warning: Dependency analysis ran into some trouble.')
|
|
427
468
|
success = False
|
|
428
469
|
dep_responses = self.threaded_deps.responses
|
|
429
470
|
|
|
@@ -453,7 +494,7 @@ class Scanner(ScanossBase):
|
|
|
453
494
|
def _merge_scan_results(
|
|
454
495
|
self,
|
|
455
496
|
scan_responses: Optional[List],
|
|
456
|
-
dep_responses: Optional[Dict[str,Any]],
|
|
497
|
+
dep_responses: Optional[Dict[str, Any]],
|
|
457
498
|
file_map: Optional[Dict[str, Any]],
|
|
458
499
|
) -> Dict[str, Any]:
|
|
459
500
|
"""Merge scan and dependency responses into a single dictionary"""
|
|
@@ -466,10 +507,10 @@ class Scanner(ScanossBase):
|
|
|
466
507
|
response = self._deobfuscate_filenames(response, file_map)
|
|
467
508
|
results.update(response)
|
|
468
509
|
|
|
469
|
-
dep_files = dep_responses.get(
|
|
510
|
+
dep_files = dep_responses.get('files', None) if dep_responses else None
|
|
470
511
|
if dep_files:
|
|
471
512
|
for dep_file in dep_files:
|
|
472
|
-
file = dep_file.pop(
|
|
513
|
+
file = dep_file.pop('file', None)
|
|
473
514
|
if file:
|
|
474
515
|
results[file] = [dep_file]
|
|
475
516
|
|
|
@@ -486,8 +527,15 @@ class Scanner(ScanossBase):
|
|
|
486
527
|
deobfuscated[key] = value
|
|
487
528
|
return deobfuscated
|
|
488
529
|
|
|
489
|
-
def scan_file_with_options(
|
|
490
|
-
|
|
530
|
+
def scan_file_with_options(
|
|
531
|
+
self,
|
|
532
|
+
file: str,
|
|
533
|
+
deps_file: str = None,
|
|
534
|
+
file_map: dict = None,
|
|
535
|
+
dep_scope: SCOPE = None,
|
|
536
|
+
dep_scope_include: str = None,
|
|
537
|
+
dep_scope_exclude: str = None,
|
|
538
|
+
) -> bool:
|
|
491
539
|
"""
|
|
492
540
|
Scan the given file for whatever scaning options that have been configured
|
|
493
541
|
:param dep_scope:
|
|
@@ -498,17 +546,23 @@ class Scanner(ScanossBase):
|
|
|
498
546
|
"""
|
|
499
547
|
success = True
|
|
500
548
|
if not file:
|
|
501
|
-
raise Exception(f
|
|
549
|
+
raise Exception(f'ERROR: Please specify a file to scan')
|
|
502
550
|
if not os.path.exists(file) or not os.path.isfile(file):
|
|
503
|
-
raise Exception(f
|
|
551
|
+
raise Exception(f'ERROR: Specified file does not exist or is not a file: {file}')
|
|
504
552
|
if not self.is_file_or_snippet_scan() and not self.is_dependency_scan():
|
|
505
|
-
raise Exception(f
|
|
553
|
+
raise Exception(f'ERROR: No scan options defined to scan file: {file}')
|
|
506
554
|
|
|
507
555
|
if self.scan_output:
|
|
508
556
|
self.print_msg(f'Writing results to {self.scan_output}...')
|
|
509
557
|
if self.is_dependency_scan():
|
|
510
|
-
if not self.threaded_deps.run(
|
|
511
|
-
|
|
558
|
+
if not self.threaded_deps.run(
|
|
559
|
+
what_to_scan=file,
|
|
560
|
+
deps_file=deps_file,
|
|
561
|
+
wait=False,
|
|
562
|
+
dep_scope=dep_scope,
|
|
563
|
+
dep_scope_include=dep_scope_include,
|
|
564
|
+
dep_scope_exclude=dep_scope_exclude,
|
|
565
|
+
): # Kick off a background dependency scan
|
|
512
566
|
success = False
|
|
513
567
|
if self.is_file_or_snippet_scan():
|
|
514
568
|
if not self.scan_file(file):
|
|
@@ -529,9 +583,9 @@ class Scanner(ScanossBase):
|
|
|
529
583
|
"""
|
|
530
584
|
success = True
|
|
531
585
|
if not file:
|
|
532
|
-
raise Exception(f
|
|
586
|
+
raise Exception(f'ERROR: Please specify a file to scan')
|
|
533
587
|
if not os.path.exists(file) or not os.path.isfile(file):
|
|
534
|
-
raise Exception(f
|
|
588
|
+
raise Exception(f'ERROR: Specified files does not exist or is not a file: {file}')
|
|
535
589
|
self.print_debug(f'Fingerprinting {file}...')
|
|
536
590
|
wfp = self.winnowing.wfp_for_file(file, file)
|
|
537
591
|
if wfp is not None and wfp != '':
|
|
@@ -554,18 +608,21 @@ class Scanner(ScanossBase):
|
|
|
554
608
|
"""
|
|
555
609
|
success = True
|
|
556
610
|
if not files:
|
|
557
|
-
raise Exception(f
|
|
558
|
-
|
|
559
|
-
file_filters = FileFilters(
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
611
|
+
raise Exception(f'ERROR: Please provide a non-empty list of filenames to scan')
|
|
612
|
+
|
|
613
|
+
file_filters = FileFilters(
|
|
614
|
+
debug=self.debug,
|
|
615
|
+
trace=self.trace,
|
|
616
|
+
quiet=self.quiet,
|
|
617
|
+
scanoss_settings=self.scan_settings,
|
|
618
|
+
all_extensions=self.all_extensions,
|
|
619
|
+
all_folders=self.all_folders,
|
|
620
|
+
hidden_files_folders=self.hidden_files_folders,
|
|
621
|
+
skip_size=self.skip_size,
|
|
622
|
+
skip_folders=self.skip_folders,
|
|
623
|
+
skip_extensions=self.skip_extensions,
|
|
624
|
+
operation_type='scanning',
|
|
625
|
+
)
|
|
569
626
|
spinner = None
|
|
570
627
|
if not self.quiet and self.isatty:
|
|
571
628
|
spinner = Spinner('Fingerprinting ')
|
|
@@ -577,7 +634,7 @@ class Scanner(ScanossBase):
|
|
|
577
634
|
file_count = 0 # count all files fingerprinted
|
|
578
635
|
wfp_file_count = 0 # count number of files in each queue post
|
|
579
636
|
scan_started = False
|
|
580
|
-
|
|
637
|
+
|
|
581
638
|
to_scan_files = file_filters.get_filtered_files_from_files(files)
|
|
582
639
|
for file in to_scan_files:
|
|
583
640
|
if self.threaded_scan and self.threaded_scan.stop_scanning():
|
|
@@ -617,7 +674,7 @@ class Scanner(ScanossBase):
|
|
|
617
674
|
f'Warning: Some errors encounted while scanning. Results might be incomplete.'
|
|
618
675
|
)
|
|
619
676
|
success = False
|
|
620
|
-
|
|
677
|
+
|
|
621
678
|
# End for loop
|
|
622
679
|
if self.threaded_scan and scan_block != '':
|
|
623
680
|
self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted
|
|
@@ -647,11 +704,13 @@ class Scanner(ScanossBase):
|
|
|
647
704
|
"""
|
|
648
705
|
success = True
|
|
649
706
|
if not files:
|
|
650
|
-
raise Exception(f
|
|
707
|
+
raise Exception(f'ERROR: Please specify a list of files to scan')
|
|
651
708
|
if not self.is_file_or_snippet_scan():
|
|
652
|
-
raise Exception(f
|
|
709
|
+
raise Exception(f'ERROR: file or snippet scan options have to be set to scan files: {files}')
|
|
653
710
|
if self.is_dependency_scan() or deps_file:
|
|
654
|
-
raise Exception(
|
|
711
|
+
raise Exception(
|
|
712
|
+
f'ERROR: The dependency scan option is currently not supported when scanning a list of files'
|
|
713
|
+
)
|
|
655
714
|
if self.scan_output:
|
|
656
715
|
self.print_msg(f'Writing results to {self.scan_output}...')
|
|
657
716
|
if self.is_file_or_snippet_scan():
|
|
@@ -672,9 +731,9 @@ class Scanner(ScanossBase):
|
|
|
672
731
|
"""
|
|
673
732
|
success = True
|
|
674
733
|
if not filename:
|
|
675
|
-
raise Exception(f
|
|
734
|
+
raise Exception(f'ERROR: Please specify a filename to scan')
|
|
676
735
|
if not contents:
|
|
677
|
-
raise Exception(f
|
|
736
|
+
raise Exception(f'ERROR: Please specify a file contents to scan')
|
|
678
737
|
|
|
679
738
|
self.print_debug(f'Fingerprinting {filename}...')
|
|
680
739
|
wfp = self.winnowing.wfp_for_contents(filename, False, contents)
|
|
@@ -700,7 +759,7 @@ class Scanner(ScanossBase):
|
|
|
700
759
|
success = True
|
|
701
760
|
wfp_file = file if file else self.wfp # If a WFP file is specified, use it, otherwise us the default
|
|
702
761
|
if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file):
|
|
703
|
-
raise Exception(f
|
|
762
|
+
raise Exception(f'ERROR: Specified WFP file does not exist or is not a file: {wfp_file}')
|
|
704
763
|
file_count = Scanner.__count_files_in_wfp_file(wfp_file)
|
|
705
764
|
cur_files = 0
|
|
706
765
|
cur_size = 0
|
|
@@ -709,7 +768,7 @@ class Scanner(ScanossBase):
|
|
|
709
768
|
max_component = {'name': '', 'hits': 0}
|
|
710
769
|
components = {}
|
|
711
770
|
self.print_debug(f'Found {file_count} files to process.')
|
|
712
|
-
raw_output =
|
|
771
|
+
raw_output = '{\n'
|
|
713
772
|
file_print = ''
|
|
714
773
|
bar = None
|
|
715
774
|
if not self.quiet and self.isatty:
|
|
@@ -720,7 +779,7 @@ class Scanner(ScanossBase):
|
|
|
720
779
|
if line.startswith(WFP_FILE_START):
|
|
721
780
|
if file_print:
|
|
722
781
|
wfp += file_print # Store the WFP for the current file
|
|
723
|
-
cur_size = len(wfp.encode(
|
|
782
|
+
cur_size = len(wfp.encode('utf-8'))
|
|
724
783
|
file_print = line # Start storing the next file
|
|
725
784
|
cur_files += 1
|
|
726
785
|
batch_files += 1
|
|
@@ -729,8 +788,10 @@ class Scanner(ScanossBase):
|
|
|
729
788
|
l_size = cur_size + len(file_print.encode('utf-8'))
|
|
730
789
|
# Hit the max post size, so sending the current batch and continue processing
|
|
731
790
|
if l_size >= self.max_post_size and wfp:
|
|
732
|
-
self.print_debug(
|
|
733
|
-
|
|
791
|
+
self.print_debug(
|
|
792
|
+
f'Sending {batch_files} ({cur_files}) of'
|
|
793
|
+
f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.'
|
|
794
|
+
)
|
|
734
795
|
if self.debug and cur_size > self.max_post_size:
|
|
735
796
|
Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}')
|
|
736
797
|
scan_resp = self.scanoss_api.scan(wfp, max_component['name']) # Scan current WFP and store
|
|
@@ -738,7 +799,7 @@ class Scanner(ScanossBase):
|
|
|
738
799
|
bar.next(batch_files)
|
|
739
800
|
if scan_resp is not None:
|
|
740
801
|
for key, value in scan_resp.items():
|
|
741
|
-
raw_output +=
|
|
802
|
+
raw_output += ' "%s":%s,' % (key, json.dumps(value, indent=2))
|
|
742
803
|
for v in value:
|
|
743
804
|
if hasattr(v, 'get'):
|
|
744
805
|
if v.get('id') != 'none':
|
|
@@ -756,8 +817,10 @@ class Scanner(ScanossBase):
|
|
|
756
817
|
if file_print:
|
|
757
818
|
wfp += file_print # Store the WFP for the current file
|
|
758
819
|
if wfp:
|
|
759
|
-
self.print_debug(
|
|
760
|
-
|
|
820
|
+
self.print_debug(
|
|
821
|
+
f'Sending {batch_files} ({cur_files}) of'
|
|
822
|
+
f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.'
|
|
823
|
+
)
|
|
761
824
|
scan_resp = self.scanoss_api.scan(wfp, max_component['name']) # Scan current WFP and store
|
|
762
825
|
if bar:
|
|
763
826
|
bar.next(batch_files)
|
|
@@ -765,13 +828,13 @@ class Scanner(ScanossBase):
|
|
|
765
828
|
if scan_resp is not None:
|
|
766
829
|
for key, value in scan_resp.items():
|
|
767
830
|
if first:
|
|
768
|
-
raw_output +=
|
|
831
|
+
raw_output += ' "%s":%s' % (key, json.dumps(value, indent=2))
|
|
769
832
|
first = False
|
|
770
833
|
else:
|
|
771
|
-
raw_output +=
|
|
834
|
+
raw_output += ',\n "%s":%s' % (key, json.dumps(value, indent=2))
|
|
772
835
|
else:
|
|
773
836
|
success = False
|
|
774
|
-
raw_output +=
|
|
837
|
+
raw_output += '\n}'
|
|
775
838
|
if bar:
|
|
776
839
|
bar.finish()
|
|
777
840
|
if self.output_format == 'plain':
|
|
@@ -802,10 +865,10 @@ class Scanner(ScanossBase):
|
|
|
802
865
|
success = True
|
|
803
866
|
wfp_file = wfp if wfp else self.wfp # If a WFP file is specified, use it, otherwise us the default
|
|
804
867
|
if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file):
|
|
805
|
-
raise Exception(f
|
|
868
|
+
raise Exception(f'ERROR: Specified WFP file does not exist or is not a file: {wfp_file}')
|
|
806
869
|
|
|
807
870
|
if not self.is_file_or_snippet_scan() and not self.is_dependency_scan():
|
|
808
|
-
raise Exception(f
|
|
871
|
+
raise Exception(f'ERROR: No scan options defined to scan WFP: {wfp}')
|
|
809
872
|
|
|
810
873
|
if self.scan_output:
|
|
811
874
|
self.print_msg(f'Writing results to {self.scan_output}...')
|
|
@@ -829,7 +892,7 @@ class Scanner(ScanossBase):
|
|
|
829
892
|
success = True
|
|
830
893
|
wfp_file = file if file else self.wfp # If a WFP file is specified, use it, otherwise us the default
|
|
831
894
|
if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file):
|
|
832
|
-
raise Exception(f
|
|
895
|
+
raise Exception(f'ERROR: Specified WFP file does not exist or is not a file: {wfp_file}')
|
|
833
896
|
cur_size = 0
|
|
834
897
|
queue_size = 0
|
|
835
898
|
file_count = 0 # count all files fingerprinted
|
|
@@ -842,7 +905,7 @@ class Scanner(ScanossBase):
|
|
|
842
905
|
if line.startswith(WFP_FILE_START):
|
|
843
906
|
if scan_block:
|
|
844
907
|
wfp += scan_block # Store the WFP for the current file
|
|
845
|
-
cur_size = len(wfp.encode(
|
|
908
|
+
cur_size = len(wfp.encode('utf-8'))
|
|
846
909
|
scan_block = line # Start storing the next file
|
|
847
910
|
file_count += 1
|
|
848
911
|
wfp_file_count += 1
|
|
@@ -861,7 +924,8 @@ class Scanner(ScanossBase):
|
|
|
861
924
|
scan_started = True
|
|
862
925
|
if not self.threaded_scan.run(wait=False):
|
|
863
926
|
self.print_stderr(
|
|
864
|
-
f'Warning: Some errors encounted while scanning. Results might be incomplete.'
|
|
927
|
+
f'Warning: Some errors encounted while scanning. Results might be incomplete.'
|
|
928
|
+
)
|
|
865
929
|
success = False
|
|
866
930
|
# End for loop
|
|
867
931
|
if scan_block:
|
|
@@ -884,15 +948,15 @@ class Scanner(ScanossBase):
|
|
|
884
948
|
"""
|
|
885
949
|
success = True
|
|
886
950
|
if not wfp:
|
|
887
|
-
raise Exception(f
|
|
888
|
-
raw_output =
|
|
951
|
+
raise Exception(f'ERROR: Please specify a WFP to scan')
|
|
952
|
+
raw_output = '{\n'
|
|
889
953
|
scan_resp = self.scanoss_api.scan(wfp)
|
|
890
954
|
if scan_resp is not None:
|
|
891
955
|
for key, value in scan_resp.items():
|
|
892
|
-
raw_output +=
|
|
956
|
+
raw_output += ' "%s":%s' % (key, json.dumps(value, indent=2))
|
|
893
957
|
else:
|
|
894
958
|
success = False
|
|
895
|
-
raw_output +=
|
|
959
|
+
raw_output += '\n}'
|
|
896
960
|
if self.output_format == 'plain':
|
|
897
961
|
self.__log_result(raw_output)
|
|
898
962
|
elif self.output_format == 'cyclonedx':
|
|
@@ -920,9 +984,9 @@ class Scanner(ScanossBase):
|
|
|
920
984
|
:return:
|
|
921
985
|
"""
|
|
922
986
|
if not filename:
|
|
923
|
-
raise Exception(f
|
|
987
|
+
raise Exception(f'ERROR: Please specify a filename to scan')
|
|
924
988
|
if not contents:
|
|
925
|
-
raise Exception(f
|
|
989
|
+
raise Exception(f'ERROR: Please specify a file contents to scan')
|
|
926
990
|
|
|
927
991
|
self.print_debug(f'Fingerprinting {filename}...')
|
|
928
992
|
wfp = self.winnowing.wfp_for_contents(filename, False, contents)
|
|
@@ -941,9 +1005,9 @@ class Scanner(ScanossBase):
|
|
|
941
1005
|
Fingerprint the specified file
|
|
942
1006
|
"""
|
|
943
1007
|
if not scan_file:
|
|
944
|
-
raise Exception(f
|
|
1008
|
+
raise Exception(f'ERROR: Please specify a file to fingerprint')
|
|
945
1009
|
if not os.path.exists(scan_file) or not os.path.isfile(scan_file):
|
|
946
|
-
raise Exception(f
|
|
1010
|
+
raise Exception(f'ERROR: Specified file does not exist or is not a file: {scan_file}')
|
|
947
1011
|
|
|
948
1012
|
self.print_debug(f'Fingerprinting {scan_file}...')
|
|
949
1013
|
wfp = self.winnowing.wfp_for_file(scan_file, scan_file)
|
|
@@ -965,16 +1029,19 @@ class Scanner(ScanossBase):
|
|
|
965
1029
|
raise Exception(f'ERROR: Please specify a folder to fingerprint')
|
|
966
1030
|
if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir):
|
|
967
1031
|
raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}')
|
|
968
|
-
file_filters = FileFilters(
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
1032
|
+
file_filters = FileFilters(
|
|
1033
|
+
debug=self.debug,
|
|
1034
|
+
trace=self.trace,
|
|
1035
|
+
quiet=self.quiet,
|
|
1036
|
+
scanoss_settings=self.scan_settings,
|
|
1037
|
+
all_extensions=self.all_extensions,
|
|
1038
|
+
all_folders=self.all_folders,
|
|
1039
|
+
hidden_files_folders=self.hidden_files_folders,
|
|
1040
|
+
skip_size=self.skip_size,
|
|
1041
|
+
skip_folders=self.skip_folders,
|
|
1042
|
+
skip_extensions=self.skip_extensions,
|
|
1043
|
+
operation_type='scanning',
|
|
1044
|
+
)
|
|
978
1045
|
wfps = ''
|
|
979
1046
|
self.print_msg(f'Searching {scan_dir} for files to fingerprint...')
|
|
980
1047
|
spinner = None
|
|
@@ -1000,6 +1067,7 @@ class Scanner(ScanossBase):
|
|
|
1000
1067
|
else:
|
|
1001
1068
|
Scanner.print_stderr(f'Warning: No files found to fingerprint in folder: {scan_dir}')
|
|
1002
1069
|
|
|
1070
|
+
|
|
1003
1071
|
#
|
|
1004
1072
|
# End of ScanOSS Class
|
|
1005
1073
|
#
|