scanoss 1.20.6__py3-none-any.whl → 1.22.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- protoc_gen_swagger/options/annotations_pb2.py +9 -12
- protoc_gen_swagger/options/annotations_pb2_grpc.py +1 -1
- protoc_gen_swagger/options/openapiv2_pb2.py +96 -98
- protoc_gen_swagger/options/openapiv2_pb2_grpc.py +1 -1
- scanoss/__init__.py +1 -1
- scanoss/api/common/v2/scanoss_common_pb2.py +20 -18
- scanoss/api/common/v2/scanoss_common_pb2_grpc.py +1 -1
- scanoss/api/components/v2/scanoss_components_pb2.py +38 -48
- scanoss/api/components/v2/scanoss_components_pb2_grpc.py +96 -142
- scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py +42 -22
- scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py +185 -75
- scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +32 -30
- scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py +83 -75
- scanoss/api/provenance/v2/scanoss_provenance_pb2.py +20 -21
- scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py +1 -1
- scanoss/api/scanning/v2/scanoss_scanning_pb2.py +20 -10
- scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +70 -40
- scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py +18 -22
- scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py +49 -71
- scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +27 -37
- scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py +72 -109
- scanoss/cli.py +384 -80
- scanoss/constants.py +12 -0
- scanoss/data/build_date.txt +1 -1
- scanoss/file_filters.py +272 -57
- scanoss/results.py +92 -109
- scanoss/scanners/__init__.py +23 -0
- scanoss/scanners/container_scanner.py +474 -0
- scanoss/scanners/folder_hasher.py +302 -0
- scanoss/scanners/scanner_config.py +73 -0
- scanoss/scanners/scanner_hfh.py +172 -0
- scanoss/scanoss_settings.py +9 -5
- scanoss/scanossbase.py +9 -3
- scanoss/scanossgrpc.py +124 -13
- scanoss/threadedscanning.py +6 -6
- scanoss/utils/abstract_presenter.py +103 -0
- scanoss/utils/crc64.py +96 -0
- scanoss/utils/simhash.py +198 -0
- {scanoss-1.20.6.dist-info → scanoss-1.22.0.dist-info}/METADATA +2 -1
- {scanoss-1.20.6.dist-info → scanoss-1.22.0.dist-info}/RECORD +44 -35
- {scanoss-1.20.6.dist-info → scanoss-1.22.0.dist-info}/WHEEL +1 -1
- {scanoss-1.20.6.dist-info → scanoss-1.22.0.dist-info}/entry_points.txt +0 -0
- {scanoss-1.20.6.dist-info → scanoss-1.22.0.dist-info}/licenses/LICENSE +0 -0
- {scanoss-1.20.6.dist-info → scanoss-1.22.0.dist-info}/top_level.txt +0 -0
scanoss/cli.py
CHANGED
|
@@ -25,13 +25,37 @@ SPDX-License-Identifier: MIT
|
|
|
25
25
|
import argparse
|
|
26
26
|
import os
|
|
27
27
|
import sys
|
|
28
|
+
from dataclasses import asdict
|
|
28
29
|
from pathlib import Path
|
|
30
|
+
from typing import List
|
|
29
31
|
|
|
30
32
|
import pypac
|
|
31
|
-
|
|
33
|
+
|
|
34
|
+
from scanoss.scanners.container_scanner import (
|
|
35
|
+
DEFAULT_SYFT_COMMAND,
|
|
36
|
+
DEFAULT_SYFT_TIMEOUT,
|
|
37
|
+
ContainerScanner,
|
|
38
|
+
create_container_scanner_config_from_args,
|
|
39
|
+
)
|
|
40
|
+
from scanoss.scanners.folder_hasher import (
|
|
41
|
+
FolderHasher,
|
|
42
|
+
create_folder_hasher_config_from_args,
|
|
43
|
+
)
|
|
44
|
+
from scanoss.scanossgrpc import (
|
|
45
|
+
ScanossGrpc,
|
|
46
|
+
ScanossGrpcError,
|
|
47
|
+
create_grpc_config_from_args,
|
|
48
|
+
)
|
|
32
49
|
|
|
33
50
|
from . import __version__
|
|
34
51
|
from .components import Components
|
|
52
|
+
from .constants import (
|
|
53
|
+
DEFAULT_POST_SIZE,
|
|
54
|
+
DEFAULT_RETRY,
|
|
55
|
+
DEFAULT_TIMEOUT,
|
|
56
|
+
MIN_TIMEOUT,
|
|
57
|
+
PYTHON_MAJOR_VERSION,
|
|
58
|
+
)
|
|
35
59
|
from .csvoutput import CsvOutput
|
|
36
60
|
from .cyclonedx import CycloneDx
|
|
37
61
|
from .filecount import FileCount
|
|
@@ -40,19 +64,17 @@ from .inspection.undeclared_component import UndeclaredComponent
|
|
|
40
64
|
from .results import Results
|
|
41
65
|
from .scancodedeps import ScancodeDeps
|
|
42
66
|
from .scanner import FAST_WINNOWING, Scanner
|
|
67
|
+
from .scanners.scanner_config import create_scanner_config_from_args
|
|
68
|
+
from .scanners.scanner_hfh import ScannerHFH
|
|
43
69
|
from .scanoss_settings import ScanossSettings, ScanossSettingsError
|
|
44
70
|
from .scantype import ScanType
|
|
45
71
|
from .spdxlite import SpdxLite
|
|
46
72
|
from .threadeddependencies import SCOPE
|
|
47
73
|
from .utils.file import validate_json_file
|
|
48
74
|
|
|
49
|
-
DEFAULT_POST_SIZE = 32
|
|
50
|
-
DEFAULT_TIMEOUT = 180
|
|
51
|
-
MIN_TIMEOUT_VALUE = 5
|
|
52
|
-
DEFAULT_RETRY = 5
|
|
53
|
-
PYTHON3_OR_LATER = 3
|
|
54
75
|
HEADER_PARTS_COUNT = 2
|
|
55
76
|
|
|
77
|
+
|
|
56
78
|
def print_stderr(*args, **kwargs):
|
|
57
79
|
"""
|
|
58
80
|
Print the given message to STDERR
|
|
@@ -60,7 +82,7 @@ def print_stderr(*args, **kwargs):
|
|
|
60
82
|
print(*args, file=sys.stderr, **kwargs)
|
|
61
83
|
|
|
62
84
|
|
|
63
|
-
def setup_args() -> None: # noqa: PLR0915
|
|
85
|
+
def setup_args() -> None: # noqa: PLR0912, PLR0915
|
|
64
86
|
"""
|
|
65
87
|
Setup all the command line arguments for processing
|
|
66
88
|
"""
|
|
@@ -95,14 +117,6 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
95
117
|
p_scan.add_argument('--files', '-e', type=str, nargs='*', help='List of files to scan.')
|
|
96
118
|
p_scan.add_argument('--identify', '-i', type=str, help='Scan and identify components in SBOM file')
|
|
97
119
|
p_scan.add_argument('--ignore', '-n', type=str, help='Ignore components specified in the SBOM file')
|
|
98
|
-
p_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
99
|
-
p_scan.add_argument(
|
|
100
|
-
'--format',
|
|
101
|
-
'-f',
|
|
102
|
-
type=str,
|
|
103
|
-
choices=['plain', 'cyclonedx', 'spdxlite', 'csv'],
|
|
104
|
-
help='Result output format (optional - default: plain)',
|
|
105
|
-
)
|
|
106
120
|
p_scan.add_argument(
|
|
107
121
|
'--threads', '-T', type=int, default=5, help='Number of threads to use while scanning (optional - default 5)'
|
|
108
122
|
)
|
|
@@ -132,15 +146,17 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
132
146
|
help='Timeout (in seconds) for API communication (optional - default 180)',
|
|
133
147
|
)
|
|
134
148
|
p_scan.add_argument(
|
|
135
|
-
'--retry',
|
|
136
|
-
|
|
149
|
+
'--retry',
|
|
150
|
+
'-R',
|
|
151
|
+
type=int,
|
|
152
|
+
default=DEFAULT_RETRY,
|
|
153
|
+
help='Retry limit for API communication (optional - default 5)',
|
|
137
154
|
)
|
|
138
155
|
p_scan.add_argument('--no-wfp-output', action='store_true', help='Skip WFP file generation')
|
|
139
156
|
p_scan.add_argument('--dependencies', '-D', action='store_true', help='Add Dependency scanning')
|
|
140
157
|
p_scan.add_argument('--dependencies-only', action='store_true', help='Run Dependency scanning only')
|
|
141
158
|
p_scan.add_argument(
|
|
142
|
-
'--sc-command', type=str,
|
|
143
|
-
help='Scancode command and path if required (optional - default scancode).'
|
|
159
|
+
'--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).'
|
|
144
160
|
)
|
|
145
161
|
p_scan.add_argument(
|
|
146
162
|
'--sc-timeout',
|
|
@@ -153,18 +169,6 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
153
169
|
)
|
|
154
170
|
p_scan.add_argument('--dep-scope-inc', '-dsi', type=str, help='Include dependencies with declared scopes')
|
|
155
171
|
p_scan.add_argument('--dep-scope-exc', '-dse', type=str, help='Exclude dependencies with declared scopes')
|
|
156
|
-
p_scan.add_argument(
|
|
157
|
-
'--settings',
|
|
158
|
-
'-st',
|
|
159
|
-
type=str,
|
|
160
|
-
help='Settings file to use for scanning (optional - default scanoss.json)',
|
|
161
|
-
)
|
|
162
|
-
p_scan.add_argument(
|
|
163
|
-
'--skip-settings-file',
|
|
164
|
-
'-stf',
|
|
165
|
-
action='store_true',
|
|
166
|
-
help='Skip default settings file (scanoss.json) if it exists',
|
|
167
|
-
)
|
|
168
172
|
|
|
169
173
|
# Sub-command: fingerprint
|
|
170
174
|
p_wfp = subparsers.add_parser(
|
|
@@ -182,19 +186,6 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
182
186
|
type=str,
|
|
183
187
|
help='Fingerprint the file contents supplied via STDIN (optional)',
|
|
184
188
|
)
|
|
185
|
-
p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
186
|
-
p_wfp.add_argument(
|
|
187
|
-
'--settings',
|
|
188
|
-
'-st',
|
|
189
|
-
type=str,
|
|
190
|
-
help='Settings file to use for fingerprinting (optional - default scanoss.json)',
|
|
191
|
-
)
|
|
192
|
-
p_wfp.add_argument(
|
|
193
|
-
'--skip-settings-file',
|
|
194
|
-
'-stf',
|
|
195
|
-
action='store_true',
|
|
196
|
-
help='Skip default settings file (scanoss.json) if it exists',
|
|
197
|
-
)
|
|
198
189
|
|
|
199
190
|
# Sub-command: dependency
|
|
200
191
|
p_dep = subparsers.add_parser(
|
|
@@ -203,9 +194,13 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
203
194
|
description=f'Produce dependency file summary: {__version__}',
|
|
204
195
|
help='Scan source code for dependencies, but do not decorate them',
|
|
205
196
|
)
|
|
206
|
-
p_dep.
|
|
207
|
-
p_dep.add_argument(
|
|
208
|
-
|
|
197
|
+
p_dep.add_argument('scan_loc', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan')
|
|
198
|
+
p_dep.add_argument(
|
|
199
|
+
'--container',
|
|
200
|
+
type=str,
|
|
201
|
+
help='Container image to scan. Supports yourrepo/yourimage:tag, Docker tar, '
|
|
202
|
+
'OCI tar, OCI directory, SIF Container, or generic filesystem directory.',
|
|
203
|
+
)
|
|
209
204
|
p_dep.add_argument(
|
|
210
205
|
'--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).'
|
|
211
206
|
)
|
|
@@ -215,6 +210,40 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
215
210
|
default=600,
|
|
216
211
|
help='Timeout (in seconds) for scancode to complete (optional - default 600)',
|
|
217
212
|
)
|
|
213
|
+
p_dep.set_defaults(func=dependency)
|
|
214
|
+
|
|
215
|
+
# Container scan sub-command
|
|
216
|
+
p_cs = subparsers.add_parser(
|
|
217
|
+
'container-scan',
|
|
218
|
+
aliases=['cs'],
|
|
219
|
+
description=f'Analyse/scan the given container image: {__version__}',
|
|
220
|
+
help='Scan container image',
|
|
221
|
+
)
|
|
222
|
+
p_cs.add_argument(
|
|
223
|
+
'scan_loc',
|
|
224
|
+
metavar='IMAGE',
|
|
225
|
+
type=str,
|
|
226
|
+
nargs='?',
|
|
227
|
+
help=(
|
|
228
|
+
'Container image to scan. Supports yourrepo/yourimage:tag, Docker tar, '
|
|
229
|
+
'OCI tar, OCI directory, SIF Container, or generic filesystem directory.'
|
|
230
|
+
),
|
|
231
|
+
)
|
|
232
|
+
p_cs.add_argument(
|
|
233
|
+
'--retry',
|
|
234
|
+
'-R',
|
|
235
|
+
type=int,
|
|
236
|
+
default=DEFAULT_RETRY,
|
|
237
|
+
help='Retry limit for API communication (optional - default 5)',
|
|
238
|
+
)
|
|
239
|
+
p_cs.add_argument(
|
|
240
|
+
'--timeout',
|
|
241
|
+
'-M',
|
|
242
|
+
type=int,
|
|
243
|
+
default=DEFAULT_TIMEOUT,
|
|
244
|
+
help='Timeout (in seconds) for API communication (optional - default 180)',
|
|
245
|
+
)
|
|
246
|
+
p_cs.set_defaults(func=container_scan)
|
|
218
247
|
|
|
219
248
|
# Sub-command: file_count
|
|
220
249
|
p_fc = subparsers.add_parser(
|
|
@@ -225,7 +254,6 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
225
254
|
)
|
|
226
255
|
p_fc.set_defaults(func=file_count)
|
|
227
256
|
p_fc.add_argument('scan_dir', metavar='DIR', type=str, nargs='?', help='A folder to search')
|
|
228
|
-
p_fc.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
229
257
|
p_fc.add_argument('--all-hidden', action='store_true', help='Scan all hidden files/folders')
|
|
230
258
|
|
|
231
259
|
# Sub-command: convert
|
|
@@ -237,7 +265,6 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
237
265
|
)
|
|
238
266
|
p_cnv.set_defaults(func=convert)
|
|
239
267
|
p_cnv.add_argument('--input', '-i', type=str, required=True, help='Input file name')
|
|
240
|
-
p_cnv.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
241
268
|
p_cnv.add_argument(
|
|
242
269
|
'--format',
|
|
243
270
|
'-f',
|
|
@@ -333,9 +360,9 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
333
360
|
for p in [c_crypto, c_vulns, c_semgrep, c_provenance]:
|
|
334
361
|
p.add_argument('--purl', '-p', type=str, nargs='*', help='Package URL - PURL to process.')
|
|
335
362
|
p.add_argument('--input', '-i', type=str, help='Input file name')
|
|
363
|
+
|
|
336
364
|
# Common Component sub-command options
|
|
337
365
|
for p in [c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]:
|
|
338
|
-
p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
339
366
|
p.add_argument(
|
|
340
367
|
'--timeout',
|
|
341
368
|
'-M',
|
|
@@ -383,7 +410,6 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
383
410
|
p_c_dwnld.add_argument(
|
|
384
411
|
'--port', '-p', required=False, type=int, default=443, help='Server port number (default: 443).'
|
|
385
412
|
)
|
|
386
|
-
p_c_dwnld.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
387
413
|
|
|
388
414
|
# Utils Sub-command: utils pac-proxy
|
|
389
415
|
p_p_proxy = utils_sub.add_parser(
|
|
@@ -491,6 +517,114 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
491
517
|
)
|
|
492
518
|
p_undeclared.set_defaults(func=inspect_undeclared)
|
|
493
519
|
|
|
520
|
+
# Sub-command: folder-scan
|
|
521
|
+
p_folder_scan = subparsers.add_parser(
|
|
522
|
+
'folder-scan',
|
|
523
|
+
aliases=['fs'],
|
|
524
|
+
description=f'Scan the given directory using folder hashing: {__version__}',
|
|
525
|
+
help='Scan the given directory using folder hashing',
|
|
526
|
+
)
|
|
527
|
+
p_folder_scan.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='The root directory to scan')
|
|
528
|
+
p_folder_scan.add_argument(
|
|
529
|
+
'--timeout',
|
|
530
|
+
'-M',
|
|
531
|
+
type=int,
|
|
532
|
+
default=600,
|
|
533
|
+
help='Timeout (in seconds) for API communication (optional - default 600)',
|
|
534
|
+
)
|
|
535
|
+
p_folder_scan.add_argument(
|
|
536
|
+
'--format',
|
|
537
|
+
'-f',
|
|
538
|
+
type=str,
|
|
539
|
+
choices=['json'],
|
|
540
|
+
default='json',
|
|
541
|
+
help='Result output format (optional - default: json)',
|
|
542
|
+
)
|
|
543
|
+
p_folder_scan.add_argument(
|
|
544
|
+
'--best-match',
|
|
545
|
+
'-bm',
|
|
546
|
+
action='store_true',
|
|
547
|
+
default=False,
|
|
548
|
+
help='Enable best match mode (optional - default: False)',
|
|
549
|
+
)
|
|
550
|
+
p_folder_scan.add_argument(
|
|
551
|
+
'--threshold',
|
|
552
|
+
type=int,
|
|
553
|
+
choices=range(1, 101),
|
|
554
|
+
metavar='1-100',
|
|
555
|
+
default=100,
|
|
556
|
+
help='Threshold for result matching (optional - default: 100)',
|
|
557
|
+
)
|
|
558
|
+
p_folder_scan.set_defaults(func=folder_hashing_scan)
|
|
559
|
+
|
|
560
|
+
# Sub-command: folder-hash
|
|
561
|
+
p_folder_hash = subparsers.add_parser(
|
|
562
|
+
'folder-hash',
|
|
563
|
+
aliases=['fh'],
|
|
564
|
+
description=f'Produce a folder hash for the given directory: {__version__}',
|
|
565
|
+
help='Produce a folder hash for the given directory',
|
|
566
|
+
)
|
|
567
|
+
p_folder_hash.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan')
|
|
568
|
+
p_folder_hash.add_argument(
|
|
569
|
+
'--format',
|
|
570
|
+
'-f',
|
|
571
|
+
type=str,
|
|
572
|
+
choices=['json'],
|
|
573
|
+
default='json',
|
|
574
|
+
help='Result output format (optional - default: json)',
|
|
575
|
+
)
|
|
576
|
+
p_folder_hash.set_defaults(func=folder_hash)
|
|
577
|
+
|
|
578
|
+
# Output options
|
|
579
|
+
for p in [
|
|
580
|
+
p_scan,
|
|
581
|
+
p_cs,
|
|
582
|
+
p_wfp,
|
|
583
|
+
p_dep,
|
|
584
|
+
p_fc,
|
|
585
|
+
p_cnv,
|
|
586
|
+
c_crypto,
|
|
587
|
+
c_vulns,
|
|
588
|
+
c_search,
|
|
589
|
+
c_versions,
|
|
590
|
+
c_semgrep,
|
|
591
|
+
c_provenance,
|
|
592
|
+
p_c_dwnld,
|
|
593
|
+
p_folder_scan,
|
|
594
|
+
p_folder_hash,
|
|
595
|
+
]:
|
|
596
|
+
p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
597
|
+
|
|
598
|
+
# Format options
|
|
599
|
+
for p in [p_scan, p_cs]:
|
|
600
|
+
choices = ['plain', 'cyclonedx', 'spdxlite', 'csv']
|
|
601
|
+
if p is p_cs:
|
|
602
|
+
choices.append('raw')
|
|
603
|
+
|
|
604
|
+
p.add_argument(
|
|
605
|
+
'--format',
|
|
606
|
+
'-f',
|
|
607
|
+
type=str,
|
|
608
|
+
choices=choices,
|
|
609
|
+
default='plain',
|
|
610
|
+
help='Result output format (optional - default: plain)',
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
# Scanoss settings options
|
|
614
|
+
for p in [p_folder_scan, p_scan, p_wfp, p_folder_hash]:
|
|
615
|
+
p.add_argument(
|
|
616
|
+
'--settings',
|
|
617
|
+
'-st',
|
|
618
|
+
type=str,
|
|
619
|
+
help='Settings file to use for scanning (optional - default scanoss.json)',
|
|
620
|
+
)
|
|
621
|
+
p.add_argument(
|
|
622
|
+
'--skip-settings-file',
|
|
623
|
+
'-stf',
|
|
624
|
+
action='store_true',
|
|
625
|
+
help='Skip default settings file (scanoss.json) if it exists',
|
|
626
|
+
)
|
|
627
|
+
|
|
494
628
|
for p in [p_copyleft, p_undeclared]:
|
|
495
629
|
p.add_argument('-i', '--input', nargs='?', help='Path to results file')
|
|
496
630
|
p.add_argument(
|
|
@@ -505,7 +639,7 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
505
639
|
p.add_argument('-s', '--status', type=str, help='Save summary data into Markdown file')
|
|
506
640
|
|
|
507
641
|
# Global Scan command options
|
|
508
|
-
for p in [p_scan]:
|
|
642
|
+
for p in [p_scan, p_cs]:
|
|
509
643
|
p.add_argument(
|
|
510
644
|
'--apiurl', type=str, help='SCANOSS API URL (optional - default: https://api.osskb.org/scan/direct)'
|
|
511
645
|
)
|
|
@@ -514,9 +648,9 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
514
648
|
# Global Scan/Fingerprint filter options
|
|
515
649
|
for p in [p_scan, p_wfp]:
|
|
516
650
|
p.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints')
|
|
517
|
-
p.add_argument('--all-extensions', action='store_true', help='Fingerprint all file extensions')
|
|
518
|
-
p.add_argument('--all-folders', action='store_true', help='Fingerprint all folders')
|
|
519
|
-
p.add_argument('--all-hidden', action='store_true', help='Fingerprint all hidden files/folders')
|
|
651
|
+
p.add_argument('--all-extensions', action='store_true', help='Fingerprint all file extensions/types...')
|
|
652
|
+
p.add_argument('--all-folders', action='store_true', help='Fingerprint all folders...')
|
|
653
|
+
p.add_argument('--all-hidden', action='store_true', help='Fingerprint all hidden files/folders...')
|
|
520
654
|
p.add_argument('--hpsm', '-H', action='store_true', help='Use High Precision Snippet Matching algorithm.')
|
|
521
655
|
p.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets')
|
|
522
656
|
p.add_argument('--skip-extension', '-E', type=str, action='append', help='File Extension to skip.')
|
|
@@ -533,7 +667,17 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
533
667
|
p.add_argument('--strip-snippet', '-N', type=str, action='append', help='Strip Snippet ID string from WFP.')
|
|
534
668
|
|
|
535
669
|
# Global Scan/GRPC options
|
|
536
|
-
for p in [
|
|
670
|
+
for p in [
|
|
671
|
+
p_scan,
|
|
672
|
+
c_crypto,
|
|
673
|
+
c_vulns,
|
|
674
|
+
c_search,
|
|
675
|
+
c_versions,
|
|
676
|
+
c_semgrep,
|
|
677
|
+
c_provenance,
|
|
678
|
+
p_folder_scan,
|
|
679
|
+
p_cs,
|
|
680
|
+
]:
|
|
537
681
|
p.add_argument(
|
|
538
682
|
'--key', '-k', type=str, help='SCANOSS API Key token (optional - not required for default OSSKB URL)'
|
|
539
683
|
)
|
|
@@ -559,7 +703,7 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
559
703
|
)
|
|
560
704
|
|
|
561
705
|
# Global GRPC options
|
|
562
|
-
for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]:
|
|
706
|
+
for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance, p_folder_scan, p_cs]:
|
|
563
707
|
p.add_argument(
|
|
564
708
|
'--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)'
|
|
565
709
|
)
|
|
@@ -570,10 +714,26 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
570
714
|
'Can also use the environment variable "grcp_proxy=<ip>:<port>"',
|
|
571
715
|
)
|
|
572
716
|
p.add_argument(
|
|
573
|
-
'--header',
|
|
717
|
+
'--header',
|
|
718
|
+
'-hdr',
|
|
574
719
|
action='append', # This allows multiple -H flags
|
|
575
720
|
type=str,
|
|
576
|
-
help='Headers to be sent on request (e.g., -hdr "Name: Value") - can be used multiple times'
|
|
721
|
+
help='Headers to be sent on request (e.g., -hdr "Name: Value") - can be used multiple times',
|
|
722
|
+
)
|
|
723
|
+
|
|
724
|
+
# Syft options
|
|
725
|
+
for p in [p_cs, p_dep]:
|
|
726
|
+
p.add_argument(
|
|
727
|
+
'--syft-command',
|
|
728
|
+
type=str,
|
|
729
|
+
help='Syft command and path if required (optional - default syft).',
|
|
730
|
+
default=DEFAULT_SYFT_COMMAND,
|
|
731
|
+
)
|
|
732
|
+
p.add_argument(
|
|
733
|
+
'--syft-timeout',
|
|
734
|
+
type=int,
|
|
735
|
+
default=DEFAULT_SYFT_TIMEOUT,
|
|
736
|
+
help='Timeout (in seconds) for syft to complete (optional - default 600)',
|
|
577
737
|
)
|
|
578
738
|
|
|
579
739
|
# Help/Trace command options
|
|
@@ -594,7 +754,10 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
594
754
|
p_results,
|
|
595
755
|
p_undeclared,
|
|
596
756
|
p_copyleft,
|
|
597
|
-
c_provenance
|
|
757
|
+
c_provenance,
|
|
758
|
+
p_folder_scan,
|
|
759
|
+
p_folder_hash,
|
|
760
|
+
p_cs,
|
|
598
761
|
]:
|
|
599
762
|
p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages')
|
|
600
763
|
p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts')
|
|
@@ -607,12 +770,12 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
607
770
|
if not args.subparser:
|
|
608
771
|
parser.print_help() # No sub command subcommand, print general help
|
|
609
772
|
sys.exit(1)
|
|
610
|
-
elif (
|
|
611
|
-
args.subparser in {'utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins'} and not args.subparsercmd):
|
|
773
|
+
elif (args.subparser in ('utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins')) and not args.subparsercmd:
|
|
612
774
|
parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed
|
|
613
775
|
sys.exit(1)
|
|
614
776
|
args.func(parser, args) # Execute the function associated with the sub-command
|
|
615
777
|
|
|
778
|
+
|
|
616
779
|
def ver(*_):
|
|
617
780
|
"""
|
|
618
781
|
Run the "ver" sub-command
|
|
@@ -691,7 +854,7 @@ def wfp(parser, args):
|
|
|
691
854
|
if not args.skip_settings_file:
|
|
692
855
|
scan_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet)
|
|
693
856
|
try:
|
|
694
|
-
scan_settings.load_json_file(args.settings)
|
|
857
|
+
scan_settings.load_json_file(args.settings, args.scan_dir)
|
|
695
858
|
except ScanossSettingsError as e:
|
|
696
859
|
print_stderr(f'Error: {e}')
|
|
697
860
|
sys.exit(1)
|
|
@@ -766,7 +929,7 @@ def get_scan_options(args):
|
|
|
766
929
|
return scan_options
|
|
767
930
|
|
|
768
931
|
|
|
769
|
-
def scan(parser, args):
|
|
932
|
+
def scan(parser, args): # noqa: PLR0912, PLR0915
|
|
770
933
|
"""
|
|
771
934
|
Run the "scan" sub-command
|
|
772
935
|
Parameters
|
|
@@ -861,7 +1024,7 @@ def scan(parser, args): # noqa: PLR0912, PLR0915
|
|
|
861
1024
|
if flags:
|
|
862
1025
|
print_stderr(f'Using flags {flags}...')
|
|
863
1026
|
elif not args.quiet:
|
|
864
|
-
if args.timeout <
|
|
1027
|
+
if args.timeout < MIN_TIMEOUT:
|
|
865
1028
|
print_stderr(f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.')
|
|
866
1029
|
if args.retry < 0:
|
|
867
1030
|
print_stderr(f'POST retry (--retry) too small: {args.retry}. Reverting to default.')
|
|
@@ -910,7 +1073,7 @@ def scan(parser, args): # noqa: PLR0912, PLR0915
|
|
|
910
1073
|
strip_hpsm_ids=args.strip_hpsm,
|
|
911
1074
|
strip_snippet_ids=args.strip_snippet,
|
|
912
1075
|
scan_settings=scan_settings,
|
|
913
|
-
req_headers=
|
|
1076
|
+
req_headers=process_req_headers(args.header),
|
|
914
1077
|
)
|
|
915
1078
|
if args.wfp:
|
|
916
1079
|
if not scanner.is_file_or_snippet_scan():
|
|
@@ -980,12 +1143,18 @@ def dependency(parser, args):
|
|
|
980
1143
|
args: Namespace
|
|
981
1144
|
Parsed arguments
|
|
982
1145
|
"""
|
|
983
|
-
if not args.
|
|
984
|
-
print_stderr('Please specify a file/folder')
|
|
1146
|
+
if not args.scan_loc and not args.container:
|
|
1147
|
+
print_stderr('Please specify a file/folder or container image')
|
|
985
1148
|
parser.parse_args([args.subparser, '-h'])
|
|
986
1149
|
sys.exit(1)
|
|
987
|
-
|
|
988
|
-
|
|
1150
|
+
|
|
1151
|
+
# Workaround to return syft scan results converted to our dependency output format
|
|
1152
|
+
if args.container:
|
|
1153
|
+
args.scan_loc = args.container
|
|
1154
|
+
return container_scan(parser, args, only_interim_results=True)
|
|
1155
|
+
|
|
1156
|
+
if not os.path.exists(args.scan_loc):
|
|
1157
|
+
print_stderr(f'Error: File or folder specified does not exist: {args.scan_loc}.')
|
|
989
1158
|
sys.exit(1)
|
|
990
1159
|
scan_output: str = None
|
|
991
1160
|
if args.output:
|
|
@@ -995,7 +1164,7 @@ def dependency(parser, args):
|
|
|
995
1164
|
sc_deps = ScancodeDeps(
|
|
996
1165
|
debug=args.debug, quiet=args.quiet, trace=args.trace, sc_command=args.sc_command, timeout=args.sc_timeout
|
|
997
1166
|
)
|
|
998
|
-
if not sc_deps.get_dependencies(what_to_scan=args.
|
|
1167
|
+
if not sc_deps.get_dependencies(what_to_scan=args.scan_loc, result_output=scan_output):
|
|
999
1168
|
sys.exit(1)
|
|
1000
1169
|
|
|
1001
1170
|
|
|
@@ -1122,7 +1291,7 @@ def utils_certloc(*_):
|
|
|
1122
1291
|
print(f'CA Cert File: {certifi.where()}')
|
|
1123
1292
|
|
|
1124
1293
|
|
|
1125
|
-
def utils_cert_download(_, args):
|
|
1294
|
+
def utils_cert_download(_, args): # pylint: disable=PLR0912 # noqa: PLR0912
|
|
1126
1295
|
"""
|
|
1127
1296
|
Run the "utils cert-download" sub-command
|
|
1128
1297
|
:param _: ignore/unused
|
|
@@ -1149,13 +1318,14 @@ def utils_cert_download(_, args): # pylint: disable=PLR0912 # noqa: PLR0912
|
|
|
1149
1318
|
certs = conn.get_peer_cert_chain()
|
|
1150
1319
|
for index, cert in enumerate(certs):
|
|
1151
1320
|
cert_components = dict(cert.get_subject().get_components())
|
|
1152
|
-
if sys.version_info[0] >=
|
|
1321
|
+
if sys.version_info[0] >= PYTHON_MAJOR_VERSION:
|
|
1153
1322
|
cn = cert_components.get(b'CN')
|
|
1154
1323
|
else:
|
|
1324
|
+
# Fallback for Python versions less than PYTHON_MAJOR_VERSION
|
|
1155
1325
|
cn = cert_components.get('CN')
|
|
1156
1326
|
if not args.quiet:
|
|
1157
1327
|
print_stderr(f'Certificate {index} - CN: {cn}')
|
|
1158
|
-
if sys.version_info[0] >=
|
|
1328
|
+
if sys.version_info[0] >= PYTHON_MAJOR_VERSION:
|
|
1159
1329
|
print(
|
|
1160
1330
|
(crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode('utf-8')).strip(), file=file
|
|
1161
1331
|
) # Print the downloaded PEM certificate
|
|
@@ -1204,7 +1374,7 @@ def get_pac_file(pac: str):
|
|
|
1204
1374
|
if pac == 'auto':
|
|
1205
1375
|
pac_file = pypac.get_pac() # try to determine the PAC file
|
|
1206
1376
|
elif pac.startswith('file://'):
|
|
1207
|
-
pac_local = pac
|
|
1377
|
+
pac_local = pac[7:] # Remove 'file://' prefix (7 characters)
|
|
1208
1378
|
if not os.path.exists(pac_local):
|
|
1209
1379
|
print_stderr(f'Error: PAC file does not exist: {pac_local}.')
|
|
1210
1380
|
sys.exit(1)
|
|
@@ -1248,7 +1418,7 @@ def comp_crypto(parser, args):
|
|
|
1248
1418
|
grpc_proxy=args.grpc_proxy,
|
|
1249
1419
|
pac=pac_file,
|
|
1250
1420
|
timeout=args.timeout,
|
|
1251
|
-
req_headers=
|
|
1421
|
+
req_headers=process_req_headers(args.header),
|
|
1252
1422
|
)
|
|
1253
1423
|
if not comps.get_crypto_details(args.input, args.purl, args.output):
|
|
1254
1424
|
sys.exit(1)
|
|
@@ -1406,6 +1576,7 @@ def comp_versions(parser, args):
|
|
|
1406
1576
|
if not comps.get_component_versions(args.output, json_file=args.input, purl=args.purl, limit=args.limit):
|
|
1407
1577
|
sys.exit(1)
|
|
1408
1578
|
|
|
1579
|
+
|
|
1409
1580
|
def comp_provenance(parser, args):
|
|
1410
1581
|
"""
|
|
1411
1582
|
Run the "component semgrep" sub-command
|
|
@@ -1424,12 +1595,23 @@ def comp_provenance(parser, args):
|
|
|
1424
1595
|
print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.')
|
|
1425
1596
|
sys.exit(1)
|
|
1426
1597
|
pac_file = get_pac_file(args.pac)
|
|
1427
|
-
comps = Components(
|
|
1428
|
-
|
|
1429
|
-
|
|
1598
|
+
comps = Components(
|
|
1599
|
+
debug=args.debug,
|
|
1600
|
+
trace=args.trace,
|
|
1601
|
+
quiet=args.quiet,
|
|
1602
|
+
grpc_url=args.api2url,
|
|
1603
|
+
api_key=args.key,
|
|
1604
|
+
ca_cert=args.ca_cert,
|
|
1605
|
+
proxy=args.proxy,
|
|
1606
|
+
grpc_proxy=args.grpc_proxy,
|
|
1607
|
+
pac=pac_file,
|
|
1608
|
+
timeout=args.timeout,
|
|
1609
|
+
req_headers=process_req_headers(args.header),
|
|
1610
|
+
)
|
|
1430
1611
|
if not comps.get_provenance_details(args.input, args.purl, args.output):
|
|
1431
1612
|
sys.exit(1)
|
|
1432
1613
|
|
|
1614
|
+
|
|
1433
1615
|
def results(parser, args):
|
|
1434
1616
|
"""
|
|
1435
1617
|
Run the "results" sub-command
|
|
@@ -1488,13 +1670,135 @@ def process_req_headers(headers_array: List[str]) -> dict:
|
|
|
1488
1670
|
dict_headers = {}
|
|
1489
1671
|
for header_str in headers_array:
|
|
1490
1672
|
# Split each "Name: Value" header
|
|
1491
|
-
parts = header_str.split(
|
|
1673
|
+
parts = header_str.split(':', 1)
|
|
1492
1674
|
if len(parts) == HEADER_PARTS_COUNT:
|
|
1493
1675
|
name = parts[0].strip()
|
|
1494
1676
|
value = parts[1].strip()
|
|
1495
1677
|
dict_headers[name] = value
|
|
1496
1678
|
return dict_headers
|
|
1497
1679
|
|
|
1680
|
+
|
|
1681
|
+
def folder_hashing_scan(parser, args):
|
|
1682
|
+
"""Run the "folder-scan" sub-command
|
|
1683
|
+
|
|
1684
|
+
Args:
|
|
1685
|
+
parser (ArgumentParser): command line parser object
|
|
1686
|
+
args (Namespace): Parsed arguments
|
|
1687
|
+
"""
|
|
1688
|
+
try:
|
|
1689
|
+
if not args.scan_dir:
|
|
1690
|
+
print_stderr('ERROR: Please specify a directory to scan')
|
|
1691
|
+
parser.parse_args([args.subparser, '-h'])
|
|
1692
|
+
sys.exit(1)
|
|
1693
|
+
|
|
1694
|
+
if not os.path.exists(args.scan_dir) or not os.path.isdir(args.scan_dir):
|
|
1695
|
+
print_stderr(f'ERROR: The specified directory {args.scan_dir} does not exist')
|
|
1696
|
+
sys.exit(1)
|
|
1697
|
+
|
|
1698
|
+
scanner_config = create_scanner_config_from_args(args)
|
|
1699
|
+
scanoss_settings = get_scanoss_settings_from_args(args)
|
|
1700
|
+
grpc_config = create_grpc_config_from_args(args)
|
|
1701
|
+
|
|
1702
|
+
client = ScanossGrpc(**asdict(grpc_config))
|
|
1703
|
+
|
|
1704
|
+
scanner = ScannerHFH(
|
|
1705
|
+
scan_dir=args.scan_dir,
|
|
1706
|
+
config=scanner_config,
|
|
1707
|
+
client=client,
|
|
1708
|
+
scanoss_settings=scanoss_settings,
|
|
1709
|
+
)
|
|
1710
|
+
|
|
1711
|
+
scanner.best_match = args.best_match
|
|
1712
|
+
scanner.threshold = args.threshold
|
|
1713
|
+
|
|
1714
|
+
scanner.scan()
|
|
1715
|
+
scanner.present(output_file=args.output, output_format=args.format)
|
|
1716
|
+
except ScanossGrpcError as e:
|
|
1717
|
+
print_stderr(f'ERROR: {e}')
|
|
1718
|
+
sys.exit(1)
|
|
1719
|
+
|
|
1720
|
+
|
|
1721
|
+
def folder_hash(parser, args):
|
|
1722
|
+
"""Run the "folder-hash" sub-command
|
|
1723
|
+
|
|
1724
|
+
Args:
|
|
1725
|
+
parser (ArgumentParser): command line parser object
|
|
1726
|
+
args (Namespace): Parsed arguments
|
|
1727
|
+
"""
|
|
1728
|
+
try:
|
|
1729
|
+
if not args.scan_dir:
|
|
1730
|
+
print_stderr('ERROR: Please specify a directory to scan')
|
|
1731
|
+
parser.parse_args([args.subparser, '-h'])
|
|
1732
|
+
sys.exit(1)
|
|
1733
|
+
|
|
1734
|
+
if not os.path.exists(args.scan_dir) or not os.path.isdir(args.scan_dir):
|
|
1735
|
+
print_stderr(f'ERROR: The specified directory {args.scan_dir} does not exist')
|
|
1736
|
+
sys.exit(1)
|
|
1737
|
+
|
|
1738
|
+
folder_hasher_config = create_folder_hasher_config_from_args(args)
|
|
1739
|
+
scanoss_settings = get_scanoss_settings_from_args(args)
|
|
1740
|
+
|
|
1741
|
+
folder_hasher = FolderHasher(
|
|
1742
|
+
scan_dir=args.scan_dir,
|
|
1743
|
+
config=folder_hasher_config,
|
|
1744
|
+
scanoss_settings=scanoss_settings,
|
|
1745
|
+
)
|
|
1746
|
+
|
|
1747
|
+
folder_hasher.hash_directory(args.scan_dir)
|
|
1748
|
+
folder_hasher.present(output_file=args.output, output_format=args.format)
|
|
1749
|
+
except Exception as e:
|
|
1750
|
+
print_stderr(f'ERROR: {e}')
|
|
1751
|
+
sys.exit(1)
|
|
1752
|
+
|
|
1753
|
+
|
|
1754
|
+
def container_scan(parser, args, only_interim_results: bool = False):
|
|
1755
|
+
"""
|
|
1756
|
+
Run the "container-scan" sub-command
|
|
1757
|
+
Parameters
|
|
1758
|
+
----------
|
|
1759
|
+
parser: ArgumentParser
|
|
1760
|
+
command line parser object
|
|
1761
|
+
args: Namespace
|
|
1762
|
+
Parsed arguments
|
|
1763
|
+
"""
|
|
1764
|
+
if not args.scan_loc:
|
|
1765
|
+
print_stderr(
|
|
1766
|
+
'Please specify a container image, Docker tar, OCI tar, OCI directory, SIF Container, or directory to scan'
|
|
1767
|
+
)
|
|
1768
|
+
parser.parse_args([args.subparser, '-h'])
|
|
1769
|
+
sys.exit(1)
|
|
1770
|
+
|
|
1771
|
+
try:
|
|
1772
|
+
config = create_container_scanner_config_from_args(args)
|
|
1773
|
+
config.only_interim_results = only_interim_results
|
|
1774
|
+
container_scanner = ContainerScanner(
|
|
1775
|
+
config=config,
|
|
1776
|
+
what_to_scan=args.scan_loc,
|
|
1777
|
+
)
|
|
1778
|
+
|
|
1779
|
+
container_scanner.scan()
|
|
1780
|
+
if only_interim_results:
|
|
1781
|
+
container_scanner.present(output_file=config.output, output_format='raw')
|
|
1782
|
+
else:
|
|
1783
|
+
container_scanner.decorate_scan_results_with_dependencies()
|
|
1784
|
+
container_scanner.present(output_file=config.output, output_format=config.format)
|
|
1785
|
+
except Exception as e:
|
|
1786
|
+
print_stderr(f'ERROR: {e}')
|
|
1787
|
+
sys.exit(1)
|
|
1788
|
+
|
|
1789
|
+
|
|
1790
|
+
def get_scanoss_settings_from_args(args):
|
|
1791
|
+
scanoss_settings = None
|
|
1792
|
+
if not args.skip_settings_file:
|
|
1793
|
+
scanoss_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet)
|
|
1794
|
+
try:
|
|
1795
|
+
scanoss_settings.load_json_file(args.settings, args.scan_dir).set_file_type('new').set_scan_type('identify')
|
|
1796
|
+
except ScanossSettingsError as e:
|
|
1797
|
+
print_stderr(f'Error: {e}')
|
|
1798
|
+
sys.exit(1)
|
|
1799
|
+
return scanoss_settings
|
|
1800
|
+
|
|
1801
|
+
|
|
1498
1802
|
def main():
|
|
1499
1803
|
"""
|
|
1500
1804
|
Run the ScanOSS CLI
|