scanoss 1.20.6__py3-none-any.whl → 1.23.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- protoc_gen_swagger/options/annotations_pb2.py +9 -12
- protoc_gen_swagger/options/annotations_pb2_grpc.py +1 -1
- protoc_gen_swagger/options/openapiv2_pb2.py +96 -98
- protoc_gen_swagger/options/openapiv2_pb2_grpc.py +1 -1
- scanoss/__init__.py +1 -1
- scanoss/api/common/v2/scanoss_common_pb2.py +20 -18
- scanoss/api/common/v2/scanoss_common_pb2_grpc.py +1 -1
- scanoss/api/components/v2/scanoss_components_pb2.py +38 -48
- scanoss/api/components/v2/scanoss_components_pb2_grpc.py +96 -142
- scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py +42 -22
- scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py +185 -75
- scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +32 -30
- scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py +83 -75
- scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py +49 -0
- scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py +142 -0
- scanoss/api/scanning/v2/scanoss_scanning_pb2.py +20 -10
- scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +70 -40
- scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py +18 -22
- scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py +49 -71
- scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +27 -37
- scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py +72 -109
- scanoss/cli.py +393 -84
- scanoss/components.py +21 -11
- scanoss/constants.py +12 -0
- scanoss/data/build_date.txt +1 -1
- scanoss/file_filters.py +272 -57
- scanoss/results.py +92 -109
- scanoss/scanners/__init__.py +23 -0
- scanoss/scanners/container_scanner.py +474 -0
- scanoss/scanners/folder_hasher.py +302 -0
- scanoss/scanners/scanner_config.py +73 -0
- scanoss/scanners/scanner_hfh.py +173 -0
- scanoss/scanoss_settings.py +9 -5
- scanoss/scanossbase.py +9 -3
- scanoss/scanossgrpc.py +143 -18
- scanoss/threadedscanning.py +6 -6
- scanoss/utils/abstract_presenter.py +103 -0
- scanoss/utils/crc64.py +96 -0
- scanoss/utils/simhash.py +198 -0
- {scanoss-1.20.6.dist-info → scanoss-1.23.0.dist-info}/METADATA +2 -1
- scanoss-1.23.0.dist-info/RECORD +83 -0
- {scanoss-1.20.6.dist-info → scanoss-1.23.0.dist-info}/WHEEL +1 -1
- scanoss/api/provenance/v2/scanoss_provenance_pb2.py +0 -42
- scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py +0 -108
- scanoss-1.20.6.dist-info/RECORD +0 -74
- /scanoss/api/{provenance → geoprovenance}/__init__.py +0 -0
- /scanoss/api/{provenance → geoprovenance}/v2/__init__.py +0 -0
- {scanoss-1.20.6.dist-info → scanoss-1.23.0.dist-info}/entry_points.txt +0 -0
- {scanoss-1.20.6.dist-info → scanoss-1.23.0.dist-info}/licenses/LICENSE +0 -0
- {scanoss-1.20.6.dist-info → scanoss-1.23.0.dist-info}/top_level.txt +0 -0
scanoss/cli.py
CHANGED
|
@@ -25,13 +25,37 @@ SPDX-License-Identifier: MIT
|
|
|
25
25
|
import argparse
|
|
26
26
|
import os
|
|
27
27
|
import sys
|
|
28
|
+
from dataclasses import asdict
|
|
28
29
|
from pathlib import Path
|
|
30
|
+
from typing import List
|
|
29
31
|
|
|
30
32
|
import pypac
|
|
31
|
-
|
|
33
|
+
|
|
34
|
+
from scanoss.scanners.container_scanner import (
|
|
35
|
+
DEFAULT_SYFT_COMMAND,
|
|
36
|
+
DEFAULT_SYFT_TIMEOUT,
|
|
37
|
+
ContainerScanner,
|
|
38
|
+
create_container_scanner_config_from_args,
|
|
39
|
+
)
|
|
40
|
+
from scanoss.scanners.folder_hasher import (
|
|
41
|
+
FolderHasher,
|
|
42
|
+
create_folder_hasher_config_from_args,
|
|
43
|
+
)
|
|
44
|
+
from scanoss.scanossgrpc import (
|
|
45
|
+
ScanossGrpc,
|
|
46
|
+
ScanossGrpcError,
|
|
47
|
+
create_grpc_config_from_args,
|
|
48
|
+
)
|
|
32
49
|
|
|
33
50
|
from . import __version__
|
|
34
51
|
from .components import Components
|
|
52
|
+
from .constants import (
|
|
53
|
+
DEFAULT_POST_SIZE,
|
|
54
|
+
DEFAULT_RETRY,
|
|
55
|
+
DEFAULT_TIMEOUT,
|
|
56
|
+
MIN_TIMEOUT,
|
|
57
|
+
PYTHON_MAJOR_VERSION,
|
|
58
|
+
)
|
|
35
59
|
from .csvoutput import CsvOutput
|
|
36
60
|
from .cyclonedx import CycloneDx
|
|
37
61
|
from .filecount import FileCount
|
|
@@ -40,19 +64,17 @@ from .inspection.undeclared_component import UndeclaredComponent
|
|
|
40
64
|
from .results import Results
|
|
41
65
|
from .scancodedeps import ScancodeDeps
|
|
42
66
|
from .scanner import FAST_WINNOWING, Scanner
|
|
67
|
+
from .scanners.scanner_config import create_scanner_config_from_args
|
|
68
|
+
from .scanners.scanner_hfh import ScannerHFH
|
|
43
69
|
from .scanoss_settings import ScanossSettings, ScanossSettingsError
|
|
44
70
|
from .scantype import ScanType
|
|
45
71
|
from .spdxlite import SpdxLite
|
|
46
72
|
from .threadeddependencies import SCOPE
|
|
47
73
|
from .utils.file import validate_json_file
|
|
48
74
|
|
|
49
|
-
DEFAULT_POST_SIZE = 32
|
|
50
|
-
DEFAULT_TIMEOUT = 180
|
|
51
|
-
MIN_TIMEOUT_VALUE = 5
|
|
52
|
-
DEFAULT_RETRY = 5
|
|
53
|
-
PYTHON3_OR_LATER = 3
|
|
54
75
|
HEADER_PARTS_COUNT = 2
|
|
55
76
|
|
|
77
|
+
|
|
56
78
|
def print_stderr(*args, **kwargs):
|
|
57
79
|
"""
|
|
58
80
|
Print the given message to STDERR
|
|
@@ -60,7 +82,7 @@ def print_stderr(*args, **kwargs):
|
|
|
60
82
|
print(*args, file=sys.stderr, **kwargs)
|
|
61
83
|
|
|
62
84
|
|
|
63
|
-
def setup_args() -> None: # noqa: PLR0915
|
|
85
|
+
def setup_args() -> None: # noqa: PLR0912, PLR0915
|
|
64
86
|
"""
|
|
65
87
|
Setup all the command line arguments for processing
|
|
66
88
|
"""
|
|
@@ -95,14 +117,6 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
95
117
|
p_scan.add_argument('--files', '-e', type=str, nargs='*', help='List of files to scan.')
|
|
96
118
|
p_scan.add_argument('--identify', '-i', type=str, help='Scan and identify components in SBOM file')
|
|
97
119
|
p_scan.add_argument('--ignore', '-n', type=str, help='Ignore components specified in the SBOM file')
|
|
98
|
-
p_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
99
|
-
p_scan.add_argument(
|
|
100
|
-
'--format',
|
|
101
|
-
'-f',
|
|
102
|
-
type=str,
|
|
103
|
-
choices=['plain', 'cyclonedx', 'spdxlite', 'csv'],
|
|
104
|
-
help='Result output format (optional - default: plain)',
|
|
105
|
-
)
|
|
106
120
|
p_scan.add_argument(
|
|
107
121
|
'--threads', '-T', type=int, default=5, help='Number of threads to use while scanning (optional - default 5)'
|
|
108
122
|
)
|
|
@@ -132,15 +146,17 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
132
146
|
help='Timeout (in seconds) for API communication (optional - default 180)',
|
|
133
147
|
)
|
|
134
148
|
p_scan.add_argument(
|
|
135
|
-
'--retry',
|
|
136
|
-
|
|
149
|
+
'--retry',
|
|
150
|
+
'-R',
|
|
151
|
+
type=int,
|
|
152
|
+
default=DEFAULT_RETRY,
|
|
153
|
+
help='Retry limit for API communication (optional - default 5)',
|
|
137
154
|
)
|
|
138
155
|
p_scan.add_argument('--no-wfp-output', action='store_true', help='Skip WFP file generation')
|
|
139
156
|
p_scan.add_argument('--dependencies', '-D', action='store_true', help='Add Dependency scanning')
|
|
140
157
|
p_scan.add_argument('--dependencies-only', action='store_true', help='Run Dependency scanning only')
|
|
141
158
|
p_scan.add_argument(
|
|
142
|
-
'--sc-command', type=str,
|
|
143
|
-
help='Scancode command and path if required (optional - default scancode).'
|
|
159
|
+
'--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).'
|
|
144
160
|
)
|
|
145
161
|
p_scan.add_argument(
|
|
146
162
|
'--sc-timeout',
|
|
@@ -153,18 +169,6 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
153
169
|
)
|
|
154
170
|
p_scan.add_argument('--dep-scope-inc', '-dsi', type=str, help='Include dependencies with declared scopes')
|
|
155
171
|
p_scan.add_argument('--dep-scope-exc', '-dse', type=str, help='Exclude dependencies with declared scopes')
|
|
156
|
-
p_scan.add_argument(
|
|
157
|
-
'--settings',
|
|
158
|
-
'-st',
|
|
159
|
-
type=str,
|
|
160
|
-
help='Settings file to use for scanning (optional - default scanoss.json)',
|
|
161
|
-
)
|
|
162
|
-
p_scan.add_argument(
|
|
163
|
-
'--skip-settings-file',
|
|
164
|
-
'-stf',
|
|
165
|
-
action='store_true',
|
|
166
|
-
help='Skip default settings file (scanoss.json) if it exists',
|
|
167
|
-
)
|
|
168
172
|
|
|
169
173
|
# Sub-command: fingerprint
|
|
170
174
|
p_wfp = subparsers.add_parser(
|
|
@@ -182,19 +186,6 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
182
186
|
type=str,
|
|
183
187
|
help='Fingerprint the file contents supplied via STDIN (optional)',
|
|
184
188
|
)
|
|
185
|
-
p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
186
|
-
p_wfp.add_argument(
|
|
187
|
-
'--settings',
|
|
188
|
-
'-st',
|
|
189
|
-
type=str,
|
|
190
|
-
help='Settings file to use for fingerprinting (optional - default scanoss.json)',
|
|
191
|
-
)
|
|
192
|
-
p_wfp.add_argument(
|
|
193
|
-
'--skip-settings-file',
|
|
194
|
-
'-stf',
|
|
195
|
-
action='store_true',
|
|
196
|
-
help='Skip default settings file (scanoss.json) if it exists',
|
|
197
|
-
)
|
|
198
189
|
|
|
199
190
|
# Sub-command: dependency
|
|
200
191
|
p_dep = subparsers.add_parser(
|
|
@@ -203,9 +194,13 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
203
194
|
description=f'Produce dependency file summary: {__version__}',
|
|
204
195
|
help='Scan source code for dependencies, but do not decorate them',
|
|
205
196
|
)
|
|
206
|
-
p_dep.
|
|
207
|
-
p_dep.add_argument(
|
|
208
|
-
|
|
197
|
+
p_dep.add_argument('scan_loc', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan')
|
|
198
|
+
p_dep.add_argument(
|
|
199
|
+
'--container',
|
|
200
|
+
type=str,
|
|
201
|
+
help='Container image to scan. Supports yourrepo/yourimage:tag, Docker tar, '
|
|
202
|
+
'OCI tar, OCI directory, SIF Container, or generic filesystem directory.',
|
|
203
|
+
)
|
|
209
204
|
p_dep.add_argument(
|
|
210
205
|
'--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).'
|
|
211
206
|
)
|
|
@@ -215,6 +210,40 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
215
210
|
default=600,
|
|
216
211
|
help='Timeout (in seconds) for scancode to complete (optional - default 600)',
|
|
217
212
|
)
|
|
213
|
+
p_dep.set_defaults(func=dependency)
|
|
214
|
+
|
|
215
|
+
# Container scan sub-command
|
|
216
|
+
p_cs = subparsers.add_parser(
|
|
217
|
+
'container-scan',
|
|
218
|
+
aliases=['cs'],
|
|
219
|
+
description=f'Analyse/scan the given container image: {__version__}',
|
|
220
|
+
help='Scan container image',
|
|
221
|
+
)
|
|
222
|
+
p_cs.add_argument(
|
|
223
|
+
'scan_loc',
|
|
224
|
+
metavar='IMAGE',
|
|
225
|
+
type=str,
|
|
226
|
+
nargs='?',
|
|
227
|
+
help=(
|
|
228
|
+
'Container image to scan. Supports yourrepo/yourimage:tag, Docker tar, '
|
|
229
|
+
'OCI tar, OCI directory, SIF Container, or generic filesystem directory.'
|
|
230
|
+
),
|
|
231
|
+
)
|
|
232
|
+
p_cs.add_argument(
|
|
233
|
+
'--retry',
|
|
234
|
+
'-R',
|
|
235
|
+
type=int,
|
|
236
|
+
default=DEFAULT_RETRY,
|
|
237
|
+
help='Retry limit for API communication (optional - default 5)',
|
|
238
|
+
)
|
|
239
|
+
p_cs.add_argument(
|
|
240
|
+
'--timeout',
|
|
241
|
+
'-M',
|
|
242
|
+
type=int,
|
|
243
|
+
default=DEFAULT_TIMEOUT,
|
|
244
|
+
help='Timeout (in seconds) for API communication (optional - default 180)',
|
|
245
|
+
)
|
|
246
|
+
p_cs.set_defaults(func=container_scan)
|
|
218
247
|
|
|
219
248
|
# Sub-command: file_count
|
|
220
249
|
p_fc = subparsers.add_parser(
|
|
@@ -225,7 +254,6 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
225
254
|
)
|
|
226
255
|
p_fc.set_defaults(func=file_count)
|
|
227
256
|
p_fc.add_argument('scan_dir', metavar='DIR', type=str, nargs='?', help='A folder to search')
|
|
228
|
-
p_fc.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
229
257
|
p_fc.add_argument('--all-hidden', action='store_true', help='Scan all hidden files/folders')
|
|
230
258
|
|
|
231
259
|
# Sub-command: convert
|
|
@@ -237,7 +265,6 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
237
265
|
)
|
|
238
266
|
p_cnv.set_defaults(func=convert)
|
|
239
267
|
p_cnv.add_argument('--input', '-i', type=str, required=True, help='Input file name')
|
|
240
|
-
p_cnv.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
241
268
|
p_cnv.add_argument(
|
|
242
269
|
'--format',
|
|
243
270
|
'-f',
|
|
@@ -296,8 +323,13 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
296
323
|
c_provenance = comp_sub.add_parser(
|
|
297
324
|
'provenance',
|
|
298
325
|
aliases=['prov', 'prv'],
|
|
299
|
-
description=f'Show Provenance findings: {__version__}',
|
|
300
|
-
help='Retrieve
|
|
326
|
+
description=f'Show GEO Provenance findings: {__version__}',
|
|
327
|
+
help='Retrieve geoprovenance for the given components',
|
|
328
|
+
)
|
|
329
|
+
c_provenance.add_argument(
|
|
330
|
+
'--origin',
|
|
331
|
+
action='store_true',
|
|
332
|
+
help='Retrieve geoprovenance using contributors origin (default: declared origin)',
|
|
301
333
|
)
|
|
302
334
|
c_provenance.set_defaults(func=comp_provenance)
|
|
303
335
|
|
|
@@ -333,9 +365,9 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
333
365
|
for p in [c_crypto, c_vulns, c_semgrep, c_provenance]:
|
|
334
366
|
p.add_argument('--purl', '-p', type=str, nargs='*', help='Package URL - PURL to process.')
|
|
335
367
|
p.add_argument('--input', '-i', type=str, help='Input file name')
|
|
368
|
+
|
|
336
369
|
# Common Component sub-command options
|
|
337
370
|
for p in [c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]:
|
|
338
|
-
p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
339
371
|
p.add_argument(
|
|
340
372
|
'--timeout',
|
|
341
373
|
'-M',
|
|
@@ -383,7 +415,6 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
383
415
|
p_c_dwnld.add_argument(
|
|
384
416
|
'--port', '-p', required=False, type=int, default=443, help='Server port number (default: 443).'
|
|
385
417
|
)
|
|
386
|
-
p_c_dwnld.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
387
418
|
|
|
388
419
|
# Utils Sub-command: utils pac-proxy
|
|
389
420
|
p_p_proxy = utils_sub.add_parser(
|
|
@@ -491,6 +522,114 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
491
522
|
)
|
|
492
523
|
p_undeclared.set_defaults(func=inspect_undeclared)
|
|
493
524
|
|
|
525
|
+
# Sub-command: folder-scan
|
|
526
|
+
p_folder_scan = subparsers.add_parser(
|
|
527
|
+
'folder-scan',
|
|
528
|
+
aliases=['fs'],
|
|
529
|
+
description=f'Scan the given directory using folder hashing: {__version__}',
|
|
530
|
+
help='Scan the given directory using folder hashing',
|
|
531
|
+
)
|
|
532
|
+
p_folder_scan.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='The root directory to scan')
|
|
533
|
+
p_folder_scan.add_argument(
|
|
534
|
+
'--timeout',
|
|
535
|
+
'-M',
|
|
536
|
+
type=int,
|
|
537
|
+
default=600,
|
|
538
|
+
help='Timeout (in seconds) for API communication (optional - default 600)',
|
|
539
|
+
)
|
|
540
|
+
p_folder_scan.add_argument(
|
|
541
|
+
'--format',
|
|
542
|
+
'-f',
|
|
543
|
+
type=str,
|
|
544
|
+
choices=['json'],
|
|
545
|
+
default='json',
|
|
546
|
+
help='Result output format (optional - default: json)',
|
|
547
|
+
)
|
|
548
|
+
p_folder_scan.add_argument(
|
|
549
|
+
'--best-match',
|
|
550
|
+
'-bm',
|
|
551
|
+
action='store_true',
|
|
552
|
+
default=False,
|
|
553
|
+
help='Enable best match mode (optional - default: False)',
|
|
554
|
+
)
|
|
555
|
+
p_folder_scan.add_argument(
|
|
556
|
+
'--threshold',
|
|
557
|
+
type=int,
|
|
558
|
+
choices=range(1, 101),
|
|
559
|
+
metavar='1-100',
|
|
560
|
+
default=100,
|
|
561
|
+
help='Threshold for result matching (optional - default: 100)',
|
|
562
|
+
)
|
|
563
|
+
p_folder_scan.set_defaults(func=folder_hashing_scan)
|
|
564
|
+
|
|
565
|
+
# Sub-command: folder-hash
|
|
566
|
+
p_folder_hash = subparsers.add_parser(
|
|
567
|
+
'folder-hash',
|
|
568
|
+
aliases=['fh'],
|
|
569
|
+
description=f'Produce a folder hash for the given directory: {__version__}',
|
|
570
|
+
help='Produce a folder hash for the given directory',
|
|
571
|
+
)
|
|
572
|
+
p_folder_hash.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan')
|
|
573
|
+
p_folder_hash.add_argument(
|
|
574
|
+
'--format',
|
|
575
|
+
'-f',
|
|
576
|
+
type=str,
|
|
577
|
+
choices=['json'],
|
|
578
|
+
default='json',
|
|
579
|
+
help='Result output format (optional - default: json)',
|
|
580
|
+
)
|
|
581
|
+
p_folder_hash.set_defaults(func=folder_hash)
|
|
582
|
+
|
|
583
|
+
# Output options
|
|
584
|
+
for p in [
|
|
585
|
+
p_scan,
|
|
586
|
+
p_cs,
|
|
587
|
+
p_wfp,
|
|
588
|
+
p_dep,
|
|
589
|
+
p_fc,
|
|
590
|
+
p_cnv,
|
|
591
|
+
c_crypto,
|
|
592
|
+
c_vulns,
|
|
593
|
+
c_search,
|
|
594
|
+
c_versions,
|
|
595
|
+
c_semgrep,
|
|
596
|
+
c_provenance,
|
|
597
|
+
p_c_dwnld,
|
|
598
|
+
p_folder_scan,
|
|
599
|
+
p_folder_hash,
|
|
600
|
+
]:
|
|
601
|
+
p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
|
|
602
|
+
|
|
603
|
+
# Format options
|
|
604
|
+
for p in [p_scan, p_cs]:
|
|
605
|
+
choices = ['plain', 'cyclonedx', 'spdxlite', 'csv']
|
|
606
|
+
if p is p_cs:
|
|
607
|
+
choices.append('raw')
|
|
608
|
+
|
|
609
|
+
p.add_argument(
|
|
610
|
+
'--format',
|
|
611
|
+
'-f',
|
|
612
|
+
type=str,
|
|
613
|
+
choices=choices,
|
|
614
|
+
default='plain',
|
|
615
|
+
help='Result output format (optional - default: plain)',
|
|
616
|
+
)
|
|
617
|
+
|
|
618
|
+
# Scanoss settings options
|
|
619
|
+
for p in [p_folder_scan, p_scan, p_wfp, p_folder_hash]:
|
|
620
|
+
p.add_argument(
|
|
621
|
+
'--settings',
|
|
622
|
+
'-st',
|
|
623
|
+
type=str,
|
|
624
|
+
help='Settings file to use for scanning (optional - default scanoss.json)',
|
|
625
|
+
)
|
|
626
|
+
p.add_argument(
|
|
627
|
+
'--skip-settings-file',
|
|
628
|
+
'-stf',
|
|
629
|
+
action='store_true',
|
|
630
|
+
help='Skip default settings file (scanoss.json) if it exists',
|
|
631
|
+
)
|
|
632
|
+
|
|
494
633
|
for p in [p_copyleft, p_undeclared]:
|
|
495
634
|
p.add_argument('-i', '--input', nargs='?', help='Path to results file')
|
|
496
635
|
p.add_argument(
|
|
@@ -505,7 +644,7 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
505
644
|
p.add_argument('-s', '--status', type=str, help='Save summary data into Markdown file')
|
|
506
645
|
|
|
507
646
|
# Global Scan command options
|
|
508
|
-
for p in [p_scan]:
|
|
647
|
+
for p in [p_scan, p_cs]:
|
|
509
648
|
p.add_argument(
|
|
510
649
|
'--apiurl', type=str, help='SCANOSS API URL (optional - default: https://api.osskb.org/scan/direct)'
|
|
511
650
|
)
|
|
@@ -514,9 +653,9 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
514
653
|
# Global Scan/Fingerprint filter options
|
|
515
654
|
for p in [p_scan, p_wfp]:
|
|
516
655
|
p.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints')
|
|
517
|
-
p.add_argument('--all-extensions', action='store_true', help='Fingerprint all file extensions')
|
|
518
|
-
p.add_argument('--all-folders', action='store_true', help='Fingerprint all folders')
|
|
519
|
-
p.add_argument('--all-hidden', action='store_true', help='Fingerprint all hidden files/folders')
|
|
656
|
+
p.add_argument('--all-extensions', action='store_true', help='Fingerprint all file extensions/types...')
|
|
657
|
+
p.add_argument('--all-folders', action='store_true', help='Fingerprint all folders...')
|
|
658
|
+
p.add_argument('--all-hidden', action='store_true', help='Fingerprint all hidden files/folders...')
|
|
520
659
|
p.add_argument('--hpsm', '-H', action='store_true', help='Use High Precision Snippet Matching algorithm.')
|
|
521
660
|
p.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets')
|
|
522
661
|
p.add_argument('--skip-extension', '-E', type=str, action='append', help='File Extension to skip.')
|
|
@@ -533,7 +672,17 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
533
672
|
p.add_argument('--strip-snippet', '-N', type=str, action='append', help='Strip Snippet ID string from WFP.')
|
|
534
673
|
|
|
535
674
|
# Global Scan/GRPC options
|
|
536
|
-
for p in [
|
|
675
|
+
for p in [
|
|
676
|
+
p_scan,
|
|
677
|
+
c_crypto,
|
|
678
|
+
c_vulns,
|
|
679
|
+
c_search,
|
|
680
|
+
c_versions,
|
|
681
|
+
c_semgrep,
|
|
682
|
+
c_provenance,
|
|
683
|
+
p_folder_scan,
|
|
684
|
+
p_cs,
|
|
685
|
+
]:
|
|
537
686
|
p.add_argument(
|
|
538
687
|
'--key', '-k', type=str, help='SCANOSS API Key token (optional - not required for default OSSKB URL)'
|
|
539
688
|
)
|
|
@@ -559,7 +708,7 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
559
708
|
)
|
|
560
709
|
|
|
561
710
|
# Global GRPC options
|
|
562
|
-
for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]:
|
|
711
|
+
for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance, p_folder_scan, p_cs]:
|
|
563
712
|
p.add_argument(
|
|
564
713
|
'--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)'
|
|
565
714
|
)
|
|
@@ -570,10 +719,26 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
570
719
|
'Can also use the environment variable "grcp_proxy=<ip>:<port>"',
|
|
571
720
|
)
|
|
572
721
|
p.add_argument(
|
|
573
|
-
'--header',
|
|
722
|
+
'--header',
|
|
723
|
+
'-hdr',
|
|
574
724
|
action='append', # This allows multiple -H flags
|
|
575
725
|
type=str,
|
|
576
|
-
help='Headers to be sent on request (e.g., -hdr "Name: Value") - can be used multiple times'
|
|
726
|
+
help='Headers to be sent on request (e.g., -hdr "Name: Value") - can be used multiple times',
|
|
727
|
+
)
|
|
728
|
+
|
|
729
|
+
# Syft options
|
|
730
|
+
for p in [p_cs, p_dep]:
|
|
731
|
+
p.add_argument(
|
|
732
|
+
'--syft-command',
|
|
733
|
+
type=str,
|
|
734
|
+
help='Syft command and path if required (optional - default syft).',
|
|
735
|
+
default=DEFAULT_SYFT_COMMAND,
|
|
736
|
+
)
|
|
737
|
+
p.add_argument(
|
|
738
|
+
'--syft-timeout',
|
|
739
|
+
type=int,
|
|
740
|
+
default=DEFAULT_SYFT_TIMEOUT,
|
|
741
|
+
help='Timeout (in seconds) for syft to complete (optional - default 600)',
|
|
577
742
|
)
|
|
578
743
|
|
|
579
744
|
# Help/Trace command options
|
|
@@ -594,7 +759,10 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
594
759
|
p_results,
|
|
595
760
|
p_undeclared,
|
|
596
761
|
p_copyleft,
|
|
597
|
-
c_provenance
|
|
762
|
+
c_provenance,
|
|
763
|
+
p_folder_scan,
|
|
764
|
+
p_folder_hash,
|
|
765
|
+
p_cs,
|
|
598
766
|
]:
|
|
599
767
|
p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages')
|
|
600
768
|
p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts')
|
|
@@ -607,12 +775,12 @@ def setup_args() -> None: # noqa: PLR0915
|
|
|
607
775
|
if not args.subparser:
|
|
608
776
|
parser.print_help() # No sub command subcommand, print general help
|
|
609
777
|
sys.exit(1)
|
|
610
|
-
elif (
|
|
611
|
-
args.subparser in {'utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins'} and not args.subparsercmd):
|
|
778
|
+
elif (args.subparser in ('utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins')) and not args.subparsercmd:
|
|
612
779
|
parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed
|
|
613
780
|
sys.exit(1)
|
|
614
781
|
args.func(parser, args) # Execute the function associated with the sub-command
|
|
615
782
|
|
|
783
|
+
|
|
616
784
|
def ver(*_):
|
|
617
785
|
"""
|
|
618
786
|
Run the "ver" sub-command
|
|
@@ -691,7 +859,7 @@ def wfp(parser, args):
|
|
|
691
859
|
if not args.skip_settings_file:
|
|
692
860
|
scan_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet)
|
|
693
861
|
try:
|
|
694
|
-
scan_settings.load_json_file(args.settings)
|
|
862
|
+
scan_settings.load_json_file(args.settings, args.scan_dir)
|
|
695
863
|
except ScanossSettingsError as e:
|
|
696
864
|
print_stderr(f'Error: {e}')
|
|
697
865
|
sys.exit(1)
|
|
@@ -766,7 +934,7 @@ def get_scan_options(args):
|
|
|
766
934
|
return scan_options
|
|
767
935
|
|
|
768
936
|
|
|
769
|
-
def scan(parser, args):
|
|
937
|
+
def scan(parser, args): # noqa: PLR0912, PLR0915
|
|
770
938
|
"""
|
|
771
939
|
Run the "scan" sub-command
|
|
772
940
|
Parameters
|
|
@@ -861,7 +1029,7 @@ def scan(parser, args): # noqa: PLR0912, PLR0915
|
|
|
861
1029
|
if flags:
|
|
862
1030
|
print_stderr(f'Using flags {flags}...')
|
|
863
1031
|
elif not args.quiet:
|
|
864
|
-
if args.timeout <
|
|
1032
|
+
if args.timeout < MIN_TIMEOUT:
|
|
865
1033
|
print_stderr(f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.')
|
|
866
1034
|
if args.retry < 0:
|
|
867
1035
|
print_stderr(f'POST retry (--retry) too small: {args.retry}. Reverting to default.')
|
|
@@ -910,7 +1078,7 @@ def scan(parser, args): # noqa: PLR0912, PLR0915
|
|
|
910
1078
|
strip_hpsm_ids=args.strip_hpsm,
|
|
911
1079
|
strip_snippet_ids=args.strip_snippet,
|
|
912
1080
|
scan_settings=scan_settings,
|
|
913
|
-
req_headers=
|
|
1081
|
+
req_headers=process_req_headers(args.header),
|
|
914
1082
|
)
|
|
915
1083
|
if args.wfp:
|
|
916
1084
|
if not scanner.is_file_or_snippet_scan():
|
|
@@ -980,12 +1148,18 @@ def dependency(parser, args):
|
|
|
980
1148
|
args: Namespace
|
|
981
1149
|
Parsed arguments
|
|
982
1150
|
"""
|
|
983
|
-
if not args.
|
|
984
|
-
print_stderr('Please specify a file/folder')
|
|
1151
|
+
if not args.scan_loc and not args.container:
|
|
1152
|
+
print_stderr('Please specify a file/folder or container image')
|
|
985
1153
|
parser.parse_args([args.subparser, '-h'])
|
|
986
1154
|
sys.exit(1)
|
|
987
|
-
|
|
988
|
-
|
|
1155
|
+
|
|
1156
|
+
# Workaround to return syft scan results converted to our dependency output format
|
|
1157
|
+
if args.container:
|
|
1158
|
+
args.scan_loc = args.container
|
|
1159
|
+
return container_scan(parser, args, only_interim_results=True)
|
|
1160
|
+
|
|
1161
|
+
if not os.path.exists(args.scan_loc):
|
|
1162
|
+
print_stderr(f'Error: File or folder specified does not exist: {args.scan_loc}.')
|
|
989
1163
|
sys.exit(1)
|
|
990
1164
|
scan_output: str = None
|
|
991
1165
|
if args.output:
|
|
@@ -995,7 +1169,7 @@ def dependency(parser, args):
|
|
|
995
1169
|
sc_deps = ScancodeDeps(
|
|
996
1170
|
debug=args.debug, quiet=args.quiet, trace=args.trace, sc_command=args.sc_command, timeout=args.sc_timeout
|
|
997
1171
|
)
|
|
998
|
-
if not sc_deps.get_dependencies(what_to_scan=args.
|
|
1172
|
+
if not sc_deps.get_dependencies(what_to_scan=args.scan_loc, result_output=scan_output):
|
|
999
1173
|
sys.exit(1)
|
|
1000
1174
|
|
|
1001
1175
|
|
|
@@ -1122,7 +1296,7 @@ def utils_certloc(*_):
|
|
|
1122
1296
|
print(f'CA Cert File: {certifi.where()}')
|
|
1123
1297
|
|
|
1124
1298
|
|
|
1125
|
-
def utils_cert_download(_, args):
|
|
1299
|
+
def utils_cert_download(_, args): # pylint: disable=PLR0912 # noqa: PLR0912
|
|
1126
1300
|
"""
|
|
1127
1301
|
Run the "utils cert-download" sub-command
|
|
1128
1302
|
:param _: ignore/unused
|
|
@@ -1149,13 +1323,14 @@ def utils_cert_download(_, args): # pylint: disable=PLR0912 # noqa: PLR0912
|
|
|
1149
1323
|
certs = conn.get_peer_cert_chain()
|
|
1150
1324
|
for index, cert in enumerate(certs):
|
|
1151
1325
|
cert_components = dict(cert.get_subject().get_components())
|
|
1152
|
-
if sys.version_info[0] >=
|
|
1326
|
+
if sys.version_info[0] >= PYTHON_MAJOR_VERSION:
|
|
1153
1327
|
cn = cert_components.get(b'CN')
|
|
1154
1328
|
else:
|
|
1329
|
+
# Fallback for Python versions less than PYTHON_MAJOR_VERSION
|
|
1155
1330
|
cn = cert_components.get('CN')
|
|
1156
1331
|
if not args.quiet:
|
|
1157
1332
|
print_stderr(f'Certificate {index} - CN: {cn}')
|
|
1158
|
-
if sys.version_info[0] >=
|
|
1333
|
+
if sys.version_info[0] >= PYTHON_MAJOR_VERSION:
|
|
1159
1334
|
print(
|
|
1160
1335
|
(crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode('utf-8')).strip(), file=file
|
|
1161
1336
|
) # Print the downloaded PEM certificate
|
|
@@ -1204,7 +1379,7 @@ def get_pac_file(pac: str):
|
|
|
1204
1379
|
if pac == 'auto':
|
|
1205
1380
|
pac_file = pypac.get_pac() # try to determine the PAC file
|
|
1206
1381
|
elif pac.startswith('file://'):
|
|
1207
|
-
pac_local = pac
|
|
1382
|
+
pac_local = pac[7:] # Remove 'file://' prefix (7 characters)
|
|
1208
1383
|
if not os.path.exists(pac_local):
|
|
1209
1384
|
print_stderr(f'Error: PAC file does not exist: {pac_local}.')
|
|
1210
1385
|
sys.exit(1)
|
|
@@ -1248,7 +1423,7 @@ def comp_crypto(parser, args):
|
|
|
1248
1423
|
grpc_proxy=args.grpc_proxy,
|
|
1249
1424
|
pac=pac_file,
|
|
1250
1425
|
timeout=args.timeout,
|
|
1251
|
-
req_headers=
|
|
1426
|
+
req_headers=process_req_headers(args.header),
|
|
1252
1427
|
)
|
|
1253
1428
|
if not comps.get_crypto_details(args.input, args.purl, args.output):
|
|
1254
1429
|
sys.exit(1)
|
|
@@ -1406,9 +1581,10 @@ def comp_versions(parser, args):
|
|
|
1406
1581
|
if not comps.get_component_versions(args.output, json_file=args.input, purl=args.purl, limit=args.limit):
|
|
1407
1582
|
sys.exit(1)
|
|
1408
1583
|
|
|
1584
|
+
|
|
1409
1585
|
def comp_provenance(parser, args):
|
|
1410
1586
|
"""
|
|
1411
|
-
Run the "component
|
|
1587
|
+
Run the "component provenance" sub-command
|
|
1412
1588
|
Parameters
|
|
1413
1589
|
----------
|
|
1414
1590
|
parser: ArgumentParser
|
|
@@ -1424,12 +1600,23 @@ def comp_provenance(parser, args):
|
|
|
1424
1600
|
print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.')
|
|
1425
1601
|
sys.exit(1)
|
|
1426
1602
|
pac_file = get_pac_file(args.pac)
|
|
1427
|
-
comps = Components(
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1603
|
+
comps = Components(
|
|
1604
|
+
debug=args.debug,
|
|
1605
|
+
trace=args.trace,
|
|
1606
|
+
quiet=args.quiet,
|
|
1607
|
+
grpc_url=args.api2url,
|
|
1608
|
+
api_key=args.key,
|
|
1609
|
+
ca_cert=args.ca_cert,
|
|
1610
|
+
proxy=args.proxy,
|
|
1611
|
+
grpc_proxy=args.grpc_proxy,
|
|
1612
|
+
pac=pac_file,
|
|
1613
|
+
timeout=args.timeout,
|
|
1614
|
+
req_headers=process_req_headers(args.header),
|
|
1615
|
+
)
|
|
1616
|
+
if not comps.get_provenance_details(args.input, args.purl, args.output, args.origin):
|
|
1431
1617
|
sys.exit(1)
|
|
1432
1618
|
|
|
1619
|
+
|
|
1433
1620
|
def results(parser, args):
|
|
1434
1621
|
"""
|
|
1435
1622
|
Run the "results" sub-command
|
|
@@ -1488,13 +1675,135 @@ def process_req_headers(headers_array: List[str]) -> dict:
|
|
|
1488
1675
|
dict_headers = {}
|
|
1489
1676
|
for header_str in headers_array:
|
|
1490
1677
|
# Split each "Name: Value" header
|
|
1491
|
-
parts = header_str.split(
|
|
1678
|
+
parts = header_str.split(':', 1)
|
|
1492
1679
|
if len(parts) == HEADER_PARTS_COUNT:
|
|
1493
1680
|
name = parts[0].strip()
|
|
1494
1681
|
value = parts[1].strip()
|
|
1495
1682
|
dict_headers[name] = value
|
|
1496
1683
|
return dict_headers
|
|
1497
1684
|
|
|
1685
|
+
|
|
1686
|
+
def folder_hashing_scan(parser, args):
|
|
1687
|
+
"""Run the "folder-scan" sub-command
|
|
1688
|
+
|
|
1689
|
+
Args:
|
|
1690
|
+
parser (ArgumentParser): command line parser object
|
|
1691
|
+
args (Namespace): Parsed arguments
|
|
1692
|
+
"""
|
|
1693
|
+
try:
|
|
1694
|
+
if not args.scan_dir:
|
|
1695
|
+
print_stderr('ERROR: Please specify a directory to scan')
|
|
1696
|
+
parser.parse_args([args.subparser, '-h'])
|
|
1697
|
+
sys.exit(1)
|
|
1698
|
+
|
|
1699
|
+
if not os.path.exists(args.scan_dir) or not os.path.isdir(args.scan_dir):
|
|
1700
|
+
print_stderr(f'ERROR: The specified directory {args.scan_dir} does not exist')
|
|
1701
|
+
sys.exit(1)
|
|
1702
|
+
|
|
1703
|
+
scanner_config = create_scanner_config_from_args(args)
|
|
1704
|
+
scanoss_settings = get_scanoss_settings_from_args(args)
|
|
1705
|
+
grpc_config = create_grpc_config_from_args(args)
|
|
1706
|
+
|
|
1707
|
+
client = ScanossGrpc(**asdict(grpc_config))
|
|
1708
|
+
|
|
1709
|
+
scanner = ScannerHFH(
|
|
1710
|
+
scan_dir=args.scan_dir,
|
|
1711
|
+
config=scanner_config,
|
|
1712
|
+
client=client,
|
|
1713
|
+
scanoss_settings=scanoss_settings,
|
|
1714
|
+
)
|
|
1715
|
+
|
|
1716
|
+
scanner.best_match = args.best_match
|
|
1717
|
+
scanner.threshold = args.threshold
|
|
1718
|
+
|
|
1719
|
+
if scanner.scan():
|
|
1720
|
+
scanner.present(output_file=args.output, output_format=args.format)
|
|
1721
|
+
except ScanossGrpcError as e:
|
|
1722
|
+
print_stderr(f'ERROR: {e}')
|
|
1723
|
+
sys.exit(1)
|
|
1724
|
+
|
|
1725
|
+
|
|
1726
|
+
def folder_hash(parser, args):
|
|
1727
|
+
"""Run the "folder-hash" sub-command
|
|
1728
|
+
|
|
1729
|
+
Args:
|
|
1730
|
+
parser (ArgumentParser): command line parser object
|
|
1731
|
+
args (Namespace): Parsed arguments
|
|
1732
|
+
"""
|
|
1733
|
+
try:
|
|
1734
|
+
if not args.scan_dir:
|
|
1735
|
+
print_stderr('ERROR: Please specify a directory to scan')
|
|
1736
|
+
parser.parse_args([args.subparser, '-h'])
|
|
1737
|
+
sys.exit(1)
|
|
1738
|
+
|
|
1739
|
+
if not os.path.exists(args.scan_dir) or not os.path.isdir(args.scan_dir):
|
|
1740
|
+
print_stderr(f'ERROR: The specified directory {args.scan_dir} does not exist')
|
|
1741
|
+
sys.exit(1)
|
|
1742
|
+
|
|
1743
|
+
folder_hasher_config = create_folder_hasher_config_from_args(args)
|
|
1744
|
+
scanoss_settings = get_scanoss_settings_from_args(args)
|
|
1745
|
+
|
|
1746
|
+
folder_hasher = FolderHasher(
|
|
1747
|
+
scan_dir=args.scan_dir,
|
|
1748
|
+
config=folder_hasher_config,
|
|
1749
|
+
scanoss_settings=scanoss_settings,
|
|
1750
|
+
)
|
|
1751
|
+
|
|
1752
|
+
folder_hasher.hash_directory(args.scan_dir)
|
|
1753
|
+
folder_hasher.present(output_file=args.output, output_format=args.format)
|
|
1754
|
+
except Exception as e:
|
|
1755
|
+
print_stderr(f'ERROR: {e}')
|
|
1756
|
+
sys.exit(1)
|
|
1757
|
+
|
|
1758
|
+
|
|
1759
|
+
def container_scan(parser, args, only_interim_results: bool = False):
|
|
1760
|
+
"""
|
|
1761
|
+
Run the "container-scan" sub-command
|
|
1762
|
+
Parameters
|
|
1763
|
+
----------
|
|
1764
|
+
parser: ArgumentParser
|
|
1765
|
+
command line parser object
|
|
1766
|
+
args: Namespace
|
|
1767
|
+
Parsed arguments
|
|
1768
|
+
"""
|
|
1769
|
+
if not args.scan_loc:
|
|
1770
|
+
print_stderr(
|
|
1771
|
+
'Please specify a container image, Docker tar, OCI tar, OCI directory, SIF Container, or directory to scan'
|
|
1772
|
+
)
|
|
1773
|
+
parser.parse_args([args.subparser, '-h'])
|
|
1774
|
+
sys.exit(1)
|
|
1775
|
+
|
|
1776
|
+
try:
|
|
1777
|
+
config = create_container_scanner_config_from_args(args)
|
|
1778
|
+
config.only_interim_results = only_interim_results
|
|
1779
|
+
container_scanner = ContainerScanner(
|
|
1780
|
+
config=config,
|
|
1781
|
+
what_to_scan=args.scan_loc,
|
|
1782
|
+
)
|
|
1783
|
+
|
|
1784
|
+
container_scanner.scan()
|
|
1785
|
+
if only_interim_results:
|
|
1786
|
+
container_scanner.present(output_file=config.output, output_format='raw')
|
|
1787
|
+
else:
|
|
1788
|
+
container_scanner.decorate_scan_results_with_dependencies()
|
|
1789
|
+
container_scanner.present(output_file=config.output, output_format=config.format)
|
|
1790
|
+
except Exception as e:
|
|
1791
|
+
print_stderr(f'ERROR: {e}')
|
|
1792
|
+
sys.exit(1)
|
|
1793
|
+
|
|
1794
|
+
|
|
1795
|
+
def get_scanoss_settings_from_args(args):
|
|
1796
|
+
scanoss_settings = None
|
|
1797
|
+
if not args.skip_settings_file:
|
|
1798
|
+
scanoss_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet)
|
|
1799
|
+
try:
|
|
1800
|
+
scanoss_settings.load_json_file(args.settings, args.scan_dir).set_file_type('new').set_scan_type('identify')
|
|
1801
|
+
except ScanossSettingsError as e:
|
|
1802
|
+
print_stderr(f'Error: {e}')
|
|
1803
|
+
sys.exit(1)
|
|
1804
|
+
return scanoss_settings
|
|
1805
|
+
|
|
1806
|
+
|
|
1498
1807
|
def main():
|
|
1499
1808
|
"""
|
|
1500
1809
|
Run the ScanOSS CLI
|