scanoss 1.27.1__py3-none-any.whl → 1.43.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- protoc_gen_swagger/options/annotations_pb2.py +18 -12
- protoc_gen_swagger/options/annotations_pb2.pyi +48 -0
- protoc_gen_swagger/options/annotations_pb2_grpc.py +20 -0
- protoc_gen_swagger/options/openapiv2_pb2.py +110 -99
- protoc_gen_swagger/options/openapiv2_pb2.pyi +1317 -0
- protoc_gen_swagger/options/openapiv2_pb2_grpc.py +20 -0
- scanoss/__init__.py +1 -1
- scanoss/api/common/v2/scanoss_common_pb2.py +49 -22
- scanoss/api/common/v2/scanoss_common_pb2_grpc.py +25 -0
- scanoss/api/components/v2/scanoss_components_pb2.py +68 -43
- scanoss/api/components/v2/scanoss_components_pb2_grpc.py +83 -22
- scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py +136 -47
- scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py +650 -33
- scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +56 -37
- scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py +64 -12
- scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py +74 -31
- scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py +252 -13
- scanoss/api/licenses/__init__.py +23 -0
- scanoss/api/licenses/v2/__init__.py +23 -0
- scanoss/api/licenses/v2/scanoss_licenses_pb2.py +84 -0
- scanoss/api/licenses/v2/scanoss_licenses_pb2_grpc.py +302 -0
- scanoss/api/scanning/v2/scanoss_scanning_pb2.py +32 -21
- scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +49 -8
- scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py +50 -23
- scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py +151 -16
- scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +78 -31
- scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py +282 -18
- scanoss/cli.py +1000 -186
- scanoss/components.py +80 -50
- scanoss/constants.py +7 -1
- scanoss/cryptography.py +89 -55
- scanoss/csvoutput.py +13 -7
- scanoss/cyclonedx.py +141 -9
- scanoss/data/build_date.txt +1 -1
- scanoss/data/osadl-copyleft.json +133 -0
- scanoss/delta.py +197 -0
- scanoss/export/__init__.py +23 -0
- scanoss/export/dependency_track.py +227 -0
- scanoss/file_filters.py +2 -163
- scanoss/filecount.py +37 -38
- scanoss/gitlabqualityreport.py +214 -0
- scanoss/header_filter.py +563 -0
- scanoss/inspection/policy_check/__init__.py +0 -0
- scanoss/inspection/policy_check/dependency_track/__init__.py +0 -0
- scanoss/inspection/policy_check/dependency_track/project_violation.py +479 -0
- scanoss/inspection/{policy_check.py → policy_check/policy_check.py} +65 -72
- scanoss/inspection/policy_check/scanoss/__init__.py +0 -0
- scanoss/inspection/{copyleft.py → policy_check/scanoss/copyleft.py} +89 -73
- scanoss/inspection/{undeclared_component.py → policy_check/scanoss/undeclared_component.py} +52 -46
- scanoss/inspection/summary/__init__.py +0 -0
- scanoss/inspection/summary/component_summary.py +170 -0
- scanoss/inspection/{license_summary.py → summary/license_summary.py} +62 -12
- scanoss/inspection/summary/match_summary.py +341 -0
- scanoss/inspection/utils/file_utils.py +44 -0
- scanoss/inspection/utils/license_utils.py +57 -71
- scanoss/inspection/utils/markdown_utils.py +63 -0
- scanoss/inspection/{inspect_base.py → utils/scan_result_processor.py} +53 -67
- scanoss/osadl.py +125 -0
- scanoss/scanner.py +135 -253
- scanoss/scanners/folder_hasher.py +47 -32
- scanoss/scanners/scanner_hfh.py +50 -18
- scanoss/scanoss_settings.py +33 -3
- scanoss/scanossapi.py +23 -25
- scanoss/scanossbase.py +1 -1
- scanoss/scanossgrpc.py +543 -289
- scanoss/services/dependency_track_service.py +132 -0
- scanoss/spdxlite.py +11 -4
- scanoss/threadeddependencies.py +19 -18
- scanoss/threadedscanning.py +10 -0
- scanoss/utils/scanoss_scan_results_utils.py +41 -0
- scanoss/winnowing.py +71 -19
- {scanoss-1.27.1.dist-info → scanoss-1.43.1.dist-info}/METADATA +8 -5
- scanoss-1.43.1.dist-info/RECORD +110 -0
- scanoss/inspection/component_summary.py +0 -94
- scanoss-1.27.1.dist-info/RECORD +0 -87
- {scanoss-1.27.1.dist-info → scanoss-1.43.1.dist-info}/WHEEL +0 -0
- {scanoss-1.27.1.dist-info → scanoss-1.43.1.dist-info}/entry_points.txt +0 -0
- {scanoss-1.27.1.dist-info → scanoss-1.43.1.dist-info}/licenses/LICENSE +0 -0
- {scanoss-1.27.1.dist-info → scanoss-1.43.1.dist-info}/top_level.txt +0 -0
scanoss/scanossgrpc.py
CHANGED
|
@@ -23,46 +23,48 @@ SPDX-License-Identifier: MIT
|
|
|
23
23
|
"""
|
|
24
24
|
|
|
25
25
|
import concurrent.futures
|
|
26
|
-
import
|
|
26
|
+
import http.client as http_client
|
|
27
|
+
import logging
|
|
27
28
|
import os
|
|
29
|
+
import sys
|
|
30
|
+
import time
|
|
28
31
|
import uuid
|
|
29
32
|
from dataclasses import dataclass
|
|
30
|
-
from enum import IntEnum
|
|
33
|
+
from enum import Enum, IntEnum
|
|
31
34
|
from typing import Dict, Optional
|
|
32
35
|
from urllib.parse import urlparse
|
|
33
36
|
|
|
34
37
|
import grpc
|
|
38
|
+
import requests
|
|
39
|
+
import urllib3
|
|
35
40
|
from google.protobuf.json_format import MessageToDict, ParseDict
|
|
41
|
+
from pypac import PACSession
|
|
36
42
|
from pypac.parser import PACFile
|
|
37
43
|
from pypac.resolver import ProxyResolver
|
|
44
|
+
from urllib3.exceptions import InsecureRequestWarning
|
|
38
45
|
|
|
46
|
+
from scanoss.api.licenses.v2.scanoss_licenses_pb2_grpc import LicenseStub
|
|
39
47
|
from scanoss.api.scanning.v2.scanoss_scanning_pb2_grpc import ScanningStub
|
|
40
48
|
from scanoss.constants import DEFAULT_TIMEOUT
|
|
41
49
|
|
|
42
50
|
from . import __version__
|
|
43
51
|
from .api.common.v2.scanoss_common_pb2 import (
|
|
52
|
+
ComponentsRequest,
|
|
44
53
|
EchoRequest,
|
|
45
|
-
EchoResponse,
|
|
46
|
-
PurlRequest,
|
|
47
54
|
StatusCode,
|
|
48
55
|
StatusResponse,
|
|
49
56
|
)
|
|
50
57
|
from .api.components.v2.scanoss_components_pb2 import (
|
|
51
58
|
CompSearchRequest,
|
|
52
|
-
CompSearchResponse,
|
|
53
59
|
CompVersionRequest,
|
|
54
|
-
CompVersionResponse,
|
|
55
60
|
)
|
|
56
61
|
from .api.components.v2.scanoss_components_pb2_grpc import ComponentsStub
|
|
57
62
|
from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub
|
|
58
63
|
from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest
|
|
59
64
|
from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub
|
|
60
|
-
from .api.geoprovenance.v2.scanoss_geoprovenance_pb2 import ContributorResponse
|
|
61
65
|
from .api.geoprovenance.v2.scanoss_geoprovenance_pb2_grpc import GeoProvenanceStub
|
|
62
66
|
from .api.scanning.v2.scanoss_scanning_pb2 import HFHRequest
|
|
63
|
-
from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse
|
|
64
67
|
from .api.semgrep.v2.scanoss_semgrep_pb2_grpc import SemgrepStub
|
|
65
|
-
from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import VulnerabilityResponse
|
|
66
68
|
from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2_grpc import VulnerabilitiesStub
|
|
67
69
|
from .scanossbase import ScanossBase
|
|
68
70
|
|
|
@@ -70,8 +72,38 @@ DEFAULT_URL = 'https://api.osskb.org' # default free service URL
|
|
|
70
72
|
DEFAULT_URL2 = 'https://api.scanoss.com' # default premium service URL
|
|
71
73
|
SCANOSS_GRPC_URL = os.environ.get('SCANOSS_GRPC_URL') if os.environ.get('SCANOSS_GRPC_URL') else DEFAULT_URL
|
|
72
74
|
SCANOSS_API_KEY = os.environ.get('SCANOSS_API_KEY') if os.environ.get('SCANOSS_API_KEY') else ''
|
|
73
|
-
|
|
74
|
-
|
|
75
|
+
DEFAULT_URI_PREFIX = '/v2'
|
|
76
|
+
|
|
77
|
+
MAX_CONCURRENT_REQUESTS = 5 # Maximum number of concurrent requests to make
|
|
78
|
+
|
|
79
|
+
# REST API endpoint mappings with HTTP methods
|
|
80
|
+
REST_ENDPOINTS = {
|
|
81
|
+
'vulnerabilities.GetComponentsVulnerabilities': {'path': '/vulnerabilities/components', 'method': 'POST'},
|
|
82
|
+
'dependencies.Echo': {'path': '/dependencies/echo', 'method': 'POST'},
|
|
83
|
+
'dependencies.GetDependencies': {'path': '/dependencies/dependencies', 'method': 'POST'},
|
|
84
|
+
'cryptography.Echo': {'path': '/cryptography/echo', 'method': 'POST'},
|
|
85
|
+
'cryptography.GetComponentsAlgorithms': {'path': '/cryptography/algorithms/components', 'method': 'POST'},
|
|
86
|
+
'cryptography.GetComponentsAlgorithmsInRange': {
|
|
87
|
+
'path': '/cryptography/algorithms/range/components',
|
|
88
|
+
'method': 'POST',
|
|
89
|
+
},
|
|
90
|
+
'cryptography.GetComponentsEncryptionHints': {'path': '/cryptography/hints/components', 'method': 'POST'},
|
|
91
|
+
'cryptography.GetComponentsHintsInRange': {'path': '/cryptography/hints/components/range', 'method': 'POST'},
|
|
92
|
+
'cryptography.GetComponentsVersionsInRange': {
|
|
93
|
+
'path': '/cryptography/algorithms/versions/range/components',
|
|
94
|
+
'method': 'POST',
|
|
95
|
+
},
|
|
96
|
+
'components.SearchComponents': {'path': '/components/search', 'method': 'GET'},
|
|
97
|
+
'components.GetComponentVersions': {'path': '/components/versions', 'method': 'GET'},
|
|
98
|
+
'geoprovenance.GetCountryContributorsByComponents': {
|
|
99
|
+
'path': '/geoprovenance/countries/components',
|
|
100
|
+
'method': 'POST',
|
|
101
|
+
},
|
|
102
|
+
'geoprovenance.GetOriginByComponents': {'path': '/geoprovenance/origin/components', 'method': 'POST'},
|
|
103
|
+
'licenses.GetComponentsLicenses': {'path': '/licenses/components', 'method': 'POST'},
|
|
104
|
+
'semgrep.GetComponentsIssues': {'path': '/semgrep/issues/components', 'method': 'POST'},
|
|
105
|
+
'scanning.FolderHashScan': {'path': '/scanning/hfh/scan', 'method': 'POST'},
|
|
106
|
+
}
|
|
75
107
|
|
|
76
108
|
|
|
77
109
|
class ScanossGrpcError(Exception):
|
|
@@ -85,31 +117,44 @@ class ScanossGrpcError(Exception):
|
|
|
85
117
|
class ScanossGrpcStatusCode(IntEnum):
|
|
86
118
|
"""Status codes for SCANOSS gRPC responses"""
|
|
87
119
|
|
|
120
|
+
UNSPECIFIED = 0
|
|
88
121
|
SUCCESS = 1
|
|
89
|
-
|
|
90
|
-
|
|
122
|
+
SUCCEEDED_WITH_WARNINGS = 2
|
|
123
|
+
WARNING = 3
|
|
91
124
|
FAILED = 4
|
|
92
125
|
|
|
93
126
|
|
|
127
|
+
class ScanossRESTStatusCode(Enum):
|
|
128
|
+
"""Status codes for SCANOSS REST responses"""
|
|
129
|
+
|
|
130
|
+
UNSPECIFIED = 'UNSPECIFIED'
|
|
131
|
+
SUCCESS = 'SUCCESS'
|
|
132
|
+
SUCCEEDED_WITH_WARNINGS = 'SUCCEEDED_WITH_WARNINGS'
|
|
133
|
+
WARNING = 'WARNING'
|
|
134
|
+
FAILED = 'FAILED'
|
|
135
|
+
|
|
136
|
+
|
|
94
137
|
class ScanossGrpc(ScanossBase):
|
|
95
138
|
"""
|
|
96
139
|
Client for gRPC functionality
|
|
97
140
|
"""
|
|
98
141
|
|
|
99
|
-
def __init__( # noqa: PLR0913, PLR0915
|
|
142
|
+
def __init__( # noqa: PLR0912, PLR0913, PLR0915
|
|
100
143
|
self,
|
|
101
|
-
url: str = None,
|
|
144
|
+
url: Optional[str] = None,
|
|
102
145
|
debug: bool = False,
|
|
103
146
|
trace: bool = False,
|
|
104
147
|
quiet: bool = False,
|
|
105
|
-
ca_cert: str = None,
|
|
106
|
-
api_key: str = None,
|
|
107
|
-
ver_details: str = None,
|
|
148
|
+
ca_cert: Optional[str] = None,
|
|
149
|
+
api_key: Optional[str] = None,
|
|
150
|
+
ver_details: Optional[str] = None,
|
|
108
151
|
timeout: int = 600,
|
|
109
|
-
proxy: str = None,
|
|
110
|
-
grpc_proxy: str = None,
|
|
111
|
-
pac: PACFile = None,
|
|
112
|
-
req_headers: dict = None,
|
|
152
|
+
proxy: Optional[str] = None,
|
|
153
|
+
grpc_proxy: Optional[str] = None,
|
|
154
|
+
pac: Optional[PACFile] = None,
|
|
155
|
+
req_headers: Optional[dict] = None,
|
|
156
|
+
ignore_cert_errors: bool = False,
|
|
157
|
+
use_grpc: Optional[bool] = False,
|
|
113
158
|
):
|
|
114
159
|
"""
|
|
115
160
|
|
|
@@ -127,27 +172,55 @@ class ScanossGrpc(ScanossBase):
|
|
|
127
172
|
grpc_proxy='http://<ip>:<port>'
|
|
128
173
|
"""
|
|
129
174
|
super().__init__(debug, trace, quiet)
|
|
130
|
-
self.url = url
|
|
131
175
|
self.api_key = api_key if api_key else SCANOSS_API_KEY
|
|
132
176
|
self.timeout = timeout
|
|
133
177
|
self.proxy = proxy
|
|
134
178
|
self.grpc_proxy = grpc_proxy
|
|
135
179
|
self.pac = pac
|
|
136
|
-
self.req_headers = req_headers
|
|
137
180
|
self.metadata = []
|
|
181
|
+
self.ignore_cert_errors = ignore_cert_errors
|
|
182
|
+
self.use_grpc = use_grpc
|
|
183
|
+
self.req_headers = req_headers if req_headers else {}
|
|
184
|
+
self.headers = {}
|
|
185
|
+
self.retry_limit = 2 # default retry limit
|
|
138
186
|
|
|
139
187
|
if self.api_key:
|
|
140
188
|
self.metadata.append(('x-api-key', api_key)) # Set API key if we have one
|
|
189
|
+
self.headers['X-Session'] = self.api_key
|
|
190
|
+
self.headers['x-api-key'] = self.api_key
|
|
141
191
|
if ver_details:
|
|
142
192
|
self.metadata.append(('x-scanoss-client', ver_details))
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
193
|
+
self.headers['x-scanoss-client'] = ver_details
|
|
194
|
+
user_agent = f'scanoss-py/{__version__}'
|
|
195
|
+
self.metadata.append(('user-agent', user_agent))
|
|
196
|
+
self.headers['User-Agent'] = user_agent
|
|
197
|
+
self.headers['user-agent'] = user_agent
|
|
198
|
+
self.headers['Content-Type'] = 'application/json'
|
|
199
|
+
# Set the correct URL/API key combination
|
|
146
200
|
self.url = url if url else SCANOSS_GRPC_URL
|
|
147
201
|
if self.api_key and not url and not os.environ.get('SCANOSS_GRPC_URL'):
|
|
148
202
|
self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium
|
|
203
|
+
self.load_generic_headers(url)
|
|
149
204
|
self.url = self.url.lower()
|
|
150
|
-
self.orig_url = self.url # Used for proxy lookup
|
|
205
|
+
self.orig_url = self.url.strip().rstrip('/') # Used for proxy lookup
|
|
206
|
+
# REST setup
|
|
207
|
+
if self.trace:
|
|
208
|
+
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
|
|
209
|
+
http_client.HTTPConnection.debuglevel = 1
|
|
210
|
+
if pac and not proxy: # Set up a PAC session if requested (and no proxy has been explicitly set)
|
|
211
|
+
self.print_debug('Setting up PAC session...')
|
|
212
|
+
self.session = PACSession(pac=pac)
|
|
213
|
+
else:
|
|
214
|
+
self.session = requests.sessions.Session()
|
|
215
|
+
if self.ignore_cert_errors:
|
|
216
|
+
self.print_debug('Ignoring cert errors...')
|
|
217
|
+
urllib3.disable_warnings(InsecureRequestWarning)
|
|
218
|
+
self.session.verify = False
|
|
219
|
+
elif ca_cert:
|
|
220
|
+
self.session.verify = ca_cert
|
|
221
|
+
self.proxies = {'https': proxy, 'http': proxy} if proxy else None
|
|
222
|
+
if self.proxies:
|
|
223
|
+
self.session.proxies = self.proxies
|
|
151
224
|
|
|
152
225
|
secure = True if self.url.startswith('https:') else False # Is it a secure connection?
|
|
153
226
|
if self.url.startswith('http'):
|
|
@@ -162,7 +235,7 @@ class ScanossGrpc(ScanossBase):
|
|
|
162
235
|
cert_data = ScanossGrpc._load_cert(ca_cert)
|
|
163
236
|
self.print_debug(f'Setting up (secure: {secure}) connection to {self.url}...')
|
|
164
237
|
self._get_proxy_config()
|
|
165
|
-
if secure
|
|
238
|
+
if not secure: # insecure connection
|
|
166
239
|
self.comp_search_stub = ComponentsStub(grpc.insecure_channel(self.url))
|
|
167
240
|
self.crypto_stub = CryptographyStub(grpc.insecure_channel(self.url))
|
|
168
241
|
self.dependencies_stub = DependenciesStub(grpc.insecure_channel(self.url))
|
|
@@ -170,6 +243,7 @@ class ScanossGrpc(ScanossBase):
|
|
|
170
243
|
self.vuln_stub = VulnerabilitiesStub(grpc.insecure_channel(self.url))
|
|
171
244
|
self.provenance_stub = GeoProvenanceStub(grpc.insecure_channel(self.url))
|
|
172
245
|
self.scanning_stub = ScanningStub(grpc.insecure_channel(self.url))
|
|
246
|
+
self.license_stub = LicenseStub(grpc.insecure_channel(self.url))
|
|
173
247
|
else:
|
|
174
248
|
if ca_cert is not None:
|
|
175
249
|
credentials = grpc.ssl_channel_credentials(cert_data) # secure with specified certificate
|
|
@@ -182,70 +256,32 @@ class ScanossGrpc(ScanossBase):
|
|
|
182
256
|
self.vuln_stub = VulnerabilitiesStub(grpc.secure_channel(self.url, credentials))
|
|
183
257
|
self.provenance_stub = GeoProvenanceStub(grpc.secure_channel(self.url, credentials))
|
|
184
258
|
self.scanning_stub = ScanningStub(grpc.secure_channel(self.url, credentials))
|
|
259
|
+
self.license_stub = LicenseStub(grpc.secure_channel(self.url, credentials))
|
|
185
260
|
|
|
186
261
|
@classmethod
|
|
187
262
|
def _load_cert(cls, cert_file: str) -> bytes:
|
|
188
263
|
with open(cert_file, 'rb') as f:
|
|
189
264
|
return f.read()
|
|
190
265
|
|
|
191
|
-
def deps_echo(self, message: str = 'Hello there!') ->
|
|
266
|
+
def deps_echo(self, message: str = 'Hello there!') -> Optional[dict]:
|
|
192
267
|
"""
|
|
193
268
|
Send Echo message to the Dependency service
|
|
194
269
|
:param self:
|
|
195
270
|
:param message: Message to send (default: Hello there!)
|
|
196
271
|
:return: echo or None
|
|
197
272
|
"""
|
|
198
|
-
|
|
199
|
-
resp: EchoResponse
|
|
200
|
-
try:
|
|
201
|
-
metadata = self.metadata[:]
|
|
202
|
-
metadata.append(('x-request-id', request_id)) # Set a Request ID
|
|
203
|
-
resp = self.dependencies_stub.Echo(EchoRequest(message=message), metadata=metadata, timeout=3)
|
|
204
|
-
except Exception as e:
|
|
205
|
-
self.print_stderr(
|
|
206
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
207
|
-
)
|
|
208
|
-
else:
|
|
209
|
-
# self.print_stderr(f'resp: {resp} - call: {call}')
|
|
210
|
-
# response_id = ""
|
|
211
|
-
# if not call:
|
|
212
|
-
# self.print_stderr(f'No call to leverage.')
|
|
213
|
-
# for key, value in call.trailing_metadata():
|
|
214
|
-
# print('Greeter client received trailing metadata: key=%s value=%s' % (key, value))
|
|
215
|
-
#
|
|
216
|
-
# for key, value in call.trailing_metadata():
|
|
217
|
-
# if key == 'x-response-id':
|
|
218
|
-
# response_id = value
|
|
219
|
-
# self.print_stderr(f'Response ID: {response_id}. Metadata: {call.trailing_metadata()}')
|
|
220
|
-
if resp:
|
|
221
|
-
return resp.message
|
|
222
|
-
self.print_stderr(f'ERROR: Problem sending Echo request ({message}) to {self.url}. rqId: {request_id}')
|
|
223
|
-
return None
|
|
273
|
+
return self._call_api('dependencies.Echo', self.dependencies_stub.Echo, {'message': message}, EchoRequest)
|
|
224
274
|
|
|
225
|
-
def crypto_echo(self, message: str = 'Hello there!') ->
|
|
275
|
+
def crypto_echo(self, message: str = 'Hello there!') -> Optional[dict]:
|
|
226
276
|
"""
|
|
227
277
|
Send Echo message to the Cryptography service
|
|
228
278
|
:param self:
|
|
229
279
|
:param message: Message to send (default: Hello there!)
|
|
230
280
|
:return: echo or None
|
|
231
281
|
"""
|
|
232
|
-
|
|
233
|
-
resp: EchoResponse
|
|
234
|
-
try:
|
|
235
|
-
metadata = self.metadata[:]
|
|
236
|
-
metadata.append(('x-request-id', request_id)) # Set a Request ID
|
|
237
|
-
resp = self.crypto_stub.Echo(EchoRequest(message=message), metadata=metadata, timeout=3)
|
|
238
|
-
except Exception as e:
|
|
239
|
-
self.print_stderr(
|
|
240
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
241
|
-
)
|
|
242
|
-
else:
|
|
243
|
-
if resp:
|
|
244
|
-
return resp.message
|
|
245
|
-
self.print_stderr(f'ERROR: Problem sending Echo request ({message}) to {self.url}. rqId: {request_id}')
|
|
246
|
-
return None
|
|
282
|
+
return self._call_api('cryptography.Echo', self.crypto_stub.Echo, {'message': message}, EchoRequest)
|
|
247
283
|
|
|
248
|
-
def get_dependencies(self, dependencies:
|
|
284
|
+
def get_dependencies(self, dependencies: Optional[dict] = None, depth: int = 1) -> Optional[dict]:
|
|
249
285
|
if not dependencies:
|
|
250
286
|
self.print_stderr('ERROR: No dependency data supplied to submit to the API.')
|
|
251
287
|
return None
|
|
@@ -254,7 +290,7 @@ class ScanossGrpc(ScanossBase):
|
|
|
254
290
|
self.print_stderr(f'ERROR: No response for dependency request: {dependencies}')
|
|
255
291
|
return resp
|
|
256
292
|
|
|
257
|
-
def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict:
|
|
293
|
+
def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> Optional[dict]:
|
|
258
294
|
"""
|
|
259
295
|
Client function to call the rpc for GetDependencies
|
|
260
296
|
:param dependencies: Message to send to the service
|
|
@@ -264,190 +300,190 @@ class ScanossGrpc(ScanossBase):
|
|
|
264
300
|
if not dependencies:
|
|
265
301
|
self.print_stderr('ERROR: No message supplied to send to gRPC service.')
|
|
266
302
|
return None
|
|
267
|
-
|
|
268
303
|
files_json = dependencies.get('files')
|
|
269
|
-
|
|
270
304
|
if files_json is None or len(files_json) == 0:
|
|
271
|
-
self.print_stderr('ERROR: No dependency data supplied to send to
|
|
305
|
+
self.print_stderr('ERROR: No dependency data supplied to send to decoration service.')
|
|
272
306
|
return None
|
|
273
|
-
|
|
274
|
-
def process_file(file):
|
|
275
|
-
request_id = str(uuid.uuid4())
|
|
276
|
-
try:
|
|
277
|
-
file_request = {'files': [file]}
|
|
278
|
-
|
|
279
|
-
request = ParseDict(file_request, DependencyRequest())
|
|
280
|
-
request.depth = depth
|
|
281
|
-
metadata = self.metadata[:]
|
|
282
|
-
metadata.append(('x-request-id', request_id))
|
|
283
|
-
self.print_debug(f'Sending dependency data for decoration (rqId: {request_id})...')
|
|
284
|
-
resp = self.dependencies_stub.GetDependencies(request, metadata=metadata, timeout=self.timeout)
|
|
285
|
-
|
|
286
|
-
return MessageToDict(resp, preserving_proto_field_name=True)
|
|
287
|
-
except Exception as e:
|
|
288
|
-
self.print_stderr(
|
|
289
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
290
|
-
)
|
|
291
|
-
return None
|
|
292
|
-
|
|
293
307
|
all_responses = []
|
|
308
|
+
# Process the dependency files in parallel
|
|
294
309
|
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_CONCURRENT_REQUESTS) as executor:
|
|
295
|
-
future_to_file = {
|
|
296
|
-
|
|
310
|
+
future_to_file = {
|
|
311
|
+
executor.submit(self._process_dep_file, file, depth, self.use_grpc): file for file in files_json
|
|
312
|
+
}
|
|
297
313
|
for future in concurrent.futures.as_completed(future_to_file):
|
|
298
314
|
response = future.result()
|
|
299
315
|
if response:
|
|
300
316
|
all_responses.append(response)
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
317
|
+
# End of concurrent processing
|
|
318
|
+
success_status = 'SUCCESS'
|
|
319
|
+
merged_response = {'files': [], 'status': {'status': success_status, 'message': 'Success'}}
|
|
320
|
+
# Merge the responses
|
|
305
321
|
for response in all_responses:
|
|
306
322
|
if response:
|
|
307
323
|
if 'files' in response and len(response['files']) > 0:
|
|
308
324
|
merged_response['files'].append(response['files'][0])
|
|
309
|
-
# Overwrite the status if
|
|
310
|
-
if 'status' in response and response['status']['status'] !=
|
|
325
|
+
# Overwrite the status if any of the responses was not successful
|
|
326
|
+
if 'status' in response and response['status']['status'] != success_status:
|
|
311
327
|
merged_response['status'] = response['status']
|
|
312
328
|
return merged_response
|
|
313
329
|
|
|
314
|
-
def
|
|
330
|
+
def _process_dep_file(self, file, depth: int = 1, use_grpc: Optional[bool] = None) -> Optional[dict]:
|
|
315
331
|
"""
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
:
|
|
332
|
+
Process a single dependency file using either gRPC or REST
|
|
333
|
+
|
|
334
|
+
Args:
|
|
335
|
+
file: dependency file purls
|
|
336
|
+
depth: depth to search (default: 1)
|
|
337
|
+
use_grpc: Whether to use gRPC or REST (None = use instance default)
|
|
338
|
+
|
|
339
|
+
Returns:
|
|
340
|
+
response JSON or None
|
|
319
341
|
"""
|
|
320
|
-
|
|
321
|
-
self.print_stderr('ERROR: No message supplied to send to gRPC service.')
|
|
322
|
-
return None
|
|
323
|
-
request_id = str(uuid.uuid4())
|
|
324
|
-
resp: VulnerabilityResponse
|
|
325
|
-
try:
|
|
326
|
-
request = ParseDict(purls, PurlRequest()) # Parse the JSON/Dict into the purl request object
|
|
327
|
-
metadata = self.metadata[:]
|
|
328
|
-
metadata.append(('x-request-id', request_id)) # Set a Request ID
|
|
329
|
-
self.print_debug(f'Sending crypto data for decoration (rqId: {request_id})...')
|
|
330
|
-
resp = self.vuln_stub.GetVulnerabilities(request, metadata=metadata, timeout=self.timeout)
|
|
331
|
-
except Exception as e:
|
|
332
|
-
self.print_stderr(
|
|
333
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
334
|
-
)
|
|
335
|
-
else:
|
|
336
|
-
if resp:
|
|
337
|
-
if not self._check_status_response(resp.status, request_id):
|
|
338
|
-
return None
|
|
339
|
-
resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict
|
|
340
|
-
del resp_dict['status']
|
|
341
|
-
return resp_dict
|
|
342
|
-
return None
|
|
342
|
+
file_request = {'files': [file], 'depth': depth}
|
|
343
343
|
|
|
344
|
-
|
|
344
|
+
return self._call_api(
|
|
345
|
+
'dependencies.GetDependencies',
|
|
346
|
+
self.dependencies_stub.GetDependencies,
|
|
347
|
+
file_request,
|
|
348
|
+
DependencyRequest,
|
|
349
|
+
'Sending dependency data for decoration (rqId: {rqId})...',
|
|
350
|
+
use_grpc=use_grpc,
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
def get_vulnerabilities_json(self, purls: Optional[dict] = None, use_grpc: Optional[bool] = None) -> Optional[dict]:
|
|
345
354
|
"""
|
|
346
|
-
Client function to call the rpc for
|
|
347
|
-
|
|
348
|
-
|
|
355
|
+
Client function to call the rpc for Vulnerability GetVulnerabilities
|
|
356
|
+
It will either use REST (default) or gRPC
|
|
357
|
+
|
|
358
|
+
Args:
|
|
359
|
+
purls (dict): Message to send to the service
|
|
360
|
+
|
|
361
|
+
Returns:
|
|
362
|
+
Server response or None
|
|
363
|
+
"""
|
|
364
|
+
return self._call_api(
|
|
365
|
+
'vulnerabilities.GetComponentsVulnerabilities',
|
|
366
|
+
self.vuln_stub.GetComponentsVulnerabilities,
|
|
367
|
+
purls,
|
|
368
|
+
ComponentsRequest,
|
|
369
|
+
'Sending vulnerability data for decoration (rqId: {rqId})...',
|
|
370
|
+
use_grpc=use_grpc,
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
def get_semgrep_json(self, purls: Optional[dict] = None, use_grpc: Optional[bool] = None) -> Optional[dict]:
|
|
349
374
|
"""
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
)
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
if not self._check_status_response(resp.status, request_id):
|
|
368
|
-
return None
|
|
369
|
-
resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict
|
|
370
|
-
del resp_dict['status']
|
|
371
|
-
return resp_dict
|
|
372
|
-
return None
|
|
375
|
+
Client function to call the rpc for Semgrep GetIssues
|
|
376
|
+
|
|
377
|
+
Args:
|
|
378
|
+
purls (dict): Message to send to the service
|
|
379
|
+
use_grpc (bool): Whether to use gRPC or REST
|
|
380
|
+
|
|
381
|
+
Returns:
|
|
382
|
+
Server response or None
|
|
383
|
+
"""
|
|
384
|
+
return self._call_api(
|
|
385
|
+
'semgrep.GetComponentsIssues',
|
|
386
|
+
self.semgrep_stub.GetComponentsIssues,
|
|
387
|
+
purls,
|
|
388
|
+
ComponentsRequest,
|
|
389
|
+
'Sending semgrep data for decoration (rqId: {rqId})...',
|
|
390
|
+
use_grpc=use_grpc,
|
|
391
|
+
)
|
|
373
392
|
|
|
374
|
-
def search_components_json(self, search: dict) -> dict:
|
|
393
|
+
def search_components_json(self, search: dict, use_grpc: Optional[bool] = None) -> Optional[dict]:
|
|
375
394
|
"""
|
|
376
395
|
Client function to call the rpc for Components SearchComponents
|
|
377
|
-
:param search: Message to send to the service
|
|
378
|
-
:return: Server response or None
|
|
379
|
-
"""
|
|
380
|
-
if not search:
|
|
381
|
-
self.print_stderr('ERROR: No message supplied to send to gRPC service.')
|
|
382
|
-
return None
|
|
383
|
-
request_id = str(uuid.uuid4())
|
|
384
|
-
resp: CompSearchResponse
|
|
385
|
-
try:
|
|
386
|
-
request = ParseDict(search, CompSearchRequest()) # Parse the JSON/Dict into the purl request object
|
|
387
|
-
metadata = self.metadata[:]
|
|
388
|
-
metadata.append(('x-request-id', request_id)) # Set a Request ID
|
|
389
|
-
self.print_debug(f'Sending component search data (rqId: {request_id})...')
|
|
390
|
-
resp = self.comp_search_stub.SearchComponents(request, metadata=metadata, timeout=self.timeout)
|
|
391
|
-
except Exception as e:
|
|
392
|
-
self.print_stderr(
|
|
393
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
394
|
-
)
|
|
395
|
-
else:
|
|
396
|
-
if resp:
|
|
397
|
-
if not self._check_status_response(resp.status, request_id):
|
|
398
|
-
return None
|
|
399
|
-
resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict
|
|
400
|
-
del resp_dict['status']
|
|
401
|
-
return resp_dict
|
|
402
|
-
return None
|
|
403
396
|
|
|
404
|
-
|
|
397
|
+
Args:
|
|
398
|
+
search (dict): Message to send to the service
|
|
399
|
+
Returns:
|
|
400
|
+
Server response or None
|
|
401
|
+
"""
|
|
402
|
+
return self._call_api(
|
|
403
|
+
'components.SearchComponents',
|
|
404
|
+
self.comp_search_stub.SearchComponents,
|
|
405
|
+
search,
|
|
406
|
+
CompSearchRequest,
|
|
407
|
+
'Sending component search data for decoration (rqId: {rqId})...',
|
|
408
|
+
use_grpc=use_grpc,
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
def get_component_versions_json(self, search: dict, use_grpc: Optional[bool] = None) -> Optional[dict]:
|
|
405
412
|
"""
|
|
406
413
|
Client function to call the rpc for Components GetComponentVersions
|
|
407
|
-
:param search: Message to send to the service
|
|
408
|
-
:return: Server response or None
|
|
409
|
-
"""
|
|
410
|
-
if not search:
|
|
411
|
-
self.print_stderr('ERROR: No message supplied to send to gRPC service.')
|
|
412
|
-
return None
|
|
413
|
-
request_id = str(uuid.uuid4())
|
|
414
|
-
resp: CompVersionResponse
|
|
415
|
-
try:
|
|
416
|
-
request = ParseDict(search, CompVersionRequest()) # Parse the JSON/Dict into the purl request object
|
|
417
|
-
metadata = self.metadata[:]
|
|
418
|
-
metadata.append(('x-request-id', request_id)) # Set a Request ID
|
|
419
|
-
self.print_debug(f'Sending component version data (rqId: {request_id})...')
|
|
420
|
-
resp = self.comp_search_stub.GetComponentVersions(request, metadata=metadata, timeout=self.timeout)
|
|
421
|
-
except Exception as e:
|
|
422
|
-
self.print_stderr(
|
|
423
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
424
|
-
)
|
|
425
|
-
else:
|
|
426
|
-
if resp:
|
|
427
|
-
if not self._check_status_response(resp.status, request_id):
|
|
428
|
-
return None
|
|
429
|
-
resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict
|
|
430
|
-
del resp_dict['status']
|
|
431
|
-
return resp_dict
|
|
432
|
-
return None
|
|
433
414
|
|
|
434
|
-
|
|
415
|
+
Args:
|
|
416
|
+
search (dict): Message to send to the service
|
|
417
|
+
Returns:
|
|
418
|
+
Server response or None
|
|
419
|
+
"""
|
|
420
|
+
return self._call_api(
|
|
421
|
+
'components.GetComponentVersions',
|
|
422
|
+
self.comp_search_stub.GetComponentVersions,
|
|
423
|
+
search,
|
|
424
|
+
CompVersionRequest,
|
|
425
|
+
'Sending component version data for decoration (rqId: {rqId})...',
|
|
426
|
+
use_grpc=use_grpc,
|
|
427
|
+
)
|
|
428
|
+
|
|
429
|
+
def folder_hash_scan(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
435
430
|
"""
|
|
436
431
|
Client function to call the rpc for Folder Hashing Scan
|
|
437
432
|
|
|
438
433
|
Args:
|
|
439
434
|
request (Dict): Folder Hash Request
|
|
435
|
+
use_grpc (Optional[bool]): Whether to use gRPC or REST API
|
|
440
436
|
|
|
441
437
|
Returns:
|
|
442
438
|
Optional[Dict]: Folder Hash Response, or None if the request was not succesfull
|
|
443
439
|
"""
|
|
444
|
-
return self.
|
|
440
|
+
return self._call_api(
|
|
441
|
+
'scanning.FolderHashScan',
|
|
445
442
|
self.scanning_stub.FolderHashScan,
|
|
446
443
|
request,
|
|
447
444
|
HFHRequest,
|
|
448
445
|
'Sending folder hash scan data (rqId: {rqId})...',
|
|
446
|
+
use_grpc=use_grpc,
|
|
449
447
|
)
|
|
450
448
|
|
|
449
|
+
def _call_api(
|
|
450
|
+
self,
|
|
451
|
+
endpoint_key: str,
|
|
452
|
+
rpc_method,
|
|
453
|
+
request_input,
|
|
454
|
+
request_type,
|
|
455
|
+
debug_msg: Optional[str] = None,
|
|
456
|
+
use_grpc: Optional[bool] = None,
|
|
457
|
+
) -> Optional[Dict]:
|
|
458
|
+
"""
|
|
459
|
+
Unified method to call either gRPC or REST API based on configuration
|
|
460
|
+
|
|
461
|
+
Args:
|
|
462
|
+
endpoint_key (str): The key to lookup the REST endpoint in REST_ENDPOINTS
|
|
463
|
+
rpc_method: The gRPC stub method (used only if use_grpc is True)
|
|
464
|
+
request_input: Either a dict or a gRPC request object
|
|
465
|
+
request_type: The type of the gRPC request object (used only if use_grpc is True)
|
|
466
|
+
debug_msg (str, optional): Debug message template that can include {rqId} placeholder
|
|
467
|
+
use_grpc (bool, optional): Override the instance's use_grpc setting. If None, uses self.use_grpc
|
|
468
|
+
|
|
469
|
+
Returns:
|
|
470
|
+
dict: The parsed response as a dictionary, or None if something went wrong
|
|
471
|
+
"""
|
|
472
|
+
if not request_input:
|
|
473
|
+
self.print_stderr('ERROR: No message supplied to send to service.')
|
|
474
|
+
return None
|
|
475
|
+
|
|
476
|
+
# Determine whether to use gRPC or REST
|
|
477
|
+
use_grpc_flag = use_grpc if use_grpc is not None else self.use_grpc
|
|
478
|
+
|
|
479
|
+
if use_grpc_flag:
|
|
480
|
+
return self._call_rpc(rpc_method, request_input, request_type, debug_msg)
|
|
481
|
+
else:
|
|
482
|
+
# For REST, we only need the dict input
|
|
483
|
+
if not isinstance(request_input, dict):
|
|
484
|
+
request_input = MessageToDict(request_input, preserving_proto_field_name=True)
|
|
485
|
+
return self._call_rest(endpoint_key, request_input, debug_msg)
|
|
486
|
+
|
|
451
487
|
def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional[str] = None) -> Optional[Dict]:
|
|
452
488
|
"""
|
|
453
489
|
Call a gRPC method and return the response as a dictionary
|
|
@@ -462,29 +498,26 @@ class ScanossGrpc(ScanossBase):
|
|
|
462
498
|
dict: The parsed gRPC response as a dictionary, or None if something went wrong
|
|
463
499
|
"""
|
|
464
500
|
request_id = str(uuid.uuid4())
|
|
465
|
-
|
|
466
501
|
if isinstance(request_input, dict):
|
|
467
502
|
request_obj = ParseDict(request_input, request_type())
|
|
468
503
|
else:
|
|
469
504
|
request_obj = request_input
|
|
470
|
-
|
|
471
505
|
metadata = self.metadata[:] + [('x-request-id', request_id)]
|
|
472
|
-
|
|
473
|
-
|
|
506
|
+
if debug_msg:
|
|
507
|
+
self.print_debug(debug_msg.format(rqId=request_id))
|
|
474
508
|
try:
|
|
475
509
|
resp = rpc_method(request_obj, metadata=metadata, timeout=self.timeout)
|
|
476
510
|
except grpc.RpcError as e:
|
|
477
511
|
raise ScanossGrpcError(
|
|
478
512
|
f'{e.__class__.__name__} while sending gRPC message (rqId: {request_id}): {e.details()}'
|
|
479
513
|
)
|
|
480
|
-
|
|
481
|
-
if resp and not self._check_status_response(resp.status, request_id):
|
|
514
|
+
if resp and not self._check_status_response_grpc(resp.status, request_id):
|
|
482
515
|
return None
|
|
483
516
|
|
|
484
517
|
resp_dict = MessageToDict(resp, preserving_proto_field_name=True)
|
|
485
518
|
return resp_dict
|
|
486
519
|
|
|
487
|
-
def
|
|
520
|
+
def _check_status_response_grpc(self, status_response: StatusResponse, request_id: str = None) -> bool:
|
|
488
521
|
"""
|
|
489
522
|
Check the response object to see if the command was successful or not
|
|
490
523
|
:param status_response: Status Response
|
|
@@ -499,15 +532,59 @@ class ScanossGrpc(ScanossBase):
|
|
|
499
532
|
if status_code > ScanossGrpcStatusCode.SUCCESS:
|
|
500
533
|
ret_val = False # default to failed
|
|
501
534
|
msg = 'Unsuccessful'
|
|
502
|
-
if status_code == ScanossGrpcStatusCode.
|
|
535
|
+
if status_code == ScanossGrpcStatusCode.SUCCEEDED_WITH_WARNINGS:
|
|
503
536
|
msg = 'Succeeded with warnings'
|
|
504
537
|
ret_val = True # No need to fail as it succeeded with warnings
|
|
505
|
-
elif status_code == ScanossGrpcStatusCode.
|
|
538
|
+
elif status_code == ScanossGrpcStatusCode.WARNING:
|
|
506
539
|
msg = 'Failed with warnings'
|
|
507
540
|
self.print_stderr(f'{msg} (rqId: {request_id} - status: {status_code}): {status_response.message}')
|
|
508
541
|
return ret_val
|
|
509
542
|
return True
|
|
510
543
|
|
|
544
|
+
def check_status_response_rest(self, status_dict: dict, request_id: Optional[str] = None) -> bool:
|
|
545
|
+
"""
|
|
546
|
+
Check the REST response dictionary to see if the command was successful or not
|
|
547
|
+
|
|
548
|
+
Args:
|
|
549
|
+
status_dict (dict): Status dictionary from REST response containing 'status' and 'message' keys
|
|
550
|
+
request_id (str, optional): Request ID for logging
|
|
551
|
+
Returns:
|
|
552
|
+
bool: True if successful, False otherwise
|
|
553
|
+
"""
|
|
554
|
+
if not status_dict:
|
|
555
|
+
self.print_stderr(f'Warning: No status response supplied (rqId: {request_id}). Assuming it was ok.')
|
|
556
|
+
return True
|
|
557
|
+
|
|
558
|
+
if request_id:
|
|
559
|
+
self.print_debug(f'Checking response status (rqId: {request_id}): {status_dict}')
|
|
560
|
+
|
|
561
|
+
# Get status from dictionary - it can be either a string or nested dict
|
|
562
|
+
status = status_dict.get('status')
|
|
563
|
+
message = status_dict.get('message', '')
|
|
564
|
+
ret_val = True
|
|
565
|
+
|
|
566
|
+
# Handle case where status might be a string directly
|
|
567
|
+
if isinstance(status, str):
|
|
568
|
+
status_str = status.upper()
|
|
569
|
+
if status_str == ScanossRESTStatusCode.SUCCESS.value:
|
|
570
|
+
ret_val = True
|
|
571
|
+
elif status_str == ScanossRESTStatusCode.SUCCEEDED_WITH_WARNINGS.value:
|
|
572
|
+
self.print_stderr(f'Succeeded with warnings (rqId: {request_id}): {message}')
|
|
573
|
+
ret_val = True
|
|
574
|
+
elif status_str == ScanossRESTStatusCode.WARNING.value:
|
|
575
|
+
self.print_stderr(f'Failed with warnings (rqId: {request_id}): {message}')
|
|
576
|
+
ret_val = False
|
|
577
|
+
elif status_str == ScanossRESTStatusCode.FAILED.value:
|
|
578
|
+
self.print_stderr(f'Unsuccessful (rqId: {request_id}): {message}')
|
|
579
|
+
ret_val = False
|
|
580
|
+
else:
|
|
581
|
+
self.print_debug(f'Unknown status "{status_str}" (rqId: {request_id}). Assuming success.')
|
|
582
|
+
ret_val = True
|
|
583
|
+
|
|
584
|
+
# Otherwise asume success
|
|
585
|
+
self.print_debug(f'Unexpected status type {type(status)} (rqId: {request_id}). Assuming success.')
|
|
586
|
+
return ret_val
|
|
587
|
+
|
|
511
588
|
def _get_proxy_config(self):
|
|
512
589
|
"""
|
|
513
590
|
Set the grpc_proxy/http_proxy/https_proxy environment variables if PAC file has been specified
|
|
@@ -530,138 +607,165 @@ class ScanossGrpc(ScanossBase):
|
|
|
530
607
|
os.environ['http_proxy'] = proxies.get('http') or ''
|
|
531
608
|
os.environ['https_proxy'] = proxies.get('https') or ''
|
|
532
609
|
|
|
533
|
-
def get_provenance_json(self, purls: dict) ->
|
|
534
|
-
"""
|
|
535
|
-
Client function to call the rpc for GetComponentProvenance
|
|
536
|
-
:param purls: Message to send to the service
|
|
537
|
-
:return: Server response or None
|
|
610
|
+
def get_provenance_json(self, purls: dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
538
611
|
"""
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
try:
|
|
545
|
-
request = ParseDict(purls, PurlRequest()) # Parse the JSON/Dict into the purl request object
|
|
546
|
-
metadata = self.metadata[:]
|
|
547
|
-
metadata.append(('x-request-id', request_id)) # Set a Request ID
|
|
548
|
-
self.print_debug(f'Sending data for provenance decoration (rqId: {request_id})...')
|
|
549
|
-
resp = self.provenance_stub.GetComponentContributors(request, metadata=metadata, timeout=self.timeout)
|
|
550
|
-
except Exception as e:
|
|
551
|
-
self.print_stderr(
|
|
552
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
553
|
-
)
|
|
554
|
-
else:
|
|
555
|
-
if resp:
|
|
556
|
-
if not self._check_status_response(resp.status, request_id):
|
|
557
|
-
return None
|
|
558
|
-
resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict
|
|
559
|
-
return resp_dict
|
|
560
|
-
return None
|
|
612
|
+
Client function to call the rpc for GetComponentContributors
|
|
613
|
+
|
|
614
|
+
Args:
|
|
615
|
+
purls (dict): ComponentsRequest
|
|
616
|
+
use_grpc (bool): Whether to use gRPC or REST (None = use instance default)
|
|
561
617
|
|
|
562
|
-
|
|
618
|
+
Returns:
|
|
619
|
+
dict: JSON response or None
|
|
620
|
+
"""
|
|
621
|
+
return self._call_api(
|
|
622
|
+
'geoprovenance.GetCountryContributorsByComponents',
|
|
623
|
+
self.provenance_stub.GetCountryContributorsByComponents,
|
|
624
|
+
purls,
|
|
625
|
+
ComponentsRequest,
|
|
626
|
+
'Sending data for provenance decoration (rqId: {rqId})...',
|
|
627
|
+
use_grpc=use_grpc,
|
|
628
|
+
)
|
|
629
|
+
|
|
630
|
+
def get_provenance_origin(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
563
631
|
"""
|
|
564
|
-
Client function to call the rpc for
|
|
632
|
+
Client function to call the rpc for GetOriginByComponents
|
|
565
633
|
|
|
566
634
|
Args:
|
|
567
|
-
request (Dict):
|
|
635
|
+
request (Dict): GetOriginByComponents Request
|
|
568
636
|
|
|
569
637
|
Returns:
|
|
570
638
|
Optional[Dict]: OriginResponse, or None if the request was not successfull
|
|
571
639
|
"""
|
|
572
|
-
return self.
|
|
573
|
-
|
|
640
|
+
return self._call_api(
|
|
641
|
+
'geoprovenance.GetOriginByComponents',
|
|
642
|
+
self.provenance_stub.GetOriginByComponents,
|
|
574
643
|
request,
|
|
575
|
-
|
|
644
|
+
ComponentsRequest,
|
|
576
645
|
'Sending data for provenance origin decoration (rqId: {rqId})...',
|
|
646
|
+
use_grpc=use_grpc,
|
|
577
647
|
)
|
|
578
648
|
|
|
579
|
-
def get_crypto_algorithms_for_purl(self, request: Dict) -> Optional[Dict]:
|
|
649
|
+
def get_crypto_algorithms_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
580
650
|
"""
|
|
581
|
-
Client function to call the rpc for
|
|
651
|
+
Client function to call the rpc for GetComponentsAlgorithms for a list of purls
|
|
582
652
|
|
|
583
653
|
Args:
|
|
584
|
-
request (Dict):
|
|
654
|
+
request (Dict): ComponentsRequest
|
|
655
|
+
use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default)
|
|
585
656
|
|
|
586
657
|
Returns:
|
|
587
658
|
Optional[Dict]: AlgorithmResponse, or None if the request was not successfull
|
|
588
659
|
"""
|
|
589
|
-
return self.
|
|
590
|
-
|
|
660
|
+
return self._call_api(
|
|
661
|
+
'cryptography.GetComponentsAlgorithms',
|
|
662
|
+
self.crypto_stub.GetComponentsAlgorithms,
|
|
591
663
|
request,
|
|
592
|
-
|
|
664
|
+
ComponentsRequest,
|
|
593
665
|
'Sending data for cryptographic algorithms decoration (rqId: {rqId})...',
|
|
666
|
+
use_grpc=use_grpc,
|
|
594
667
|
)
|
|
595
668
|
|
|
596
|
-
def get_crypto_algorithms_in_range_for_purl(self, request: Dict) -> Optional[Dict]:
|
|
669
|
+
def get_crypto_algorithms_in_range_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
597
670
|
"""
|
|
598
|
-
Client function to call the rpc for
|
|
671
|
+
Client function to call the rpc for GetComponentsAlgorithmsInRange for a list of purls
|
|
599
672
|
|
|
600
673
|
Args:
|
|
601
|
-
request (Dict):
|
|
674
|
+
request (Dict): ComponentsRequest
|
|
675
|
+
use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default)
|
|
602
676
|
|
|
603
677
|
Returns:
|
|
604
678
|
Optional[Dict]: AlgorithmsInRangeResponse, or None if the request was not successfull
|
|
605
679
|
"""
|
|
606
|
-
return self.
|
|
607
|
-
|
|
680
|
+
return self._call_api(
|
|
681
|
+
'cryptography.GetComponentsAlgorithmsInRange',
|
|
682
|
+
self.crypto_stub.GetComponentsAlgorithmsInRange,
|
|
608
683
|
request,
|
|
609
|
-
|
|
684
|
+
ComponentsRequest,
|
|
610
685
|
'Sending data for cryptographic algorithms in range decoration (rqId: {rqId})...',
|
|
686
|
+
use_grpc=use_grpc,
|
|
611
687
|
)
|
|
612
688
|
|
|
613
|
-
def get_encryption_hints_for_purl(self, request: Dict) -> Optional[Dict]:
|
|
689
|
+
def get_encryption_hints_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
614
690
|
"""
|
|
615
|
-
Client function to call the rpc for
|
|
691
|
+
Client function to call the rpc for GetComponentsEncryptionHints for a list of purls
|
|
616
692
|
|
|
617
693
|
Args:
|
|
618
|
-
request (Dict):
|
|
694
|
+
request (Dict): ComponentsRequest
|
|
695
|
+
use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default)
|
|
619
696
|
|
|
620
697
|
Returns:
|
|
621
698
|
Optional[Dict]: HintsResponse, or None if the request was not successfull
|
|
622
699
|
"""
|
|
623
|
-
return self.
|
|
624
|
-
|
|
700
|
+
return self._call_api(
|
|
701
|
+
'cryptography.GetComponentsEncryptionHints',
|
|
702
|
+
self.crypto_stub.GetComponentsEncryptionHints,
|
|
625
703
|
request,
|
|
626
|
-
|
|
704
|
+
ComponentsRequest,
|
|
627
705
|
'Sending data for encryption hints decoration (rqId: {rqId})...',
|
|
706
|
+
use_grpc=use_grpc,
|
|
628
707
|
)
|
|
629
708
|
|
|
630
|
-
def get_encryption_hints_in_range_for_purl(self, request: Dict) -> Optional[Dict]:
|
|
709
|
+
def get_encryption_hints_in_range_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
631
710
|
"""
|
|
632
|
-
Client function to call the rpc for
|
|
711
|
+
Client function to call the rpc for GetComponentsHintsInRange for a list of purls
|
|
633
712
|
|
|
634
713
|
Args:
|
|
635
|
-
request (Dict):
|
|
714
|
+
request (Dict): ComponentsRequest
|
|
715
|
+
use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default)
|
|
636
716
|
|
|
637
717
|
Returns:
|
|
638
718
|
Optional[Dict]: HintsInRangeResponse, or None if the request was not successfull
|
|
639
719
|
"""
|
|
640
|
-
return self.
|
|
641
|
-
|
|
720
|
+
return self._call_api(
|
|
721
|
+
'cryptography.GetComponentsHintsInRange',
|
|
722
|
+
self.crypto_stub.GetComponentsHintsInRange,
|
|
642
723
|
request,
|
|
643
|
-
|
|
724
|
+
ComponentsRequest,
|
|
644
725
|
'Sending data for encryption hints in range decoration (rqId: {rqId})...',
|
|
726
|
+
use_grpc=use_grpc,
|
|
645
727
|
)
|
|
646
728
|
|
|
647
|
-
def get_versions_in_range_for_purl(self, request: Dict) -> Optional[Dict]:
|
|
729
|
+
def get_versions_in_range_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
648
730
|
"""
|
|
649
|
-
Client function to call the rpc for
|
|
731
|
+
Client function to call the rpc for GetComponentsVersionsInRange for a list of purls
|
|
650
732
|
|
|
651
733
|
Args:
|
|
652
|
-
request (Dict):
|
|
734
|
+
request (Dict): ComponentsRequest
|
|
735
|
+
use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default)
|
|
653
736
|
|
|
654
737
|
Returns:
|
|
655
738
|
Optional[Dict]: VersionsInRangeResponse, or None if the request was not successfull
|
|
656
739
|
"""
|
|
657
|
-
return self.
|
|
658
|
-
|
|
740
|
+
return self._call_api(
|
|
741
|
+
'cryptography.GetComponentsVersionsInRange',
|
|
742
|
+
self.crypto_stub.GetComponentsVersionsInRange,
|
|
659
743
|
request,
|
|
660
|
-
|
|
744
|
+
ComponentsRequest,
|
|
661
745
|
'Sending data for cryptographic versions in range decoration (rqId: {rqId})...',
|
|
746
|
+
use_grpc=use_grpc,
|
|
747
|
+
)
|
|
748
|
+
|
|
749
|
+
def get_licenses(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
750
|
+
"""
|
|
751
|
+
Client function to call the rpc for Licenses GetComponentsLicenses
|
|
752
|
+
It will either use REST (default) or gRPC depending on the use_grpc flag
|
|
753
|
+
|
|
754
|
+
Args:
|
|
755
|
+
request (Dict): ComponentsRequest
|
|
756
|
+
Returns:
|
|
757
|
+
Optional[Dict]: ComponentsLicenseResponse, or None if the request was not successfull
|
|
758
|
+
"""
|
|
759
|
+
return self._call_api(
|
|
760
|
+
'licenses.GetComponentsLicenses',
|
|
761
|
+
self.license_stub.GetComponentsLicenses,
|
|
762
|
+
request,
|
|
763
|
+
ComponentsRequest,
|
|
764
|
+
'Sending data for license decoration (rqId: {rqId})...',
|
|
765
|
+
use_grpc=use_grpc,
|
|
662
766
|
)
|
|
663
767
|
|
|
664
|
-
def load_generic_headers(self):
|
|
768
|
+
def load_generic_headers(self, url: Optional[str] = None):
|
|
665
769
|
"""
|
|
666
770
|
Adds custom headers from req_headers to metadata.
|
|
667
771
|
|
|
@@ -671,10 +775,155 @@ class ScanossGrpc(ScanossBase):
|
|
|
671
775
|
if self.req_headers: # Load generic headers
|
|
672
776
|
for key, value in self.req_headers.items():
|
|
673
777
|
if key == 'x-api-key': # Set premium URL if x-api-key header is set
|
|
674
|
-
if not
|
|
778
|
+
if not url and not os.environ.get('SCANOSS_GRPC_URL'):
|
|
675
779
|
self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium
|
|
676
780
|
self.api_key = value
|
|
677
781
|
self.metadata.append((key, value))
|
|
782
|
+
self.headers[key] = value
|
|
783
|
+
|
|
784
|
+
#
|
|
785
|
+
# End of gRPC Client Functions
|
|
786
|
+
#
|
|
787
|
+
# Start of REST Client Functions
|
|
788
|
+
#
|
|
789
|
+
|
|
790
|
+
def _rest_get(self, uri: str, request_id: str, params: Optional[dict] = None) -> Optional[dict]:
|
|
791
|
+
"""
|
|
792
|
+
Send a GET request to the specified URI with optional query parameters.
|
|
793
|
+
|
|
794
|
+
Args:
|
|
795
|
+
uri (str): URI to send GET request to
|
|
796
|
+
request_id (str): request id
|
|
797
|
+
params (dict, optional): Optional query parameters as dictionary
|
|
798
|
+
|
|
799
|
+
Returns:
|
|
800
|
+
dict: JSON response or None
|
|
801
|
+
"""
|
|
802
|
+
if not uri:
|
|
803
|
+
self.print_stderr('Error: Missing URI. Cannot perform GET request.')
|
|
804
|
+
return None
|
|
805
|
+
self.print_trace(f'Sending REST GET request to {uri}...')
|
|
806
|
+
headers = self.headers.copy()
|
|
807
|
+
headers['x-request-id'] = request_id
|
|
808
|
+
retry = 0
|
|
809
|
+
while retry <= self.retry_limit:
|
|
810
|
+
retry += 1
|
|
811
|
+
try:
|
|
812
|
+
response = self.session.get(uri, headers=headers, params=params, timeout=self.timeout)
|
|
813
|
+
response.raise_for_status() # Raises an HTTPError for bad responses
|
|
814
|
+
return response.json()
|
|
815
|
+
except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as e:
|
|
816
|
+
self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) sending GET request - {e}.')
|
|
817
|
+
raise Exception(f'ERROR: The SCANOSS API GET request failed for {uri}') from e
|
|
818
|
+
except requests.exceptions.HTTPError as e:
|
|
819
|
+
self.print_stderr(f'ERROR: HTTP error sending GET request ({request_id}): {e}')
|
|
820
|
+
raise Exception(
|
|
821
|
+
f'ERROR: The SCANOSS API GET request failed with status {e.response.status_code} for {uri}'
|
|
822
|
+
) from e
|
|
823
|
+
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
|
|
824
|
+
if retry > self.retry_limit: # Timed out retry_limit or more times, fail
|
|
825
|
+
self.print_stderr(f'ERROR: {e.__class__.__name__} sending GET request ({request_id}): {e}')
|
|
826
|
+
raise Exception(
|
|
827
|
+
f'ERROR: The SCANOSS API GET request timed out ({e.__class__.__name__}) for {uri}'
|
|
828
|
+
) from e
|
|
829
|
+
else:
|
|
830
|
+
self.print_stderr(f'Warning: {e.__class__.__name__} communicating with {self.url}. Retrying...')
|
|
831
|
+
time.sleep(5)
|
|
832
|
+
except requests.exceptions.RequestException as e:
|
|
833
|
+
self.print_stderr(f'Error: Problem sending GET request to {uri}: {e}')
|
|
834
|
+
raise Exception(f'ERROR: The SCANOSS API GET request failed for {uri}') from e
|
|
835
|
+
except Exception as e:
|
|
836
|
+
self.print_stderr(
|
|
837
|
+
f'ERROR: Exception ({e.__class__.__name__}) sending GET request ({request_id}) to {uri}: {e}'
|
|
838
|
+
)
|
|
839
|
+
raise Exception(f'ERROR: The SCANOSS API GET request failed for {uri}') from e
|
|
840
|
+
return None
|
|
841
|
+
|
|
842
|
+
def _rest_post(self, uri: str, request_id: str, data: dict) -> Optional[dict]:
|
|
843
|
+
"""
|
|
844
|
+
Post the specified data to the given URI.
|
|
845
|
+
|
|
846
|
+
Args:
|
|
847
|
+
uri (str): URI to post to
|
|
848
|
+
request_id (str): request id
|
|
849
|
+
data (dict): json data to post
|
|
850
|
+
|
|
851
|
+
Returns:
|
|
852
|
+
dict: JSON response or None
|
|
853
|
+
"""
|
|
854
|
+
if not uri:
|
|
855
|
+
self.print_stderr('Error: Missing URI. Cannot search for project.')
|
|
856
|
+
return None
|
|
857
|
+
self.print_trace(f'Sending REST POST data to {uri}...')
|
|
858
|
+
headers = self.headers.copy()
|
|
859
|
+
headers['x-request-id'] = request_id
|
|
860
|
+
retry = 0
|
|
861
|
+
while retry <= self.retry_limit:
|
|
862
|
+
retry += 1
|
|
863
|
+
try:
|
|
864
|
+
response = self.session.post(uri, headers=headers, json=data, timeout=self.timeout)
|
|
865
|
+
response.raise_for_status() # Raises an HTTPError for bad responses
|
|
866
|
+
return response.json()
|
|
867
|
+
except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as e:
|
|
868
|
+
self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}.')
|
|
869
|
+
raise Exception(f'ERROR: The SCANOSS Decoration API request failed for {uri}') from e
|
|
870
|
+
except requests.exceptions.HTTPError as e:
|
|
871
|
+
self.print_stderr(f'ERROR: HTTP error POSTing data ({request_id}): {e}')
|
|
872
|
+
raise Exception(
|
|
873
|
+
f'ERROR: The SCANOSS Decoration API request failed with status {e.response.status_code} for {uri}'
|
|
874
|
+
) from e
|
|
875
|
+
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
|
|
876
|
+
if retry > self.retry_limit: # Timed out retry_limit or more times, fail
|
|
877
|
+
self.print_stderr(f'ERROR: {e.__class__.__name__} POSTing decoration data ({request_id}): {e}')
|
|
878
|
+
raise Exception(
|
|
879
|
+
f'ERROR: The SCANOSS Decoration API request timed out ({e.__class__.__name__}) for {uri}'
|
|
880
|
+
) from e
|
|
881
|
+
else:
|
|
882
|
+
self.print_stderr(f'Warning: {e.__class__.__name__} communicating with {self.url}. Retrying...')
|
|
883
|
+
time.sleep(5)
|
|
884
|
+
except requests.exceptions.RequestException as e:
|
|
885
|
+
self.print_stderr(f'Error: Problem posting data to {uri}: {e}')
|
|
886
|
+
raise Exception(f'ERROR: The SCANOSS Decoration API request failed for {uri}') from e
|
|
887
|
+
except Exception as e:
|
|
888
|
+
self.print_stderr(
|
|
889
|
+
f'ERROR: Exception ({e.__class__.__name__}) POSTing data ({request_id}) to {uri}: {e}'
|
|
890
|
+
)
|
|
891
|
+
raise Exception(f'ERROR: The SCANOSS Decoration API request failed for {uri}') from e
|
|
892
|
+
return None
|
|
893
|
+
|
|
894
|
+
def _call_rest(self, endpoint_key: str, request_input: dict, debug_msg: Optional[str] = None) -> Optional[Dict]:
|
|
895
|
+
"""
|
|
896
|
+
Call a REST endpoint and return the response as a dictionary
|
|
897
|
+
|
|
898
|
+
Args:
|
|
899
|
+
endpoint_key (str): The key to lookup the REST endpoint in REST_ENDPOINTS
|
|
900
|
+
request_input (dict): The request data to send
|
|
901
|
+
debug_msg (str, optional): Debug message template that can include {rqId} placeholder.
|
|
902
|
+
|
|
903
|
+
Returns:
|
|
904
|
+
dict: The parsed REST response as a dictionary, or None if something went wrong
|
|
905
|
+
"""
|
|
906
|
+
if endpoint_key not in REST_ENDPOINTS:
|
|
907
|
+
raise ScanossGrpcError(f'Unknown REST endpoint key: {endpoint_key}')
|
|
908
|
+
|
|
909
|
+
endpoint_config = REST_ENDPOINTS[endpoint_key]
|
|
910
|
+
endpoint_path = endpoint_config['path']
|
|
911
|
+
method = endpoint_config['method']
|
|
912
|
+
endpoint_url = f'{self.orig_url}{DEFAULT_URI_PREFIX}{endpoint_path}'
|
|
913
|
+
request_id = str(uuid.uuid4())
|
|
914
|
+
|
|
915
|
+
if debug_msg:
|
|
916
|
+
self.print_debug(debug_msg.format(rqId=request_id))
|
|
917
|
+
|
|
918
|
+
if method == 'GET':
|
|
919
|
+
response = self._rest_get(endpoint_url, request_id, params=request_input)
|
|
920
|
+
else: # POST
|
|
921
|
+
response = self._rest_post(endpoint_url, request_id, request_input)
|
|
922
|
+
|
|
923
|
+
if response and 'status' in response and not self.check_status_response_rest(response['status'], request_id):
|
|
924
|
+
return None
|
|
925
|
+
|
|
926
|
+
return response
|
|
678
927
|
|
|
679
928
|
|
|
680
929
|
#
|
|
@@ -711,3 +960,8 @@ def create_grpc_config_from_args(args) -> GrpcConfig:
|
|
|
711
960
|
proxy=getattr(args, 'proxy', None),
|
|
712
961
|
grpc_proxy=getattr(args, 'grpc_proxy', None),
|
|
713
962
|
)
|
|
963
|
+
|
|
964
|
+
|
|
965
|
+
#
|
|
966
|
+
# End of GrpcConfig class
|
|
967
|
+
#
|