scanoss 1.31.5__py3-none-any.whl → 1.34.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- protoc_gen_swagger/options/annotations_pb2.py +18 -12
- protoc_gen_swagger/options/annotations_pb2.pyi +48 -0
- protoc_gen_swagger/options/annotations_pb2_grpc.py +20 -0
- protoc_gen_swagger/options/openapiv2_pb2.py +110 -99
- protoc_gen_swagger/options/openapiv2_pb2.pyi +1317 -0
- protoc_gen_swagger/options/openapiv2_pb2_grpc.py +20 -0
- scanoss/__init__.py +1 -1
- scanoss/api/common/v2/scanoss_common_pb2.py +8 -6
- scanoss/api/common/v2/scanoss_common_pb2_grpc.py +5 -1
- scanoss/api/components/v2/scanoss_components_pb2.py +46 -32
- scanoss/api/components/v2/scanoss_components_pb2_grpc.py +6 -6
- scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py +107 -29
- scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py +545 -9
- scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +29 -21
- scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py +1 -0
- scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py +51 -19
- scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py +189 -1
- scanoss/api/licenses/v2/scanoss_licenses_pb2.py +27 -27
- scanoss/api/scanning/v2/scanoss_scanning_pb2.py +18 -18
- scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py +29 -13
- scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py +102 -8
- scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +21 -21
- scanoss/cli.py +196 -144
- scanoss/components.py +80 -50
- scanoss/cryptography.py +64 -44
- scanoss/cyclonedx.py +22 -0
- scanoss/data/build_date.txt +1 -1
- scanoss/scanner.py +3 -0
- scanoss/scanossapi.py +22 -24
- scanoss/scanossgrpc.py +538 -287
- {scanoss-1.31.5.dist-info → scanoss-1.34.0.dist-info}/METADATA +4 -3
- {scanoss-1.31.5.dist-info → scanoss-1.34.0.dist-info}/RECORD +36 -34
- {scanoss-1.31.5.dist-info → scanoss-1.34.0.dist-info}/WHEEL +0 -0
- {scanoss-1.31.5.dist-info → scanoss-1.34.0.dist-info}/entry_points.txt +0 -0
- {scanoss-1.31.5.dist-info → scanoss-1.34.0.dist-info}/licenses/LICENSE +0 -0
- {scanoss-1.31.5.dist-info → scanoss-1.34.0.dist-info}/top_level.txt +0 -0
scanoss/scanossgrpc.py
CHANGED
|
@@ -23,46 +23,48 @@ SPDX-License-Identifier: MIT
|
|
|
23
23
|
"""
|
|
24
24
|
|
|
25
25
|
import concurrent.futures
|
|
26
|
-
import
|
|
26
|
+
import http.client as http_client
|
|
27
|
+
import logging
|
|
27
28
|
import os
|
|
29
|
+
import sys
|
|
30
|
+
import time
|
|
28
31
|
import uuid
|
|
29
32
|
from dataclasses import dataclass
|
|
30
|
-
from enum import IntEnum
|
|
33
|
+
from enum import Enum, IntEnum
|
|
31
34
|
from typing import Dict, Optional
|
|
32
35
|
from urllib.parse import urlparse
|
|
33
36
|
|
|
34
37
|
import grpc
|
|
38
|
+
import requests
|
|
39
|
+
import urllib3
|
|
35
40
|
from google.protobuf.json_format import MessageToDict, ParseDict
|
|
41
|
+
from pypac import PACSession
|
|
36
42
|
from pypac.parser import PACFile
|
|
37
43
|
from pypac.resolver import ProxyResolver
|
|
44
|
+
from urllib3.exceptions import InsecureRequestWarning
|
|
38
45
|
|
|
46
|
+
from scanoss.api.licenses.v2.scanoss_licenses_pb2_grpc import LicenseStub
|
|
39
47
|
from scanoss.api.scanning.v2.scanoss_scanning_pb2_grpc import ScanningStub
|
|
40
48
|
from scanoss.constants import DEFAULT_TIMEOUT
|
|
41
49
|
|
|
42
50
|
from . import __version__
|
|
43
51
|
from .api.common.v2.scanoss_common_pb2 import (
|
|
52
|
+
ComponentsRequest,
|
|
44
53
|
EchoRequest,
|
|
45
|
-
EchoResponse,
|
|
46
|
-
PurlRequest,
|
|
47
54
|
StatusCode,
|
|
48
55
|
StatusResponse,
|
|
49
56
|
)
|
|
50
57
|
from .api.components.v2.scanoss_components_pb2 import (
|
|
51
58
|
CompSearchRequest,
|
|
52
|
-
CompSearchResponse,
|
|
53
59
|
CompVersionRequest,
|
|
54
|
-
CompVersionResponse,
|
|
55
60
|
)
|
|
56
61
|
from .api.components.v2.scanoss_components_pb2_grpc import ComponentsStub
|
|
57
62
|
from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub
|
|
58
63
|
from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest
|
|
59
64
|
from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub
|
|
60
|
-
from .api.geoprovenance.v2.scanoss_geoprovenance_pb2 import ContributorResponse
|
|
61
65
|
from .api.geoprovenance.v2.scanoss_geoprovenance_pb2_grpc import GeoProvenanceStub
|
|
62
66
|
from .api.scanning.v2.scanoss_scanning_pb2 import HFHRequest
|
|
63
|
-
from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse
|
|
64
67
|
from .api.semgrep.v2.scanoss_semgrep_pb2_grpc import SemgrepStub
|
|
65
|
-
from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import VulnerabilityResponse
|
|
66
68
|
from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2_grpc import VulnerabilitiesStub
|
|
67
69
|
from .scanossbase import ScanossBase
|
|
68
70
|
|
|
@@ -70,8 +72,38 @@ DEFAULT_URL = 'https://api.osskb.org' # default free service URL
|
|
|
70
72
|
DEFAULT_URL2 = 'https://api.scanoss.com' # default premium service URL
|
|
71
73
|
SCANOSS_GRPC_URL = os.environ.get('SCANOSS_GRPC_URL') if os.environ.get('SCANOSS_GRPC_URL') else DEFAULT_URL
|
|
72
74
|
SCANOSS_API_KEY = os.environ.get('SCANOSS_API_KEY') if os.environ.get('SCANOSS_API_KEY') else ''
|
|
73
|
-
|
|
74
|
-
|
|
75
|
+
DEFAULT_URI_PREFIX = '/v2'
|
|
76
|
+
|
|
77
|
+
MAX_CONCURRENT_REQUESTS = 5 # Maximum number of concurrent requests to make
|
|
78
|
+
|
|
79
|
+
# REST API endpoint mappings with HTTP methods
|
|
80
|
+
REST_ENDPOINTS = {
|
|
81
|
+
'vulnerabilities.GetComponentsVulnerabilities': {'path': '/vulnerabilities/components', 'method': 'POST'},
|
|
82
|
+
'dependencies.Echo': {'path': '/dependencies/echo', 'method': 'POST'},
|
|
83
|
+
'dependencies.GetDependencies': {'path': '/dependencies/dependencies', 'method': 'POST'},
|
|
84
|
+
'cryptography.Echo': {'path': '/cryptography/echo', 'method': 'POST'},
|
|
85
|
+
'cryptography.GetComponentsAlgorithms': {'path': '/cryptography/algorithms/components', 'method': 'POST'},
|
|
86
|
+
'cryptography.GetComponentsAlgorithmsInRange': {
|
|
87
|
+
'path': '/cryptography/algorithms/range/components',
|
|
88
|
+
'method': 'POST',
|
|
89
|
+
},
|
|
90
|
+
'cryptography.GetComponentsEncryptionHints': {'path': '/cryptography/hints/components', 'method': 'POST'},
|
|
91
|
+
'cryptography.GetComponentsHintsInRange': {'path': '/cryptography/hints/components/range', 'method': 'POST'},
|
|
92
|
+
'cryptography.GetComponentsVersionsInRange': {
|
|
93
|
+
'path': '/cryptography/algorithms/versions/range/components',
|
|
94
|
+
'method': 'POST',
|
|
95
|
+
},
|
|
96
|
+
'components.SearchComponents': {'path': '/components/search', 'method': 'GET'},
|
|
97
|
+
'components.GetComponentVersions': {'path': '/components/versions', 'method': 'GET'},
|
|
98
|
+
'geoprovenance.GetCountryContributorsByComponents': {
|
|
99
|
+
'path': '/geoprovenance/countries/components',
|
|
100
|
+
'method': 'POST',
|
|
101
|
+
},
|
|
102
|
+
'geoprovenance.GetOriginByComponents': {'path': '/geoprovenance/origin/components', 'method': 'POST'},
|
|
103
|
+
'licenses.GetComponentsLicenses': {'path': '/licenses/components', 'method': 'POST'},
|
|
104
|
+
'semgrep.GetComponentsIssues': {'path': '/semgrep/issues/components', 'method': 'POST'},
|
|
105
|
+
'scanning.FolderHashScan': {'path': '/scanning/hfh/scan', 'method': 'POST'},
|
|
106
|
+
}
|
|
75
107
|
|
|
76
108
|
|
|
77
109
|
class ScanossGrpcError(Exception):
|
|
@@ -85,31 +117,44 @@ class ScanossGrpcError(Exception):
|
|
|
85
117
|
class ScanossGrpcStatusCode(IntEnum):
|
|
86
118
|
"""Status codes for SCANOSS gRPC responses"""
|
|
87
119
|
|
|
120
|
+
UNSPECIFIED = 0
|
|
88
121
|
SUCCESS = 1
|
|
89
|
-
|
|
90
|
-
|
|
122
|
+
SUCCEEDED_WITH_WARNINGS = 2
|
|
123
|
+
WARNING = 3
|
|
91
124
|
FAILED = 4
|
|
92
125
|
|
|
93
126
|
|
|
127
|
+
class ScanossRESTStatusCode(Enum):
|
|
128
|
+
"""Status codes for SCANOSS REST responses"""
|
|
129
|
+
|
|
130
|
+
UNSPECIFIED = 'UNSPECIFIED'
|
|
131
|
+
SUCCESS = 'SUCCESS'
|
|
132
|
+
SUCCEEDED_WITH_WARNINGS = 'SUCCEEDED_WITH_WARNINGS'
|
|
133
|
+
WARNING = 'WARNING'
|
|
134
|
+
FAILED = 'FAILED'
|
|
135
|
+
|
|
136
|
+
|
|
94
137
|
class ScanossGrpc(ScanossBase):
|
|
95
138
|
"""
|
|
96
139
|
Client for gRPC functionality
|
|
97
140
|
"""
|
|
98
141
|
|
|
99
|
-
def __init__( # noqa: PLR0913, PLR0915
|
|
142
|
+
def __init__( # noqa: PLR0912, PLR0913, PLR0915
|
|
100
143
|
self,
|
|
101
|
-
url: str = None,
|
|
144
|
+
url: Optional[str] = None,
|
|
102
145
|
debug: bool = False,
|
|
103
146
|
trace: bool = False,
|
|
104
147
|
quiet: bool = False,
|
|
105
|
-
ca_cert: str = None,
|
|
106
|
-
api_key: str = None,
|
|
107
|
-
ver_details: str = None,
|
|
148
|
+
ca_cert: Optional[str] = None,
|
|
149
|
+
api_key: Optional[str] = None,
|
|
150
|
+
ver_details: Optional[str] = None,
|
|
108
151
|
timeout: int = 600,
|
|
109
|
-
proxy: str = None,
|
|
110
|
-
grpc_proxy: str = None,
|
|
111
|
-
pac: PACFile = None,
|
|
112
|
-
req_headers: dict = None,
|
|
152
|
+
proxy: Optional[str] = None,
|
|
153
|
+
grpc_proxy: Optional[str] = None,
|
|
154
|
+
pac: Optional[PACFile] = None,
|
|
155
|
+
req_headers: Optional[dict] = None,
|
|
156
|
+
ignore_cert_errors: bool = False,
|
|
157
|
+
use_grpc: Optional[bool] = False,
|
|
113
158
|
):
|
|
114
159
|
"""
|
|
115
160
|
|
|
@@ -127,27 +172,55 @@ class ScanossGrpc(ScanossBase):
|
|
|
127
172
|
grpc_proxy='http://<ip>:<port>'
|
|
128
173
|
"""
|
|
129
174
|
super().__init__(debug, trace, quiet)
|
|
130
|
-
self.url = url
|
|
131
175
|
self.api_key = api_key if api_key else SCANOSS_API_KEY
|
|
132
176
|
self.timeout = timeout
|
|
133
177
|
self.proxy = proxy
|
|
134
178
|
self.grpc_proxy = grpc_proxy
|
|
135
179
|
self.pac = pac
|
|
136
|
-
self.req_headers = req_headers
|
|
137
180
|
self.metadata = []
|
|
181
|
+
self.ignore_cert_errors = ignore_cert_errors
|
|
182
|
+
self.use_grpc = use_grpc
|
|
183
|
+
self.req_headers = req_headers if req_headers else {}
|
|
184
|
+
self.headers = {}
|
|
185
|
+
self.retry_limit = 2 # default retry limit
|
|
138
186
|
|
|
139
187
|
if self.api_key:
|
|
140
188
|
self.metadata.append(('x-api-key', api_key)) # Set API key if we have one
|
|
189
|
+
self.headers['X-Session'] = self.api_key
|
|
190
|
+
self.headers['x-api-key'] = self.api_key
|
|
141
191
|
if ver_details:
|
|
142
192
|
self.metadata.append(('x-scanoss-client', ver_details))
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
193
|
+
self.headers['x-scanoss-client'] = ver_details
|
|
194
|
+
user_agent = f'scanoss-py/{__version__}'
|
|
195
|
+
self.metadata.append(('user-agent', user_agent))
|
|
196
|
+
self.headers['User-Agent'] = user_agent
|
|
197
|
+
self.headers['user-agent'] = user_agent
|
|
198
|
+
self.headers['Content-Type'] = 'application/json'
|
|
199
|
+
# Set the correct URL/API key combination
|
|
146
200
|
self.url = url if url else SCANOSS_GRPC_URL
|
|
147
201
|
if self.api_key and not url and not os.environ.get('SCANOSS_GRPC_URL'):
|
|
148
202
|
self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium
|
|
203
|
+
self.load_generic_headers(url)
|
|
149
204
|
self.url = self.url.lower()
|
|
150
|
-
self.orig_url = self.url # Used for proxy lookup
|
|
205
|
+
self.orig_url = self.url.strip().rstrip('/') # Used for proxy lookup
|
|
206
|
+
# REST setup
|
|
207
|
+
if self.trace:
|
|
208
|
+
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
|
|
209
|
+
http_client.HTTPConnection.debuglevel = 1
|
|
210
|
+
if pac and not proxy: # Set up a PAC session if requested (and no proxy has been explicitly set)
|
|
211
|
+
self.print_debug('Setting up PAC session...')
|
|
212
|
+
self.session = PACSession(pac=pac)
|
|
213
|
+
else:
|
|
214
|
+
self.session = requests.sessions.Session()
|
|
215
|
+
if self.ignore_cert_errors:
|
|
216
|
+
self.print_debug('Ignoring cert errors...')
|
|
217
|
+
urllib3.disable_warnings(InsecureRequestWarning)
|
|
218
|
+
self.session.verify = False
|
|
219
|
+
elif ca_cert:
|
|
220
|
+
self.session.verify = ca_cert
|
|
221
|
+
self.proxies = {'https': proxy, 'http': proxy} if proxy else None
|
|
222
|
+
if self.proxies:
|
|
223
|
+
self.session.proxies = self.proxies
|
|
151
224
|
|
|
152
225
|
secure = True if self.url.startswith('https:') else False # Is it a secure connection?
|
|
153
226
|
if self.url.startswith('http'):
|
|
@@ -162,7 +235,7 @@ class ScanossGrpc(ScanossBase):
|
|
|
162
235
|
cert_data = ScanossGrpc._load_cert(ca_cert)
|
|
163
236
|
self.print_debug(f'Setting up (secure: {secure}) connection to {self.url}...')
|
|
164
237
|
self._get_proxy_config()
|
|
165
|
-
if secure
|
|
238
|
+
if not secure: # insecure connection
|
|
166
239
|
self.comp_search_stub = ComponentsStub(grpc.insecure_channel(self.url))
|
|
167
240
|
self.crypto_stub = CryptographyStub(grpc.insecure_channel(self.url))
|
|
168
241
|
self.dependencies_stub = DependenciesStub(grpc.insecure_channel(self.url))
|
|
@@ -170,6 +243,7 @@ class ScanossGrpc(ScanossBase):
|
|
|
170
243
|
self.vuln_stub = VulnerabilitiesStub(grpc.insecure_channel(self.url))
|
|
171
244
|
self.provenance_stub = GeoProvenanceStub(grpc.insecure_channel(self.url))
|
|
172
245
|
self.scanning_stub = ScanningStub(grpc.insecure_channel(self.url))
|
|
246
|
+
self.license_stub = LicenseStub(grpc.insecure_channel(self.url))
|
|
173
247
|
else:
|
|
174
248
|
if ca_cert is not None:
|
|
175
249
|
credentials = grpc.ssl_channel_credentials(cert_data) # secure with specified certificate
|
|
@@ -182,70 +256,32 @@ class ScanossGrpc(ScanossBase):
|
|
|
182
256
|
self.vuln_stub = VulnerabilitiesStub(grpc.secure_channel(self.url, credentials))
|
|
183
257
|
self.provenance_stub = GeoProvenanceStub(grpc.secure_channel(self.url, credentials))
|
|
184
258
|
self.scanning_stub = ScanningStub(grpc.secure_channel(self.url, credentials))
|
|
259
|
+
self.license_stub = LicenseStub(grpc.secure_channel(self.url, credentials))
|
|
185
260
|
|
|
186
261
|
@classmethod
|
|
187
262
|
def _load_cert(cls, cert_file: str) -> bytes:
|
|
188
263
|
with open(cert_file, 'rb') as f:
|
|
189
264
|
return f.read()
|
|
190
265
|
|
|
191
|
-
def deps_echo(self, message: str = 'Hello there!') ->
|
|
266
|
+
def deps_echo(self, message: str = 'Hello there!') -> Optional[dict]:
|
|
192
267
|
"""
|
|
193
268
|
Send Echo message to the Dependency service
|
|
194
269
|
:param self:
|
|
195
270
|
:param message: Message to send (default: Hello there!)
|
|
196
271
|
:return: echo or None
|
|
197
272
|
"""
|
|
198
|
-
|
|
199
|
-
resp: EchoResponse
|
|
200
|
-
try:
|
|
201
|
-
metadata = self.metadata[:]
|
|
202
|
-
metadata.append(('x-request-id', request_id)) # Set a Request ID
|
|
203
|
-
resp = self.dependencies_stub.Echo(EchoRequest(message=message), metadata=metadata, timeout=3)
|
|
204
|
-
except Exception as e:
|
|
205
|
-
self.print_stderr(
|
|
206
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
207
|
-
)
|
|
208
|
-
else:
|
|
209
|
-
# self.print_stderr(f'resp: {resp} - call: {call}')
|
|
210
|
-
# response_id = ""
|
|
211
|
-
# if not call:
|
|
212
|
-
# self.print_stderr(f'No call to leverage.')
|
|
213
|
-
# for key, value in call.trailing_metadata():
|
|
214
|
-
# print('Greeter client received trailing metadata: key=%s value=%s' % (key, value))
|
|
215
|
-
#
|
|
216
|
-
# for key, value in call.trailing_metadata():
|
|
217
|
-
# if key == 'x-response-id':
|
|
218
|
-
# response_id = value
|
|
219
|
-
# self.print_stderr(f'Response ID: {response_id}. Metadata: {call.trailing_metadata()}')
|
|
220
|
-
if resp:
|
|
221
|
-
return resp.message
|
|
222
|
-
self.print_stderr(f'ERROR: Problem sending Echo request ({message}) to {self.url}. rqId: {request_id}')
|
|
223
|
-
return None
|
|
273
|
+
return self._call_api('dependencies.Echo', self.dependencies_stub.Echo, {'message': message}, EchoRequest)
|
|
224
274
|
|
|
225
|
-
def crypto_echo(self, message: str = 'Hello there!') ->
|
|
275
|
+
def crypto_echo(self, message: str = 'Hello there!') -> Optional[dict]:
|
|
226
276
|
"""
|
|
227
277
|
Send Echo message to the Cryptography service
|
|
228
278
|
:param self:
|
|
229
279
|
:param message: Message to send (default: Hello there!)
|
|
230
280
|
:return: echo or None
|
|
231
281
|
"""
|
|
232
|
-
|
|
233
|
-
resp: EchoResponse
|
|
234
|
-
try:
|
|
235
|
-
metadata = self.metadata[:]
|
|
236
|
-
metadata.append(('x-request-id', request_id)) # Set a Request ID
|
|
237
|
-
resp = self.crypto_stub.Echo(EchoRequest(message=message), metadata=metadata, timeout=3)
|
|
238
|
-
except Exception as e:
|
|
239
|
-
self.print_stderr(
|
|
240
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
241
|
-
)
|
|
242
|
-
else:
|
|
243
|
-
if resp:
|
|
244
|
-
return resp.message
|
|
245
|
-
self.print_stderr(f'ERROR: Problem sending Echo request ({message}) to {self.url}. rqId: {request_id}')
|
|
246
|
-
return None
|
|
282
|
+
return self._call_api('cryptography.Echo', self.crypto_stub.Echo, {'message': message}, EchoRequest)
|
|
247
283
|
|
|
248
|
-
def get_dependencies(self, dependencies:
|
|
284
|
+
def get_dependencies(self, dependencies: Optional[dict] = None, depth: int = 1) -> Optional[dict]:
|
|
249
285
|
if not dependencies:
|
|
250
286
|
self.print_stderr('ERROR: No dependency data supplied to submit to the API.')
|
|
251
287
|
return None
|
|
@@ -254,7 +290,7 @@ class ScanossGrpc(ScanossBase):
|
|
|
254
290
|
self.print_stderr(f'ERROR: No response for dependency request: {dependencies}')
|
|
255
291
|
return resp
|
|
256
292
|
|
|
257
|
-
def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict:
|
|
293
|
+
def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> Optional[dict]:
|
|
258
294
|
"""
|
|
259
295
|
Client function to call the rpc for GetDependencies
|
|
260
296
|
:param dependencies: Message to send to the service
|
|
@@ -264,172 +300,131 @@ class ScanossGrpc(ScanossBase):
|
|
|
264
300
|
if not dependencies:
|
|
265
301
|
self.print_stderr('ERROR: No message supplied to send to gRPC service.')
|
|
266
302
|
return None
|
|
267
|
-
|
|
268
303
|
files_json = dependencies.get('files')
|
|
269
|
-
|
|
270
304
|
if files_json is None or len(files_json) == 0:
|
|
271
|
-
self.print_stderr('ERROR: No dependency data supplied to send to
|
|
305
|
+
self.print_stderr('ERROR: No dependency data supplied to send to decoration service.')
|
|
272
306
|
return None
|
|
273
|
-
|
|
274
|
-
def process_file(file):
|
|
275
|
-
request_id = str(uuid.uuid4())
|
|
276
|
-
try:
|
|
277
|
-
file_request = {'files': [file]}
|
|
278
|
-
|
|
279
|
-
request = ParseDict(file_request, DependencyRequest())
|
|
280
|
-
request.depth = depth
|
|
281
|
-
metadata = self.metadata[:]
|
|
282
|
-
metadata.append(('x-request-id', request_id))
|
|
283
|
-
self.print_debug(f'Sending dependency data for decoration (rqId: {request_id})...')
|
|
284
|
-
resp = self.dependencies_stub.GetDependencies(request, metadata=metadata, timeout=self.timeout)
|
|
285
|
-
|
|
286
|
-
return MessageToDict(resp, preserving_proto_field_name=True)
|
|
287
|
-
except Exception as e:
|
|
288
|
-
self.print_stderr(
|
|
289
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
290
|
-
)
|
|
291
|
-
return None
|
|
292
|
-
|
|
293
307
|
all_responses = []
|
|
308
|
+
# Process the dependency files in parallel
|
|
294
309
|
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_CONCURRENT_REQUESTS) as executor:
|
|
295
|
-
future_to_file = {
|
|
296
|
-
|
|
310
|
+
future_to_file = {
|
|
311
|
+
executor.submit(self._process_dep_file, file, depth, self.use_grpc): file for file in files_json
|
|
312
|
+
}
|
|
297
313
|
for future in concurrent.futures.as_completed(future_to_file):
|
|
298
314
|
response = future.result()
|
|
299
315
|
if response:
|
|
300
316
|
all_responses.append(response)
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
317
|
+
# End of concurrent processing
|
|
318
|
+
success_status = 'SUCCESS'
|
|
319
|
+
merged_response = {'files': [], 'status': {'status': success_status, 'message': 'Success'}}
|
|
320
|
+
# Merge the responses
|
|
305
321
|
for response in all_responses:
|
|
306
322
|
if response:
|
|
307
323
|
if 'files' in response and len(response['files']) > 0:
|
|
308
324
|
merged_response['files'].append(response['files'][0])
|
|
309
|
-
# Overwrite the status if
|
|
310
|
-
if 'status' in response and response['status']['status'] !=
|
|
325
|
+
# Overwrite the status if any of the responses was not successful
|
|
326
|
+
if 'status' in response and response['status']['status'] != success_status:
|
|
311
327
|
merged_response['status'] = response['status']
|
|
312
328
|
return merged_response
|
|
313
329
|
|
|
314
|
-
def
|
|
330
|
+
def _process_dep_file(self, file, depth: int = 1, use_grpc: Optional[bool] = None) -> Optional[dict]:
|
|
315
331
|
"""
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
:
|
|
332
|
+
Process a single dependency file using either gRPC or REST
|
|
333
|
+
|
|
334
|
+
Args:
|
|
335
|
+
file: dependency file purls
|
|
336
|
+
depth: depth to search (default: 1)
|
|
337
|
+
use_grpc: Whether to use gRPC or REST (None = use instance default)
|
|
338
|
+
|
|
339
|
+
Returns:
|
|
340
|
+
response JSON or None
|
|
319
341
|
"""
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
resp = self.vuln_stub.GetVulnerabilities(request, metadata=metadata, timeout=self.timeout)
|
|
331
|
-
except Exception as e:
|
|
332
|
-
self.print_stderr(
|
|
333
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
334
|
-
)
|
|
335
|
-
else:
|
|
336
|
-
if resp:
|
|
337
|
-
if not self._check_status_response(resp.status, request_id):
|
|
338
|
-
return None
|
|
339
|
-
resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict
|
|
340
|
-
del resp_dict['status']
|
|
341
|
-
return resp_dict
|
|
342
|
-
return None
|
|
342
|
+
file_request = {'files': [file], 'depth': depth}
|
|
343
|
+
|
|
344
|
+
return self._call_api(
|
|
345
|
+
'dependencies.GetDependencies',
|
|
346
|
+
self.dependencies_stub.GetDependencies,
|
|
347
|
+
file_request,
|
|
348
|
+
DependencyRequest,
|
|
349
|
+
'Sending dependency data for decoration (rqId: {rqId})...',
|
|
350
|
+
use_grpc=use_grpc,
|
|
351
|
+
)
|
|
343
352
|
|
|
344
|
-
def
|
|
353
|
+
def get_vulnerabilities_json(self, purls: Optional[dict] = None, use_grpc: Optional[bool] = None) -> Optional[dict]:
|
|
345
354
|
"""
|
|
346
|
-
Client function to call the rpc for
|
|
347
|
-
|
|
348
|
-
|
|
355
|
+
Client function to call the rpc for Vulnerability GetVulnerabilities
|
|
356
|
+
It will either use REST (default) or gRPC
|
|
357
|
+
|
|
358
|
+
Args:
|
|
359
|
+
purls (dict): Message to send to the service
|
|
360
|
+
|
|
361
|
+
Returns:
|
|
362
|
+
Server response or None
|
|
363
|
+
"""
|
|
364
|
+
return self._call_api(
|
|
365
|
+
'vulnerabilities.GetComponentsVulnerabilities',
|
|
366
|
+
self.vuln_stub.GetComponentsVulnerabilities,
|
|
367
|
+
purls,
|
|
368
|
+
ComponentsRequest,
|
|
369
|
+
'Sending vulnerability data for decoration (rqId: {rqId})...',
|
|
370
|
+
use_grpc=use_grpc,
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
def get_semgrep_json(self, purls: Optional[dict] = None, use_grpc: Optional[bool] = None) -> Optional[dict]:
|
|
349
374
|
"""
|
|
350
|
-
|
|
351
|
-
self.print_stderr('ERROR: No message supplied to send to gRPC service.')
|
|
352
|
-
return None
|
|
353
|
-
request_id = str(uuid.uuid4())
|
|
354
|
-
resp: SemgrepResponse
|
|
355
|
-
try:
|
|
356
|
-
request = ParseDict(purls, PurlRequest()) # Parse the JSON/Dict into the purl request object
|
|
357
|
-
metadata = self.metadata[:]
|
|
358
|
-
metadata.append(('x-request-id', request_id)) # Set a Request ID
|
|
359
|
-
self.print_debug(f'Sending semgrep data for decoration (rqId: {request_id})...')
|
|
360
|
-
resp = self.semgrep_stub.GetIssues(request, metadata=metadata, timeout=self.timeout)
|
|
361
|
-
except Exception as e:
|
|
362
|
-
self.print_stderr(
|
|
363
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
364
|
-
)
|
|
365
|
-
else:
|
|
366
|
-
if resp:
|
|
367
|
-
if not self._check_status_response(resp.status, request_id):
|
|
368
|
-
return None
|
|
369
|
-
resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict
|
|
370
|
-
del resp_dict['status']
|
|
371
|
-
return resp_dict
|
|
372
|
-
return None
|
|
375
|
+
Client function to call the rpc for Semgrep GetIssues
|
|
373
376
|
|
|
374
|
-
|
|
377
|
+
Args:
|
|
378
|
+
purls (dict): Message to send to the service
|
|
379
|
+
use_grpc (bool): Whether to use gRPC or REST
|
|
380
|
+
|
|
381
|
+
Returns:
|
|
382
|
+
Server response or None
|
|
383
|
+
"""
|
|
384
|
+
return self._call_api(
|
|
385
|
+
'semgrep.GetComponentsIssues',
|
|
386
|
+
self.semgrep_stub.GetComponentsIssues,
|
|
387
|
+
purls,
|
|
388
|
+
ComponentsRequest,
|
|
389
|
+
'Sending semgrep data for decoration (rqId: {rqId})...',
|
|
390
|
+
use_grpc=use_grpc,
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
def search_components_json(self, search: dict, use_grpc: Optional[bool] = None) -> Optional[dict]:
|
|
375
394
|
"""
|
|
376
395
|
Client function to call the rpc for Components SearchComponents
|
|
377
|
-
:param search: Message to send to the service
|
|
378
|
-
:return: Server response or None
|
|
379
|
-
"""
|
|
380
|
-
if not search:
|
|
381
|
-
self.print_stderr('ERROR: No message supplied to send to gRPC service.')
|
|
382
|
-
return None
|
|
383
|
-
request_id = str(uuid.uuid4())
|
|
384
|
-
resp: CompSearchResponse
|
|
385
|
-
try:
|
|
386
|
-
request = ParseDict(search, CompSearchRequest()) # Parse the JSON/Dict into the purl request object
|
|
387
|
-
metadata = self.metadata[:]
|
|
388
|
-
metadata.append(('x-request-id', request_id)) # Set a Request ID
|
|
389
|
-
self.print_debug(f'Sending component search data (rqId: {request_id})...')
|
|
390
|
-
resp = self.comp_search_stub.SearchComponents(request, metadata=metadata, timeout=self.timeout)
|
|
391
|
-
except Exception as e:
|
|
392
|
-
self.print_stderr(
|
|
393
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
394
|
-
)
|
|
395
|
-
else:
|
|
396
|
-
if resp:
|
|
397
|
-
if not self._check_status_response(resp.status, request_id):
|
|
398
|
-
return None
|
|
399
|
-
resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict
|
|
400
|
-
del resp_dict['status']
|
|
401
|
-
return resp_dict
|
|
402
|
-
return None
|
|
403
396
|
|
|
404
|
-
|
|
397
|
+
Args:
|
|
398
|
+
search (dict): Message to send to the service
|
|
399
|
+
Returns:
|
|
400
|
+
Server response or None
|
|
401
|
+
"""
|
|
402
|
+
return self._call_api(
|
|
403
|
+
'components.SearchComponents',
|
|
404
|
+
self.comp_search_stub.SearchComponents,
|
|
405
|
+
search,
|
|
406
|
+
CompSearchRequest,
|
|
407
|
+
'Sending component search data for decoration (rqId: {rqId})...',
|
|
408
|
+
use_grpc=use_grpc,
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
def get_component_versions_json(self, search: dict, use_grpc: Optional[bool] = None) -> Optional[dict]:
|
|
405
412
|
"""
|
|
406
413
|
Client function to call the rpc for Components GetComponentVersions
|
|
407
|
-
|
|
408
|
-
:
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
except Exception as e:
|
|
422
|
-
self.print_stderr(
|
|
423
|
-
f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}'
|
|
424
|
-
)
|
|
425
|
-
else:
|
|
426
|
-
if resp:
|
|
427
|
-
if not self._check_status_response(resp.status, request_id):
|
|
428
|
-
return None
|
|
429
|
-
resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict
|
|
430
|
-
del resp_dict['status']
|
|
431
|
-
return resp_dict
|
|
432
|
-
return None
|
|
414
|
+
|
|
415
|
+
Args:
|
|
416
|
+
search (dict): Message to send to the service
|
|
417
|
+
Returns:
|
|
418
|
+
Server response or None
|
|
419
|
+
"""
|
|
420
|
+
return self._call_api(
|
|
421
|
+
'components.GetComponentVersions',
|
|
422
|
+
self.comp_search_stub.GetComponentVersions,
|
|
423
|
+
search,
|
|
424
|
+
CompVersionRequest,
|
|
425
|
+
'Sending component version data for decoration (rqId: {rqId})...',
|
|
426
|
+
use_grpc=use_grpc,
|
|
427
|
+
)
|
|
433
428
|
|
|
434
429
|
def folder_hash_scan(self, request: Dict) -> Optional[Dict]:
|
|
435
430
|
"""
|
|
@@ -448,6 +443,44 @@ class ScanossGrpc(ScanossBase):
|
|
|
448
443
|
'Sending folder hash scan data (rqId: {rqId})...',
|
|
449
444
|
)
|
|
450
445
|
|
|
446
|
+
def _call_api(
|
|
447
|
+
self,
|
|
448
|
+
endpoint_key: str,
|
|
449
|
+
rpc_method,
|
|
450
|
+
request_input,
|
|
451
|
+
request_type,
|
|
452
|
+
debug_msg: Optional[str] = None,
|
|
453
|
+
use_grpc: Optional[bool] = None,
|
|
454
|
+
) -> Optional[Dict]:
|
|
455
|
+
"""
|
|
456
|
+
Unified method to call either gRPC or REST API based on configuration
|
|
457
|
+
|
|
458
|
+
Args:
|
|
459
|
+
endpoint_key (str): The key to lookup the REST endpoint in REST_ENDPOINTS
|
|
460
|
+
rpc_method: The gRPC stub method (used only if use_grpc is True)
|
|
461
|
+
request_input: Either a dict or a gRPC request object
|
|
462
|
+
request_type: The type of the gRPC request object (used only if use_grpc is True)
|
|
463
|
+
debug_msg (str, optional): Debug message template that can include {rqId} placeholder
|
|
464
|
+
use_grpc (bool, optional): Override the instance's use_grpc setting. If None, uses self.use_grpc
|
|
465
|
+
|
|
466
|
+
Returns:
|
|
467
|
+
dict: The parsed response as a dictionary, or None if something went wrong
|
|
468
|
+
"""
|
|
469
|
+
if not request_input:
|
|
470
|
+
self.print_stderr('ERROR: No message supplied to send to service.')
|
|
471
|
+
return None
|
|
472
|
+
|
|
473
|
+
# Determine whether to use gRPC or REST
|
|
474
|
+
use_grpc_flag = use_grpc if use_grpc is not None else self.use_grpc
|
|
475
|
+
|
|
476
|
+
if use_grpc_flag:
|
|
477
|
+
return self._call_rpc(rpc_method, request_input, request_type, debug_msg)
|
|
478
|
+
else:
|
|
479
|
+
# For REST, we only need the dict input
|
|
480
|
+
if not isinstance(request_input, dict):
|
|
481
|
+
request_input = MessageToDict(request_input, preserving_proto_field_name=True)
|
|
482
|
+
return self._call_rest(endpoint_key, request_input, debug_msg)
|
|
483
|
+
|
|
451
484
|
def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional[str] = None) -> Optional[Dict]:
|
|
452
485
|
"""
|
|
453
486
|
Call a gRPC method and return the response as a dictionary
|
|
@@ -462,29 +495,26 @@ class ScanossGrpc(ScanossBase):
|
|
|
462
495
|
dict: The parsed gRPC response as a dictionary, or None if something went wrong
|
|
463
496
|
"""
|
|
464
497
|
request_id = str(uuid.uuid4())
|
|
465
|
-
|
|
466
498
|
if isinstance(request_input, dict):
|
|
467
499
|
request_obj = ParseDict(request_input, request_type())
|
|
468
500
|
else:
|
|
469
501
|
request_obj = request_input
|
|
470
|
-
|
|
471
502
|
metadata = self.metadata[:] + [('x-request-id', request_id)]
|
|
472
|
-
|
|
473
|
-
|
|
503
|
+
if debug_msg:
|
|
504
|
+
self.print_debug(debug_msg.format(rqId=request_id))
|
|
474
505
|
try:
|
|
475
506
|
resp = rpc_method(request_obj, metadata=metadata, timeout=self.timeout)
|
|
476
507
|
except grpc.RpcError as e:
|
|
477
508
|
raise ScanossGrpcError(
|
|
478
509
|
f'{e.__class__.__name__} while sending gRPC message (rqId: {request_id}): {e.details()}'
|
|
479
510
|
)
|
|
480
|
-
|
|
481
|
-
if resp and not self._check_status_response(resp.status, request_id):
|
|
511
|
+
if resp and not self._check_status_response_grpc(resp.status, request_id):
|
|
482
512
|
return None
|
|
483
513
|
|
|
484
514
|
resp_dict = MessageToDict(resp, preserving_proto_field_name=True)
|
|
485
515
|
return resp_dict
|
|
486
516
|
|
|
487
|
-
def
|
|
517
|
+
def _check_status_response_grpc(self, status_response: StatusResponse, request_id: str = None) -> bool:
|
|
488
518
|
"""
|
|
489
519
|
Check the response object to see if the command was successful or not
|
|
490
520
|
:param status_response: Status Response
|
|
@@ -499,15 +529,59 @@ class ScanossGrpc(ScanossBase):
|
|
|
499
529
|
if status_code > ScanossGrpcStatusCode.SUCCESS:
|
|
500
530
|
ret_val = False # default to failed
|
|
501
531
|
msg = 'Unsuccessful'
|
|
502
|
-
if status_code == ScanossGrpcStatusCode.
|
|
532
|
+
if status_code == ScanossGrpcStatusCode.SUCCEEDED_WITH_WARNINGS:
|
|
503
533
|
msg = 'Succeeded with warnings'
|
|
504
534
|
ret_val = True # No need to fail as it succeeded with warnings
|
|
505
|
-
elif status_code == ScanossGrpcStatusCode.
|
|
535
|
+
elif status_code == ScanossGrpcStatusCode.WARNING:
|
|
506
536
|
msg = 'Failed with warnings'
|
|
507
537
|
self.print_stderr(f'{msg} (rqId: {request_id} - status: {status_code}): {status_response.message}')
|
|
508
538
|
return ret_val
|
|
509
539
|
return True
|
|
510
540
|
|
|
541
|
+
def check_status_response_rest(self, status_dict: dict, request_id: Optional[str] = None) -> bool:
|
|
542
|
+
"""
|
|
543
|
+
Check the REST response dictionary to see if the command was successful or not
|
|
544
|
+
|
|
545
|
+
Args:
|
|
546
|
+
status_dict (dict): Status dictionary from REST response containing 'status' and 'message' keys
|
|
547
|
+
request_id (str, optional): Request ID for logging
|
|
548
|
+
Returns:
|
|
549
|
+
bool: True if successful, False otherwise
|
|
550
|
+
"""
|
|
551
|
+
if not status_dict:
|
|
552
|
+
self.print_stderr(f'Warning: No status response supplied (rqId: {request_id}). Assuming it was ok.')
|
|
553
|
+
return True
|
|
554
|
+
|
|
555
|
+
if request_id:
|
|
556
|
+
self.print_debug(f'Checking response status (rqId: {request_id}): {status_dict}')
|
|
557
|
+
|
|
558
|
+
# Get status from dictionary - it can be either a string or nested dict
|
|
559
|
+
status = status_dict.get('status')
|
|
560
|
+
message = status_dict.get('message', '')
|
|
561
|
+
ret_val = True
|
|
562
|
+
|
|
563
|
+
# Handle case where status might be a string directly
|
|
564
|
+
if isinstance(status, str):
|
|
565
|
+
status_str = status.upper()
|
|
566
|
+
if status_str == ScanossRESTStatusCode.SUCCESS.value:
|
|
567
|
+
ret_val = True
|
|
568
|
+
elif status_str == ScanossRESTStatusCode.SUCCEEDED_WITH_WARNINGS.value:
|
|
569
|
+
self.print_stderr(f'Succeeded with warnings (rqId: {request_id}): {message}')
|
|
570
|
+
ret_val = True
|
|
571
|
+
elif status_str == ScanossRESTStatusCode.WARNING.value:
|
|
572
|
+
self.print_stderr(f'Failed with warnings (rqId: {request_id}): {message}')
|
|
573
|
+
ret_val = False
|
|
574
|
+
elif status_str == ScanossRESTStatusCode.FAILED.value:
|
|
575
|
+
self.print_stderr(f'Unsuccessful (rqId: {request_id}): {message}')
|
|
576
|
+
ret_val = False
|
|
577
|
+
else:
|
|
578
|
+
self.print_debug(f'Unknown status "{status_str}" (rqId: {request_id}). Assuming success.')
|
|
579
|
+
ret_val = True
|
|
580
|
+
|
|
581
|
+
# Otherwise asume success
|
|
582
|
+
self.print_debug(f'Unexpected status type {type(status)} (rqId: {request_id}). Assuming success.')
|
|
583
|
+
return ret_val
|
|
584
|
+
|
|
511
585
|
def _get_proxy_config(self):
|
|
512
586
|
"""
|
|
513
587
|
Set the grpc_proxy/http_proxy/https_proxy environment variables if PAC file has been specified
|
|
@@ -530,138 +604,165 @@ class ScanossGrpc(ScanossBase):
|
|
|
530
604
|
os.environ['http_proxy'] = proxies.get('http') or ''
|
|
531
605
|
os.environ['https_proxy'] = proxies.get('https') or ''
|
|
532
606
|
|
|
533
|
-
def get_provenance_json(self, purls: dict) ->
|
|
534
|
-
"""
|
|
535
|
-
Client function to call the rpc for GetComponentProvenance
|
|
536
|
-
:param purls: Message to send to the service
|
|
537
|
-
:return: Server response or None
|
|
607
|
+
def get_provenance_json(self, purls: dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
538
608
|
"""
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
)
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
if not self._check_status_response(resp.status, request_id):
|
|
557
|
-
return None
|
|
558
|
-
resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict
|
|
559
|
-
return resp_dict
|
|
560
|
-
return None
|
|
609
|
+
Client function to call the rpc for GetComponentContributors
|
|
610
|
+
|
|
611
|
+
Args:
|
|
612
|
+
purls (dict): ComponentsRequest
|
|
613
|
+
use_grpc (bool): Whether to use gRPC or REST (None = use instance default)
|
|
614
|
+
|
|
615
|
+
Returns:
|
|
616
|
+
dict: JSON response or None
|
|
617
|
+
"""
|
|
618
|
+
return self._call_api(
|
|
619
|
+
'geoprovenance.GetCountryContributorsByComponents',
|
|
620
|
+
self.provenance_stub.GetCountryContributorsByComponents,
|
|
621
|
+
purls,
|
|
622
|
+
ComponentsRequest,
|
|
623
|
+
'Sending data for provenance decoration (rqId: {rqId})...',
|
|
624
|
+
use_grpc=use_grpc,
|
|
625
|
+
)
|
|
561
626
|
|
|
562
|
-
def get_provenance_origin(self, request: Dict) -> Optional[Dict]:
|
|
627
|
+
def get_provenance_origin(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
563
628
|
"""
|
|
564
|
-
Client function to call the rpc for
|
|
629
|
+
Client function to call the rpc for GetOriginByComponents
|
|
565
630
|
|
|
566
631
|
Args:
|
|
567
|
-
request (Dict):
|
|
632
|
+
request (Dict): GetOriginByComponents Request
|
|
568
633
|
|
|
569
634
|
Returns:
|
|
570
635
|
Optional[Dict]: OriginResponse, or None if the request was not successfull
|
|
571
636
|
"""
|
|
572
|
-
return self.
|
|
573
|
-
|
|
637
|
+
return self._call_api(
|
|
638
|
+
'geoprovenance.GetOriginByComponents',
|
|
639
|
+
self.provenance_stub.GetOriginByComponents,
|
|
574
640
|
request,
|
|
575
|
-
|
|
641
|
+
ComponentsRequest,
|
|
576
642
|
'Sending data for provenance origin decoration (rqId: {rqId})...',
|
|
643
|
+
use_grpc=use_grpc,
|
|
577
644
|
)
|
|
578
645
|
|
|
579
|
-
def get_crypto_algorithms_for_purl(self, request: Dict) -> Optional[Dict]:
|
|
646
|
+
def get_crypto_algorithms_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
580
647
|
"""
|
|
581
|
-
Client function to call the rpc for
|
|
648
|
+
Client function to call the rpc for GetComponentsAlgorithms for a list of purls
|
|
582
649
|
|
|
583
650
|
Args:
|
|
584
|
-
request (Dict):
|
|
651
|
+
request (Dict): ComponentsRequest
|
|
652
|
+
use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default)
|
|
585
653
|
|
|
586
654
|
Returns:
|
|
587
655
|
Optional[Dict]: AlgorithmResponse, or None if the request was not successfull
|
|
588
656
|
"""
|
|
589
|
-
return self.
|
|
590
|
-
|
|
657
|
+
return self._call_api(
|
|
658
|
+
'cryptography.GetComponentsAlgorithms',
|
|
659
|
+
self.crypto_stub.GetComponentsAlgorithms,
|
|
591
660
|
request,
|
|
592
|
-
|
|
661
|
+
ComponentsRequest,
|
|
593
662
|
'Sending data for cryptographic algorithms decoration (rqId: {rqId})...',
|
|
663
|
+
use_grpc=use_grpc,
|
|
594
664
|
)
|
|
595
665
|
|
|
596
|
-
def get_crypto_algorithms_in_range_for_purl(self, request: Dict) -> Optional[Dict]:
|
|
666
|
+
def get_crypto_algorithms_in_range_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
597
667
|
"""
|
|
598
|
-
Client function to call the rpc for
|
|
668
|
+
Client function to call the rpc for GetComponentsAlgorithmsInRange for a list of purls
|
|
599
669
|
|
|
600
670
|
Args:
|
|
601
|
-
request (Dict):
|
|
671
|
+
request (Dict): ComponentsRequest
|
|
672
|
+
use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default)
|
|
602
673
|
|
|
603
674
|
Returns:
|
|
604
675
|
Optional[Dict]: AlgorithmsInRangeResponse, or None if the request was not successfull
|
|
605
676
|
"""
|
|
606
|
-
return self.
|
|
607
|
-
|
|
677
|
+
return self._call_api(
|
|
678
|
+
'cryptography.GetComponentsAlgorithmsInRange',
|
|
679
|
+
self.crypto_stub.GetComponentsAlgorithmsInRange,
|
|
608
680
|
request,
|
|
609
|
-
|
|
681
|
+
ComponentsRequest,
|
|
610
682
|
'Sending data for cryptographic algorithms in range decoration (rqId: {rqId})...',
|
|
683
|
+
use_grpc=use_grpc,
|
|
611
684
|
)
|
|
612
685
|
|
|
613
|
-
def get_encryption_hints_for_purl(self, request: Dict) -> Optional[Dict]:
|
|
686
|
+
def get_encryption_hints_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
614
687
|
"""
|
|
615
|
-
Client function to call the rpc for
|
|
688
|
+
Client function to call the rpc for GetComponentsEncryptionHints for a list of purls
|
|
616
689
|
|
|
617
690
|
Args:
|
|
618
|
-
request (Dict):
|
|
691
|
+
request (Dict): ComponentsRequest
|
|
692
|
+
use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default)
|
|
619
693
|
|
|
620
694
|
Returns:
|
|
621
695
|
Optional[Dict]: HintsResponse, or None if the request was not successfull
|
|
622
696
|
"""
|
|
623
|
-
return self.
|
|
624
|
-
|
|
697
|
+
return self._call_api(
|
|
698
|
+
'cryptography.GetComponentsEncryptionHints',
|
|
699
|
+
self.crypto_stub.GetComponentsEncryptionHints,
|
|
625
700
|
request,
|
|
626
|
-
|
|
701
|
+
ComponentsRequest,
|
|
627
702
|
'Sending data for encryption hints decoration (rqId: {rqId})...',
|
|
703
|
+
use_grpc=use_grpc,
|
|
628
704
|
)
|
|
629
705
|
|
|
630
|
-
def get_encryption_hints_in_range_for_purl(self, request: Dict) -> Optional[Dict]:
|
|
706
|
+
def get_encryption_hints_in_range_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
631
707
|
"""
|
|
632
|
-
Client function to call the rpc for
|
|
708
|
+
Client function to call the rpc for GetComponentsHintsInRange for a list of purls
|
|
633
709
|
|
|
634
710
|
Args:
|
|
635
|
-
request (Dict):
|
|
711
|
+
request (Dict): ComponentsRequest
|
|
712
|
+
use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default)
|
|
636
713
|
|
|
637
714
|
Returns:
|
|
638
715
|
Optional[Dict]: HintsInRangeResponse, or None if the request was not successfull
|
|
639
716
|
"""
|
|
640
|
-
return self.
|
|
641
|
-
|
|
717
|
+
return self._call_api(
|
|
718
|
+
'cryptography.GetComponentsHintsInRange',
|
|
719
|
+
self.crypto_stub.GetComponentsHintsInRange,
|
|
642
720
|
request,
|
|
643
|
-
|
|
721
|
+
ComponentsRequest,
|
|
644
722
|
'Sending data for encryption hints in range decoration (rqId: {rqId})...',
|
|
723
|
+
use_grpc=use_grpc,
|
|
645
724
|
)
|
|
646
725
|
|
|
647
|
-
def get_versions_in_range_for_purl(self, request: Dict) -> Optional[Dict]:
|
|
726
|
+
def get_versions_in_range_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
648
727
|
"""
|
|
649
|
-
Client function to call the rpc for
|
|
728
|
+
Client function to call the rpc for GetComponentsVersionsInRange for a list of purls
|
|
650
729
|
|
|
651
730
|
Args:
|
|
652
|
-
request (Dict):
|
|
731
|
+
request (Dict): ComponentsRequest
|
|
732
|
+
use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default)
|
|
653
733
|
|
|
654
734
|
Returns:
|
|
655
735
|
Optional[Dict]: VersionsInRangeResponse, or None if the request was not successfull
|
|
656
736
|
"""
|
|
657
|
-
return self.
|
|
658
|
-
|
|
737
|
+
return self._call_api(
|
|
738
|
+
'cryptography.GetComponentsVersionsInRange',
|
|
739
|
+
self.crypto_stub.GetComponentsVersionsInRange,
|
|
659
740
|
request,
|
|
660
|
-
|
|
741
|
+
ComponentsRequest,
|
|
661
742
|
'Sending data for cryptographic versions in range decoration (rqId: {rqId})...',
|
|
743
|
+
use_grpc=use_grpc,
|
|
744
|
+
)
|
|
745
|
+
|
|
746
|
+
def get_licenses(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]:
|
|
747
|
+
"""
|
|
748
|
+
Client function to call the rpc for Licenses GetComponentsLicenses
|
|
749
|
+
It will either use REST (default) or gRPC depending on the use_grpc flag
|
|
750
|
+
|
|
751
|
+
Args:
|
|
752
|
+
request (Dict): ComponentsRequest
|
|
753
|
+
Returns:
|
|
754
|
+
Optional[Dict]: ComponentsLicenseResponse, or None if the request was not successfull
|
|
755
|
+
"""
|
|
756
|
+
return self._call_api(
|
|
757
|
+
'licenses.GetComponentsLicenses',
|
|
758
|
+
self.license_stub.GetComponentsLicenses,
|
|
759
|
+
request,
|
|
760
|
+
ComponentsRequest,
|
|
761
|
+
'Sending data for license decoration (rqId: {rqId})...',
|
|
762
|
+
use_grpc=use_grpc,
|
|
662
763
|
)
|
|
663
764
|
|
|
664
|
-
def load_generic_headers(self):
|
|
765
|
+
def load_generic_headers(self, url: Optional[str] = None):
|
|
665
766
|
"""
|
|
666
767
|
Adds custom headers from req_headers to metadata.
|
|
667
768
|
|
|
@@ -671,10 +772,155 @@ class ScanossGrpc(ScanossBase):
|
|
|
671
772
|
if self.req_headers: # Load generic headers
|
|
672
773
|
for key, value in self.req_headers.items():
|
|
673
774
|
if key == 'x-api-key': # Set premium URL if x-api-key header is set
|
|
674
|
-
if not
|
|
775
|
+
if not url and not os.environ.get('SCANOSS_GRPC_URL'):
|
|
675
776
|
self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium
|
|
676
777
|
self.api_key = value
|
|
677
778
|
self.metadata.append((key, value))
|
|
779
|
+
self.headers[key] = value
|
|
780
|
+
|
|
781
|
+
#
|
|
782
|
+
# End of gRPC Client Functions
|
|
783
|
+
#
|
|
784
|
+
# Start of REST Client Functions
|
|
785
|
+
#
|
|
786
|
+
|
|
787
|
+
def _rest_get(self, uri: str, request_id: str, params: Optional[dict] = None) -> Optional[dict]:
|
|
788
|
+
"""
|
|
789
|
+
Send a GET request to the specified URI with optional query parameters.
|
|
790
|
+
|
|
791
|
+
Args:
|
|
792
|
+
uri (str): URI to send GET request to
|
|
793
|
+
request_id (str): request id
|
|
794
|
+
params (dict, optional): Optional query parameters as dictionary
|
|
795
|
+
|
|
796
|
+
Returns:
|
|
797
|
+
dict: JSON response or None
|
|
798
|
+
"""
|
|
799
|
+
if not uri:
|
|
800
|
+
self.print_stderr('Error: Missing URI. Cannot perform GET request.')
|
|
801
|
+
return None
|
|
802
|
+
self.print_trace(f'Sending REST GET request to {uri}...')
|
|
803
|
+
headers = self.headers.copy()
|
|
804
|
+
headers['x-request-id'] = request_id
|
|
805
|
+
retry = 0
|
|
806
|
+
while retry <= self.retry_limit:
|
|
807
|
+
retry += 1
|
|
808
|
+
try:
|
|
809
|
+
response = self.session.get(uri, headers=headers, params=params, timeout=self.timeout)
|
|
810
|
+
response.raise_for_status() # Raises an HTTPError for bad responses
|
|
811
|
+
return response.json()
|
|
812
|
+
except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as e:
|
|
813
|
+
self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) sending GET request - {e}.')
|
|
814
|
+
raise Exception(f'ERROR: The SCANOSS API GET request failed for {uri}') from e
|
|
815
|
+
except requests.exceptions.HTTPError as e:
|
|
816
|
+
self.print_stderr(f'ERROR: HTTP error sending GET request ({request_id}): {e}')
|
|
817
|
+
raise Exception(
|
|
818
|
+
f'ERROR: The SCANOSS API GET request failed with status {e.response.status_code} for {uri}'
|
|
819
|
+
) from e
|
|
820
|
+
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
|
|
821
|
+
if retry > self.retry_limit: # Timed out retry_limit or more times, fail
|
|
822
|
+
self.print_stderr(f'ERROR: {e.__class__.__name__} sending GET request ({request_id}): {e}')
|
|
823
|
+
raise Exception(
|
|
824
|
+
f'ERROR: The SCANOSS API GET request timed out ({e.__class__.__name__}) for {uri}'
|
|
825
|
+
) from e
|
|
826
|
+
else:
|
|
827
|
+
self.print_stderr(f'Warning: {e.__class__.__name__} communicating with {self.url}. Retrying...')
|
|
828
|
+
time.sleep(5)
|
|
829
|
+
except requests.exceptions.RequestException as e:
|
|
830
|
+
self.print_stderr(f'Error: Problem sending GET request to {uri}: {e}')
|
|
831
|
+
raise Exception(f'ERROR: The SCANOSS API GET request failed for {uri}') from e
|
|
832
|
+
except Exception as e:
|
|
833
|
+
self.print_stderr(
|
|
834
|
+
f'ERROR: Exception ({e.__class__.__name__}) sending GET request ({request_id}) to {uri}: {e}'
|
|
835
|
+
)
|
|
836
|
+
raise Exception(f'ERROR: The SCANOSS API GET request failed for {uri}') from e
|
|
837
|
+
return None
|
|
838
|
+
|
|
839
|
+
def _rest_post(self, uri: str, request_id: str, data: dict) -> Optional[dict]:
|
|
840
|
+
"""
|
|
841
|
+
Post the specified data to the given URI.
|
|
842
|
+
|
|
843
|
+
Args:
|
|
844
|
+
uri (str): URI to post to
|
|
845
|
+
request_id (str): request id
|
|
846
|
+
data (dict): json data to post
|
|
847
|
+
|
|
848
|
+
Returns:
|
|
849
|
+
dict: JSON response or None
|
|
850
|
+
"""
|
|
851
|
+
if not uri:
|
|
852
|
+
self.print_stderr('Error: Missing URI. Cannot search for project.')
|
|
853
|
+
return None
|
|
854
|
+
self.print_trace(f'Sending REST POST data to {uri}...')
|
|
855
|
+
headers = self.headers.copy()
|
|
856
|
+
headers['x-request-id'] = request_id
|
|
857
|
+
retry = 0
|
|
858
|
+
while retry <= self.retry_limit:
|
|
859
|
+
retry += 1
|
|
860
|
+
try:
|
|
861
|
+
response = self.session.post(uri, headers=headers, json=data, timeout=self.timeout)
|
|
862
|
+
response.raise_for_status() # Raises an HTTPError for bad responses
|
|
863
|
+
return response.json()
|
|
864
|
+
except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as e:
|
|
865
|
+
self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}.')
|
|
866
|
+
raise Exception(f'ERROR: The SCANOSS Decoration API request failed for {uri}') from e
|
|
867
|
+
except requests.exceptions.HTTPError as e:
|
|
868
|
+
self.print_stderr(f'ERROR: HTTP error POSTing data ({request_id}): {e}')
|
|
869
|
+
raise Exception(
|
|
870
|
+
f'ERROR: The SCANOSS Decoration API request failed with status {e.response.status_code} for {uri}'
|
|
871
|
+
) from e
|
|
872
|
+
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
|
|
873
|
+
if retry > self.retry_limit: # Timed out retry_limit or more times, fail
|
|
874
|
+
self.print_stderr(f'ERROR: {e.__class__.__name__} POSTing decoration data ({request_id}): {e}')
|
|
875
|
+
raise Exception(
|
|
876
|
+
f'ERROR: The SCANOSS Decoration API request timed out ({e.__class__.__name__}) for {uri}'
|
|
877
|
+
) from e
|
|
878
|
+
else:
|
|
879
|
+
self.print_stderr(f'Warning: {e.__class__.__name__} communicating with {self.url}. Retrying...')
|
|
880
|
+
time.sleep(5)
|
|
881
|
+
except requests.exceptions.RequestException as e:
|
|
882
|
+
self.print_stderr(f'Error: Problem posting data to {uri}: {e}')
|
|
883
|
+
raise Exception(f'ERROR: The SCANOSS Decoration API request failed for {uri}') from e
|
|
884
|
+
except Exception as e:
|
|
885
|
+
self.print_stderr(
|
|
886
|
+
f'ERROR: Exception ({e.__class__.__name__}) POSTing data ({request_id}) to {uri}: {e}'
|
|
887
|
+
)
|
|
888
|
+
raise Exception(f'ERROR: The SCANOSS Decoration API request failed for {uri}') from e
|
|
889
|
+
return None
|
|
890
|
+
|
|
891
|
+
def _call_rest(self, endpoint_key: str, request_input: dict, debug_msg: Optional[str] = None) -> Optional[Dict]:
|
|
892
|
+
"""
|
|
893
|
+
Call a REST endpoint and return the response as a dictionary
|
|
894
|
+
|
|
895
|
+
Args:
|
|
896
|
+
endpoint_key (str): The key to lookup the REST endpoint in REST_ENDPOINTS
|
|
897
|
+
request_input (dict): The request data to send
|
|
898
|
+
debug_msg (str, optional): Debug message template that can include {rqId} placeholder.
|
|
899
|
+
|
|
900
|
+
Returns:
|
|
901
|
+
dict: The parsed REST response as a dictionary, or None if something went wrong
|
|
902
|
+
"""
|
|
903
|
+
if endpoint_key not in REST_ENDPOINTS:
|
|
904
|
+
raise ScanossGrpcError(f'Unknown REST endpoint key: {endpoint_key}')
|
|
905
|
+
|
|
906
|
+
endpoint_config = REST_ENDPOINTS[endpoint_key]
|
|
907
|
+
endpoint_path = endpoint_config['path']
|
|
908
|
+
method = endpoint_config['method']
|
|
909
|
+
endpoint_url = f'{self.orig_url}{DEFAULT_URI_PREFIX}{endpoint_path}'
|
|
910
|
+
request_id = str(uuid.uuid4())
|
|
911
|
+
|
|
912
|
+
if debug_msg:
|
|
913
|
+
self.print_debug(debug_msg.format(rqId=request_id))
|
|
914
|
+
|
|
915
|
+
if method == 'GET':
|
|
916
|
+
response = self._rest_get(endpoint_url, request_id, params=request_input)
|
|
917
|
+
else: # POST
|
|
918
|
+
response = self._rest_post(endpoint_url, request_id, request_input)
|
|
919
|
+
|
|
920
|
+
if response and 'status' in response and not self.check_status_response_rest(response['status'], request_id):
|
|
921
|
+
return None
|
|
922
|
+
|
|
923
|
+
return response
|
|
678
924
|
|
|
679
925
|
|
|
680
926
|
#
|
|
@@ -711,3 +957,8 @@ def create_grpc_config_from_args(args) -> GrpcConfig:
|
|
|
711
957
|
proxy=getattr(args, 'proxy', None),
|
|
712
958
|
grpc_proxy=getattr(args, 'grpc_proxy', None),
|
|
713
959
|
)
|
|
960
|
+
|
|
961
|
+
|
|
962
|
+
#
|
|
963
|
+
# End of GrpcConfig class
|
|
964
|
+
#
|