fosslight-binary 5.1.31__tar.gz → 5.1.33__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/PKG-INFO +2 -2
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/pyproject.toml +2 -2
- fosslight_binary-5.1.33/src/fosslight_binary/_exclude.py +15 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary/_jar_analysis.py +141 -89
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary/_simple_mode.py +11 -4
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary/binary_analysis.py +14 -9
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary.egg-info/PKG-INFO +2 -2
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary.egg-info/SOURCES.txt +1 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary.egg-info/requires.txt +1 -1
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/LICENSE +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/LICENSES/Apache-2.0.txt +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/LICENSES/LicenseRef-3rd-party-licenses.txt +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/MANIFEST.in +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/README.md +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/setup.cfg +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary/__init__.py +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary/_binary.py +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary/_binary_dao.py +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary/_help.py +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary/cli.py +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary.egg-info/dependency_links.txt +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary.egg-info/entry_points.txt +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary.egg-info/top_level.txt +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/tests/test_fosslight_binary.py +0 -0
- {fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/tests/test_kb_ssl.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fosslight_binary
|
|
3
|
-
Version: 5.1.
|
|
3
|
+
Version: 5.1.33
|
|
4
4
|
Summary: FOSSLight Binary Scanner
|
|
5
5
|
Author: LG Electronics
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -29,7 +29,7 @@ Requires-Dist: py-tlsh
|
|
|
29
29
|
Requires-Dist: pytz
|
|
30
30
|
Requires-Dist: XlsxWriter
|
|
31
31
|
Requires-Dist: PyYAML
|
|
32
|
-
Requires-Dist: fosslight_util>=2.2.
|
|
32
|
+
Requires-Dist: fosslight_util>=2.2.14
|
|
33
33
|
Requires-Dist: truststore
|
|
34
34
|
Requires-Dist: python-magic-bin; sys_platform == "win32"
|
|
35
35
|
Requires-Dist: python-magic; "darwin" in sys_platform
|
|
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "fosslight_binary"
|
|
10
|
-
version = "5.1.
|
|
10
|
+
version = "5.1.33"
|
|
11
11
|
description = "FOSSLight Binary Scanner"
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
license = "Apache-2.0"
|
|
@@ -36,7 +36,7 @@ dependencies = [
|
|
|
36
36
|
"pytz",
|
|
37
37
|
"XlsxWriter",
|
|
38
38
|
"PyYAML",
|
|
39
|
-
"fosslight_util>=2.2.
|
|
39
|
+
"fosslight_util>=2.2.14",
|
|
40
40
|
"truststore",
|
|
41
41
|
"python-magic-bin; sys_platform == 'win32'",
|
|
42
42
|
"python-magic; 'darwin' in sys_platform",
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Copyright (c) 2026 LG Electronics Inc.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Binary-scanner filename excludes."""
|
|
5
|
+
|
|
6
|
+
from fosslight_util.exclude import is_excluded_filename
|
|
7
|
+
|
|
8
|
+
# Scanner output binaries that should not be re-analyzed.
|
|
9
|
+
EXCLUDE_FILENAME_BINARY = frozenset({
|
|
10
|
+
"fosslight_bin", "fosslight_bin.exe",
|
|
11
|
+
})
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def is_excluded_binary_filename(file_path: str) -> bool:
|
|
15
|
+
return is_excluded_filename(file_path, EXCLUDE_FILENAME_BINARY)
|
|
@@ -19,14 +19,38 @@ from fosslight_util.oss_item import OssItem
|
|
|
19
19
|
|
|
20
20
|
logger = logging.getLogger(constant.LOGGER_NAME)
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
# The Central Portal host serves the Central Solr index and its g/a/v fields, so
|
|
23
|
+
# its answer is authoritative for the whole index. The legacy search.maven.org
|
|
24
|
+
# endpoint is no longer queried: it often accepts the TCP connect and then never
|
|
25
|
+
# starts the response, so querying it only spends the timeout budget.
|
|
26
|
+
_CENTRAL_SEARCH_URL = "https://central.sonatype.com/solrsearch/select"
|
|
27
|
+
|
|
28
|
+
# (connect, read) seconds for the remote POM fetch. A dead host fails on connect
|
|
29
|
+
# in 3s, while a POM that is merely slow still has 7s to arrive - the license is
|
|
30
|
+
# taken from whatever answers, and the fallback behind it is final. Kept separate
|
|
31
|
+
# from the search budget below: that one is deliberately tight to abandon a
|
|
32
|
+
# stalled Solr host, which is the wrong bound for downloading a file.
|
|
33
|
+
_POM_DOWNLOAD_TIMEOUT = (3, 7)
|
|
34
|
+
_CENTRAL_SEARCH_TIMEOUT = 2.5 # seconds - search budget
|
|
25
35
|
_MAVEN_JAR_HTTP_TIMEOUT = (2, 2) # match Util probe timeouts for multi-repo jar checks
|
|
26
|
-
_MAX_RETRY = 3 # maximum Central API retry attempts per JAR
|
|
27
36
|
_central_network_warned = False # Flag to suppress repeated network-unavailable warnings within one run
|
|
37
|
+
_central_lookup_warned = False # Flag to suppress repeated lookup-failure warnings within one run
|
|
28
38
|
_COORD_TOKEN = re.compile(r'[A-Za-z0-9._+-]+') # shape of a Maven groupId / artifactId / version
|
|
29
39
|
|
|
40
|
+
# Outcome of one SHA-1 search. Only a 200 reply carries an answer from the
|
|
41
|
+
# Central index, so only _SEARCH_MISS states that a SHA-1 is not published
|
|
42
|
+
# there. _SEARCH_FAILED means the question was never answered: a JAR the index
|
|
43
|
+
# did not identify is not the same as a JAR with no OSS to report, and reporting
|
|
44
|
+
# the two alike lets a rate-limited scan look like a clean one.
|
|
45
|
+
_SEARCH_HIT = 'hit'
|
|
46
|
+
_SEARCH_MISS = 'miss'
|
|
47
|
+
_SEARCH_FAILED = 'failed'
|
|
48
|
+
_SEARCH_TIMED_OUT = 'timed_out'
|
|
49
|
+
|
|
50
|
+
# Reported alongside the coordinate source when a SHA-1 miss suppresses the
|
|
51
|
+
# Download Location, so the empty column has a stated reason in the report.
|
|
52
|
+
_SHA1_MISS_COMMENT = 'SHA-1 not in Maven Central'
|
|
53
|
+
|
|
30
54
|
|
|
31
55
|
def _sha1_of_file(filepath):
|
|
32
56
|
h = hashlib.sha1()
|
|
@@ -115,44 +139,82 @@ def _warn_network_once(context=""):
|
|
|
115
139
|
_central_network_warned = True
|
|
116
140
|
|
|
117
141
|
|
|
142
|
+
def _warn_lookup_failed_once(reason, context=""):
|
|
143
|
+
"""Warn once per run that a Central lookup produced no answer at all."""
|
|
144
|
+
global _central_lookup_warned
|
|
145
|
+
if not _central_lookup_warned:
|
|
146
|
+
msg = f"Maven Central 조회에 실패했습니다 ({reason}) - 일부 JAR의 OSS 식별이 누락될 수 있습니다"
|
|
147
|
+
if context:
|
|
148
|
+
msg += f" [{context}]"
|
|
149
|
+
logger.warning(msg)
|
|
150
|
+
_central_lookup_warned = True
|
|
151
|
+
|
|
152
|
+
|
|
118
153
|
def _search_central_by_sha1(sha1, timeout=None):
|
|
154
|
+
"""Look up Maven coordinates by JAR checksum on the Central Portal endpoint.
|
|
155
|
+
|
|
156
|
+
Returns ``(coordinates, outcome)``. A 200 reply is final: a complete
|
|
157
|
+
coordinate triple is ``_SEARCH_HIT``, and no document or an incomplete one
|
|
158
|
+
is ``_SEARCH_MISS``, a normal result that stays quiet. A reply carrying no
|
|
159
|
+
index answer is ``_SEARCH_FAILED`` and warns once per run so it cannot pass
|
|
160
|
+
as a miss - non-200 and unparseable bodies through
|
|
161
|
+
``_warn_lookup_failed_once``, a refused connection through
|
|
162
|
+
``_warn_network_once``, whose flags are independent. A timeout is separate:
|
|
163
|
+
``_SEARCH_TIMED_OUT`` with a debug log and no warning, because the retry
|
|
164
|
+
policy meant to warn after exhausting its rounds is not implemented yet.
|
|
165
|
+
"""
|
|
119
166
|
if not sha1:
|
|
120
|
-
return {},
|
|
167
|
+
return {}, _SEARCH_MISS
|
|
121
168
|
_timeout = timeout if timeout is not None else _CENTRAL_SEARCH_TIMEOUT
|
|
169
|
+
params = {"q": f"1:{sha1}", "rows": 1, "wt": "json"}
|
|
170
|
+
where = f"SHA-1 search: {sha1[:10]}..."
|
|
171
|
+
|
|
122
172
|
try:
|
|
123
|
-
params = {"q": f"1:{sha1}", "rows": 1, "wt": "json"}
|
|
124
173
|
resp = requests.get(_CENTRAL_SEARCH_URL, params=params, timeout=_timeout)
|
|
125
|
-
resp.raise_for_status()
|
|
126
|
-
docs = resp.json().get("response", {}).get("docs", [])
|
|
127
|
-
if not docs:
|
|
128
|
-
return {}, False
|
|
129
|
-
doc = docs[0]
|
|
130
|
-
groupId = doc.get("g", "")
|
|
131
|
-
artifactId = doc.get("a", "")
|
|
132
|
-
version = doc.get("v") or doc.get("latestVersion", "")
|
|
133
|
-
|
|
134
|
-
if not (groupId and artifactId and version):
|
|
135
|
-
logger.debug(f"Maven Central returned an incomplete document for {sha1}: {doc}")
|
|
136
|
-
return {}, False
|
|
137
|
-
|
|
138
|
-
return {
|
|
139
|
-
"groupId": groupId,
|
|
140
|
-
"artifactId": artifactId,
|
|
141
|
-
"version": version,
|
|
142
|
-
}, False
|
|
143
174
|
except requests.exceptions.Timeout:
|
|
144
|
-
logger.debug(f"Maven Central SHA-1 search timed out ({sha1})
|
|
145
|
-
return {},
|
|
175
|
+
logger.debug(f"Maven Central SHA-1 search timed out ({sha1}) at {_CENTRAL_SEARCH_URL}")
|
|
176
|
+
return {}, _SEARCH_TIMED_OUT
|
|
146
177
|
except Exception as ex:
|
|
147
178
|
if _is_network_error(ex):
|
|
148
|
-
_warn_network_once(
|
|
179
|
+
_warn_network_once(where)
|
|
149
180
|
else:
|
|
150
|
-
|
|
151
|
-
|
|
181
|
+
_warn_lookup_failed_once(type(ex).__name__, where)
|
|
182
|
+
logger.debug(f"Maven Central SHA-1 search failed ({sha1}) at {_CENTRAL_SEARCH_URL}: {ex}")
|
|
183
|
+
return {}, _SEARCH_FAILED
|
|
184
|
+
|
|
185
|
+
if resp.status_code != 200:
|
|
186
|
+
_warn_lookup_failed_once(f"HTTP {resp.status_code}", where)
|
|
187
|
+
logger.debug(f"Maven Central SHA-1 search got HTTP {resp.status_code} ({sha1})")
|
|
188
|
+
return {}, _SEARCH_FAILED
|
|
189
|
+
|
|
190
|
+
try:
|
|
191
|
+
docs = resp.json().get("response", {}).get("docs", [])
|
|
192
|
+
except Exception as ex:
|
|
193
|
+
_warn_lookup_failed_once("응답을 해석할 수 없음", where)
|
|
194
|
+
logger.debug(f"Maven Central SHA-1 search reply was unusable ({sha1}): {ex}")
|
|
195
|
+
return {}, _SEARCH_FAILED
|
|
196
|
+
|
|
197
|
+
if not docs:
|
|
198
|
+
return {}, _SEARCH_MISS
|
|
199
|
+
|
|
200
|
+
doc = docs[0]
|
|
201
|
+
groupId = doc.get("g", "")
|
|
202
|
+
artifactId = doc.get("a", "")
|
|
203
|
+
version = doc.get("v") or doc.get("latestVersion", "")
|
|
204
|
+
|
|
205
|
+
if not (groupId and artifactId and version):
|
|
206
|
+
logger.debug(f"Maven Central returned an incomplete document for {sha1}: {doc}")
|
|
207
|
+
return {}, _SEARCH_MISS
|
|
208
|
+
|
|
209
|
+
return {
|
|
210
|
+
"groupId": groupId,
|
|
211
|
+
"artifactId": artifactId,
|
|
212
|
+
"version": version,
|
|
213
|
+
}, _SEARCH_HIT
|
|
152
214
|
|
|
153
215
|
|
|
154
216
|
def _download_pom_to_tempfile(group_id, artifact_id, version, timeout=None):
|
|
155
|
-
_timeout = timeout if timeout is not None else
|
|
217
|
+
_timeout = timeout if timeout is not None else _POM_DOWNLOAD_TIMEOUT
|
|
156
218
|
group_path = group_id.replace('.', '/')
|
|
157
219
|
pom_name = f"{artifact_id}-{version}.pom"
|
|
158
220
|
urls = [
|
|
@@ -169,7 +231,7 @@ def _download_pom_to_tempfile(group_id, artifact_id, version, timeout=None):
|
|
|
169
231
|
logger.debug(f"POM downloaded to {tmp.name} from {url}")
|
|
170
232
|
return tmp.name, False
|
|
171
233
|
except requests.exceptions.Timeout:
|
|
172
|
-
logger.debug(f"POM download timed out from {url} -
|
|
234
|
+
logger.debug(f"POM download timed out from {url} - trying next repository")
|
|
173
235
|
any_timeout = True
|
|
174
236
|
except Exception as ex:
|
|
175
237
|
if _is_network_error(ex):
|
|
@@ -229,20 +291,17 @@ def _find_jar_download_url(group_id, artifact_id, version):
|
|
|
229
291
|
return ""
|
|
230
292
|
|
|
231
293
|
|
|
232
|
-
def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None
|
|
294
|
+
def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None):
|
|
233
295
|
groupId = artifactId = version = project_url = license_str = ''
|
|
234
296
|
confirmed_in_central = False
|
|
235
297
|
trusted_coordinates = False
|
|
236
298
|
source = ''
|
|
237
299
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
if timed_out:
|
|
244
|
-
logger.debug(f"{rel_path}: Central SHA-1 search timed out - will retry")
|
|
245
|
-
return None, True
|
|
300
|
+
central_info, search_outcome = _search_central_by_sha1(sha1, timeout=search_timeout)
|
|
301
|
+
if search_outcome == _SEARCH_TIMED_OUT:
|
|
302
|
+
logger.debug(f"{rel_path}: Central SHA-1 search timed out - falling back to JAR internals")
|
|
303
|
+
elif search_outcome == _SEARCH_FAILED:
|
|
304
|
+
logger.debug(f"{rel_path}: Central SHA-1 lookup failed - falling back to JAR internals")
|
|
246
305
|
|
|
247
306
|
g2, a2, v2, url2, pom_tmp_path = _read_pom_from_jar(jar_path)
|
|
248
307
|
|
|
@@ -258,7 +317,7 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
|
|
|
258
317
|
if names_match:
|
|
259
318
|
logger.debug(f"{rel_path}: Central and JAR pom.xml match ({central_oss_name} {c_version}) - using JAR pom.xml for license")
|
|
260
319
|
groupId, artifactId, version, project_url = g2, a2, v2, url2
|
|
261
|
-
source = 'pom.xml'
|
|
320
|
+
source = 'pom.xml (Central verified)'
|
|
262
321
|
confirmed_in_central = True
|
|
263
322
|
trusted_coordinates = True
|
|
264
323
|
|
|
@@ -283,16 +342,19 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
|
|
|
283
342
|
confirmed_in_central = True
|
|
284
343
|
trusted_coordinates = True
|
|
285
344
|
|
|
286
|
-
tmp_path, timed_out = _download_pom_to_tempfile(
|
|
287
|
-
groupId, artifactId, version, timeout=search_timeout)
|
|
345
|
+
tmp_path, timed_out = _download_pom_to_tempfile(groupId, artifactId, version)
|
|
288
346
|
if timed_out:
|
|
289
|
-
logger.
|
|
347
|
+
logger.warning(
|
|
348
|
+
f"{rel_path}: POM download timed out for {groupId}:{artifactId}:{version}"
|
|
349
|
+
" - using JAR pom.xml license")
|
|
290
350
|
if pom_tmp_path:
|
|
291
351
|
try:
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
352
|
+
license_str = get_license_from_pom(
|
|
353
|
+
group_id=g2, artifact_id=a2, version=v2,
|
|
354
|
+
pom_path=pom_tmp_path, check_parent=True)
|
|
355
|
+
logger.debug(f"{rel_path}: license from JAR pom.xml after Central POM timeout={license_str!r}")
|
|
356
|
+
except Exception as ex:
|
|
357
|
+
logger.debug(f"get_license_from_pom (jar pom_path) failed: {ex}")
|
|
296
358
|
if tmp_path:
|
|
297
359
|
try:
|
|
298
360
|
license_str = get_license_from_pom(
|
|
@@ -314,11 +376,12 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
|
|
|
314
376
|
pass
|
|
315
377
|
|
|
316
378
|
else:
|
|
317
|
-
|
|
379
|
+
if search_outcome == _SEARCH_MISS:
|
|
380
|
+
logger.debug(f"{rel_path}: not found in Maven Central - falling back to JAR internals")
|
|
318
381
|
|
|
319
382
|
if g2 or a2:
|
|
320
383
|
groupId, artifactId, version, project_url = g2, a2, v2, url2
|
|
321
|
-
source = 'pom.xml'
|
|
384
|
+
source = 'pom.xml (Local)'
|
|
322
385
|
trusted_coordinates = True
|
|
323
386
|
|
|
324
387
|
if pom_tmp_path:
|
|
@@ -348,9 +411,22 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
|
|
|
348
411
|
trusted_coordinates = False
|
|
349
412
|
|
|
350
413
|
if not (groupId or artifactId):
|
|
351
|
-
return None
|
|
352
|
-
|
|
353
|
-
|
|
414
|
+
return None
|
|
415
|
+
|
|
416
|
+
# A miss is Central's answer that these exact bytes are not published. The
|
|
417
|
+
# JAR's own pom.xml may still name real coordinates, so a URL built from them
|
|
418
|
+
# would resolve - but it would serve the upstream artifact, not this file. A
|
|
419
|
+
# locally patched JAR keeps the official pom.xml, so filling the URL would
|
|
420
|
+
# report a modified binary as the official one. Omit it and say why; the
|
|
421
|
+
# probe is skipped because its result could not be reported anyway. A lookup
|
|
422
|
+
# that never answered (failed / timed out) is not a miss and still fills the
|
|
423
|
+
# URL, so an unreachable Central does not blank the whole report.
|
|
424
|
+
unverified_after_miss = (search_outcome == _SEARCH_MISS
|
|
425
|
+
and trusted_coordinates and not confirmed_in_central)
|
|
426
|
+
|
|
427
|
+
if unverified_after_miss:
|
|
428
|
+
dl_url = ""
|
|
429
|
+
elif confirmed_in_central or trusted_coordinates:
|
|
354
430
|
dl_url = _find_jar_download_url(groupId, artifactId, version)
|
|
355
431
|
else:
|
|
356
432
|
dl_url = ""
|
|
@@ -359,15 +435,18 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
|
|
|
359
435
|
|
|
360
436
|
oss = OssItem(oss_name, version, license_str, dl_url)
|
|
361
437
|
oss.comment = source
|
|
438
|
+
if unverified_after_miss:
|
|
439
|
+
oss.comment = _SHA1_MISS_COMMENT
|
|
362
440
|
|
|
363
441
|
logger.debug(
|
|
364
442
|
f"Result: {rel_path} | {oss_name} {version} | [{license_str}] | dl={dl_url} | source={source}")
|
|
365
443
|
|
|
366
|
-
return {"oss_list": [oss]}
|
|
444
|
+
return {"oss_list": [oss]}
|
|
367
445
|
|
|
368
446
|
|
|
369
447
|
def _has_valid_jar_oss(oss_list):
|
|
370
|
-
|
|
448
|
+
"""True if any entry is identified. A missing license is reported as empty."""
|
|
449
|
+
return any(oss.name for oss in oss_list)
|
|
371
450
|
|
|
372
451
|
|
|
373
452
|
def _store_jar_result(jar_items, sha1, result):
|
|
@@ -379,12 +458,12 @@ def _store_jar_result(jar_items, sha1, result):
|
|
|
379
458
|
|
|
380
459
|
|
|
381
460
|
def analyze_jar_file(path_to_find_bin, path_to_exclude):
|
|
382
|
-
global _central_network_warned
|
|
461
|
+
global _central_network_warned, _central_lookup_warned
|
|
383
462
|
_central_network_warned = False
|
|
463
|
+
_central_lookup_warned = False
|
|
384
464
|
jar_items = {}
|
|
465
|
+
analyzed = set()
|
|
385
466
|
success = True
|
|
386
|
-
retry_queue = []
|
|
387
|
-
pending_sha1s = set()
|
|
388
467
|
|
|
389
468
|
jar_files = []
|
|
390
469
|
for root_dir, _dirs, files in os.walk(path_to_find_bin):
|
|
@@ -402,42 +481,15 @@ def analyze_jar_file(path_to_find_bin, path_to_exclude):
|
|
|
402
481
|
continue
|
|
403
482
|
|
|
404
483
|
sha1 = _sha1_of_file(jar_path)
|
|
405
|
-
if not sha1 or sha1 in
|
|
484
|
+
if not sha1 or sha1 in analyzed:
|
|
406
485
|
continue
|
|
486
|
+
analyzed.add(sha1)
|
|
407
487
|
|
|
408
|
-
result
|
|
488
|
+
result = _process_one_jar(
|
|
409
489
|
jar_path, rel_path, sha1, search_timeout=_CENTRAL_SEARCH_TIMEOUT)
|
|
410
|
-
|
|
411
|
-
if needs_retry:
|
|
412
|
-
logger.debug(f"{rel_path}: Central API timed out - queued for retry (attempt 1/{_MAX_RETRY})")
|
|
413
|
-
pending_sha1s.add(sha1)
|
|
414
|
-
retry_queue.append((jar_path, rel_path, sha1, 1))
|
|
415
|
-
elif result is not None:
|
|
490
|
+
if result is not None:
|
|
416
491
|
_store_jar_result(jar_items, sha1, result)
|
|
417
492
|
|
|
418
|
-
while retry_queue:
|
|
419
|
-
next_queue = []
|
|
420
|
-
for jar_path, rel_path, sha1, attempt in retry_queue:
|
|
421
|
-
if attempt >= _MAX_RETRY:
|
|
422
|
-
logger.warning(
|
|
423
|
-
f"{rel_path}: Maven Central API timed out after {_MAX_RETRY} attempts"
|
|
424
|
-
" - falling back to JAR internals")
|
|
425
|
-
result, _ = _process_one_jar(jar_path, rel_path, sha1, skip_central_search=True)
|
|
426
|
-
if result is not None:
|
|
427
|
-
_store_jar_result(jar_items, sha1, result)
|
|
428
|
-
continue
|
|
429
|
-
|
|
430
|
-
logger.debug(f"{rel_path}: retrying Central API (attempt {attempt + 1}/{_MAX_RETRY})")
|
|
431
|
-
result, needs_retry = _process_one_jar(
|
|
432
|
-
jar_path, rel_path, sha1, search_timeout=_CENTRAL_SEARCH_TIMEOUT)
|
|
433
|
-
|
|
434
|
-
if needs_retry:
|
|
435
|
-
next_queue.append((jar_path, rel_path, sha1, attempt + 1))
|
|
436
|
-
elif result is not None:
|
|
437
|
-
_store_jar_result(jar_items, sha1, result)
|
|
438
|
-
|
|
439
|
-
retry_queue = next_queue
|
|
440
|
-
|
|
441
493
|
return jar_items, success
|
|
442
494
|
|
|
443
495
|
|
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
# Copyright (c) 2024 LG Electronics Inc.
|
|
5
5
|
# SPDX-License-Identifier: Apache-2.0
|
|
6
6
|
import os
|
|
7
|
-
import re
|
|
8
7
|
import logging
|
|
9
8
|
import zipfile
|
|
10
9
|
import tarfile
|
|
10
|
+
import magic
|
|
11
11
|
import yaml
|
|
12
12
|
import fosslight_util.constant as constant
|
|
13
13
|
from fosslight_util.set_log import init_log
|
|
@@ -18,7 +18,14 @@ logger = logging.getLogger(constant.LOGGER_NAME)
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
def is_compressed_file(filename):
|
|
21
|
-
|
|
21
|
+
if zipfile.is_zipfile(filename) or tarfile.is_tarfile(filename):
|
|
22
|
+
return True
|
|
23
|
+
|
|
24
|
+
try:
|
|
25
|
+
file_type = magic.from_file(filename).lower()
|
|
26
|
+
return 'archive' in file_type or 'compressed' in file_type
|
|
27
|
+
except Exception:
|
|
28
|
+
return False
|
|
22
29
|
|
|
23
30
|
|
|
24
31
|
def is_jar_file(filename):
|
|
@@ -35,9 +42,9 @@ def exclude_bin_for_simple_mode(binary_list):
|
|
|
35
42
|
|
|
36
43
|
path = bin.bin_name_with_path
|
|
37
44
|
|
|
38
|
-
if is_jar_file(path) and not
|
|
45
|
+
if is_jar_file(path) and not path.lower().endswith('-sources.jar'):
|
|
39
46
|
bin_list_exclude_compressed.append(path)
|
|
40
|
-
elif
|
|
47
|
+
elif path.lower().endswith('-sources.jar') or is_compressed_file(path):
|
|
41
48
|
compressed_list.append(path)
|
|
42
49
|
else:
|
|
43
50
|
bin_list_exclude_compressed.append(path)
|
|
@@ -22,6 +22,7 @@ from fosslight_util.time import current_timestamp_utc, format_running_time, time
|
|
|
22
22
|
from fosslight_util.correct import correct_with_yaml
|
|
23
23
|
from fosslight_util.oss_item import ScannerItem
|
|
24
24
|
from fosslight_util.exclude import get_excluded_paths
|
|
25
|
+
from ._exclude import EXCLUDE_FILENAME_BINARY, is_excluded_binary_filename
|
|
25
26
|
import hashlib
|
|
26
27
|
import tlsh
|
|
27
28
|
from io import open
|
|
@@ -93,15 +94,14 @@ def init(path_to_find_bin, output_file_name, formats, path_to_exclude=[]):
|
|
|
93
94
|
for i, output_extension in enumerate(output_extensions):
|
|
94
95
|
if output_files[i] is None or output_files[i] == "":
|
|
95
96
|
if formats:
|
|
96
|
-
if formats[i].startswith('spdx')
|
|
97
|
+
if formats[i].startswith('spdx'):
|
|
97
98
|
if platform.system() == 'Windows':
|
|
98
99
|
logger.warning(f'{formats[i]} is not supported on Windows. Please remove {formats[i]} from format.')
|
|
99
100
|
to_remove.append(i)
|
|
100
101
|
else:
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
output_files[i] = f'fosslight_cyclonedx_bin_{file_time}'
|
|
102
|
+
output_files[i] = f"fosslight_spdx_bin_{file_time}"
|
|
103
|
+
elif formats[i].startswith('cyclonedx'):
|
|
104
|
+
output_files[i] = f'fosslight_cyclonedx_bin_{file_time}'
|
|
105
105
|
else:
|
|
106
106
|
if output_extension == _json_ext:
|
|
107
107
|
output_files[i] = f"fosslight_opossum_bin_{file_time}"
|
|
@@ -147,7 +147,7 @@ def get_file_list(path_to_find, excluded_files):
|
|
|
147
147
|
for file in files:
|
|
148
148
|
bin_with_path = os.path.join(root, file)
|
|
149
149
|
rel_path_file = os.path.relpath(bin_with_path, path_to_find).replace('\\', '/')
|
|
150
|
-
if rel_path_file in excluded_files:
|
|
150
|
+
if rel_path_file in excluded_files or is_excluded_binary_filename(rel_path_file):
|
|
151
151
|
continue
|
|
152
152
|
file_lower_case = file.lower()
|
|
153
153
|
extension = os.path.splitext(file_lower_case)[1][1:].strip()
|
|
@@ -200,10 +200,14 @@ def find_binaries(path_to_find_bin, output_dir, formats, kb_url="", kb_token="",
|
|
|
200
200
|
excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped = all_exclude_mode
|
|
201
201
|
elif simple_mode:
|
|
202
202
|
excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped \
|
|
203
|
-
= get_excluded_paths(
|
|
203
|
+
= get_excluded_paths(
|
|
204
|
+
path_to_find_bin, path_to_exclude, REMOVE_FILE_EXTENSION_SIMPLE,
|
|
205
|
+
exclude_filenames=EXCLUDE_FILENAME_BINARY)
|
|
204
206
|
else:
|
|
205
207
|
excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped \
|
|
206
|
-
= get_excluded_paths(
|
|
208
|
+
= get_excluded_paths(
|
|
209
|
+
path_to_find_bin, path_to_exclude,
|
|
210
|
+
exclude_filenames=EXCLUDE_FILENAME_BINARY)
|
|
207
211
|
logger.debug(f"Skipped paths: {excluded_path_with_default_exclusion}")
|
|
208
212
|
|
|
209
213
|
if not os.path.isdir(path_to_find_bin):
|
|
@@ -273,7 +277,8 @@ def find_binaries(path_to_find_bin, output_dir, formats, kb_url="", kb_token="",
|
|
|
273
277
|
|
|
274
278
|
for combined_path_and_file, output_extension, output_format in zip(result_reports, output_extensions, formats):
|
|
275
279
|
results.append(write_output_file(combined_path_and_file, output_extension, scan_item,
|
|
276
|
-
BIN_EXT_HEADER, HIDE_HEADER, output_format
|
|
280
|
+
BIN_EXT_HEADER, HIDE_HEADER, output_format,
|
|
281
|
+
scanner_covers=[scan_item.cover]))
|
|
277
282
|
|
|
278
283
|
except Exception as ex:
|
|
279
284
|
error_occured(error_msg=str(ex), exit=False)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fosslight_binary
|
|
3
|
-
Version: 5.1.
|
|
3
|
+
Version: 5.1.33
|
|
4
4
|
Summary: FOSSLight Binary Scanner
|
|
5
5
|
Author: LG Electronics
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -29,7 +29,7 @@ Requires-Dist: py-tlsh
|
|
|
29
29
|
Requires-Dist: pytz
|
|
30
30
|
Requires-Dist: XlsxWriter
|
|
31
31
|
Requires-Dist: PyYAML
|
|
32
|
-
Requires-Dist: fosslight_util>=2.2.
|
|
32
|
+
Requires-Dist: fosslight_util>=2.2.14
|
|
33
33
|
Requires-Dist: truststore
|
|
34
34
|
Requires-Dist: python-magic-bin; sys_platform == "win32"
|
|
35
35
|
Requires-Dist: python-magic; "darwin" in sys_platform
|
{fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary.egg-info/SOURCES.txt
RENAMED
|
@@ -8,6 +8,7 @@ LICENSES/LicenseRef-3rd-party-licenses.txt
|
|
|
8
8
|
src/fosslight_binary/__init__.py
|
|
9
9
|
src/fosslight_binary/_binary.py
|
|
10
10
|
src/fosslight_binary/_binary_dao.py
|
|
11
|
+
src/fosslight_binary/_exclude.py
|
|
11
12
|
src/fosslight_binary/_help.py
|
|
12
13
|
src/fosslight_binary/_jar_analysis.py
|
|
13
14
|
src/fosslight_binary/_simple_mode.py
|
|
File without changes
|
|
File without changes
|
{fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/LICENSES/LicenseRef-3rd-party-licenses.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary.egg-info/entry_points.txt
RENAMED
|
File without changes
|
{fosslight_binary-5.1.31 → fosslight_binary-5.1.33}/src/fosslight_binary.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|