fosslight-binary 5.1.32__tar.gz → 5.1.34__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/PKG-INFO +1 -1
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/pyproject.toml +1 -1
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/_jar_analysis.py +137 -110
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/binary_analysis.py +125 -108
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/PKG-INFO +1 -1
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/LICENSE +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/LICENSES/Apache-2.0.txt +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/LICENSES/LicenseRef-3rd-party-licenses.txt +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/MANIFEST.in +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/README.md +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/setup.cfg +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/__init__.py +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/_binary.py +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/_binary_dao.py +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/_exclude.py +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/_help.py +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/_simple_mode.py +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/cli.py +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/SOURCES.txt +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/dependency_links.txt +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/entry_points.txt +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/requires.txt +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/top_level.txt +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/tests/test_fosslight_binary.py +0 -0
- {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/tests/test_kb_ssl.py +0 -0
|
@@ -19,21 +19,38 @@ from fosslight_util.oss_item import OssItem
|
|
|
19
19
|
|
|
20
20
|
logger = logging.getLogger(constant.LOGGER_NAME)
|
|
21
21
|
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
22
|
+
# The Central Portal host serves the Central Solr index and its g/a/v fields, so
|
|
23
|
+
# its answer is authoritative for the whole index. The legacy search.maven.org
|
|
24
|
+
# endpoint is no longer queried: it often accepts the TCP connect and then never
|
|
25
|
+
# starts the response, so querying it only spends the timeout budget.
|
|
26
|
+
_CENTRAL_SEARCH_URL = "https://central.sonatype.com/solrsearch/select"
|
|
27
|
+
|
|
28
|
+
# (connect, read) seconds for the remote POM fetch. A dead host fails on connect
|
|
29
|
+
# in 3s, while a POM that is merely slow still has 7s to arrive - the license is
|
|
30
|
+
# taken from whatever answers, and the fallback behind it is final. Kept separate
|
|
31
|
+
# from the search budget below: that one is deliberately tight to abandon a
|
|
32
|
+
# stalled Solr host, which is the wrong bound for downloading a file.
|
|
33
|
+
_POM_DOWNLOAD_TIMEOUT = (3, 7)
|
|
34
|
+
_CENTRAL_SEARCH_TIMEOUT = 2.5 # seconds - search budget
|
|
32
35
|
_MAVEN_JAR_HTTP_TIMEOUT = (2, 2) # match Util probe timeouts for multi-repo jar checks
|
|
33
|
-
_MAX_RETRY = 3 # maximum Central API retry attempts per JAR
|
|
34
36
|
_central_network_warned = False # Flag to suppress repeated network-unavailable warnings within one run
|
|
37
|
+
_central_lookup_warned = False # Flag to suppress repeated lookup-failure warnings within one run
|
|
35
38
|
_COORD_TOKEN = re.compile(r'[A-Za-z0-9._+-]+') # shape of a Maven groupId / artifactId / version
|
|
36
39
|
|
|
40
|
+
# Outcome of one SHA-1 search. Only a 200 reply carries an answer from the
|
|
41
|
+
# Central index, so only _SEARCH_MISS states that a SHA-1 is not published
|
|
42
|
+
# there. _SEARCH_FAILED means the question was never answered: a JAR the index
|
|
43
|
+
# did not identify is not the same as a JAR with no OSS to report, and reporting
|
|
44
|
+
# the two alike lets a rate-limited scan look like a clean one.
|
|
45
|
+
_SEARCH_HIT = 'hit'
|
|
46
|
+
_SEARCH_MISS = 'miss'
|
|
47
|
+
_SEARCH_FAILED = 'failed'
|
|
48
|
+
_SEARCH_TIMED_OUT = 'timed_out'
|
|
49
|
+
|
|
50
|
+
# Reported alongside the coordinate source when a SHA-1 miss suppresses the
|
|
51
|
+
# Download Location, so the empty column has a stated reason in the report.
|
|
52
|
+
_SHA1_MISS_COMMENT = 'SHA-1 not in Maven Central'
|
|
53
|
+
|
|
37
54
|
|
|
38
55
|
def _sha1_of_file(filepath):
|
|
39
56
|
h = hashlib.sha1()
|
|
@@ -122,61 +139,82 @@ def _warn_network_once(context=""):
|
|
|
122
139
|
_central_network_warned = True
|
|
123
140
|
|
|
124
141
|
|
|
125
|
-
def
|
|
126
|
-
"""
|
|
142
|
+
def _warn_lookup_failed_once(reason, context=""):
|
|
143
|
+
"""Warn once per run that a Central lookup produced no answer at all."""
|
|
144
|
+
global _central_lookup_warned
|
|
145
|
+
if not _central_lookup_warned:
|
|
146
|
+
msg = f"Maven Central 조회에 실패했습니다 ({reason}) - 일부 JAR의 OSS 식별이 누락될 수 있습니다"
|
|
147
|
+
if context:
|
|
148
|
+
msg += f" [{context}]"
|
|
149
|
+
logger.warning(msg)
|
|
150
|
+
_central_lookup_warned = True
|
|
127
151
|
|
|
128
|
-
|
|
129
|
-
|
|
152
|
+
|
|
153
|
+
def _search_central_by_sha1(sha1, timeout=None):
|
|
154
|
+
"""Look up Maven coordinates by JAR checksum on the Central Portal endpoint.
|
|
155
|
+
|
|
156
|
+
Returns ``(coordinates, outcome)``. A 200 reply is final: a complete
|
|
157
|
+
coordinate triple is ``_SEARCH_HIT``, and no document or an incomplete one
|
|
158
|
+
is ``_SEARCH_MISS``, a normal result that stays quiet. A reply carrying no
|
|
159
|
+
index answer is ``_SEARCH_FAILED`` and warns once per run so it cannot pass
|
|
160
|
+
as a miss - non-200 and unparseable bodies through
|
|
161
|
+
``_warn_lookup_failed_once``, a refused connection through
|
|
162
|
+
``_warn_network_once``, whose flags are independent. A timeout is separate:
|
|
163
|
+
``_SEARCH_TIMED_OUT`` with a debug log and no warning, because the retry
|
|
164
|
+
policy meant to warn after exhausting its rounds is not implemented yet.
|
|
130
165
|
"""
|
|
131
166
|
if not sha1:
|
|
132
|
-
return {},
|
|
167
|
+
return {}, _SEARCH_MISS
|
|
133
168
|
_timeout = timeout if timeout is not None else _CENTRAL_SEARCH_TIMEOUT
|
|
134
169
|
params = {"q": f"1:{sha1}", "rows": 1, "wt": "json"}
|
|
135
|
-
|
|
136
|
-
unreachable = False
|
|
170
|
+
where = f"SHA-1 search: {sha1[:10]}..."
|
|
137
171
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
else:
|
|
151
|
-
logger.debug(f"Maven Central SHA-1 search failed ({sha1}) at {url}: {ex}")
|
|
152
|
-
continue
|
|
172
|
+
try:
|
|
173
|
+
resp = requests.get(_CENTRAL_SEARCH_URL, params=params, timeout=_timeout)
|
|
174
|
+
except requests.exceptions.Timeout:
|
|
175
|
+
logger.debug(f"Maven Central SHA-1 search timed out ({sha1}) at {_CENTRAL_SEARCH_URL}")
|
|
176
|
+
return {}, _SEARCH_TIMED_OUT
|
|
177
|
+
except Exception as ex:
|
|
178
|
+
if _is_network_error(ex):
|
|
179
|
+
_warn_network_once(where)
|
|
180
|
+
else:
|
|
181
|
+
_warn_lookup_failed_once(type(ex).__name__, where)
|
|
182
|
+
logger.debug(f"Maven Central SHA-1 search failed ({sha1}) at {_CENTRAL_SEARCH_URL}: {ex}")
|
|
183
|
+
return {}, _SEARCH_FAILED
|
|
153
184
|
|
|
154
|
-
|
|
155
|
-
|
|
185
|
+
if resp.status_code != 200:
|
|
186
|
+
_warn_lookup_failed_once(f"HTTP {resp.status_code}", where)
|
|
187
|
+
logger.debug(f"Maven Central SHA-1 search got HTTP {resp.status_code} ({sha1})")
|
|
188
|
+
return {}, _SEARCH_FAILED
|
|
156
189
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
190
|
+
try:
|
|
191
|
+
docs = resp.json().get("response", {}).get("docs", [])
|
|
192
|
+
except Exception as ex:
|
|
193
|
+
_warn_lookup_failed_once("응답을 해석할 수 없음", where)
|
|
194
|
+
logger.debug(f"Maven Central SHA-1 search reply was unusable ({sha1}): {ex}")
|
|
195
|
+
return {}, _SEARCH_FAILED
|
|
161
196
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
return {}, False
|
|
197
|
+
if not docs:
|
|
198
|
+
return {}, _SEARCH_MISS
|
|
165
199
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
}, False
|
|
200
|
+
doc = docs[0]
|
|
201
|
+
groupId = doc.get("g", "")
|
|
202
|
+
artifactId = doc.get("a", "")
|
|
203
|
+
version = doc.get("v") or doc.get("latestVersion", "")
|
|
171
204
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
205
|
+
if not (groupId and artifactId and version):
|
|
206
|
+
logger.debug(f"Maven Central returned an incomplete document for {sha1}: {doc}")
|
|
207
|
+
return {}, _SEARCH_MISS
|
|
208
|
+
|
|
209
|
+
return {
|
|
210
|
+
"groupId": groupId,
|
|
211
|
+
"artifactId": artifactId,
|
|
212
|
+
"version": version,
|
|
213
|
+
}, _SEARCH_HIT
|
|
176
214
|
|
|
177
215
|
|
|
178
216
|
def _download_pom_to_tempfile(group_id, artifact_id, version, timeout=None):
|
|
179
|
-
_timeout = timeout if timeout is not None else
|
|
217
|
+
_timeout = timeout if timeout is not None else _POM_DOWNLOAD_TIMEOUT
|
|
180
218
|
group_path = group_id.replace('.', '/')
|
|
181
219
|
pom_name = f"{artifact_id}-{version}.pom"
|
|
182
220
|
urls = [
|
|
@@ -193,7 +231,7 @@ def _download_pom_to_tempfile(group_id, artifact_id, version, timeout=None):
|
|
|
193
231
|
logger.debug(f"POM downloaded to {tmp.name} from {url}")
|
|
194
232
|
return tmp.name, False
|
|
195
233
|
except requests.exceptions.Timeout:
|
|
196
|
-
logger.debug(f"POM download timed out from {url} -
|
|
234
|
+
logger.debug(f"POM download timed out from {url} - trying next repository")
|
|
197
235
|
any_timeout = True
|
|
198
236
|
except Exception as ex:
|
|
199
237
|
if _is_network_error(ex):
|
|
@@ -253,20 +291,17 @@ def _find_jar_download_url(group_id, artifact_id, version):
|
|
|
253
291
|
return ""
|
|
254
292
|
|
|
255
293
|
|
|
256
|
-
def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None
|
|
294
|
+
def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None):
|
|
257
295
|
groupId = artifactId = version = project_url = license_str = ''
|
|
258
296
|
confirmed_in_central = False
|
|
259
297
|
trusted_coordinates = False
|
|
260
298
|
source = ''
|
|
261
299
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
if timed_out:
|
|
268
|
-
logger.debug(f"{rel_path}: Central SHA-1 search timed out - will retry")
|
|
269
|
-
return None, True
|
|
300
|
+
central_info, search_outcome = _search_central_by_sha1(sha1, timeout=search_timeout)
|
|
301
|
+
if search_outcome == _SEARCH_TIMED_OUT:
|
|
302
|
+
logger.debug(f"{rel_path}: Central SHA-1 search timed out - falling back to JAR internals")
|
|
303
|
+
elif search_outcome == _SEARCH_FAILED:
|
|
304
|
+
logger.debug(f"{rel_path}: Central SHA-1 lookup failed - falling back to JAR internals")
|
|
270
305
|
|
|
271
306
|
g2, a2, v2, url2, pom_tmp_path = _read_pom_from_jar(jar_path)
|
|
272
307
|
|
|
@@ -282,7 +317,7 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
|
|
|
282
317
|
if names_match:
|
|
283
318
|
logger.debug(f"{rel_path}: Central and JAR pom.xml match ({central_oss_name} {c_version}) - using JAR pom.xml for license")
|
|
284
319
|
groupId, artifactId, version, project_url = g2, a2, v2, url2
|
|
285
|
-
source = 'pom.xml'
|
|
320
|
+
source = 'pom.xml (Central verified)'
|
|
286
321
|
confirmed_in_central = True
|
|
287
322
|
trusted_coordinates = True
|
|
288
323
|
|
|
@@ -307,16 +342,19 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
|
|
|
307
342
|
confirmed_in_central = True
|
|
308
343
|
trusted_coordinates = True
|
|
309
344
|
|
|
310
|
-
tmp_path, timed_out = _download_pom_to_tempfile(
|
|
311
|
-
groupId, artifactId, version, timeout=search_timeout)
|
|
345
|
+
tmp_path, timed_out = _download_pom_to_tempfile(groupId, artifactId, version)
|
|
312
346
|
if timed_out:
|
|
313
|
-
logger.
|
|
347
|
+
logger.warning(
|
|
348
|
+
f"{rel_path}: POM download timed out for {groupId}:{artifactId}:{version}"
|
|
349
|
+
" - using JAR pom.xml license")
|
|
314
350
|
if pom_tmp_path:
|
|
315
351
|
try:
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
352
|
+
license_str = get_license_from_pom(
|
|
353
|
+
group_id=g2, artifact_id=a2, version=v2,
|
|
354
|
+
pom_path=pom_tmp_path, check_parent=True)
|
|
355
|
+
logger.debug(f"{rel_path}: license from JAR pom.xml after Central POM timeout={license_str!r}")
|
|
356
|
+
except Exception as ex:
|
|
357
|
+
logger.debug(f"get_license_from_pom (jar pom_path) failed: {ex}")
|
|
320
358
|
if tmp_path:
|
|
321
359
|
try:
|
|
322
360
|
license_str = get_license_from_pom(
|
|
@@ -338,11 +376,12 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
|
|
|
338
376
|
pass
|
|
339
377
|
|
|
340
378
|
else:
|
|
341
|
-
|
|
379
|
+
if search_outcome == _SEARCH_MISS:
|
|
380
|
+
logger.debug(f"{rel_path}: not found in Maven Central - falling back to JAR internals")
|
|
342
381
|
|
|
343
382
|
if g2 or a2:
|
|
344
383
|
groupId, artifactId, version, project_url = g2, a2, v2, url2
|
|
345
|
-
source = 'pom.xml'
|
|
384
|
+
source = 'pom.xml (Local)'
|
|
346
385
|
trusted_coordinates = True
|
|
347
386
|
|
|
348
387
|
if pom_tmp_path:
|
|
@@ -372,9 +411,22 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
|
|
|
372
411
|
trusted_coordinates = False
|
|
373
412
|
|
|
374
413
|
if not (groupId or artifactId):
|
|
375
|
-
return None
|
|
376
|
-
|
|
377
|
-
|
|
414
|
+
return None
|
|
415
|
+
|
|
416
|
+
# A miss is Central's answer that these exact bytes are not published. The
|
|
417
|
+
# JAR's own pom.xml may still name real coordinates, so a URL built from them
|
|
418
|
+
# would resolve - but it would serve the upstream artifact, not this file. A
|
|
419
|
+
# locally patched JAR keeps the official pom.xml, so filling the URL would
|
|
420
|
+
# report a modified binary as the official one. Omit it and say why; the
|
|
421
|
+
# probe is skipped because its result could not be reported anyway. A lookup
|
|
422
|
+
# that never answered (failed / timed out) is not a miss and still fills the
|
|
423
|
+
# URL, so an unreachable Central does not blank the whole report.
|
|
424
|
+
unverified_after_miss = (search_outcome == _SEARCH_MISS
|
|
425
|
+
and trusted_coordinates and not confirmed_in_central)
|
|
426
|
+
|
|
427
|
+
if unverified_after_miss:
|
|
428
|
+
dl_url = ""
|
|
429
|
+
elif confirmed_in_central or trusted_coordinates:
|
|
378
430
|
dl_url = _find_jar_download_url(groupId, artifactId, version)
|
|
379
431
|
else:
|
|
380
432
|
dl_url = ""
|
|
@@ -383,11 +435,13 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
|
|
|
383
435
|
|
|
384
436
|
oss = OssItem(oss_name, version, license_str, dl_url)
|
|
385
437
|
oss.comment = source
|
|
438
|
+
if unverified_after_miss:
|
|
439
|
+
oss.comment = _SHA1_MISS_COMMENT
|
|
386
440
|
|
|
387
441
|
logger.debug(
|
|
388
442
|
f"Result: {rel_path} | {oss_name} {version} | [{license_str}] | dl={dl_url} | source={source}")
|
|
389
443
|
|
|
390
|
-
return {"oss_list": [oss]}
|
|
444
|
+
return {"oss_list": [oss]}
|
|
391
445
|
|
|
392
446
|
|
|
393
447
|
def _has_valid_jar_oss(oss_list):
|
|
@@ -404,12 +458,12 @@ def _store_jar_result(jar_items, sha1, result):
|
|
|
404
458
|
|
|
405
459
|
|
|
406
460
|
def analyze_jar_file(path_to_find_bin, path_to_exclude):
|
|
407
|
-
global _central_network_warned
|
|
461
|
+
global _central_network_warned, _central_lookup_warned
|
|
408
462
|
_central_network_warned = False
|
|
463
|
+
_central_lookup_warned = False
|
|
409
464
|
jar_items = {}
|
|
465
|
+
analyzed = set()
|
|
410
466
|
success = True
|
|
411
|
-
retry_queue = []
|
|
412
|
-
pending_sha1s = set()
|
|
413
467
|
|
|
414
468
|
jar_files = []
|
|
415
469
|
for root_dir, _dirs, files in os.walk(path_to_find_bin):
|
|
@@ -427,42 +481,15 @@ def analyze_jar_file(path_to_find_bin, path_to_exclude):
|
|
|
427
481
|
continue
|
|
428
482
|
|
|
429
483
|
sha1 = _sha1_of_file(jar_path)
|
|
430
|
-
if not sha1 or sha1 in
|
|
484
|
+
if not sha1 or sha1 in analyzed:
|
|
431
485
|
continue
|
|
486
|
+
analyzed.add(sha1)
|
|
432
487
|
|
|
433
|
-
result
|
|
488
|
+
result = _process_one_jar(
|
|
434
489
|
jar_path, rel_path, sha1, search_timeout=_CENTRAL_SEARCH_TIMEOUT)
|
|
435
|
-
|
|
436
|
-
if needs_retry:
|
|
437
|
-
logger.debug(f"{rel_path}: Central API timed out - queued for retry (attempt 1/{_MAX_RETRY})")
|
|
438
|
-
pending_sha1s.add(sha1)
|
|
439
|
-
retry_queue.append((jar_path, rel_path, sha1, 1))
|
|
440
|
-
elif result is not None:
|
|
490
|
+
if result is not None:
|
|
441
491
|
_store_jar_result(jar_items, sha1, result)
|
|
442
492
|
|
|
443
|
-
while retry_queue:
|
|
444
|
-
next_queue = []
|
|
445
|
-
for jar_path, rel_path, sha1, attempt in retry_queue:
|
|
446
|
-
if attempt >= _MAX_RETRY:
|
|
447
|
-
logger.warning(
|
|
448
|
-
f"{rel_path}: Maven Central API timed out after {_MAX_RETRY} attempts"
|
|
449
|
-
" - falling back to JAR internals")
|
|
450
|
-
result, _ = _process_one_jar(jar_path, rel_path, sha1, skip_central_search=True)
|
|
451
|
-
if result is not None:
|
|
452
|
-
_store_jar_result(jar_items, sha1, result)
|
|
453
|
-
continue
|
|
454
|
-
|
|
455
|
-
logger.debug(f"{rel_path}: retrying Central API (attempt {attempt + 1}/{_MAX_RETRY})")
|
|
456
|
-
result, needs_retry = _process_one_jar(
|
|
457
|
-
jar_path, rel_path, sha1, search_timeout=_CENTRAL_SEARCH_TIMEOUT)
|
|
458
|
-
|
|
459
|
-
if needs_retry:
|
|
460
|
-
next_queue.append((jar_path, rel_path, sha1, attempt + 1))
|
|
461
|
-
elif result is not None:
|
|
462
|
-
_store_jar_result(jar_items, sha1, result)
|
|
463
|
-
|
|
464
|
-
retry_queue = next_queue
|
|
465
|
-
|
|
466
493
|
return jar_items, success
|
|
467
494
|
|
|
468
495
|
|
|
@@ -33,7 +33,7 @@ import shutil
|
|
|
33
33
|
PKG_NAME = "fosslight_binary"
|
|
34
34
|
logger = logging.getLogger(constant.LOGGER_NAME)
|
|
35
35
|
|
|
36
|
-
_REMOVE_FILE_EXTENSION = ['json', 'js']
|
|
36
|
+
_REMOVE_FILE_EXTENSION = ['json', 'js', 'xlsx', 'xls', 'xlsm']
|
|
37
37
|
_REMOVE_FILE_COMMAND_RESULT = ['timezone data', 'apple binary property list']
|
|
38
38
|
INCLUDE_FILE_COMMAND_RESULT = ['current ar archive']
|
|
39
39
|
|
|
@@ -86,7 +86,9 @@ def init(path_to_find_bin, output_file_name, formats, path_to_exclude=[]):
|
|
|
86
86
|
output_path = os.path.abspath(output_path)
|
|
87
87
|
|
|
88
88
|
original_output_path = output_path
|
|
89
|
-
output_path = os.path.join(output_path, '.
|
|
89
|
+
output_path = os.path.join(output_path, f'.fosslight_temp_{file_time}')
|
|
90
|
+
if os.path.isdir(output_path):
|
|
91
|
+
shutil.rmtree(output_path, ignore_errors=True)
|
|
90
92
|
|
|
91
93
|
while len(output_files) < len(output_extensions):
|
|
92
94
|
output_files.append(None)
|
|
@@ -181,12 +183,18 @@ def find_binaries(path_to_find_bin, output_dir, formats, kb_url="", kb_token="",
|
|
|
181
183
|
if not path_to_find_bin.endswith(os.path.sep):
|
|
182
184
|
_root_path += os.path.sep
|
|
183
185
|
|
|
186
|
+
output_path = ""
|
|
187
|
+
original_output_path = ""
|
|
188
|
+
log_file = ""
|
|
189
|
+
final_log_path = ""
|
|
190
|
+
|
|
184
191
|
if simple_mode:
|
|
185
192
|
mode = "Simple Mode"
|
|
186
193
|
_result_log, binary_yaml_file, compressed_yaml_file = init_simple(output_dir, PKG_NAME, start_time)
|
|
187
194
|
else:
|
|
188
195
|
_result_log, result_reports, output_extensions, formats, output_path, original_output_path, log_file = init(
|
|
189
196
|
path_to_find_bin, output_dir, formats, path_to_exclude)
|
|
197
|
+
final_log_path = os.path.join(original_output_path, f"fosslight_log_bin_{timestamp_for_filename(start_time)}.txt")
|
|
190
198
|
|
|
191
199
|
total_bin_cnt = 0
|
|
192
200
|
db_loaded_cnt = 0
|
|
@@ -196,130 +204,139 @@ def find_binaries(path_to_find_bin, output_dir, formats, kb_url="", kb_token="",
|
|
|
196
204
|
bin_list = []
|
|
197
205
|
scan_item = ScannerItem(PKG_NAME, "")
|
|
198
206
|
|
|
199
|
-
if all_exclude_mode and len(all_exclude_mode) == 4:
|
|
200
|
-
excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped = all_exclude_mode
|
|
201
|
-
elif simple_mode:
|
|
202
|
-
excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped \
|
|
203
|
-
= get_excluded_paths(
|
|
204
|
-
path_to_find_bin, path_to_exclude, REMOVE_FILE_EXTENSION_SIMPLE,
|
|
205
|
-
exclude_filenames=EXCLUDE_FILENAME_BINARY)
|
|
206
|
-
else:
|
|
207
|
-
excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped \
|
|
208
|
-
= get_excluded_paths(
|
|
209
|
-
path_to_find_bin, path_to_exclude,
|
|
210
|
-
exclude_filenames=EXCLUDE_FILENAME_BINARY)
|
|
211
|
-
logger.debug(f"Skipped paths: {excluded_path_with_default_exclusion}")
|
|
212
|
-
|
|
213
|
-
if not os.path.isdir(path_to_find_bin):
|
|
214
|
-
error_occured(error_msg=f"(-p option) Can't find the directory: {path_to_find_bin}",
|
|
215
|
-
result_log=_result_log,
|
|
216
|
-
exit=True,
|
|
217
|
-
mode=mode)
|
|
218
|
-
if not correct_filepath:
|
|
219
|
-
correct_filepath = path_to_find_bin
|
|
220
207
|
try:
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
208
|
+
if all_exclude_mode and len(all_exclude_mode) == 4:
|
|
209
|
+
excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped = all_exclude_mode
|
|
210
|
+
elif simple_mode:
|
|
211
|
+
excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped \
|
|
212
|
+
= get_excluded_paths(
|
|
213
|
+
path_to_find_bin, path_to_exclude, REMOVE_FILE_EXTENSION_SIMPLE,
|
|
214
|
+
exclude_filenames=EXCLUDE_FILENAME_BINARY)
|
|
215
|
+
else:
|
|
216
|
+
excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped \
|
|
217
|
+
= get_excluded_paths(
|
|
218
|
+
path_to_find_bin, path_to_exclude,
|
|
219
|
+
exclude_filenames=EXCLUDE_FILENAME_BINARY)
|
|
220
|
+
logger.debug(f"Skipped paths: {excluded_path_with_default_exclusion}")
|
|
221
|
+
|
|
222
|
+
if not os.path.isdir(path_to_find_bin):
|
|
223
|
+
error_occured(error_msg=f"(-p option) Can't find the directory: {path_to_find_bin}",
|
|
224
|
+
result_log=_result_log,
|
|
225
|
+
exit=True,
|
|
226
|
+
mode=mode)
|
|
227
|
+
|
|
228
|
+
if not correct_filepath:
|
|
229
|
+
correct_filepath = path_to_find_bin
|
|
229
230
|
try:
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
total_bin_cnt = len(filtered_bin_list) + len(compressed_list)
|
|
231
|
+
_, file_list, found_jar = get_file_list(path_to_find_bin, excluded_files)
|
|
232
|
+
return_list = list(return_bin_only(file_list))
|
|
233
233
|
except Exception as ex:
|
|
234
|
-
error_occured(error_msg=f"Failed to
|
|
234
|
+
error_occured(error_msg=f"Failed to check whether it is binary or not : {ex}",
|
|
235
235
|
result_log=_result_log,
|
|
236
236
|
exit=True,
|
|
237
|
-
mode=
|
|
237
|
+
mode=mode)
|
|
238
238
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
if success:
|
|
257
|
-
return_list = merge_binary_list(jar_items, return_list)
|
|
258
|
-
else:
|
|
259
|
-
logger.warning("Could not find OSS information for some jar files.")
|
|
260
|
-
|
|
261
|
-
return_list, db_loaded_cnt, kb_cover_msg = get_oss_info_from_db(return_list, kb_url, kb_token)
|
|
262
|
-
return_list = sorted(return_list, key=lambda row: (row.bin_name_with_path))
|
|
263
|
-
scan_item.append_file_items(return_list, PKG_NAME)
|
|
264
|
-
if correct_mode:
|
|
265
|
-
success, msg_correct, correct_list = correct_with_yaml(correct_filepath, path_to_find_bin, scan_item)
|
|
266
|
-
if not success:
|
|
267
|
-
logger.info(f"No correction with yaml: {msg_correct}")
|
|
239
|
+
if simple_mode:
|
|
240
|
+
try:
|
|
241
|
+
compressed_list, filtered_bin_list = filter_binary(return_list)
|
|
242
|
+
results = print_simple_mode(binary_yaml_file, compressed_yaml_file, compressed_list, filtered_bin_list)
|
|
243
|
+
total_bin_cnt = len(filtered_bin_list) + len(compressed_list)
|
|
244
|
+
except Exception as ex:
|
|
245
|
+
error_occured(error_msg=f"Failed to run simple mode: {ex}",
|
|
246
|
+
result_log=_result_log,
|
|
247
|
+
exit=True,
|
|
248
|
+
mode="Simple mode")
|
|
249
|
+
|
|
250
|
+
for success_to_write, writing_msg, result_file in results:
|
|
251
|
+
if success_to_write:
|
|
252
|
+
if result_file:
|
|
253
|
+
logger.info(f"Output file :{result_file}")
|
|
254
|
+
else:
|
|
255
|
+
logger.warning(f"{writing_msg}")
|
|
268
256
|
else:
|
|
269
|
-
|
|
270
|
-
|
|
257
|
+
logger.error(f"Fail to generate result file.:{writing_msg}")
|
|
258
|
+
else:
|
|
259
|
+
total_bin_cnt = len(return_list)
|
|
260
|
+
scan_item = ScannerItem(PKG_NAME, start_time)
|
|
261
|
+
scan_item.set_cover_pathinfo(path_to_find_bin, excluded_path_without_dot)
|
|
262
|
+
try:
|
|
263
|
+
if found_jar:
|
|
264
|
+
logger.info("Run to analyze .jar file")
|
|
265
|
+
jar_items, success = analyze_jar_file(path_to_find_bin, excluded_files)
|
|
266
|
+
if success:
|
|
267
|
+
return_list = merge_binary_list(jar_items, return_list)
|
|
268
|
+
else:
|
|
269
|
+
logger.warning("Could not find OSS information for some jar files.")
|
|
270
|
+
|
|
271
|
+
return_list, db_loaded_cnt, kb_cover_msg = get_oss_info_from_db(return_list, kb_url, kb_token)
|
|
272
|
+
return_list = sorted(return_list, key=lambda row: (row.bin_name_with_path))
|
|
273
|
+
scan_item.append_file_items(return_list, PKG_NAME)
|
|
274
|
+
if correct_mode:
|
|
275
|
+
success, msg_correct, correct_list = correct_with_yaml(correct_filepath, path_to_find_bin, scan_item)
|
|
276
|
+
if not success:
|
|
277
|
+
logger.info(f"No correction with yaml: {msg_correct}")
|
|
278
|
+
else:
|
|
279
|
+
return_list = correct_list
|
|
280
|
+
logger.info("Success to correct with yaml.")
|
|
271
281
|
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
282
|
+
finish_time = current_timestamp_utc()
|
|
283
|
+
scan_item.set_cover_comment(f"Detected binaries: {len(return_list)} (Scanned Files : {cnt_file_except_skipped})")
|
|
284
|
+
if kb_cover_msg:
|
|
285
|
+
scan_item.set_cover_comment(kb_cover_msg)
|
|
286
|
+
scan_item.set_cover_finish_time(finish_time)
|
|
277
287
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
288
|
+
for combined_path_and_file, output_extension, output_format in zip(result_reports, output_extensions, formats):
|
|
289
|
+
results.append(write_output_file(combined_path_and_file, output_extension, scan_item,
|
|
290
|
+
BIN_EXT_HEADER, HIDE_HEADER, output_format,
|
|
291
|
+
scanner_covers=[scan_item.cover]))
|
|
282
292
|
|
|
283
|
-
|
|
284
|
-
|
|
293
|
+
except Exception as ex:
|
|
294
|
+
error_occured(error_msg=str(ex), exit=False)
|
|
285
295
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
296
|
+
for success_to_write, writing_msg, result_file in results:
|
|
297
|
+
if success_to_write:
|
|
298
|
+
if result_file:
|
|
299
|
+
logger.info(f"Output file :{os.path.join(original_output_path, os.path.basename(result_file))}")
|
|
300
|
+
else:
|
|
301
|
+
logger.warning(f"{writing_msg}")
|
|
302
|
+
for row in scan_item.get_cover_comment():
|
|
303
|
+
logger.info(row)
|
|
290
304
|
else:
|
|
291
|
-
logger.
|
|
292
|
-
for row in scan_item.get_cover_comment():
|
|
293
|
-
logger.info(row)
|
|
294
|
-
else:
|
|
295
|
-
logger.error(f"Fail to generate result file.:{writing_msg}")
|
|
305
|
+
logger.error(f"Fail to generate result file.:{writing_msg}")
|
|
296
306
|
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
307
|
+
try:
|
|
308
|
+
if os.path.isdir(output_path):
|
|
309
|
+
shutil.copytree(output_path, original_output_path, dirs_exist_ok=True)
|
|
310
|
+
else:
|
|
311
|
+
logger.debug(f"Temp directory not found, skip moving: {output_path}")
|
|
312
|
+
except Exception as ex:
|
|
313
|
+
logger.debug(f"Failed to move temp files: {ex}")
|
|
314
|
+
success_to_write = False
|
|
304
315
|
|
|
305
316
|
try:
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
logger.debug(f"Temp directory not found, skip moving: {output_path}")
|
|
317
|
+
print_result_log(mode=mode, success=True, result_log=_result_log,
|
|
318
|
+
file_cnt=str(cnt_file_except_skipped),
|
|
319
|
+
bin_file_cnt=str(total_bin_cnt),
|
|
320
|
+
auto_bin_cnt=str(db_loaded_cnt), bin_list=bin_list)
|
|
311
321
|
except Exception as ex:
|
|
312
|
-
|
|
322
|
+
error_occured(error_msg=f"Print log : {ex}", exit=False)
|
|
313
323
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
324
|
+
return success_to_write, scan_item
|
|
325
|
+
finally:
|
|
326
|
+
if output_path:
|
|
327
|
+
try:
|
|
328
|
+
if log_file and final_log_path and os.path.isfile(log_file):
|
|
329
|
+
move_log_file(log_file, final_log_path)
|
|
330
|
+
elif log_file:
|
|
331
|
+
logger.debug("Moving binary analysis log file is skipped")
|
|
332
|
+
except Exception as ex:
|
|
333
|
+
logger.debug(f"Failed to move log file: {ex}")
|
|
321
334
|
|
|
322
|
-
|
|
335
|
+
try:
|
|
336
|
+
if os.path.isdir(output_path):
|
|
337
|
+
shutil.rmtree(output_path)
|
|
338
|
+
except Exception as ex:
|
|
339
|
+
logger.debug(f"Failed to cleanup temp files: {ex}")
|
|
323
340
|
|
|
324
341
|
|
|
325
342
|
def return_bin_only(file_list, need_checksum_tlsh=True):
|
|
File without changes
|
|
File without changes
|
{fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/LICENSES/LicenseRef-3rd-party-licenses.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
{fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/entry_points.txt
RENAMED
|
File without changes
|
{fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/requires.txt
RENAMED
|
File without changes
|
{fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|