fosslight-binary 5.1.32__tar.gz → 5.1.34__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/PKG-INFO +1 -1
  2. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/pyproject.toml +1 -1
  3. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/_jar_analysis.py +137 -110
  4. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/binary_analysis.py +125 -108
  5. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/PKG-INFO +1 -1
  6. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/LICENSE +0 -0
  7. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/LICENSES/Apache-2.0.txt +0 -0
  8. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/LICENSES/LicenseRef-3rd-party-licenses.txt +0 -0
  9. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/MANIFEST.in +0 -0
  10. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/README.md +0 -0
  11. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/setup.cfg +0 -0
  12. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/__init__.py +0 -0
  13. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/_binary.py +0 -0
  14. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/_binary_dao.py +0 -0
  15. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/_exclude.py +0 -0
  16. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/_help.py +0 -0
  17. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/_simple_mode.py +0 -0
  18. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary/cli.py +0 -0
  19. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/SOURCES.txt +0 -0
  20. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/dependency_links.txt +0 -0
  21. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/entry_points.txt +0 -0
  22. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/requires.txt +0 -0
  23. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/src/fosslight_binary.egg-info/top_level.txt +0 -0
  24. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/tests/test_fosslight_binary.py +0 -0
  25. {fosslight_binary-5.1.32 → fosslight_binary-5.1.34}/tests/test_kb_ssl.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fosslight_binary
3
- Version: 5.1.32
3
+ Version: 5.1.34
4
4
  Summary: FOSSLight Binary Scanner
5
5
  Author: LG Electronics
6
6
  License-Expression: Apache-2.0
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
7
7
 
8
8
  [project]
9
9
  name = "fosslight_binary"
10
- version = "5.1.32"
10
+ version = "5.1.34"
11
11
  description = "FOSSLight Binary Scanner"
12
12
  readme = "README.md"
13
13
  license = "Apache-2.0"
@@ -19,21 +19,38 @@ from fosslight_util.oss_item import OssItem
19
19
 
20
20
  logger = logging.getLogger(constant.LOGGER_NAME)
21
21
 
22
- # Both hosts serve the same Central Solr index and the same g/a/v fields, but
23
- # search.maven.org answers only about one query in six: the TCP connect always
24
- # succeeds and then the response never starts, so no timeout budget rescues it.
25
- # The Portal host answers every query in about half a second, so it goes first.
26
- _CENTRAL_SEARCH_URLS = (
27
- "https://central.sonatype.com/solrsearch/select",
28
- "https://search.maven.org/solrsearch/select",
29
- )
30
- _REQUEST_TIMEOUT = 10 # seconds - used for HEAD / POM download
31
- _CENTRAL_SEARCH_TIMEOUT = 2.5 # seconds - per-host budget, retried on timeout
22
+ # The Central Portal host serves the Central Solr index and its g/a/v fields, so
23
+ # its answer is authoritative for the whole index. The legacy search.maven.org
24
+ # endpoint is no longer queried: it often accepts the TCP connect and then never
25
+ # starts the response, so querying it only spends the timeout budget.
26
+ _CENTRAL_SEARCH_URL = "https://central.sonatype.com/solrsearch/select"
27
+
28
+ # (connect, read) seconds for the remote POM fetch. A dead host fails on connect
29
+ # in 3s, while a POM that is merely slow still has 7s to arrive - the license is
30
+ # taken from whatever answers, and the fallback behind it is final. Kept separate
31
+ # from the search budget below: that one is deliberately tight to abandon a
32
+ # stalled Solr host, which is the wrong bound for downloading a file.
33
+ _POM_DOWNLOAD_TIMEOUT = (3, 7)
34
+ _CENTRAL_SEARCH_TIMEOUT = 2.5 # seconds - search budget
32
35
  _MAVEN_JAR_HTTP_TIMEOUT = (2, 2) # match Util probe timeouts for multi-repo jar checks
33
- _MAX_RETRY = 3 # maximum Central API retry attempts per JAR
34
36
  _central_network_warned = False # Flag to suppress repeated network-unavailable warnings within one run
37
+ _central_lookup_warned = False # Flag to suppress repeated lookup-failure warnings within one run
35
38
  _COORD_TOKEN = re.compile(r'[A-Za-z0-9._+-]+') # shape of a Maven groupId / artifactId / version
36
39
 
40
+ # Outcome of one SHA-1 search. Only a 200 reply carries an answer from the
41
+ # Central index, so only _SEARCH_MISS states that a SHA-1 is not published
42
+ # there. _SEARCH_FAILED means the question was never answered: a JAR the index
43
+ # did not identify is not the same as a JAR with no OSS to report, and reporting
44
+ # the two alike lets a rate-limited scan look like a clean one.
45
+ _SEARCH_HIT = 'hit'
46
+ _SEARCH_MISS = 'miss'
47
+ _SEARCH_FAILED = 'failed'
48
+ _SEARCH_TIMED_OUT = 'timed_out'
49
+
50
+ # Reported alongside the coordinate source when a SHA-1 miss suppresses the
51
+ # Download Location, so the empty column has a stated reason in the report.
52
+ _SHA1_MISS_COMMENT = 'SHA-1 not in Maven Central'
53
+
37
54
 
38
55
  def _sha1_of_file(filepath):
39
56
  h = hashlib.sha1()
@@ -122,61 +139,82 @@ def _warn_network_once(context=""):
122
139
  _central_network_warned = True
123
140
 
124
141
 
125
- def _search_central_by_sha1(sha1, timeout=None):
126
- """Look up Maven coordinates by JAR checksum, trying each Central host in turn.
142
+ def _warn_lookup_failed_once(reason, context=""):
143
+ """Warn once per run that a Central lookup produced no answer at all."""
144
+ global _central_lookup_warned
145
+ if not _central_lookup_warned:
146
+ msg = f"Maven Central 조회에 실패했습니다 ({reason}) - 일부 JAR의 OSS 식별이 누락될 수 있습니다"
147
+ if context:
148
+ msg += f" [{context}]"
149
+ logger.warning(msg)
150
+ _central_lookup_warned = True
127
151
 
128
- An answer from any host is authoritative for the whole index, so a miss ends
129
- the search. ``timed_out`` is reported only when no host answered at all.
152
+
153
+ def _search_central_by_sha1(sha1, timeout=None):
154
+ """Look up Maven coordinates by JAR checksum on the Central Portal endpoint.
155
+
156
+ Returns ``(coordinates, outcome)``. A 200 reply is final: a complete
157
+ coordinate triple is ``_SEARCH_HIT``, and no document or an incomplete one
158
+ is ``_SEARCH_MISS``, a normal result that stays quiet. A reply carrying no
159
+ index answer is ``_SEARCH_FAILED`` and warns once per run so it cannot pass
160
+ as a miss - non-200 and unparseable bodies through
161
+ ``_warn_lookup_failed_once``, a refused connection through
162
+ ``_warn_network_once``, whose flags are independent. A timeout is separate:
163
+ ``_SEARCH_TIMED_OUT`` with a debug log and no warning, because the retry
164
+ policy meant to warn after exhausting its rounds is not implemented yet.
130
165
  """
131
166
  if not sha1:
132
- return {}, False
167
+ return {}, _SEARCH_MISS
133
168
  _timeout = timeout if timeout is not None else _CENTRAL_SEARCH_TIMEOUT
134
169
  params = {"q": f"1:{sha1}", "rows": 1, "wt": "json"}
135
- any_timeout = False
136
- unreachable = False
170
+ where = f"SHA-1 search: {sha1[:10]}..."
137
171
 
138
- for url in _CENTRAL_SEARCH_URLS:
139
- try:
140
- resp = requests.get(url, params=params, timeout=_timeout)
141
- resp.raise_for_status()
142
- docs = resp.json().get("response", {}).get("docs", [])
143
- except requests.exceptions.Timeout:
144
- logger.debug(f"Maven Central SHA-1 search timed out ({sha1}) at {url} - will retry")
145
- any_timeout = True
146
- continue
147
- except Exception as ex:
148
- if _is_network_error(ex):
149
- unreachable = True
150
- else:
151
- logger.debug(f"Maven Central SHA-1 search failed ({sha1}) at {url}: {ex}")
152
- continue
172
+ try:
173
+ resp = requests.get(_CENTRAL_SEARCH_URL, params=params, timeout=_timeout)
174
+ except requests.exceptions.Timeout:
175
+ logger.debug(f"Maven Central SHA-1 search timed out ({sha1}) at {_CENTRAL_SEARCH_URL}")
176
+ return {}, _SEARCH_TIMED_OUT
177
+ except Exception as ex:
178
+ if _is_network_error(ex):
179
+ _warn_network_once(where)
180
+ else:
181
+ _warn_lookup_failed_once(type(ex).__name__, where)
182
+ logger.debug(f"Maven Central SHA-1 search failed ({sha1}) at {_CENTRAL_SEARCH_URL}: {ex}")
183
+ return {}, _SEARCH_FAILED
153
184
 
154
- if not docs:
155
- return {}, False
185
+ if resp.status_code != 200:
186
+ _warn_lookup_failed_once(f"HTTP {resp.status_code}", where)
187
+ logger.debug(f"Maven Central SHA-1 search got HTTP {resp.status_code} ({sha1})")
188
+ return {}, _SEARCH_FAILED
156
189
 
157
- doc = docs[0]
158
- groupId = doc.get("g", "")
159
- artifactId = doc.get("a", "")
160
- version = doc.get("v") or doc.get("latestVersion", "")
190
+ try:
191
+ docs = resp.json().get("response", {}).get("docs", [])
192
+ except Exception as ex:
193
+ _warn_lookup_failed_once("응답을 해석할 수 없음", where)
194
+ logger.debug(f"Maven Central SHA-1 search reply was unusable ({sha1}): {ex}")
195
+ return {}, _SEARCH_FAILED
161
196
 
162
- if not (groupId and artifactId and version):
163
- logger.debug(f"Maven Central returned an incomplete document for {sha1}: {doc}")
164
- return {}, False
197
+ if not docs:
198
+ return {}, _SEARCH_MISS
165
199
 
166
- return {
167
- "groupId": groupId,
168
- "artifactId": artifactId,
169
- "version": version,
170
- }, False
200
+ doc = docs[0]
201
+ groupId = doc.get("g", "")
202
+ artifactId = doc.get("a", "")
203
+ version = doc.get("v") or doc.get("latestVersion", "")
171
204
 
172
- # Warn only once every host has failed: a fallback answer is not a failure.
173
- if unreachable:
174
- _warn_network_once(f"SHA-1 search: {sha1[:10]}...")
175
- return {}, any_timeout
205
+ if not (groupId and artifactId and version):
206
+ logger.debug(f"Maven Central returned an incomplete document for {sha1}: {doc}")
207
+ return {}, _SEARCH_MISS
208
+
209
+ return {
210
+ "groupId": groupId,
211
+ "artifactId": artifactId,
212
+ "version": version,
213
+ }, _SEARCH_HIT
176
214
 
177
215
 
178
216
  def _download_pom_to_tempfile(group_id, artifact_id, version, timeout=None):
179
- _timeout = timeout if timeout is not None else _REQUEST_TIMEOUT
217
+ _timeout = timeout if timeout is not None else _POM_DOWNLOAD_TIMEOUT
180
218
  group_path = group_id.replace('.', '/')
181
219
  pom_name = f"{artifact_id}-{version}.pom"
182
220
  urls = [
@@ -193,7 +231,7 @@ def _download_pom_to_tempfile(group_id, artifact_id, version, timeout=None):
193
231
  logger.debug(f"POM downloaded to {tmp.name} from {url}")
194
232
  return tmp.name, False
195
233
  except requests.exceptions.Timeout:
196
- logger.debug(f"POM download timed out from {url} - will retry")
234
+ logger.debug(f"POM download timed out from {url} - trying next repository")
197
235
  any_timeout = True
198
236
  except Exception as ex:
199
237
  if _is_network_error(ex):
@@ -253,20 +291,17 @@ def _find_jar_download_url(group_id, artifact_id, version):
253
291
  return ""
254
292
 
255
293
 
256
- def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central_search=False):
294
+ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None):
257
295
  groupId = artifactId = version = project_url = license_str = ''
258
296
  confirmed_in_central = False
259
297
  trusted_coordinates = False
260
298
  source = ''
261
299
 
262
- if skip_central_search:
263
- central_info = {}
264
- timed_out = False
265
- else:
266
- central_info, timed_out = _search_central_by_sha1(sha1, timeout=search_timeout)
267
- if timed_out:
268
- logger.debug(f"{rel_path}: Central SHA-1 search timed out - will retry")
269
- return None, True
300
+ central_info, search_outcome = _search_central_by_sha1(sha1, timeout=search_timeout)
301
+ if search_outcome == _SEARCH_TIMED_OUT:
302
+ logger.debug(f"{rel_path}: Central SHA-1 search timed out - falling back to JAR internals")
303
+ elif search_outcome == _SEARCH_FAILED:
304
+ logger.debug(f"{rel_path}: Central SHA-1 lookup failed - falling back to JAR internals")
270
305
 
271
306
  g2, a2, v2, url2, pom_tmp_path = _read_pom_from_jar(jar_path)
272
307
 
@@ -282,7 +317,7 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
282
317
  if names_match:
283
318
  logger.debug(f"{rel_path}: Central and JAR pom.xml match ({central_oss_name} {c_version}) - using JAR pom.xml for license")
284
319
  groupId, artifactId, version, project_url = g2, a2, v2, url2
285
- source = 'pom.xml'
320
+ source = 'pom.xml (Central verified)'
286
321
  confirmed_in_central = True
287
322
  trusted_coordinates = True
288
323
 
@@ -307,16 +342,19 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
307
342
  confirmed_in_central = True
308
343
  trusted_coordinates = True
309
344
 
310
- tmp_path, timed_out = _download_pom_to_tempfile(
311
- groupId, artifactId, version, timeout=search_timeout)
345
+ tmp_path, timed_out = _download_pom_to_tempfile(groupId, artifactId, version)
312
346
  if timed_out:
313
- logger.debug(f"{rel_path}: POM download timed out - will retry")
347
+ logger.warning(
348
+ f"{rel_path}: POM download timed out for {groupId}:{artifactId}:{version}"
349
+ " - using JAR pom.xml license")
314
350
  if pom_tmp_path:
315
351
  try:
316
- os.remove(pom_tmp_path)
317
- except Exception:
318
- pass
319
- return None, True
352
+ license_str = get_license_from_pom(
353
+ group_id=g2, artifact_id=a2, version=v2,
354
+ pom_path=pom_tmp_path, check_parent=True)
355
+ logger.debug(f"{rel_path}: license from JAR pom.xml after Central POM timeout={license_str!r}")
356
+ except Exception as ex:
357
+ logger.debug(f"get_license_from_pom (jar pom_path) failed: {ex}")
320
358
  if tmp_path:
321
359
  try:
322
360
  license_str = get_license_from_pom(
@@ -338,11 +376,12 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
338
376
  pass
339
377
 
340
378
  else:
341
- logger.debug(f"{rel_path}: not found in Maven Central - falling back to JAR internals")
379
+ if search_outcome == _SEARCH_MISS:
380
+ logger.debug(f"{rel_path}: not found in Maven Central - falling back to JAR internals")
342
381
 
343
382
  if g2 or a2:
344
383
  groupId, artifactId, version, project_url = g2, a2, v2, url2
345
- source = 'pom.xml'
384
+ source = 'pom.xml (Local)'
346
385
  trusted_coordinates = True
347
386
 
348
387
  if pom_tmp_path:
@@ -372,9 +411,22 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
372
411
  trusted_coordinates = False
373
412
 
374
413
  if not (groupId or artifactId):
375
- return None, False
376
-
377
- if confirmed_in_central or trusted_coordinates:
414
+ return None
415
+
416
+ # A miss is Central's answer that these exact bytes are not published. The
417
+ # JAR's own pom.xml may still name real coordinates, so a URL built from them
418
+ # would resolve - but it would serve the upstream artifact, not this file. A
419
+ # locally patched JAR keeps the official pom.xml, so filling the URL would
420
+ # report a modified binary as the official one. Omit it and say why; the
421
+ # probe is skipped because its result could not be reported anyway. A lookup
422
+ # that never answered (failed / timed out) is not a miss and still fills the
423
+ # URL, so an unreachable Central does not blank the whole report.
424
+ unverified_after_miss = (search_outcome == _SEARCH_MISS
425
+ and trusted_coordinates and not confirmed_in_central)
426
+
427
+ if unverified_after_miss:
428
+ dl_url = ""
429
+ elif confirmed_in_central or trusted_coordinates:
378
430
  dl_url = _find_jar_download_url(groupId, artifactId, version)
379
431
  else:
380
432
  dl_url = ""
@@ -383,11 +435,13 @@ def _process_one_jar(jar_path, rel_path, sha1, search_timeout=None, skip_central
383
435
 
384
436
  oss = OssItem(oss_name, version, license_str, dl_url)
385
437
  oss.comment = source
438
+ if unverified_after_miss:
439
+ oss.comment = _SHA1_MISS_COMMENT
386
440
 
387
441
  logger.debug(
388
442
  f"Result: {rel_path} | {oss_name} {version} | [{license_str}] | dl={dl_url} | source={source}")
389
443
 
390
- return {"oss_list": [oss]}, False
444
+ return {"oss_list": [oss]}
391
445
 
392
446
 
393
447
  def _has_valid_jar_oss(oss_list):
@@ -404,12 +458,12 @@ def _store_jar_result(jar_items, sha1, result):
404
458
 
405
459
 
406
460
  def analyze_jar_file(path_to_find_bin, path_to_exclude):
407
- global _central_network_warned
461
+ global _central_network_warned, _central_lookup_warned
408
462
  _central_network_warned = False
463
+ _central_lookup_warned = False
409
464
  jar_items = {}
465
+ analyzed = set()
410
466
  success = True
411
- retry_queue = []
412
- pending_sha1s = set()
413
467
 
414
468
  jar_files = []
415
469
  for root_dir, _dirs, files in os.walk(path_to_find_bin):
@@ -427,42 +481,15 @@ def analyze_jar_file(path_to_find_bin, path_to_exclude):
427
481
  continue
428
482
 
429
483
  sha1 = _sha1_of_file(jar_path)
430
- if not sha1 or sha1 in jar_items or sha1 in pending_sha1s:
484
+ if not sha1 or sha1 in analyzed:
431
485
  continue
486
+ analyzed.add(sha1)
432
487
 
433
- result, needs_retry = _process_one_jar(
488
+ result = _process_one_jar(
434
489
  jar_path, rel_path, sha1, search_timeout=_CENTRAL_SEARCH_TIMEOUT)
435
-
436
- if needs_retry:
437
- logger.debug(f"{rel_path}: Central API timed out - queued for retry (attempt 1/{_MAX_RETRY})")
438
- pending_sha1s.add(sha1)
439
- retry_queue.append((jar_path, rel_path, sha1, 1))
440
- elif result is not None:
490
+ if result is not None:
441
491
  _store_jar_result(jar_items, sha1, result)
442
492
 
443
- while retry_queue:
444
- next_queue = []
445
- for jar_path, rel_path, sha1, attempt in retry_queue:
446
- if attempt >= _MAX_RETRY:
447
- logger.warning(
448
- f"{rel_path}: Maven Central API timed out after {_MAX_RETRY} attempts"
449
- " - falling back to JAR internals")
450
- result, _ = _process_one_jar(jar_path, rel_path, sha1, skip_central_search=True)
451
- if result is not None:
452
- _store_jar_result(jar_items, sha1, result)
453
- continue
454
-
455
- logger.debug(f"{rel_path}: retrying Central API (attempt {attempt + 1}/{_MAX_RETRY})")
456
- result, needs_retry = _process_one_jar(
457
- jar_path, rel_path, sha1, search_timeout=_CENTRAL_SEARCH_TIMEOUT)
458
-
459
- if needs_retry:
460
- next_queue.append((jar_path, rel_path, sha1, attempt + 1))
461
- elif result is not None:
462
- _store_jar_result(jar_items, sha1, result)
463
-
464
- retry_queue = next_queue
465
-
466
493
  return jar_items, success
467
494
 
468
495
 
@@ -33,7 +33,7 @@ import shutil
33
33
  PKG_NAME = "fosslight_binary"
34
34
  logger = logging.getLogger(constant.LOGGER_NAME)
35
35
 
36
- _REMOVE_FILE_EXTENSION = ['json', 'js']
36
+ _REMOVE_FILE_EXTENSION = ['json', 'js', 'xlsx', 'xls', 'xlsm']
37
37
  _REMOVE_FILE_COMMAND_RESULT = ['timezone data', 'apple binary property list']
38
38
  INCLUDE_FILE_COMMAND_RESULT = ['current ar archive']
39
39
 
@@ -86,7 +86,9 @@ def init(path_to_find_bin, output_file_name, formats, path_to_exclude=[]):
86
86
  output_path = os.path.abspath(output_path)
87
87
 
88
88
  original_output_path = output_path
89
- output_path = os.path.join(output_path, '.fosslight_temp')
89
+ output_path = os.path.join(output_path, f'.fosslight_temp_{file_time}')
90
+ if os.path.isdir(output_path):
91
+ shutil.rmtree(output_path, ignore_errors=True)
90
92
 
91
93
  while len(output_files) < len(output_extensions):
92
94
  output_files.append(None)
@@ -181,12 +183,18 @@ def find_binaries(path_to_find_bin, output_dir, formats, kb_url="", kb_token="",
181
183
  if not path_to_find_bin.endswith(os.path.sep):
182
184
  _root_path += os.path.sep
183
185
 
186
+ output_path = ""
187
+ original_output_path = ""
188
+ log_file = ""
189
+ final_log_path = ""
190
+
184
191
  if simple_mode:
185
192
  mode = "Simple Mode"
186
193
  _result_log, binary_yaml_file, compressed_yaml_file = init_simple(output_dir, PKG_NAME, start_time)
187
194
  else:
188
195
  _result_log, result_reports, output_extensions, formats, output_path, original_output_path, log_file = init(
189
196
  path_to_find_bin, output_dir, formats, path_to_exclude)
197
+ final_log_path = os.path.join(original_output_path, f"fosslight_log_bin_{timestamp_for_filename(start_time)}.txt")
190
198
 
191
199
  total_bin_cnt = 0
192
200
  db_loaded_cnt = 0
@@ -196,130 +204,139 @@ def find_binaries(path_to_find_bin, output_dir, formats, kb_url="", kb_token="",
196
204
  bin_list = []
197
205
  scan_item = ScannerItem(PKG_NAME, "")
198
206
 
199
- if all_exclude_mode and len(all_exclude_mode) == 4:
200
- excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped = all_exclude_mode
201
- elif simple_mode:
202
- excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped \
203
- = get_excluded_paths(
204
- path_to_find_bin, path_to_exclude, REMOVE_FILE_EXTENSION_SIMPLE,
205
- exclude_filenames=EXCLUDE_FILENAME_BINARY)
206
- else:
207
- excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped \
208
- = get_excluded_paths(
209
- path_to_find_bin, path_to_exclude,
210
- exclude_filenames=EXCLUDE_FILENAME_BINARY)
211
- logger.debug(f"Skipped paths: {excluded_path_with_default_exclusion}")
212
-
213
- if not os.path.isdir(path_to_find_bin):
214
- error_occured(error_msg=f"(-p option) Can't find the directory: {path_to_find_bin}",
215
- result_log=_result_log,
216
- exit=True,
217
- mode=mode)
218
- if not correct_filepath:
219
- correct_filepath = path_to_find_bin
220
207
  try:
221
- _, file_list, found_jar = get_file_list(path_to_find_bin, excluded_files)
222
- return_list = list(return_bin_only(file_list))
223
- except Exception as ex:
224
- error_occured(error_msg=f"Failed to check whether it is binary or not : {ex}",
225
- result_log=_result_log,
226
- exit=True,
227
- mode=mode)
228
- if simple_mode:
208
+ if all_exclude_mode and len(all_exclude_mode) == 4:
209
+ excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped = all_exclude_mode
210
+ elif simple_mode:
211
+ excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped \
212
+ = get_excluded_paths(
213
+ path_to_find_bin, path_to_exclude, REMOVE_FILE_EXTENSION_SIMPLE,
214
+ exclude_filenames=EXCLUDE_FILENAME_BINARY)
215
+ else:
216
+ excluded_path_with_default_exclusion, excluded_path_without_dot, excluded_files, cnt_file_except_skipped \
217
+ = get_excluded_paths(
218
+ path_to_find_bin, path_to_exclude,
219
+ exclude_filenames=EXCLUDE_FILENAME_BINARY)
220
+ logger.debug(f"Skipped paths: {excluded_path_with_default_exclusion}")
221
+
222
+ if not os.path.isdir(path_to_find_bin):
223
+ error_occured(error_msg=f"(-p option) Can't find the directory: {path_to_find_bin}",
224
+ result_log=_result_log,
225
+ exit=True,
226
+ mode=mode)
227
+
228
+ if not correct_filepath:
229
+ correct_filepath = path_to_find_bin
229
230
  try:
230
- compressed_list, filtered_bin_list = filter_binary(return_list)
231
- results = print_simple_mode(binary_yaml_file, compressed_yaml_file, compressed_list, filtered_bin_list)
232
- total_bin_cnt = len(filtered_bin_list) + len(compressed_list)
231
+ _, file_list, found_jar = get_file_list(path_to_find_bin, excluded_files)
232
+ return_list = list(return_bin_only(file_list))
233
233
  except Exception as ex:
234
- error_occured(error_msg=f"Failed to run simple mode: {ex}",
234
+ error_occured(error_msg=f"Failed to check whether it is binary or not : {ex}",
235
235
  result_log=_result_log,
236
236
  exit=True,
237
- mode="Simple mode")
237
+ mode=mode)
238
238
 
239
- for success_to_write, writing_msg, result_file in results:
240
- if success_to_write:
241
- if result_file:
242
- logger.info(f"Output file :{result_file}")
243
- else:
244
- logger.warning(f"{writing_msg}")
245
- else:
246
- logger.error(f"Fail to generate result file.:{writing_msg}")
247
- else:
248
- total_bin_cnt = len(return_list)
249
- scan_item = ScannerItem(PKG_NAME, start_time)
250
- scan_item.set_cover_pathinfo(path_to_find_bin, excluded_path_without_dot)
251
- try:
252
- # Run JAR analysis via Maven Central API
253
- if found_jar:
254
- logger.info("Run to analyze .jar file")
255
- jar_items, success = analyze_jar_file(path_to_find_bin, excluded_files)
256
- if success:
257
- return_list = merge_binary_list(jar_items, return_list)
258
- else:
259
- logger.warning("Could not find OSS information for some jar files.")
260
-
261
- return_list, db_loaded_cnt, kb_cover_msg = get_oss_info_from_db(return_list, kb_url, kb_token)
262
- return_list = sorted(return_list, key=lambda row: (row.bin_name_with_path))
263
- scan_item.append_file_items(return_list, PKG_NAME)
264
- if correct_mode:
265
- success, msg_correct, correct_list = correct_with_yaml(correct_filepath, path_to_find_bin, scan_item)
266
- if not success:
267
- logger.info(f"No correction with yaml: {msg_correct}")
239
+ if simple_mode:
240
+ try:
241
+ compressed_list, filtered_bin_list = filter_binary(return_list)
242
+ results = print_simple_mode(binary_yaml_file, compressed_yaml_file, compressed_list, filtered_bin_list)
243
+ total_bin_cnt = len(filtered_bin_list) + len(compressed_list)
244
+ except Exception as ex:
245
+ error_occured(error_msg=f"Failed to run simple mode: {ex}",
246
+ result_log=_result_log,
247
+ exit=True,
248
+ mode="Simple mode")
249
+
250
+ for success_to_write, writing_msg, result_file in results:
251
+ if success_to_write:
252
+ if result_file:
253
+ logger.info(f"Output file :{result_file}")
254
+ else:
255
+ logger.warning(f"{writing_msg}")
268
256
  else:
269
- return_list = correct_list
270
- logger.info("Success to correct with yaml.")
257
+ logger.error(f"Fail to generate result file.:{writing_msg}")
258
+ else:
259
+ total_bin_cnt = len(return_list)
260
+ scan_item = ScannerItem(PKG_NAME, start_time)
261
+ scan_item.set_cover_pathinfo(path_to_find_bin, excluded_path_without_dot)
262
+ try:
263
+ if found_jar:
264
+ logger.info("Run to analyze .jar file")
265
+ jar_items, success = analyze_jar_file(path_to_find_bin, excluded_files)
266
+ if success:
267
+ return_list = merge_binary_list(jar_items, return_list)
268
+ else:
269
+ logger.warning("Could not find OSS information for some jar files.")
270
+
271
+ return_list, db_loaded_cnt, kb_cover_msg = get_oss_info_from_db(return_list, kb_url, kb_token)
272
+ return_list = sorted(return_list, key=lambda row: (row.bin_name_with_path))
273
+ scan_item.append_file_items(return_list, PKG_NAME)
274
+ if correct_mode:
275
+ success, msg_correct, correct_list = correct_with_yaml(correct_filepath, path_to_find_bin, scan_item)
276
+ if not success:
277
+ logger.info(f"No correction with yaml: {msg_correct}")
278
+ else:
279
+ return_list = correct_list
280
+ logger.info("Success to correct with yaml.")
271
281
 
272
- finish_time = current_timestamp_utc()
273
- scan_item.set_cover_comment(f"Detected binaries: {len(return_list)} (Scanned Files : {cnt_file_except_skipped})")
274
- if kb_cover_msg:
275
- scan_item.set_cover_comment(kb_cover_msg)
276
- scan_item.set_cover_finish_time(finish_time)
282
+ finish_time = current_timestamp_utc()
283
+ scan_item.set_cover_comment(f"Detected binaries: {len(return_list)} (Scanned Files : {cnt_file_except_skipped})")
284
+ if kb_cover_msg:
285
+ scan_item.set_cover_comment(kb_cover_msg)
286
+ scan_item.set_cover_finish_time(finish_time)
277
287
 
278
- for combined_path_and_file, output_extension, output_format in zip(result_reports, output_extensions, formats):
279
- results.append(write_output_file(combined_path_and_file, output_extension, scan_item,
280
- BIN_EXT_HEADER, HIDE_HEADER, output_format,
281
- scanner_covers=[scan_item.cover]))
288
+ for combined_path_and_file, output_extension, output_format in zip(result_reports, output_extensions, formats):
289
+ results.append(write_output_file(combined_path_and_file, output_extension, scan_item,
290
+ BIN_EXT_HEADER, HIDE_HEADER, output_format,
291
+ scanner_covers=[scan_item.cover]))
282
292
 
283
- except Exception as ex:
284
- error_occured(error_msg=str(ex), exit=False)
293
+ except Exception as ex:
294
+ error_occured(error_msg=str(ex), exit=False)
285
295
 
286
- for success_to_write, writing_msg, result_file in results:
287
- if success_to_write:
288
- if result_file:
289
- logger.info(f"Output file :{os.path.join(original_output_path, os.path.basename(result_file))}")
296
+ for success_to_write, writing_msg, result_file in results:
297
+ if success_to_write:
298
+ if result_file:
299
+ logger.info(f"Output file :{os.path.join(original_output_path, os.path.basename(result_file))}")
300
+ else:
301
+ logger.warning(f"{writing_msg}")
302
+ for row in scan_item.get_cover_comment():
303
+ logger.info(row)
290
304
  else:
291
- logger.warning(f"{writing_msg}")
292
- for row in scan_item.get_cover_comment():
293
- logger.info(row)
294
- else:
295
- logger.error(f"Fail to generate result file.:{writing_msg}")
305
+ logger.error(f"Fail to generate result file.:{writing_msg}")
296
306
 
297
- try:
298
- if os.path.isfile(log_file):
299
- move_log_file(log_file, os.path.join(original_output_path, f"fosslight_log_bin_{timestamp_for_filename(start_time)}.txt"))
300
- else:
301
- logger.debug("Moving binary analysis log file is skipped")
302
- except Exception as ex:
303
- logger.debug(f"Failed to move log file: {ex}")
307
+ try:
308
+ if os.path.isdir(output_path):
309
+ shutil.copytree(output_path, original_output_path, dirs_exist_ok=True)
310
+ else:
311
+ logger.debug(f"Temp directory not found, skip moving: {output_path}")
312
+ except Exception as ex:
313
+ logger.debug(f"Failed to move temp files: {ex}")
314
+ success_to_write = False
304
315
 
305
316
  try:
306
- if os.path.isdir(output_path):
307
- shutil.copytree(output_path, original_output_path, dirs_exist_ok=True)
308
- shutil.rmtree(output_path)
309
- else:
310
- logger.debug(f"Temp directory not found, skip moving: {output_path}")
317
+ print_result_log(mode=mode, success=True, result_log=_result_log,
318
+ file_cnt=str(cnt_file_except_skipped),
319
+ bin_file_cnt=str(total_bin_cnt),
320
+ auto_bin_cnt=str(db_loaded_cnt), bin_list=bin_list)
311
321
  except Exception as ex:
312
- logger.debug(f"Failed to move temp files: {ex}")
322
+ error_occured(error_msg=f"Print log : {ex}", exit=False)
313
323
 
314
- try:
315
- print_result_log(mode=mode, success=True, result_log=_result_log,
316
- file_cnt=str(cnt_file_except_skipped),
317
- bin_file_cnt=str(total_bin_cnt),
318
- auto_bin_cnt=str(db_loaded_cnt), bin_list=bin_list)
319
- except Exception as ex:
320
- error_occured(error_msg=f"Print log : {ex}", exit=False)
324
+ return success_to_write, scan_item
325
+ finally:
326
+ if output_path:
327
+ try:
328
+ if log_file and final_log_path and os.path.isfile(log_file):
329
+ move_log_file(log_file, final_log_path)
330
+ elif log_file:
331
+ logger.debug("Moving binary analysis log file is skipped")
332
+ except Exception as ex:
333
+ logger.debug(f"Failed to move log file: {ex}")
321
334
 
322
- return success_to_write, scan_item
335
+ try:
336
+ if os.path.isdir(output_path):
337
+ shutil.rmtree(output_path)
338
+ except Exception as ex:
339
+ logger.debug(f"Failed to cleanup temp files: {ex}")
323
340
 
324
341
 
325
342
  def return_bin_only(file_list, need_checksum_tlsh=True):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fosslight_binary
3
- Version: 5.1.32
3
+ Version: 5.1.34
4
4
  Summary: FOSSLight Binary Scanner
5
5
  Author: LG Electronics
6
6
  License-Expression: Apache-2.0