fosslight-binary 5.1.29__tar.gz → 5.1.30__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/PKG-INFO +1 -2
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/pyproject.toml +1 -2
- fosslight_binary-5.1.30/src/fosslight_binary/_binary_dao.py +279 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/_help.py +4 -3
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/binary_analysis.py +4 -2
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/cli.py +13 -5
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/PKG-INFO +1 -2
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/requires.txt +0 -1
- fosslight_binary-5.1.29/src/fosslight_binary/_binary_dao.py +0 -186
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/LICENSE +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/LICENSES/Apache-2.0.txt +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/LICENSES/LicenseRef-3rd-party-licenses.txt +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/MANIFEST.in +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/README.md +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/setup.cfg +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/__init__.py +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/_binary.py +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/_jar_analysis.py +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/_simple_mode.py +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/SOURCES.txt +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/dependency_links.txt +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/entry_points.txt +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/top_level.txt +0 -0
- {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/tests/test_fosslight_binary.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fosslight_binary
|
|
3
|
-
Version: 5.1.
|
|
3
|
+
Version: 5.1.30
|
|
4
4
|
Summary: FOSSLight Binary Scanner
|
|
5
5
|
Author: LG Electronics
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -24,7 +24,6 @@ Requires-Dist: defusedxml
|
|
|
24
24
|
Requires-Dist: numpy
|
|
25
25
|
Requires-Dist: pandas
|
|
26
26
|
Requires-Dist: parmap
|
|
27
|
-
Requires-Dist: psycopg2-binary
|
|
28
27
|
Requires-Dist: python-dateutil
|
|
29
28
|
Requires-Dist: py-tlsh
|
|
30
29
|
Requires-Dist: pytz
|
|
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "fosslight_binary"
|
|
10
|
-
version = "5.1.
|
|
10
|
+
version = "5.1.30"
|
|
11
11
|
description = "FOSSLight Binary Scanner"
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
license = "Apache-2.0"
|
|
@@ -31,7 +31,6 @@ dependencies = [
|
|
|
31
31
|
"numpy",
|
|
32
32
|
"pandas",
|
|
33
33
|
"parmap",
|
|
34
|
-
"psycopg2-binary",
|
|
35
34
|
"python-dateutil",
|
|
36
35
|
"py-tlsh",
|
|
37
36
|
"pytz",
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
# Copyright (c) 2020 LG Electronics Inc.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
"""Binary DB lookup via ldb_service POST /binary/match."""
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
import os
|
|
10
|
+
import urllib.error
|
|
11
|
+
import urllib.request
|
|
12
|
+
from typing import Dict, List, Optional, Tuple
|
|
13
|
+
from urllib.parse import urlparse
|
|
14
|
+
|
|
15
|
+
from fosslight_binary._binary import TLSH_CHECKSUM_NULL
|
|
16
|
+
from fosslight_util.oss_item import OssItem
|
|
17
|
+
import fosslight_util.constant as constant
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(constant.LOGGER_NAME)
|
|
20
|
+
|
|
21
|
+
DEFAULT_KB_URL = "http://fosslight-kb.lge.com/"
|
|
22
|
+
_BINARY_MATCH_PATH = "/binary/match"
|
|
23
|
+
_HTTP_TIMEOUT_SEC = 120
|
|
24
|
+
_PROBE_TIMEOUT_SEC = 10
|
|
25
|
+
_DEFAULT_CHUNK_SIZE = 3000
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _get_chunk_size() -> int:
|
|
29
|
+
raw = os.environ.get("BINARY_MATCH_CHUNK_SIZE", str(_DEFAULT_CHUNK_SIZE))
|
|
30
|
+
try:
|
|
31
|
+
chunk_size = int(raw)
|
|
32
|
+
except ValueError:
|
|
33
|
+
logger.warning(f"Invalid BINARY_MATCH_CHUNK_SIZE={raw!r}; using {_DEFAULT_CHUNK_SIZE}")
|
|
34
|
+
return _DEFAULT_CHUNK_SIZE
|
|
35
|
+
if chunk_size <= 0:
|
|
36
|
+
logger.warning(
|
|
37
|
+
f"BINARY_MATCH_CHUNK_SIZE must be > 0 (got {chunk_size}); using {_DEFAULT_CHUNK_SIZE}"
|
|
38
|
+
)
|
|
39
|
+
return _DEFAULT_CHUNK_SIZE
|
|
40
|
+
return chunk_size
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
_CHUNK_SIZE = _get_chunk_size()
|
|
44
|
+
|
|
45
|
+
MatchKey = Tuple[str, str]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def resolve_kb_config(kb_url: str = "", kb_token: str = "") -> Tuple[str, str]:
|
|
49
|
+
url = (kb_url or os.environ.get("KB_URL", DEFAULT_KB_URL)).strip() or DEFAULT_KB_URL
|
|
50
|
+
token = (kb_token or "").strip() or (os.environ.get("KB_TOKEN") or "").strip()
|
|
51
|
+
return f"{url.rstrip('/')}/", token
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _is_valid_kb_url_format(kb_url: str) -> bool:
|
|
55
|
+
parsed = urlparse(kb_url)
|
|
56
|
+
return parsed.scheme in ("http", "https") and bool(parsed.netloc)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def check_binary_match_endpoint(kb_url: str, kb_token: str = "") -> Tuple[bool, str]:
|
|
60
|
+
"""
|
|
61
|
+
Return (available, cover_comment).
|
|
62
|
+
|
|
63
|
+
cover_comment:
|
|
64
|
+
- ``KB({kb_url}) Unreachable`` when the host cannot be reached
|
|
65
|
+
- ``KB({kb_url}) Skipped`` when /binary/match is missing (HTTP 404)
|
|
66
|
+
- empty for other skip cases (invalid URL)
|
|
67
|
+
"""
|
|
68
|
+
if not _is_valid_kb_url_format(kb_url):
|
|
69
|
+
logger.warning(f"Invalid KB URL format: {kb_url}")
|
|
70
|
+
return False, ""
|
|
71
|
+
|
|
72
|
+
endpoint = f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}"
|
|
73
|
+
data = json.dumps({"items": []}).encode("utf-8")
|
|
74
|
+
request = urllib.request.Request(endpoint, data=data, method="POST")
|
|
75
|
+
request.add_header("Accept", "application/json")
|
|
76
|
+
request.add_header("Content-Type", "application/json")
|
|
77
|
+
if kb_token:
|
|
78
|
+
request.add_header("Authorization", f"Bearer {kb_token}")
|
|
79
|
+
|
|
80
|
+
try:
|
|
81
|
+
with urllib.request.urlopen(request, timeout=_PROBE_TIMEOUT_SEC) as response:
|
|
82
|
+
response.read()
|
|
83
|
+
return True, ""
|
|
84
|
+
except urllib.error.HTTPError as ex:
|
|
85
|
+
if ex.code == 404:
|
|
86
|
+
logger.warning(
|
|
87
|
+
f"KB binary match endpoint not found (HTTP 404): {endpoint}. "
|
|
88
|
+
"Skipping binary match API."
|
|
89
|
+
)
|
|
90
|
+
return False, f"KB({kb_url}) Skipped"
|
|
91
|
+
logger.debug(
|
|
92
|
+
f"KB binary match endpoint responded with HTTP {ex.code}; treated as available"
|
|
93
|
+
)
|
|
94
|
+
return True, ""
|
|
95
|
+
except urllib.error.URLError as ex:
|
|
96
|
+
logger.warning(f"KB binary match endpoint unreachable: {ex}. Skipping binary match API.")
|
|
97
|
+
return False, f"KB({kb_url}) Unreachable"
|
|
98
|
+
except Exception as ex:
|
|
99
|
+
logger.warning(
|
|
100
|
+
f"Failed to check KB binary match endpoint: {ex}. Skipping binary match API."
|
|
101
|
+
)
|
|
102
|
+
return False, f"KB({kb_url}) Unreachable"
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _is_unknown_checksum(checksum: str) -> bool:
|
|
106
|
+
"""True when checksum was not computed (empty or TLSH_CHECKSUM_NULL)."""
|
|
107
|
+
return (not checksum) or checksum == TLSH_CHECKSUM_NULL
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _is_unknown_tlsh(tlsh: str) -> bool:
|
|
111
|
+
"""True when tlsh was not computed (empty or TLSH_CHECKSUM_NULL)."""
|
|
112
|
+
return (not tlsh) or tlsh == TLSH_CHECKSUM_NULL
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _match_key(filename: str, checksum: str, index: int) -> MatchKey:
|
|
116
|
+
"""Dedupe key. Unknown checksums stay unique per list index (no filename-only merge)."""
|
|
117
|
+
if _is_unknown_checksum(checksum):
|
|
118
|
+
return filename, f"__unknown_{index}"
|
|
119
|
+
return filename, checksum
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _build_deduped_payload(
|
|
123
|
+
bin_info_list,
|
|
124
|
+
tlsh_null: str,
|
|
125
|
+
) -> Tuple[List[dict], Dict[MatchKey, str]]:
|
|
126
|
+
"""Deduplicate by filename+checksum; return API payload and key→api_id map.
|
|
127
|
+
|
|
128
|
+
Items with empty/\"0\" checksum are not deduped — each keeps its own API entry
|
|
129
|
+
(and its own tlsh) so distinct files that share a basename are not conflated.
|
|
130
|
+
Items with both checksum and tlsh unknown are omitted (nothing to match).
|
|
131
|
+
"""
|
|
132
|
+
key_to_id: Dict[MatchKey, str] = {}
|
|
133
|
+
items_payload: List[dict] = []
|
|
134
|
+
|
|
135
|
+
for index, item in enumerate(bin_info_list):
|
|
136
|
+
if item.exclude:
|
|
137
|
+
continue
|
|
138
|
+
filename = item.binary_name_without_path
|
|
139
|
+
checksum = item.checksum or ""
|
|
140
|
+
tlsh = item.tlsh or tlsh_null
|
|
141
|
+
if _is_unknown_checksum(checksum) and _is_unknown_tlsh(tlsh):
|
|
142
|
+
continue
|
|
143
|
+
key = _match_key(filename, checksum, index)
|
|
144
|
+
if not _is_unknown_checksum(checksum) and key in key_to_id:
|
|
145
|
+
continue
|
|
146
|
+
api_id = str(len(items_payload))
|
|
147
|
+
key_to_id[key] = api_id
|
|
148
|
+
items_payload.append({
|
|
149
|
+
"id": api_id,
|
|
150
|
+
"filename": filename,
|
|
151
|
+
"checksum": checksum,
|
|
152
|
+
"tlsh": tlsh,
|
|
153
|
+
})
|
|
154
|
+
|
|
155
|
+
return items_payload, key_to_id
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _apply_match_result_to_item(item, result: dict) -> bool:
|
|
159
|
+
"""Apply a /binary/match result to one binary item. Returns True if matched."""
|
|
160
|
+
if not result or not result.get("matched"):
|
|
161
|
+
return False
|
|
162
|
+
oss_rows = result.get("oss_items") or []
|
|
163
|
+
if not oss_rows:
|
|
164
|
+
return False
|
|
165
|
+
|
|
166
|
+
if not item.found_in_jar_analysis and item.oss_items:
|
|
167
|
+
item.oss_items = []
|
|
168
|
+
|
|
169
|
+
bin_oss_items = []
|
|
170
|
+
for row in oss_rows:
|
|
171
|
+
if item.found_in_jar_analysis:
|
|
172
|
+
break
|
|
173
|
+
oss_from_db = OssItem(
|
|
174
|
+
row.get("oss_name") or "",
|
|
175
|
+
row.get("oss_version") or "",
|
|
176
|
+
row.get("license") or "",
|
|
177
|
+
)
|
|
178
|
+
if bin_oss_items:
|
|
179
|
+
if not any(
|
|
180
|
+
oss_item.name == oss_from_db.name
|
|
181
|
+
and oss_item.version == oss_from_db.version
|
|
182
|
+
and oss_item.license == oss_from_db.license
|
|
183
|
+
for oss_item in bin_oss_items
|
|
184
|
+
):
|
|
185
|
+
bin_oss_items.append(oss_from_db)
|
|
186
|
+
else:
|
|
187
|
+
bin_oss_items.append(oss_from_db)
|
|
188
|
+
|
|
189
|
+
if bin_oss_items:
|
|
190
|
+
item.set_oss_items(bin_oss_items)
|
|
191
|
+
item.comment = "Binary DB result"
|
|
192
|
+
item.found_in_bin_db = True
|
|
193
|
+
return True
|
|
194
|
+
return False
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""):
|
|
198
|
+
"""
|
|
199
|
+
Call ldb_service /binary/match and attach OSS info to binary items.
|
|
200
|
+
Deduplicates by filename+checksum before the API call and maps results back.
|
|
201
|
+
Returns (bin_info_list, matched_count, kb_cover_comment).
|
|
202
|
+
"""
|
|
203
|
+
_cnt_auto_identified = 0
|
|
204
|
+
if not bin_info_list:
|
|
205
|
+
return bin_info_list, _cnt_auto_identified, ""
|
|
206
|
+
|
|
207
|
+
base_url, token = resolve_kb_config(kb_url, kb_token)
|
|
208
|
+
available, kb_cover_comment = check_binary_match_endpoint(base_url, token)
|
|
209
|
+
if not available:
|
|
210
|
+
return bin_info_list, _cnt_auto_identified, kb_cover_comment
|
|
211
|
+
|
|
212
|
+
items_payload, key_to_id = _build_deduped_payload(bin_info_list, TLSH_CHECKSUM_NULL)
|
|
213
|
+
if not items_payload:
|
|
214
|
+
return bin_info_list, _cnt_auto_identified, ""
|
|
215
|
+
|
|
216
|
+
results_by_id = {}
|
|
217
|
+
try:
|
|
218
|
+
for chunk_start in range(0, len(items_payload), _CHUNK_SIZE):
|
|
219
|
+
chunk = items_payload[chunk_start: chunk_start + _CHUNK_SIZE]
|
|
220
|
+
response = _post_binary_match(base_url, token, chunk)
|
|
221
|
+
if response is None:
|
|
222
|
+
logger.warning(
|
|
223
|
+
f"Binary match chunk failed "
|
|
224
|
+
f"({chunk_start}:{chunk_start + len(chunk)}); "
|
|
225
|
+
"keeping results so far and continuing with next chunks."
|
|
226
|
+
)
|
|
227
|
+
continue
|
|
228
|
+
for result in response.get("results", []):
|
|
229
|
+
results_by_id[str(result.get("id"))] = result
|
|
230
|
+
except Exception as ex:
|
|
231
|
+
logger.warning(f"Binary match API failed: {ex}")
|
|
232
|
+
|
|
233
|
+
for index, item in enumerate(bin_info_list):
|
|
234
|
+
if item.exclude:
|
|
235
|
+
continue
|
|
236
|
+
key = _match_key(item.binary_name_without_path, item.checksum or "", index)
|
|
237
|
+
api_id = key_to_id.get(key)
|
|
238
|
+
if api_id is None:
|
|
239
|
+
continue
|
|
240
|
+
result = results_by_id.get(api_id)
|
|
241
|
+
if _apply_match_result_to_item(item, result):
|
|
242
|
+
_cnt_auto_identified += 1
|
|
243
|
+
|
|
244
|
+
return bin_info_list, _cnt_auto_identified, ""
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _post_binary_match(kb_url: str, kb_token: str, items: list) -> Optional[dict]:
|
|
248
|
+
data = json.dumps({"items": items}).encode("utf-8")
|
|
249
|
+
request = urllib.request.Request(
|
|
250
|
+
f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}",
|
|
251
|
+
data=data,
|
|
252
|
+
method="POST",
|
|
253
|
+
)
|
|
254
|
+
request.add_header("Accept", "application/json")
|
|
255
|
+
request.add_header("Content-Type", "application/json")
|
|
256
|
+
if kb_token:
|
|
257
|
+
request.add_header("Authorization", f"Bearer {kb_token}")
|
|
258
|
+
|
|
259
|
+
try:
|
|
260
|
+
with urllib.request.urlopen(request, timeout=_HTTP_TIMEOUT_SEC) as response:
|
|
261
|
+
body = response.read().decode()
|
|
262
|
+
return json.loads(body) if body else {}
|
|
263
|
+
except urllib.error.HTTPError as ex:
|
|
264
|
+
body = ""
|
|
265
|
+
try:
|
|
266
|
+
body = ex.read().decode()
|
|
267
|
+
except Exception:
|
|
268
|
+
pass
|
|
269
|
+
if ex.code == 404:
|
|
270
|
+
logger.warning(
|
|
271
|
+
f"Binary match endpoint not found (HTTP 404): "
|
|
272
|
+
f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}"
|
|
273
|
+
)
|
|
274
|
+
else:
|
|
275
|
+
logger.warning(f"Binary match HTTP {ex.code}: {body or ex.reason}")
|
|
276
|
+
return None
|
|
277
|
+
except urllib.error.URLError as ex:
|
|
278
|
+
logger.debug(f"Binary match unreachable: {ex}")
|
|
279
|
+
return None
|
|
@@ -32,7 +32,8 @@ _HELP_MESSAGE_BINARY = f"""
|
|
|
32
32
|
|
|
33
33
|
🔍 Scanner-Specific Options
|
|
34
34
|
────────────────────────────────────────────────────────────────────
|
|
35
|
-
|
|
35
|
+
--kb_url <url> KB API URL (priority: parameter > KB_URL env > default)
|
|
36
|
+
--kb_token <token> KB bearer token (priority: parameter > KB_TOKEN env)
|
|
36
37
|
--notice Print the open source license notice text
|
|
37
38
|
--no_correction Skip OSS information correction with sbom-info.yaml
|
|
38
39
|
--correct_fpath <path> Path to custom sbom-info.yaml file
|
|
@@ -48,8 +49,8 @@ _HELP_MESSAGE_BINARY = f"""
|
|
|
48
49
|
# Generate output in specific format
|
|
49
50
|
fosslight_binary -f excel -o results/
|
|
50
51
|
|
|
51
|
-
#
|
|
52
|
-
fosslight_binary
|
|
52
|
+
# Binary DB lookup via ldb_service
|
|
53
|
+
fosslight_binary --kb_url http://fosslight-kb.lge.com/ --kb_token <token>
|
|
53
54
|
"""
|
|
54
55
|
|
|
55
56
|
|
|
@@ -169,7 +169,7 @@ def get_file_list(path_to_find, excluded_files):
|
|
|
169
169
|
return file_cnt, bin_list, found_jar
|
|
170
170
|
|
|
171
171
|
|
|
172
|
-
def find_binaries(path_to_find_bin, output_dir, formats,
|
|
172
|
+
def find_binaries(path_to_find_bin, output_dir, formats, kb_url="", kb_token="", simple_mode=False,
|
|
173
173
|
correct_mode=True, correct_filepath="", path_to_exclude=[],
|
|
174
174
|
all_exclude_mode=()):
|
|
175
175
|
global start_time, finish_time, _root_path, _result_log
|
|
@@ -254,7 +254,7 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F
|
|
|
254
254
|
else:
|
|
255
255
|
logger.warning("Could not find OSS information for some jar files.")
|
|
256
256
|
|
|
257
|
-
return_list, db_loaded_cnt = get_oss_info_from_db(return_list,
|
|
257
|
+
return_list, db_loaded_cnt, kb_cover_msg = get_oss_info_from_db(return_list, kb_url, kb_token)
|
|
258
258
|
return_list = sorted(return_list, key=lambda row: (row.bin_name_with_path))
|
|
259
259
|
scan_item.append_file_items(return_list, PKG_NAME)
|
|
260
260
|
if correct_mode:
|
|
@@ -267,6 +267,8 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F
|
|
|
267
267
|
|
|
268
268
|
finish_time = current_timestamp_utc()
|
|
269
269
|
scan_item.set_cover_comment(f"Detected binaries: {len(return_list)} (Scanned Files : {cnt_file_except_skipped})")
|
|
270
|
+
if kb_cover_msg:
|
|
271
|
+
scan_item.set_cover_comment(kb_cover_msg)
|
|
270
272
|
scan_item.set_cover_finish_time(finish_time)
|
|
271
273
|
|
|
272
274
|
for combined_path_and_file, output_extension, output_format in zip(result_reports, output_extensions, formats):
|
|
@@ -36,7 +36,8 @@ def main():
|
|
|
36
36
|
path_to_exclude = []
|
|
37
37
|
output_dir = ""
|
|
38
38
|
format = []
|
|
39
|
-
|
|
39
|
+
kb_url = ""
|
|
40
|
+
kb_token = ""
|
|
40
41
|
simple_mode = False
|
|
41
42
|
correct_mode = True
|
|
42
43
|
|
|
@@ -46,9 +47,10 @@ def main():
|
|
|
46
47
|
parser.add_argument('-s', '--simple', action='store_true', required=False, help=argparse.SUPPRESS)
|
|
47
48
|
parser.add_argument('-p', '--path', type=str, required=False)
|
|
48
49
|
parser.add_argument('-o', '--output', type=str, required=False)
|
|
49
|
-
parser.add_argument('-d', '--dburl', type=str, default='', required=False)
|
|
50
50
|
parser.add_argument('-f', '--formats', type=str, required=False, nargs="*")
|
|
51
51
|
parser.add_argument('-e', '--exclude', nargs="*", required=False, default=[])
|
|
52
|
+
parser.add_argument('--kb_url', type=str, required=False, default="")
|
|
53
|
+
parser.add_argument('--kb_token', type=str, required=False, default="")
|
|
52
54
|
parser.add_argument('--notice', action='store_true', required=False)
|
|
53
55
|
parser.add_argument('--no_correction', action='store_true', required=False)
|
|
54
56
|
parser.add_argument('--correct_fpath', nargs=1, type=str, required=False)
|
|
@@ -81,8 +83,11 @@ def main():
|
|
|
81
83
|
if args.output: # -o option
|
|
82
84
|
output_dir = args.output
|
|
83
85
|
|
|
84
|
-
if args.
|
|
85
|
-
|
|
86
|
+
if args.kb_url:
|
|
87
|
+
kb_url = args.kb_url
|
|
88
|
+
|
|
89
|
+
if args.kb_token:
|
|
90
|
+
kb_token = args.kb_token
|
|
86
91
|
|
|
87
92
|
if args.formats: # -f option
|
|
88
93
|
format = list(args.formats)
|
|
@@ -113,7 +118,10 @@ def main():
|
|
|
113
118
|
timer.setDaemon(True)
|
|
114
119
|
timer.start()
|
|
115
120
|
|
|
116
|
-
find_binaries(
|
|
121
|
+
find_binaries(
|
|
122
|
+
path_to_find_bin, output_dir, format, kb_url, kb_token,
|
|
123
|
+
simple_mode, correct_mode, correct_filepath, path_to_exclude,
|
|
124
|
+
)
|
|
117
125
|
|
|
118
126
|
|
|
119
127
|
if __name__ == '__main__':
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fosslight_binary
|
|
3
|
-
Version: 5.1.
|
|
3
|
+
Version: 5.1.30
|
|
4
4
|
Summary: FOSSLight Binary Scanner
|
|
5
5
|
Author: LG Electronics
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -24,7 +24,6 @@ Requires-Dist: defusedxml
|
|
|
24
24
|
Requires-Dist: numpy
|
|
25
25
|
Requires-Dist: pandas
|
|
26
26
|
Requires-Dist: parmap
|
|
27
|
-
Requires-Dist: psycopg2-binary
|
|
28
27
|
Requires-Dist: python-dateutil
|
|
29
28
|
Requires-Dist: py-tlsh
|
|
30
29
|
Requires-Dist: pytz
|
|
@@ -1,186 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
|
-
# Copyright (c) 2020 LG Electronics Inc.
|
|
4
|
-
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
-
|
|
6
|
-
import tlsh
|
|
7
|
-
import logging
|
|
8
|
-
import psycopg2
|
|
9
|
-
import pandas as pd
|
|
10
|
-
import socket
|
|
11
|
-
from urllib.parse import urlparse
|
|
12
|
-
from fosslight_binary._binary import TLSH_CHECKSUM_NULL
|
|
13
|
-
from fosslight_util.oss_item import OssItem
|
|
14
|
-
import fosslight_util.constant as constant
|
|
15
|
-
|
|
16
|
-
columns = ['filename', 'pathname', 'checksum', 'tlshchecksum', 'ossname',
|
|
17
|
-
'ossversion', 'license', 'platformname',
|
|
18
|
-
'platformversion']
|
|
19
|
-
conn = ""
|
|
20
|
-
cur = ""
|
|
21
|
-
logger = logging.getLogger(constant.LOGGER_NAME)
|
|
22
|
-
DB_URL_DEFAULT = "postgresql://bin_analysis_script_user:script_123@bat.lge.com:5432/bat"
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
def get_oss_info_from_db(bin_info_list, dburl=""):
|
|
26
|
-
_cnt_auto_identified = 0
|
|
27
|
-
conn_str, dbc = get_connection_string(dburl)
|
|
28
|
-
# DB URL에서 host 추출
|
|
29
|
-
try:
|
|
30
|
-
db_host = dbc.hostname
|
|
31
|
-
except Exception as ex:
|
|
32
|
-
logger.warning(f"Failed to parse DB URL for host: {ex}")
|
|
33
|
-
db_host = None
|
|
34
|
-
|
|
35
|
-
is_internal = False
|
|
36
|
-
if db_host:
|
|
37
|
-
try:
|
|
38
|
-
# DNS lookup 시도
|
|
39
|
-
socket.gethostbyname(db_host)
|
|
40
|
-
is_internal = True
|
|
41
|
-
except Exception:
|
|
42
|
-
is_internal = False
|
|
43
|
-
|
|
44
|
-
if is_internal:
|
|
45
|
-
connect_to_lge_bin_db(conn_str)
|
|
46
|
-
if conn != "" and cur != "":
|
|
47
|
-
for item in bin_info_list:
|
|
48
|
-
bin_oss_items = []
|
|
49
|
-
tlsh_value = item.tlsh
|
|
50
|
-
checksum_value = item.checksum
|
|
51
|
-
bin_file_name = item.binary_name_without_path
|
|
52
|
-
|
|
53
|
-
df_result = get_oss_info_by_tlsh_and_filename(
|
|
54
|
-
bin_file_name, checksum_value, tlsh_value)
|
|
55
|
-
if df_result is not None and len(df_result) > 0:
|
|
56
|
-
_cnt_auto_identified += 1
|
|
57
|
-
# Initialize the saved contents at .jar analyzing only once
|
|
58
|
-
if not item.found_in_jar_analysis and item.oss_items:
|
|
59
|
-
item.oss_items = []
|
|
60
|
-
|
|
61
|
-
for idx, row in df_result.iterrows():
|
|
62
|
-
if not item.found_in_jar_analysis:
|
|
63
|
-
oss_from_db = OssItem(row['ossname'], row['ossversion'], row['license'])
|
|
64
|
-
|
|
65
|
-
if bin_oss_items:
|
|
66
|
-
if not any(oss_item.name == oss_from_db.name
|
|
67
|
-
and oss_item.version == oss_from_db.version
|
|
68
|
-
and oss_item.license == oss_from_db.license
|
|
69
|
-
for oss_item in bin_oss_items):
|
|
70
|
-
bin_oss_items.append(oss_from_db)
|
|
71
|
-
else:
|
|
72
|
-
bin_oss_items.append(oss_from_db)
|
|
73
|
-
|
|
74
|
-
if bin_oss_items:
|
|
75
|
-
item.set_oss_items(bin_oss_items)
|
|
76
|
-
item.comment = "Binary DB result"
|
|
77
|
-
item.found_in_binary = True
|
|
78
|
-
else:
|
|
79
|
-
logger.warning(f"Internal network detected, but DB connection to '{db_host}' failed. Skipping DB query.")
|
|
80
|
-
disconnect_lge_bin_db()
|
|
81
|
-
else:
|
|
82
|
-
logger.debug(f"Binary DB host '{db_host}' is not reachable. Skipping DB query.")
|
|
83
|
-
return bin_info_list, _cnt_auto_identified
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
def get_connection_string(dburl):
|
|
87
|
-
# dburl format : 'postgresql://username:password@host:port/database_name'
|
|
88
|
-
connection_string = ""
|
|
89
|
-
user_dburl = True
|
|
90
|
-
dbc = ""
|
|
91
|
-
if dburl == "" or dburl is None:
|
|
92
|
-
user_dburl = False
|
|
93
|
-
dburl = DB_URL_DEFAULT
|
|
94
|
-
try:
|
|
95
|
-
if user_dburl:
|
|
96
|
-
logger.debug("DB URL:" + dburl)
|
|
97
|
-
dbc = urlparse(dburl)
|
|
98
|
-
connection_string = "dbname={dbname} user={user} host={host} password={password} port={port}" \
|
|
99
|
-
.format(dbname=dbc.path.lstrip('/'),
|
|
100
|
-
user=dbc.username,
|
|
101
|
-
host=dbc.hostname,
|
|
102
|
-
password=dbc.password,
|
|
103
|
-
port=dbc.port)
|
|
104
|
-
except Exception as ex:
|
|
105
|
-
if user_dburl:
|
|
106
|
-
logger.warning(f"(Minor) Failed to parsing db url : {ex}")
|
|
107
|
-
|
|
108
|
-
return connection_string, dbc
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
def get_oss_info_by_tlsh_and_filename(file_name, checksum_value, tlsh_value):
|
|
112
|
-
sql_statement = "SELECT filename,pathname,checksum,tlshchecksum,ossname,ossversion,\
|
|
113
|
-
license,platformname,platformversion FROM lgematching "
|
|
114
|
-
sql_statement_checksum = " WHERE filename=%s AND checksum=%s;" # Using parameterized query
|
|
115
|
-
sql_statement_filename = "SELECT DISTINCT ON (tlshchecksum) tlshchecksum FROM lgematching WHERE filename=%s;" # Using parameterized query
|
|
116
|
-
|
|
117
|
-
final_result_item = ""
|
|
118
|
-
|
|
119
|
-
df_result = get_list_by_using_query(
|
|
120
|
-
sql_statement + sql_statement_checksum, columns, (file_name, checksum_value))
|
|
121
|
-
# Found a file with the same checksum.
|
|
122
|
-
if df_result is not None and len(df_result) > 0:
|
|
123
|
-
final_result_item = df_result
|
|
124
|
-
else:
|
|
125
|
-
# Match tlsh and fileName
|
|
126
|
-
df_result = get_list_by_using_query(
|
|
127
|
-
sql_statement_filename, ['tlshchecksum'], (file_name,))
|
|
128
|
-
if df_result is None or len(df_result) <= 0:
|
|
129
|
-
final_result_item = ""
|
|
130
|
-
elif tlsh_value == TLSH_CHECKSUM_NULL: # Couldn't get the tlsh of a file.
|
|
131
|
-
final_result_item = ""
|
|
132
|
-
else:
|
|
133
|
-
matched_tlsh = ""
|
|
134
|
-
matched_tlsh_diff = -1
|
|
135
|
-
for row in df_result.tlshchecksum:
|
|
136
|
-
try:
|
|
137
|
-
if row != TLSH_CHECKSUM_NULL:
|
|
138
|
-
tlsh_diff = tlsh.diff(row, tlsh_value)
|
|
139
|
-
if tlsh_diff <= 120: # MATCHED
|
|
140
|
-
if (matched_tlsh_diff < 0) or (tlsh_diff < matched_tlsh_diff):
|
|
141
|
-
matched_tlsh_diff = tlsh_diff
|
|
142
|
-
matched_tlsh = row
|
|
143
|
-
except Exception as ex:
|
|
144
|
-
logger.warning(f"* (Minor) Error_tlsh_comparison: {ex}")
|
|
145
|
-
if matched_tlsh != "":
|
|
146
|
-
final_result_item = get_list_by_using_query(
|
|
147
|
-
sql_statement + " WHERE filename=%s AND tlshchecksum=%s;", columns, (file_name, matched_tlsh))
|
|
148
|
-
|
|
149
|
-
return final_result_item
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
def get_list_by_using_query(sql_query, columns, params=None):
|
|
153
|
-
result_rows = "" # DataFrame
|
|
154
|
-
try:
|
|
155
|
-
if params:
|
|
156
|
-
cur.execute(sql_query, params)
|
|
157
|
-
else:
|
|
158
|
-
cur.execute(sql_query)
|
|
159
|
-
rows = cur.fetchall()
|
|
160
|
-
|
|
161
|
-
if rows is not None and len(rows) > 0:
|
|
162
|
-
result_rows = pd.DataFrame(data=rows, columns=columns)
|
|
163
|
-
except Exception as ex:
|
|
164
|
-
logger.error(f"Database query error: {ex}")
|
|
165
|
-
result_rows = ""
|
|
166
|
-
return result_rows
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
def disconnect_lge_bin_db():
|
|
170
|
-
try:
|
|
171
|
-
cur.close()
|
|
172
|
-
conn.close()
|
|
173
|
-
except:
|
|
174
|
-
pass
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
def connect_to_lge_bin_db(connection_string):
|
|
178
|
-
global conn, cur
|
|
179
|
-
|
|
180
|
-
try:
|
|
181
|
-
conn = psycopg2.connect(connection_string)
|
|
182
|
-
cur = conn.cursor()
|
|
183
|
-
except Exception as ex:
|
|
184
|
-
logger.debug(f"(Minor) Can't connect to Binary DB. : {ex}")
|
|
185
|
-
conn = ""
|
|
186
|
-
cur = ""
|
|
File without changes
|
|
File without changes
|
{fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/LICENSES/LicenseRef-3rd-party-licenses.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
{fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/entry_points.txt
RENAMED
|
File without changes
|
{fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|