fosslight-binary 5.1.29__tar.gz → 5.1.30__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/PKG-INFO +1 -2
  2. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/pyproject.toml +1 -2
  3. fosslight_binary-5.1.30/src/fosslight_binary/_binary_dao.py +279 -0
  4. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/_help.py +4 -3
  5. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/binary_analysis.py +4 -2
  6. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/cli.py +13 -5
  7. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/PKG-INFO +1 -2
  8. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/requires.txt +0 -1
  9. fosslight_binary-5.1.29/src/fosslight_binary/_binary_dao.py +0 -186
  10. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/LICENSE +0 -0
  11. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/LICENSES/Apache-2.0.txt +0 -0
  12. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/LICENSES/LicenseRef-3rd-party-licenses.txt +0 -0
  13. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/MANIFEST.in +0 -0
  14. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/README.md +0 -0
  15. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/setup.cfg +0 -0
  16. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/__init__.py +0 -0
  17. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/_binary.py +0 -0
  18. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/_jar_analysis.py +0 -0
  19. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary/_simple_mode.py +0 -0
  20. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/SOURCES.txt +0 -0
  21. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/dependency_links.txt +0 -0
  22. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/entry_points.txt +0 -0
  23. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/src/fosslight_binary.egg-info/top_level.txt +0 -0
  24. {fosslight_binary-5.1.29 → fosslight_binary-5.1.30}/tests/test_fosslight_binary.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fosslight_binary
3
- Version: 5.1.29
3
+ Version: 5.1.30
4
4
  Summary: FOSSLight Binary Scanner
5
5
  Author: LG Electronics
6
6
  License-Expression: Apache-2.0
@@ -24,7 +24,6 @@ Requires-Dist: defusedxml
24
24
  Requires-Dist: numpy
25
25
  Requires-Dist: pandas
26
26
  Requires-Dist: parmap
27
- Requires-Dist: psycopg2-binary
28
27
  Requires-Dist: python-dateutil
29
28
  Requires-Dist: py-tlsh
30
29
  Requires-Dist: pytz
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
7
7
 
8
8
  [project]
9
9
  name = "fosslight_binary"
10
- version = "5.1.29"
10
+ version = "5.1.30"
11
11
  description = "FOSSLight Binary Scanner"
12
12
  readme = "README.md"
13
13
  license = "Apache-2.0"
@@ -31,7 +31,6 @@ dependencies = [
31
31
  "numpy",
32
32
  "pandas",
33
33
  "parmap",
34
- "psycopg2-binary",
35
34
  "python-dateutil",
36
35
  "py-tlsh",
37
36
  "pytz",
@@ -0,0 +1,279 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ # Copyright (c) 2020 LG Electronics Inc.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+ """Binary DB lookup via ldb_service POST /binary/match."""
6
+
7
+ import json
8
+ import logging
9
+ import os
10
+ import urllib.error
11
+ import urllib.request
12
+ from typing import Dict, List, Optional, Tuple
13
+ from urllib.parse import urlparse
14
+
15
+ from fosslight_binary._binary import TLSH_CHECKSUM_NULL
16
+ from fosslight_util.oss_item import OssItem
17
+ import fosslight_util.constant as constant
18
+
19
+ logger = logging.getLogger(constant.LOGGER_NAME)
20
+
21
+ DEFAULT_KB_URL = "http://fosslight-kb.lge.com/"
22
+ _BINARY_MATCH_PATH = "/binary/match"
23
+ _HTTP_TIMEOUT_SEC = 120
24
+ _PROBE_TIMEOUT_SEC = 10
25
+ _DEFAULT_CHUNK_SIZE = 3000
26
+
27
+
28
+ def _get_chunk_size() -> int:
29
+ raw = os.environ.get("BINARY_MATCH_CHUNK_SIZE", str(_DEFAULT_CHUNK_SIZE))
30
+ try:
31
+ chunk_size = int(raw)
32
+ except ValueError:
33
+ logger.warning(f"Invalid BINARY_MATCH_CHUNK_SIZE={raw!r}; using {_DEFAULT_CHUNK_SIZE}")
34
+ return _DEFAULT_CHUNK_SIZE
35
+ if chunk_size <= 0:
36
+ logger.warning(
37
+ f"BINARY_MATCH_CHUNK_SIZE must be > 0 (got {chunk_size}); using {_DEFAULT_CHUNK_SIZE}"
38
+ )
39
+ return _DEFAULT_CHUNK_SIZE
40
+ return chunk_size
41
+
42
+
43
+ _CHUNK_SIZE = _get_chunk_size()
44
+
45
+ MatchKey = Tuple[str, str]
46
+
47
+
48
+ def resolve_kb_config(kb_url: str = "", kb_token: str = "") -> Tuple[str, str]:
49
+ url = (kb_url or os.environ.get("KB_URL", DEFAULT_KB_URL)).strip() or DEFAULT_KB_URL
50
+ token = (kb_token or "").strip() or (os.environ.get("KB_TOKEN") or "").strip()
51
+ return f"{url.rstrip('/')}/", token
52
+
53
+
54
+ def _is_valid_kb_url_format(kb_url: str) -> bool:
55
+ parsed = urlparse(kb_url)
56
+ return parsed.scheme in ("http", "https") and bool(parsed.netloc)
57
+
58
+
59
+ def check_binary_match_endpoint(kb_url: str, kb_token: str = "") -> Tuple[bool, str]:
60
+ """
61
+ Return (available, cover_comment).
62
+
63
+ cover_comment:
64
+ - ``KB({kb_url}) Unreachable`` when the host cannot be reached
65
+ - ``KB({kb_url}) Skipped`` when /binary/match is missing (HTTP 404)
66
+ - empty for other skip cases (invalid URL)
67
+ """
68
+ if not _is_valid_kb_url_format(kb_url):
69
+ logger.warning(f"Invalid KB URL format: {kb_url}")
70
+ return False, ""
71
+
72
+ endpoint = f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}"
73
+ data = json.dumps({"items": []}).encode("utf-8")
74
+ request = urllib.request.Request(endpoint, data=data, method="POST")
75
+ request.add_header("Accept", "application/json")
76
+ request.add_header("Content-Type", "application/json")
77
+ if kb_token:
78
+ request.add_header("Authorization", f"Bearer {kb_token}")
79
+
80
+ try:
81
+ with urllib.request.urlopen(request, timeout=_PROBE_TIMEOUT_SEC) as response:
82
+ response.read()
83
+ return True, ""
84
+ except urllib.error.HTTPError as ex:
85
+ if ex.code == 404:
86
+ logger.warning(
87
+ f"KB binary match endpoint not found (HTTP 404): {endpoint}. "
88
+ "Skipping binary match API."
89
+ )
90
+ return False, f"KB({kb_url}) Skipped"
91
+ logger.debug(
92
+ f"KB binary match endpoint responded with HTTP {ex.code}; treated as available"
93
+ )
94
+ return True, ""
95
+ except urllib.error.URLError as ex:
96
+ logger.warning(f"KB binary match endpoint unreachable: {ex}. Skipping binary match API.")
97
+ return False, f"KB({kb_url}) Unreachable"
98
+ except Exception as ex:
99
+ logger.warning(
100
+ f"Failed to check KB binary match endpoint: {ex}. Skipping binary match API."
101
+ )
102
+ return False, f"KB({kb_url}) Unreachable"
103
+
104
+
105
+ def _is_unknown_checksum(checksum: str) -> bool:
106
+ """True when checksum was not computed (empty or TLSH_CHECKSUM_NULL)."""
107
+ return (not checksum) or checksum == TLSH_CHECKSUM_NULL
108
+
109
+
110
+ def _is_unknown_tlsh(tlsh: str) -> bool:
111
+ """True when tlsh was not computed (empty or TLSH_CHECKSUM_NULL)."""
112
+ return (not tlsh) or tlsh == TLSH_CHECKSUM_NULL
113
+
114
+
115
+ def _match_key(filename: str, checksum: str, index: int) -> MatchKey:
116
+ """Dedupe key. Unknown checksums stay unique per list index (no filename-only merge)."""
117
+ if _is_unknown_checksum(checksum):
118
+ return filename, f"__unknown_{index}"
119
+ return filename, checksum
120
+
121
+
122
+ def _build_deduped_payload(
123
+ bin_info_list,
124
+ tlsh_null: str,
125
+ ) -> Tuple[List[dict], Dict[MatchKey, str]]:
126
+ """Deduplicate by filename+checksum; return API payload and key→api_id map.
127
+
128
+ Items with empty/\"0\" checksum are not deduped — each keeps its own API entry
129
+ (and its own tlsh) so distinct files that share a basename are not conflated.
130
+ Items with both checksum and tlsh unknown are omitted (nothing to match).
131
+ """
132
+ key_to_id: Dict[MatchKey, str] = {}
133
+ items_payload: List[dict] = []
134
+
135
+ for index, item in enumerate(bin_info_list):
136
+ if item.exclude:
137
+ continue
138
+ filename = item.binary_name_without_path
139
+ checksum = item.checksum or ""
140
+ tlsh = item.tlsh or tlsh_null
141
+ if _is_unknown_checksum(checksum) and _is_unknown_tlsh(tlsh):
142
+ continue
143
+ key = _match_key(filename, checksum, index)
144
+ if not _is_unknown_checksum(checksum) and key in key_to_id:
145
+ continue
146
+ api_id = str(len(items_payload))
147
+ key_to_id[key] = api_id
148
+ items_payload.append({
149
+ "id": api_id,
150
+ "filename": filename,
151
+ "checksum": checksum,
152
+ "tlsh": tlsh,
153
+ })
154
+
155
+ return items_payload, key_to_id
156
+
157
+
158
+ def _apply_match_result_to_item(item, result: dict) -> bool:
159
+ """Apply a /binary/match result to one binary item. Returns True if matched."""
160
+ if not result or not result.get("matched"):
161
+ return False
162
+ oss_rows = result.get("oss_items") or []
163
+ if not oss_rows:
164
+ return False
165
+
166
+ if not item.found_in_jar_analysis and item.oss_items:
167
+ item.oss_items = []
168
+
169
+ bin_oss_items = []
170
+ for row in oss_rows:
171
+ if item.found_in_jar_analysis:
172
+ break
173
+ oss_from_db = OssItem(
174
+ row.get("oss_name") or "",
175
+ row.get("oss_version") or "",
176
+ row.get("license") or "",
177
+ )
178
+ if bin_oss_items:
179
+ if not any(
180
+ oss_item.name == oss_from_db.name
181
+ and oss_item.version == oss_from_db.version
182
+ and oss_item.license == oss_from_db.license
183
+ for oss_item in bin_oss_items
184
+ ):
185
+ bin_oss_items.append(oss_from_db)
186
+ else:
187
+ bin_oss_items.append(oss_from_db)
188
+
189
+ if bin_oss_items:
190
+ item.set_oss_items(bin_oss_items)
191
+ item.comment = "Binary DB result"
192
+ item.found_in_bin_db = True
193
+ return True
194
+ return False
195
+
196
+
197
+ def get_oss_info_from_db(bin_info_list, kb_url: str = "", kb_token: str = ""):
198
+ """
199
+ Call ldb_service /binary/match and attach OSS info to binary items.
200
+ Deduplicates by filename+checksum before the API call and maps results back.
201
+ Returns (bin_info_list, matched_count, kb_cover_comment).
202
+ """
203
+ _cnt_auto_identified = 0
204
+ if not bin_info_list:
205
+ return bin_info_list, _cnt_auto_identified, ""
206
+
207
+ base_url, token = resolve_kb_config(kb_url, kb_token)
208
+ available, kb_cover_comment = check_binary_match_endpoint(base_url, token)
209
+ if not available:
210
+ return bin_info_list, _cnt_auto_identified, kb_cover_comment
211
+
212
+ items_payload, key_to_id = _build_deduped_payload(bin_info_list, TLSH_CHECKSUM_NULL)
213
+ if not items_payload:
214
+ return bin_info_list, _cnt_auto_identified, ""
215
+
216
+ results_by_id = {}
217
+ try:
218
+ for chunk_start in range(0, len(items_payload), _CHUNK_SIZE):
219
+ chunk = items_payload[chunk_start: chunk_start + _CHUNK_SIZE]
220
+ response = _post_binary_match(base_url, token, chunk)
221
+ if response is None:
222
+ logger.warning(
223
+ f"Binary match chunk failed "
224
+ f"({chunk_start}:{chunk_start + len(chunk)}); "
225
+ "keeping results so far and continuing with next chunks."
226
+ )
227
+ continue
228
+ for result in response.get("results", []):
229
+ results_by_id[str(result.get("id"))] = result
230
+ except Exception as ex:
231
+ logger.warning(f"Binary match API failed: {ex}")
232
+
233
+ for index, item in enumerate(bin_info_list):
234
+ if item.exclude:
235
+ continue
236
+ key = _match_key(item.binary_name_without_path, item.checksum or "", index)
237
+ api_id = key_to_id.get(key)
238
+ if api_id is None:
239
+ continue
240
+ result = results_by_id.get(api_id)
241
+ if _apply_match_result_to_item(item, result):
242
+ _cnt_auto_identified += 1
243
+
244
+ return bin_info_list, _cnt_auto_identified, ""
245
+
246
+
247
+ def _post_binary_match(kb_url: str, kb_token: str, items: list) -> Optional[dict]:
248
+ data = json.dumps({"items": items}).encode("utf-8")
249
+ request = urllib.request.Request(
250
+ f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}",
251
+ data=data,
252
+ method="POST",
253
+ )
254
+ request.add_header("Accept", "application/json")
255
+ request.add_header("Content-Type", "application/json")
256
+ if kb_token:
257
+ request.add_header("Authorization", f"Bearer {kb_token}")
258
+
259
+ try:
260
+ with urllib.request.urlopen(request, timeout=_HTTP_TIMEOUT_SEC) as response:
261
+ body = response.read().decode()
262
+ return json.loads(body) if body else {}
263
+ except urllib.error.HTTPError as ex:
264
+ body = ""
265
+ try:
266
+ body = ex.read().decode()
267
+ except Exception:
268
+ pass
269
+ if ex.code == 404:
270
+ logger.warning(
271
+ f"Binary match endpoint not found (HTTP 404): "
272
+ f"{kb_url.rstrip('/')}{_BINARY_MATCH_PATH}"
273
+ )
274
+ else:
275
+ logger.warning(f"Binary match HTTP {ex.code}: {body or ex.reason}")
276
+ return None
277
+ except urllib.error.URLError as ex:
278
+ logger.debug(f"Binary match unreachable: {ex}")
279
+ return None
@@ -32,7 +32,8 @@ _HELP_MESSAGE_BINARY = f"""
32
32
 
33
33
  🔍 Scanner-Specific Options
34
34
  ────────────────────────────────────────────────────────────────────
35
- -d <db_url> DB Connection (format: 'postgresql://user:pass@host:port/db')
35
+ --kb_url <url> KB API URL (priority: parameter > KB_URL env > default)
36
+ --kb_token <token> KB bearer token (priority: parameter > KB_TOKEN env)
36
37
  --notice Print the open source license notice text
37
38
  --no_correction Skip OSS information correction with sbom-info.yaml
38
39
  --correct_fpath <path> Path to custom sbom-info.yaml file
@@ -48,8 +49,8 @@ _HELP_MESSAGE_BINARY = f"""
48
49
  # Generate output in specific format
49
50
  fosslight_binary -f excel -o results/
50
51
 
51
- # Connect to Binary DB for OSS information
52
- fosslight_binary -d "postgresql://user:pass@localhost:5432/exampledb"
52
+ # Binary DB lookup via ldb_service
53
+ fosslight_binary --kb_url http://fosslight-kb.lge.com/ --kb_token <token>
53
54
  """
54
55
 
55
56
 
@@ -169,7 +169,7 @@ def get_file_list(path_to_find, excluded_files):
169
169
  return file_cnt, bin_list, found_jar
170
170
 
171
171
 
172
- def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=False,
172
+ def find_binaries(path_to_find_bin, output_dir, formats, kb_url="", kb_token="", simple_mode=False,
173
173
  correct_mode=True, correct_filepath="", path_to_exclude=[],
174
174
  all_exclude_mode=()):
175
175
  global start_time, finish_time, _root_path, _result_log
@@ -254,7 +254,7 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F
254
254
  else:
255
255
  logger.warning("Could not find OSS information for some jar files.")
256
256
 
257
- return_list, db_loaded_cnt = get_oss_info_from_db(return_list, dburl)
257
+ return_list, db_loaded_cnt, kb_cover_msg = get_oss_info_from_db(return_list, kb_url, kb_token)
258
258
  return_list = sorted(return_list, key=lambda row: (row.bin_name_with_path))
259
259
  scan_item.append_file_items(return_list, PKG_NAME)
260
260
  if correct_mode:
@@ -267,6 +267,8 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F
267
267
 
268
268
  finish_time = current_timestamp_utc()
269
269
  scan_item.set_cover_comment(f"Detected binaries: {len(return_list)} (Scanned Files : {cnt_file_except_skipped})")
270
+ if kb_cover_msg:
271
+ scan_item.set_cover_comment(kb_cover_msg)
270
272
  scan_item.set_cover_finish_time(finish_time)
271
273
 
272
274
  for combined_path_and_file, output_extension, output_format in zip(result_reports, output_extensions, formats):
@@ -36,7 +36,8 @@ def main():
36
36
  path_to_exclude = []
37
37
  output_dir = ""
38
38
  format = []
39
- db_url = ""
39
+ kb_url = ""
40
+ kb_token = ""
40
41
  simple_mode = False
41
42
  correct_mode = True
42
43
 
@@ -46,9 +47,10 @@ def main():
46
47
  parser.add_argument('-s', '--simple', action='store_true', required=False, help=argparse.SUPPRESS)
47
48
  parser.add_argument('-p', '--path', type=str, required=False)
48
49
  parser.add_argument('-o', '--output', type=str, required=False)
49
- parser.add_argument('-d', '--dburl', type=str, default='', required=False)
50
50
  parser.add_argument('-f', '--formats', type=str, required=False, nargs="*")
51
51
  parser.add_argument('-e', '--exclude', nargs="*", required=False, default=[])
52
+ parser.add_argument('--kb_url', type=str, required=False, default="")
53
+ parser.add_argument('--kb_token', type=str, required=False, default="")
52
54
  parser.add_argument('--notice', action='store_true', required=False)
53
55
  parser.add_argument('--no_correction', action='store_true', required=False)
54
56
  parser.add_argument('--correct_fpath', nargs=1, type=str, required=False)
@@ -81,8 +83,11 @@ def main():
81
83
  if args.output: # -o option
82
84
  output_dir = args.output
83
85
 
84
- if args.dburl: # -d option
85
- db_url = args.dburl
86
+ if args.kb_url:
87
+ kb_url = args.kb_url
88
+
89
+ if args.kb_token:
90
+ kb_token = args.kb_token
86
91
 
87
92
  if args.formats: # -f option
88
93
  format = list(args.formats)
@@ -113,7 +118,10 @@ def main():
113
118
  timer.setDaemon(True)
114
119
  timer.start()
115
120
 
116
- find_binaries(path_to_find_bin, output_dir, format, db_url, simple_mode, correct_mode, correct_filepath, path_to_exclude)
121
+ find_binaries(
122
+ path_to_find_bin, output_dir, format, kb_url, kb_token,
123
+ simple_mode, correct_mode, correct_filepath, path_to_exclude,
124
+ )
117
125
 
118
126
 
119
127
  if __name__ == '__main__':
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fosslight_binary
3
- Version: 5.1.29
3
+ Version: 5.1.30
4
4
  Summary: FOSSLight Binary Scanner
5
5
  Author: LG Electronics
6
6
  License-Expression: Apache-2.0
@@ -24,7 +24,6 @@ Requires-Dist: defusedxml
24
24
  Requires-Dist: numpy
25
25
  Requires-Dist: pandas
26
26
  Requires-Dist: parmap
27
- Requires-Dist: psycopg2-binary
28
27
  Requires-Dist: python-dateutil
29
28
  Requires-Dist: py-tlsh
30
29
  Requires-Dist: pytz
@@ -4,7 +4,6 @@ defusedxml
4
4
  numpy
5
5
  pandas
6
6
  parmap
7
- psycopg2-binary
8
7
  python-dateutil
9
8
  py-tlsh
10
9
  pytz
@@ -1,186 +0,0 @@
1
- #!/usr/bin/env python
2
- # -*- coding: utf-8 -*-
3
- # Copyright (c) 2020 LG Electronics Inc.
4
- # SPDX-License-Identifier: Apache-2.0
5
-
6
- import tlsh
7
- import logging
8
- import psycopg2
9
- import pandas as pd
10
- import socket
11
- from urllib.parse import urlparse
12
- from fosslight_binary._binary import TLSH_CHECKSUM_NULL
13
- from fosslight_util.oss_item import OssItem
14
- import fosslight_util.constant as constant
15
-
16
- columns = ['filename', 'pathname', 'checksum', 'tlshchecksum', 'ossname',
17
- 'ossversion', 'license', 'platformname',
18
- 'platformversion']
19
- conn = ""
20
- cur = ""
21
- logger = logging.getLogger(constant.LOGGER_NAME)
22
- DB_URL_DEFAULT = "postgresql://bin_analysis_script_user:script_123@bat.lge.com:5432/bat"
23
-
24
-
25
- def get_oss_info_from_db(bin_info_list, dburl=""):
26
- _cnt_auto_identified = 0
27
- conn_str, dbc = get_connection_string(dburl)
28
- # DB URL에서 host 추출
29
- try:
30
- db_host = dbc.hostname
31
- except Exception as ex:
32
- logger.warning(f"Failed to parse DB URL for host: {ex}")
33
- db_host = None
34
-
35
- is_internal = False
36
- if db_host:
37
- try:
38
- # DNS lookup 시도
39
- socket.gethostbyname(db_host)
40
- is_internal = True
41
- except Exception:
42
- is_internal = False
43
-
44
- if is_internal:
45
- connect_to_lge_bin_db(conn_str)
46
- if conn != "" and cur != "":
47
- for item in bin_info_list:
48
- bin_oss_items = []
49
- tlsh_value = item.tlsh
50
- checksum_value = item.checksum
51
- bin_file_name = item.binary_name_without_path
52
-
53
- df_result = get_oss_info_by_tlsh_and_filename(
54
- bin_file_name, checksum_value, tlsh_value)
55
- if df_result is not None and len(df_result) > 0:
56
- _cnt_auto_identified += 1
57
- # Initialize the saved contents at .jar analyzing only once
58
- if not item.found_in_jar_analysis and item.oss_items:
59
- item.oss_items = []
60
-
61
- for idx, row in df_result.iterrows():
62
- if not item.found_in_jar_analysis:
63
- oss_from_db = OssItem(row['ossname'], row['ossversion'], row['license'])
64
-
65
- if bin_oss_items:
66
- if not any(oss_item.name == oss_from_db.name
67
- and oss_item.version == oss_from_db.version
68
- and oss_item.license == oss_from_db.license
69
- for oss_item in bin_oss_items):
70
- bin_oss_items.append(oss_from_db)
71
- else:
72
- bin_oss_items.append(oss_from_db)
73
-
74
- if bin_oss_items:
75
- item.set_oss_items(bin_oss_items)
76
- item.comment = "Binary DB result"
77
- item.found_in_binary = True
78
- else:
79
- logger.warning(f"Internal network detected, but DB connection to '{db_host}' failed. Skipping DB query.")
80
- disconnect_lge_bin_db()
81
- else:
82
- logger.debug(f"Binary DB host '{db_host}' is not reachable. Skipping DB query.")
83
- return bin_info_list, _cnt_auto_identified
84
-
85
-
86
- def get_connection_string(dburl):
87
- # dburl format : 'postgresql://username:password@host:port/database_name'
88
- connection_string = ""
89
- user_dburl = True
90
- dbc = ""
91
- if dburl == "" or dburl is None:
92
- user_dburl = False
93
- dburl = DB_URL_DEFAULT
94
- try:
95
- if user_dburl:
96
- logger.debug("DB URL:" + dburl)
97
- dbc = urlparse(dburl)
98
- connection_string = "dbname={dbname} user={user} host={host} password={password} port={port}" \
99
- .format(dbname=dbc.path.lstrip('/'),
100
- user=dbc.username,
101
- host=dbc.hostname,
102
- password=dbc.password,
103
- port=dbc.port)
104
- except Exception as ex:
105
- if user_dburl:
106
- logger.warning(f"(Minor) Failed to parsing db url : {ex}")
107
-
108
- return connection_string, dbc
109
-
110
-
111
- def get_oss_info_by_tlsh_and_filename(file_name, checksum_value, tlsh_value):
112
- sql_statement = "SELECT filename,pathname,checksum,tlshchecksum,ossname,ossversion,\
113
- license,platformname,platformversion FROM lgematching "
114
- sql_statement_checksum = " WHERE filename=%s AND checksum=%s;" # Using parameterized query
115
- sql_statement_filename = "SELECT DISTINCT ON (tlshchecksum) tlshchecksum FROM lgematching WHERE filename=%s;" # Using parameterized query
116
-
117
- final_result_item = ""
118
-
119
- df_result = get_list_by_using_query(
120
- sql_statement + sql_statement_checksum, columns, (file_name, checksum_value))
121
- # Found a file with the same checksum.
122
- if df_result is not None and len(df_result) > 0:
123
- final_result_item = df_result
124
- else:
125
- # Match tlsh and fileName
126
- df_result = get_list_by_using_query(
127
- sql_statement_filename, ['tlshchecksum'], (file_name,))
128
- if df_result is None or len(df_result) <= 0:
129
- final_result_item = ""
130
- elif tlsh_value == TLSH_CHECKSUM_NULL: # Couldn't get the tlsh of a file.
131
- final_result_item = ""
132
- else:
133
- matched_tlsh = ""
134
- matched_tlsh_diff = -1
135
- for row in df_result.tlshchecksum:
136
- try:
137
- if row != TLSH_CHECKSUM_NULL:
138
- tlsh_diff = tlsh.diff(row, tlsh_value)
139
- if tlsh_diff <= 120: # MATCHED
140
- if (matched_tlsh_diff < 0) or (tlsh_diff < matched_tlsh_diff):
141
- matched_tlsh_diff = tlsh_diff
142
- matched_tlsh = row
143
- except Exception as ex:
144
- logger.warning(f"* (Minor) Error_tlsh_comparison: {ex}")
145
- if matched_tlsh != "":
146
- final_result_item = get_list_by_using_query(
147
- sql_statement + " WHERE filename=%s AND tlshchecksum=%s;", columns, (file_name, matched_tlsh))
148
-
149
- return final_result_item
150
-
151
-
152
- def get_list_by_using_query(sql_query, columns, params=None):
153
- result_rows = "" # DataFrame
154
- try:
155
- if params:
156
- cur.execute(sql_query, params)
157
- else:
158
- cur.execute(sql_query)
159
- rows = cur.fetchall()
160
-
161
- if rows is not None and len(rows) > 0:
162
- result_rows = pd.DataFrame(data=rows, columns=columns)
163
- except Exception as ex:
164
- logger.error(f"Database query error: {ex}")
165
- result_rows = ""
166
- return result_rows
167
-
168
-
169
- def disconnect_lge_bin_db():
170
- try:
171
- cur.close()
172
- conn.close()
173
- except:
174
- pass
175
-
176
-
177
- def connect_to_lge_bin_db(connection_string):
178
- global conn, cur
179
-
180
- try:
181
- conn = psycopg2.connect(connection_string)
182
- cur = conn.cursor()
183
- except Exception as ex:
184
- logger.debug(f"(Minor) Can't connect to Binary DB. : {ex}")
185
- conn = ""
186
- cur = ""