libidentifier 0.2.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- libidentifier/__init__.py +36 -0
- libidentifier/api.py +90 -0
- libidentifier/archive.py +133 -0
- libidentifier/cli.py +340 -0
- libidentifier/cli_resources.py +36 -0
- libidentifier/cli_setup.py +177 -0
- libidentifier/config.py +54 -0
- libidentifier/detect.py +30 -0
- libidentifier/errors.py +30 -0
- libidentifier/extractors/__init__.py +2 -0
- libidentifier/extractors/dex.py +121 -0
- libidentifier/extractors/dex_tables.py +137 -0
- libidentifier/extractors/elf.py +66 -0
- libidentifier/extractors/instructions.py +77 -0
- libidentifier/extractors/metadata.py +99 -0
- libidentifier/extractors/version_metadata.py +72 -0
- libidentifier/fingerprint/__init__.py +2 -0
- libidentifier/fingerprint/dex_v2.py +97 -0
- libidentifier/fingerprint/elf_v1.py +16 -0
- libidentifier/grouping.py +74 -0
- libidentifier/inventory.py +260 -0
- libidentifier/matcher.py +381 -0
- libidentifier/models.py +423 -0
- libidentifier/profiles/__init__.py +2 -0
- libidentifier/profiles/builder.py +182 -0
- libidentifier/profiles/catalog.py +135 -0
- libidentifier/profiles/distribution.py +303 -0
- libidentifier/profiles/fetcher.py +141 -0
- libidentifier/profiles/pom.py +66 -0
- libidentifier/profiles/schema.py +137 -0
- libidentifier/profiles/store.py +212 -0
- libidentifier/purl.py +36 -0
- libidentifier/relationships.py +168 -0
- libidentifier/sbom.py +62 -0
- libidentifier/serialization.py +53 -0
- libidentifier/table.py +254 -0
- libidentifier/version.py +2 -0
- libidentifier/vulnerabilities/__init__.py +2 -0
- libidentifier/vulnerabilities/correlate.py +164 -0
- libidentifier/vulnerabilities/database.py +213 -0
- libidentifier/vulnerabilities/models.py +155 -0
- libidentifier/vulnerabilities/runner.py +148 -0
- libidentifier/vulnerabilities/setup.py +74 -0
- libidentifier-0.2.0rc1.dist-info/METADATA +156 -0
- libidentifier-0.2.0rc1.dist-info/RECORD +50 -0
- libidentifier-0.2.0rc1.dist-info/WHEEL +5 -0
- libidentifier-0.2.0rc1.dist-info/entry_points.txt +2 -0
- libidentifier-0.2.0rc1.dist-info/licenses/LICENSE +62 -0
- libidentifier-0.2.0rc1.dist-info/licenses/THIRD_PARTY_NOTICES.md +15 -0
- libidentifier-0.2.0rc1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
"""Input inspection and feature aggregation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import os
|
|
7
|
+
from dataclasses import dataclass, field, replace
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from .archive import read_selected_members
|
|
11
|
+
from .config import FINGERPRINT_ALGORITHM, SCHEMA_VERSION, IdentifyOptions
|
|
12
|
+
from .detect import detect_bytes, detect_file
|
|
13
|
+
from .errors import InputError
|
|
14
|
+
from .extractors.dex import ClassFeature, parse_dex
|
|
15
|
+
from .extractors.elf import ElfFeature, parse_elf
|
|
16
|
+
from .extractors.metadata import detect_frameworks
|
|
17
|
+
from .models import (
|
|
18
|
+
AnalysisError,
|
|
19
|
+
ArtifactRecord,
|
|
20
|
+
CoverageRecord,
|
|
21
|
+
FrameworkHint,
|
|
22
|
+
InputRecord,
|
|
23
|
+
ProfileDatabaseRecord,
|
|
24
|
+
ScanResult,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class Inventory:
|
|
30
|
+
input: InputRecord
|
|
31
|
+
analyzed_files: list[ArtifactRecord] = field(default_factory=list)
|
|
32
|
+
dex_classes: list[ClassFeature] = field(default_factory=list)
|
|
33
|
+
elf_features: list[tuple[ElfFeature, ArtifactRecord]] = field(default_factory=list)
|
|
34
|
+
frameworks_detected: list[FrameworkHint] = field(default_factory=list)
|
|
35
|
+
errors: list[AnalysisError] = field(default_factory=list)
|
|
36
|
+
coverage: CoverageRecord = field(default_factory=CoverageRecord)
|
|
37
|
+
selected_bytes: int = 0
|
|
38
|
+
ownership_assertions: list = field(default_factory=list)
|
|
39
|
+
dex_tables: dict = field(default_factory=dict)
|
|
40
|
+
version_metadata: list = field(default_factory=list)
|
|
41
|
+
metadata_issues: list = field(default_factory=list)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def sha256_bytes(data: bytes) -> str:
|
|
45
|
+
return hashlib.sha256(data).hexdigest()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def sha256_file(path: Path) -> str:
|
|
49
|
+
h = hashlib.sha256()
|
|
50
|
+
with path.open("rb") as fh:
|
|
51
|
+
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
|
52
|
+
h.update(chunk)
|
|
53
|
+
return h.hexdigest()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _coverage(inv: Inventory) -> None:
|
|
57
|
+
dex_seen = any(a.format == "dex" for a in inv.analyzed_files)
|
|
58
|
+
elf_seen = any(a.format == "elf" for a in inv.analyzed_files)
|
|
59
|
+
incomplete = any(e.code in {
|
|
60
|
+
"member_too_large", "archive_budget", "encrypted_member", "unsupported_compression",
|
|
61
|
+
"compression_ratio", "unsafe_member_path", "unsupported_member", "file_read_error",
|
|
62
|
+
"member_read_error", "duplicate_member", "unsupported_file", "directory_depth_limit",
|
|
63
|
+
} for e in inv.errors)
|
|
64
|
+
dex_errors = incomplete or any(e.code.startswith("dex_") or e.code == "parse_error" for e in inv.errors)
|
|
65
|
+
elf_errors = incomplete or any(e.code.startswith("elf_") for e in inv.errors)
|
|
66
|
+
analyzed_count = {
|
|
67
|
+
"dex": sum(1 for a in inv.analyzed_files if a.format == "dex" and a.status == "analyzed"),
|
|
68
|
+
"elf": sum(1 for a in inv.analyzed_files if a.format == "elf" and a.status == "analyzed"),
|
|
69
|
+
"classes": len(inv.dex_classes),
|
|
70
|
+
"native_binaries": sum(1 for a in inv.analyzed_files if a.format == "elf" and a.status == "analyzed"),
|
|
71
|
+
}
|
|
72
|
+
skipped = dict(inv.coverage.skipped_count)
|
|
73
|
+
for e in inv.errors:
|
|
74
|
+
if e.code in {"member_too_large", "archive_budget"}:
|
|
75
|
+
skipped["size_limit"] += 1
|
|
76
|
+
elif e.code == "encrypted_member":
|
|
77
|
+
skipped["encrypted"] += 1
|
|
78
|
+
elif "unsupported" in e.code:
|
|
79
|
+
skipped["unsupported"] += 1
|
|
80
|
+
elif "parse" in e.code:
|
|
81
|
+
skipped["parse_error"] += 1
|
|
82
|
+
inv.coverage = CoverageRecord(
|
|
83
|
+
dex="partial" if dex_errors else ("analyzed" if dex_seen else "absent"),
|
|
84
|
+
elf="partial" if elf_errors else ("analyzed" if elf_seen else "absent"),
|
|
85
|
+
metadata="partial" if inv.frameworks_detected or inv.version_metadata or inv.metadata_issues else "absent",
|
|
86
|
+
analyzed_count=analyzed_count,
|
|
87
|
+
skipped_count=skipped,
|
|
88
|
+
limitations=(["Some input members were skipped; absence of detections is inconclusive"] if incomplete else [])
|
|
89
|
+
+ ["max_scan_seconds is not enforced in-process; use a host-managed subprocess for a hard timeout"],
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _analyze_blob(inv: Inventory, data: bytes, *, fmt: str, container: str, member: str | None, path: str | None) -> None:
|
|
94
|
+
digest = sha256_bytes(data)
|
|
95
|
+
inv.selected_bytes += len(data)
|
|
96
|
+
record = ArtifactRecord(member=member, format=fmt, sha256=digest, status="analyzed", path=path, container=container)
|
|
97
|
+
if fmt == "dex":
|
|
98
|
+
try:
|
|
99
|
+
from .extractors.dex_tables import DexTables
|
|
100
|
+
tables = DexTables(data)
|
|
101
|
+
classes = parse_dex(data, container=container, member=member, sha256=digest, tables=tables)
|
|
102
|
+
inv.dex_classes.extend(classes)
|
|
103
|
+
inv.dex_tables[(container, member)] = tables
|
|
104
|
+
inv.analyzed_files.append(record)
|
|
105
|
+
except Exception as exc:
|
|
106
|
+
inv.analyzed_files.append(replace(record, status="failed"))
|
|
107
|
+
inv.errors.append(AnalysisError("dex_parse_error", member, str(exc), "error"))
|
|
108
|
+
elif fmt == "elf":
|
|
109
|
+
try:
|
|
110
|
+
elf = parse_elf(data)
|
|
111
|
+
inv.elf_features.append((elf, record))
|
|
112
|
+
inv.analyzed_files.append(record)
|
|
113
|
+
except Exception as exc:
|
|
114
|
+
inv.analyzed_files.append(replace(record, status="failed"))
|
|
115
|
+
inv.errors.append(AnalysisError("elf_parse_error", member, str(exc), "error"))
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _scan_file(path: Path, options: IdentifyOptions, *, top_input: Path | None = None, member_prefix: str | None = None) -> Inventory:
|
|
119
|
+
fmt = detect_file(path)
|
|
120
|
+
if fmt == "unknown":
|
|
121
|
+
raise InputError(f"unsupported input format: {path}")
|
|
122
|
+
size = path.stat().st_size
|
|
123
|
+
if fmt in {"dex", "elf"} and size > min(options.max_member_uncompressed_bytes, options.max_total_selected_uncompressed_bytes):
|
|
124
|
+
raise InputError("standalone binary exceeds scan byte limit")
|
|
125
|
+
input_path = top_input or path
|
|
126
|
+
inv = Inventory(InputRecord(path=str(input_path), format=fmt, sha256=sha256_file(path), size=size))
|
|
127
|
+
if fmt in {"dex", "elf"}:
|
|
128
|
+
limit = min(options.max_member_uncompressed_bytes, options.max_total_selected_uncompressed_bytes)
|
|
129
|
+
if path.stat().st_size > limit:
|
|
130
|
+
raise InputError("standalone binary exceeds scan byte limit")
|
|
131
|
+
with path.open("rb") as stream:
|
|
132
|
+
data = stream.read(limit + 1)
|
|
133
|
+
if len(data) > limit:
|
|
134
|
+
raise InputError("standalone binary exceeds scan byte limit")
|
|
135
|
+
_analyze_blob(inv, data, fmt=fmt, container=str(input_path), member=member_prefix, path=str(path))
|
|
136
|
+
inv.frameworks_detected = detect_frameworks([], inv.dex_classes, inv.elf_features, container=str(input_path))
|
|
137
|
+
elif fmt in {"apk", "zip", "jar", "aar"}:
|
|
138
|
+
try:
|
|
139
|
+
members, errors = read_selected_members(path, options)
|
|
140
|
+
inv.errors.extend(errors)
|
|
141
|
+
except Exception as exc:
|
|
142
|
+
raise InputError(f"cannot inspect ZIP/APK: {exc}") from exc
|
|
143
|
+
for member in members:
|
|
144
|
+
if member.format in {"dex", "elf"}:
|
|
145
|
+
_analyze_blob(inv, member.data, fmt=member.format, container=str(path), member=member.name, path=None)
|
|
146
|
+
elif member.format == 'metadata':
|
|
147
|
+
from .extractors.version_metadata import read_properties
|
|
148
|
+
from .models import CodeLocator
|
|
149
|
+
props = read_properties(member.data, member.name)
|
|
150
|
+
if props:
|
|
151
|
+
inv.version_metadata.append((CodeLocator(str(path), member.name, sha256=member.sha256), props))
|
|
152
|
+
else:
|
|
153
|
+
inv.metadata_issues.append(AnalysisError('version_metadata_unsupported', member.name,
|
|
154
|
+
'Unsupported or inconsistent Maven properties; no version inferred'))
|
|
155
|
+
inv.frameworks_detected = detect_frameworks(members, inv.dex_classes, inv.elf_features, container=str(path))
|
|
156
|
+
inv.selected_bytes = sum(len(member.data) for member in members)
|
|
157
|
+
_coverage(inv)
|
|
158
|
+
return inv
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _merge(base: Inventory, child: Inventory) -> None:
|
|
162
|
+
base.analyzed_files.extend(child.analyzed_files)
|
|
163
|
+
base.dex_classes.extend(child.dex_classes)
|
|
164
|
+
base.elf_features.extend(child.elf_features)
|
|
165
|
+
base.frameworks_detected.extend(child.frameworks_detected)
|
|
166
|
+
base.errors.extend(child.errors)
|
|
167
|
+
base.selected_bytes += child.selected_bytes
|
|
168
|
+
base.dex_tables.update(child.dex_tables)
|
|
169
|
+
base.version_metadata.extend(child.version_metadata)
|
|
170
|
+
base.metadata_issues.extend(child.metadata_issues)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def inspect_path(path, options: IdentifyOptions) -> Inventory:
|
|
174
|
+
try:
|
|
175
|
+
return _inspect_path(path, options)
|
|
176
|
+
except OSError as exc:
|
|
177
|
+
raise InputError(f"cannot read input {path}: {exc}") from exc
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _inspect_path(path, options: IdentifyOptions) -> Inventory:
|
|
181
|
+
p = Path(path)
|
|
182
|
+
if not p.exists():
|
|
183
|
+
raise InputError(f"input does not exist: {p}")
|
|
184
|
+
if p.is_symlink():
|
|
185
|
+
raise InputError("top-level symlink inputs are not supported")
|
|
186
|
+
if p.is_file():
|
|
187
|
+
return _scan_file(p, options)
|
|
188
|
+
if not p.is_dir():
|
|
189
|
+
raise InputError("input must be a file or directory")
|
|
190
|
+
|
|
191
|
+
files = []
|
|
192
|
+
traversal_errors = []
|
|
193
|
+
root_depth = len(p.resolve().parts)
|
|
194
|
+
total = 0
|
|
195
|
+
def walk_error(exc):
|
|
196
|
+
traversal_errors.append(AnalysisError("file_read_error", str(exc.filename), str(exc)))
|
|
197
|
+
|
|
198
|
+
for folder, directories, names in os.walk(p, followlinks=False, onerror=walk_error):
|
|
199
|
+
directories[:] = sorted(d for d in directories if not (Path(folder) / d).is_symlink())
|
|
200
|
+
depth = len(Path(folder).resolve().parts) - root_depth
|
|
201
|
+
if depth + 1 >= options.max_directory_depth and directories:
|
|
202
|
+
traversal_errors.append(AnalysisError("directory_depth_limit", str(Path(folder).relative_to(p)), "Deeper directories skipped"))
|
|
203
|
+
directories[:] = []
|
|
204
|
+
for name in sorted(names):
|
|
205
|
+
child = Path(folder) / name
|
|
206
|
+
if child.is_symlink() or not child.is_file():
|
|
207
|
+
continue
|
|
208
|
+
files.append(child)
|
|
209
|
+
total += child.stat().st_size
|
|
210
|
+
if len(files) > options.max_directory_files:
|
|
211
|
+
raise InputError("directory contains too many files")
|
|
212
|
+
if total > options.max_total_selected_uncompressed_bytes:
|
|
213
|
+
raise InputError("directory exceeds scan byte budget")
|
|
214
|
+
files.sort()
|
|
215
|
+
|
|
216
|
+
root_hash = hashlib.sha256()
|
|
217
|
+
for child in files:
|
|
218
|
+
root_hash.update(str(child.relative_to(p)).replace("\\", "/").encode())
|
|
219
|
+
root_hash.update(sha256_file(child).encode())
|
|
220
|
+
inv = Inventory(InputRecord(path=str(p), format="directory", sha256=root_hash.hexdigest(), size=total))
|
|
221
|
+
inv.errors.extend(traversal_errors)
|
|
222
|
+
for child in files:
|
|
223
|
+
try:
|
|
224
|
+
fmt = detect_file(child)
|
|
225
|
+
except OSError as exc:
|
|
226
|
+
inv.errors.append(AnalysisError("file_read_error", str(child.relative_to(p)), str(exc), "warning"))
|
|
227
|
+
continue
|
|
228
|
+
if fmt in {"unknown", "zip", "jar", "aar"}:
|
|
229
|
+
continue
|
|
230
|
+
try:
|
|
231
|
+
remaining = options.max_total_selected_uncompressed_bytes - inv.selected_bytes
|
|
232
|
+
if remaining <= 0:
|
|
233
|
+
inv.errors.append(AnalysisError("archive_budget", str(child.relative_to(p)), "Directory selected-byte budget exhausted"))
|
|
234
|
+
break
|
|
235
|
+
child_inv = _scan_file(child, replace(options, max_total_selected_uncompressed_bytes=remaining), top_input=p, member_prefix=str(child.relative_to(p)).replace("\\", "/"))
|
|
236
|
+
child_inv.errors = [replace(e, member=f"{child.relative_to(p).as_posix()}!/{e.member}" if e.member else child.relative_to(p).as_posix()) for e in child_inv.errors]
|
|
237
|
+
_merge(inv, child_inv)
|
|
238
|
+
except InputError as exc:
|
|
239
|
+
inv.errors.append(AnalysisError("unsupported_file", str(child.relative_to(p)), str(exc), "warning"))
|
|
240
|
+
inv.frameworks_detected = sorted(inv.frameworks_detected, key=lambda h: (h.name, h.evidence))
|
|
241
|
+
_coverage(inv)
|
|
242
|
+
return inv
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def empty_scan_result(inv: Inventory, tool_version: str) -> ScanResult:
|
|
246
|
+
return ScanResult(
|
|
247
|
+
schema_version=SCHEMA_VERSION,
|
|
248
|
+
tool_name="libidentifier",
|
|
249
|
+
tool_version=tool_version,
|
|
250
|
+
fingerprint_algorithm=FINGERPRINT_ALGORITHM,
|
|
251
|
+
profile_database=ProfileDatabaseRecord(),
|
|
252
|
+
input=inv.input,
|
|
253
|
+
analyzed_files=sorted(inv.analyzed_files, key=lambda a: (a.member or "", a.sha256)),
|
|
254
|
+
frameworks_detected=inv.frameworks_detected,
|
|
255
|
+
components=[],
|
|
256
|
+
class_ownership_hints=[],
|
|
257
|
+
unknown_classes=sorted((c.locator for c in inv.dex_classes), key=lambda x: (x.member or "", x.class_descriptor or "")),
|
|
258
|
+
coverage=inv.coverage,
|
|
259
|
+
errors=inv.errors,
|
|
260
|
+
)
|
libidentifier/matcher.py
ADDED
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
"""Generic profile matching engine."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import math
|
|
7
|
+
import sqlite3
|
|
8
|
+
from collections import Counter, defaultdict
|
|
9
|
+
|
|
10
|
+
from .config import IdentifyOptions
|
|
11
|
+
from .extractors.dex import ClassFeature
|
|
12
|
+
from .extractors.elf import ElfFeature
|
|
13
|
+
from .fingerprint.dex_v2 import digest_class, is_trivial_shape
|
|
14
|
+
from .models import CodeLocator, CodeOwnerHint, ComponentMatch
|
|
15
|
+
from .profiles.store import dex_class_rows, release_rows
|
|
16
|
+
from .purl import family_purl
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _component_id(name: str, family_id: str, input_sha: str) -> str:
|
|
20
|
+
return "cmp-" + hashlib.sha256(f"{name}:{family_id}:{input_sha}".encode()).hexdigest()[:16]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _dex_matches(
|
|
24
|
+
conn: sqlite3.Connection,
|
|
25
|
+
classes: list[ClassFeature],
|
|
26
|
+
*,
|
|
27
|
+
input_sha: str,
|
|
28
|
+
options: IdentifyOptions,
|
|
29
|
+
) -> tuple[list[ComponentMatch], list[CodeOwnerHint], set[tuple[str, str | None, str | None]]]:
|
|
30
|
+
observed = Counter()
|
|
31
|
+
locators: dict[str, list[CodeLocator]] = defaultdict(list)
|
|
32
|
+
observed_descriptors: dict[str, Counter[str]] = defaultdict(Counter)
|
|
33
|
+
discriminative_observed: set[str] = set()
|
|
34
|
+
for feature in classes:
|
|
35
|
+
digest = digest_class(feature)
|
|
36
|
+
observed[digest] += 1
|
|
37
|
+
locators[digest].append(feature.locator)
|
|
38
|
+
observed_descriptors[digest][feature.descriptor] += 1
|
|
39
|
+
if not is_trivial_shape(feature):
|
|
40
|
+
discriminative_observed.add(digest)
|
|
41
|
+
if not observed:
|
|
42
|
+
return [], [], set()
|
|
43
|
+
|
|
44
|
+
placeholders = ",".join("?" for _ in observed)
|
|
45
|
+
rows = list(
|
|
46
|
+
conn.execute(
|
|
47
|
+
f"SELECT digest, release_id, multiplicity FROM dex_shape WHERE digest IN ({placeholders})",
|
|
48
|
+
tuple(sorted(observed)),
|
|
49
|
+
)
|
|
50
|
+
)
|
|
51
|
+
release_ids = {row[1] for row in rows}
|
|
52
|
+
releases = release_rows(conn, release_ids)
|
|
53
|
+
if not rows:
|
|
54
|
+
return [], [], set()
|
|
55
|
+
|
|
56
|
+
digest_family_counts = {
|
|
57
|
+
digest: count
|
|
58
|
+
for digest, count in conn.execute(
|
|
59
|
+
f"""
|
|
60
|
+
SELECT ds.digest, COUNT(DISTINCT lr.family_id)
|
|
61
|
+
FROM dex_shape ds JOIN library_release lr USING(release_id)
|
|
62
|
+
WHERE ds.digest IN ({placeholders})
|
|
63
|
+
GROUP BY ds.digest
|
|
64
|
+
""",
|
|
65
|
+
tuple(sorted(observed)),
|
|
66
|
+
)
|
|
67
|
+
}
|
|
68
|
+
family_count = conn.execute("SELECT COUNT(*) FROM library_family").fetchone()[0] or 1
|
|
69
|
+
weights = {d: 1.0 + math.log((1.0 + family_count) / (1.0 + digest_family_counts.get(d, family_count))) for d in observed}
|
|
70
|
+
|
|
71
|
+
reference_counts: dict[str, Counter[str]] = defaultdict(Counter)
|
|
72
|
+
for digest, rid, multiplicity in rows:
|
|
73
|
+
reference_counts[rid][digest] = int(multiplicity)
|
|
74
|
+
totals: dict[str, float] = {}
|
|
75
|
+
for rid in reference_counts:
|
|
76
|
+
totals[rid] = sum(
|
|
77
|
+
weights.get(d, 1.0) * count
|
|
78
|
+
for d, count in conn.execute("SELECT digest, multiplicity FROM dex_shape WHERE release_id=?", (rid,))
|
|
79
|
+
)
|
|
80
|
+
profile_class_totals = {
|
|
81
|
+
rid: sum(
|
|
82
|
+
count
|
|
83
|
+
for _digest, count in conn.execute("SELECT digest, multiplicity FROM dex_shape WHERE release_id=?", (rid,))
|
|
84
|
+
)
|
|
85
|
+
for rid in reference_counts
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
candidates = []
|
|
89
|
+
for rid, counts in reference_counts.items():
|
|
90
|
+
structural_matched = sum(min(observed[d], c) for d, c in counts.items())
|
|
91
|
+
structural_weighted = sum(weights.get(d, 1.0) * min(observed[d], c) for d, c in counts.items())
|
|
92
|
+
reference_descriptors = dex_class_rows(conn, rid, set(counts))
|
|
93
|
+
exact_matched = 0
|
|
94
|
+
exact_weighted = 0.0
|
|
95
|
+
for digest, descriptor_counts in reference_descriptors.items():
|
|
96
|
+
for descriptor, ref_count in descriptor_counts.items():
|
|
97
|
+
count = min(observed_descriptors[digest][descriptor], ref_count)
|
|
98
|
+
exact_matched += count
|
|
99
|
+
exact_weighted += weights.get(digest, 1.0) * count
|
|
100
|
+
matched = exact_matched or structural_matched
|
|
101
|
+
weighted = exact_weighted or structural_weighted
|
|
102
|
+
match_mode = "exact_descriptor" if exact_matched else "structural_shape"
|
|
103
|
+
total = totals.get(rid) or 1.0
|
|
104
|
+
discriminative = sum(min(observed[d], c) for d, c in counts.items() if d in discriminative_observed)
|
|
105
|
+
unique_anchors = sum(1 for d in counts if d in discriminative_observed and digest_family_counts.get(d, 99) <= 1)
|
|
106
|
+
candidates.append(
|
|
107
|
+
{
|
|
108
|
+
"release_id": rid,
|
|
109
|
+
"family_id": releases[rid]["family_id"],
|
|
110
|
+
"coverage": weighted / total,
|
|
111
|
+
"matched": matched,
|
|
112
|
+
"structural_matched": structural_matched,
|
|
113
|
+
"exact_descriptor_matches": exact_matched,
|
|
114
|
+
"profile_classes": profile_class_totals.get(rid, sum(counts.values())),
|
|
115
|
+
"discriminative": discriminative,
|
|
116
|
+
"unique_anchors": unique_anchors,
|
|
117
|
+
"digests": set(counts),
|
|
118
|
+
"counts": counts,
|
|
119
|
+
"reference_descriptors": reference_descriptors,
|
|
120
|
+
"match_mode": match_mode,
|
|
121
|
+
"weighted_total": total,
|
|
122
|
+
}
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
by_family: dict[str, list[dict]] = defaultdict(list)
|
|
126
|
+
for cand in candidates:
|
|
127
|
+
by_family[cand["family_id"]].append(cand)
|
|
128
|
+
family_best = []
|
|
129
|
+
for fid, cands in by_family.items():
|
|
130
|
+
best = sorted(cands, key=lambda c: (-c["coverage"], -c["matched"], c["release_id"]))[0]
|
|
131
|
+
family_best.append(best)
|
|
132
|
+
family_best.sort(key=lambda c: (-c["coverage"], -c["matched"], c["family_id"]))
|
|
133
|
+
|
|
134
|
+
components: list[ComponentMatch] = []
|
|
135
|
+
hints: list[CodeOwnerHint] = []
|
|
136
|
+
assigned: set[tuple[str, str | None, str | None]] = set()
|
|
137
|
+
for index, cand in enumerate(family_best):
|
|
138
|
+
runner = 0.0
|
|
139
|
+
for j, other in enumerate(family_best):
|
|
140
|
+
if j == index:
|
|
141
|
+
continue
|
|
142
|
+
overlap = cand["digests"].intersection(other["digests"])
|
|
143
|
+
if not overlap:
|
|
144
|
+
continue
|
|
145
|
+
overlap_weight = sum(
|
|
146
|
+
weights.get(d, 1.0) * min(cand["counts"].get(d, 0), other["counts"].get(d, 0))
|
|
147
|
+
for d in overlap
|
|
148
|
+
)
|
|
149
|
+
material_overlap = overlap_weight / max(min(cand["weighted_total"], other["weighted_total"]), 1.0)
|
|
150
|
+
if material_overlap >= options.min_material_runner_overlap:
|
|
151
|
+
runner = max(runner, other["coverage"])
|
|
152
|
+
runner_delta = cand["coverage"] - runner
|
|
153
|
+
if cand["discriminative"] >= options.high_min_discriminative_classes and cand["coverage"] >= options.high_min_weighted_coverage and cand["unique_anchors"] >= options.high_min_unique_anchors and runner_delta >= options.high_min_runner_up_delta:
|
|
154
|
+
confidence = "high"
|
|
155
|
+
partial_match = False
|
|
156
|
+
elif cand["discriminative"] >= options.medium_min_discriminative_classes and cand["coverage"] >= options.medium_min_weighted_coverage and runner_delta >= 0:
|
|
157
|
+
confidence = "medium"
|
|
158
|
+
partial_match = False
|
|
159
|
+
elif (
|
|
160
|
+
cand["match_mode"] == "exact_descriptor"
|
|
161
|
+
and cand["exact_descriptor_matches"] >= options.medium_partial_min_exact_classes
|
|
162
|
+
and cand["coverage"] >= options.medium_partial_min_weighted_coverage
|
|
163
|
+
and cand["unique_anchors"] >= options.medium_partial_min_unique_anchors
|
|
164
|
+
and runner_delta >= 0
|
|
165
|
+
):
|
|
166
|
+
confidence = "medium"
|
|
167
|
+
partial_match = True
|
|
168
|
+
else:
|
|
169
|
+
continue
|
|
170
|
+
family_releases = sorted(by_family[cand["family_id"]], key=lambda c: (-c["coverage"], c["release_id"]))
|
|
171
|
+
supported_releases = [c for c in family_releases if c is cand or (
|
|
172
|
+
c["coverage"] >= options.medium_min_weighted_coverage
|
|
173
|
+
and c["discriminative"] >= options.medium_min_discriminative_classes
|
|
174
|
+
) or (
|
|
175
|
+
c["match_mode"] == "exact_descriptor"
|
|
176
|
+
and c["coverage"] >= options.medium_partial_min_weighted_coverage
|
|
177
|
+
and c["exact_descriptor_matches"] >= options.medium_partial_min_exact_classes
|
|
178
|
+
and c["unique_anchors"] >= options.medium_partial_min_unique_anchors
|
|
179
|
+
)]
|
|
180
|
+
all_versions = sorted({releases[c["release_id"]]["version"] for c in supported_releases})
|
|
181
|
+
candidate_versions = all_versions[: options.max_version_candidates]
|
|
182
|
+
exact = [
|
|
183
|
+
releases[c["release_id"]]
|
|
184
|
+
for c in family_releases
|
|
185
|
+
if input_sha in releases[c["release_id"]]['source_hashes']
|
|
186
|
+
]
|
|
187
|
+
ambiguous_exact = False
|
|
188
|
+
if exact:
|
|
189
|
+
families = conn.execute('SELECT COUNT(DISTINCT r.family_id) FROM reference_artifact a JOIN library_release r USING(release_id) WHERE a.sha256=?', (input_sha,)).fetchone()[0]
|
|
190
|
+
if families != 1:
|
|
191
|
+
ambiguous_exact = True
|
|
192
|
+
exact = []
|
|
193
|
+
version = exact[0]["version"] if len(exact) == 1 else None
|
|
194
|
+
purl = exact[0]["purl"] if len(exact) == 1 else None
|
|
195
|
+
version_confidence = "high" if version else "unknown"
|
|
196
|
+
matched_locs = []
|
|
197
|
+
ambiguous_locs = []
|
|
198
|
+
for digest in sorted(cand["digests"]):
|
|
199
|
+
remaining = cand["counts"][digest]
|
|
200
|
+
descriptor_counts = Counter(cand["reference_descriptors"].get(digest, {}))
|
|
201
|
+
exact_locators = [
|
|
202
|
+
locator
|
|
203
|
+
for locator in locators[digest]
|
|
204
|
+
if locator.class_descriptor in descriptor_counts and descriptor_counts[locator.class_descriptor] > 0
|
|
205
|
+
]
|
|
206
|
+
if cand["exact_descriptor_matches"]:
|
|
207
|
+
locator_pool = exact_locators
|
|
208
|
+
else:
|
|
209
|
+
locator_pool = locators[digest]
|
|
210
|
+
if len(locator_pool) > remaining:
|
|
211
|
+
# A shape/count cannot choose which physical definition belongs.
|
|
212
|
+
ambiguous_locs.extend(locator_pool)
|
|
213
|
+
matched_locs.extend(locator_pool)
|
|
214
|
+
continue
|
|
215
|
+
for locator in locator_pool:
|
|
216
|
+
if remaining <= 0:
|
|
217
|
+
break
|
|
218
|
+
matched_locs.append(locator)
|
|
219
|
+
if locator.class_descriptor in descriptor_counts:
|
|
220
|
+
descriptor_counts[locator.class_descriptor] -= 1
|
|
221
|
+
remaining -= 1
|
|
222
|
+
if not matched_locs:
|
|
223
|
+
continue
|
|
224
|
+
release = releases[cand["release_id"]]
|
|
225
|
+
cid = _component_id(release["name"], cand["family_id"], input_sha)
|
|
226
|
+
for locator in matched_locs:
|
|
227
|
+
key = (locator.container, locator.member, locator.class_descriptor)
|
|
228
|
+
assigned.add(key)
|
|
229
|
+
hints.append(
|
|
230
|
+
CodeOwnerHint(
|
|
231
|
+
subject=locator.class_descriptor or "",
|
|
232
|
+
origin="third_party",
|
|
233
|
+
component_id=cid,
|
|
234
|
+
confidence=confidence,
|
|
235
|
+
method="structural_fingerprint",
|
|
236
|
+
)
|
|
237
|
+
)
|
|
238
|
+
components.append(
|
|
239
|
+
ComponentMatch(
|
|
240
|
+
component_id=cid,
|
|
241
|
+
name=release["name"],
|
|
242
|
+
ecosystem=release["ecosystem"],
|
|
243
|
+
purl=purl,
|
|
244
|
+
family_purl=family_purl(release["purl"]),
|
|
245
|
+
version_candidates_total=len(all_versions),
|
|
246
|
+
version_candidates_truncated=len(candidate_versions) < len(all_versions),
|
|
247
|
+
version=version,
|
|
248
|
+
version_candidates=candidate_versions,
|
|
249
|
+
identity_confidence=confidence,
|
|
250
|
+
version_confidence=version_confidence,
|
|
251
|
+
identification_methods=["dex_shape_v1"],
|
|
252
|
+
score=round(cand["coverage"], 6),
|
|
253
|
+
score_details={
|
|
254
|
+
"version_candidate_scores": {releases[c["release_id"]]["version"]: {"weighted_coverage": round(c["coverage"], 6), "exact_descriptor_matches": c["exact_descriptor_matches"], "supported": c in supported_releases} for c in family_releases},
|
|
255
|
+
"matched_classes": cand["matched"],
|
|
256
|
+
"profile_classes": cand["profile_classes"],
|
|
257
|
+
"exact_descriptor_matches": cand["exact_descriptor_matches"],
|
|
258
|
+
"structural_matched_classes": cand["structural_matched"],
|
|
259
|
+
"match_mode": cand["match_mode"],
|
|
260
|
+
"partial_match": partial_match,
|
|
261
|
+
"weighted_coverage": round(cand["coverage"], 6),
|
|
262
|
+
"unique_anchors": cand["unique_anchors"],
|
|
263
|
+
"runner_up_delta": round(runner_delta, 6),
|
|
264
|
+
"ambiguous_code": [x.to_dict() for x in ambiguous_locs],
|
|
265
|
+
},
|
|
266
|
+
matched_code=sorted(matched_locs, key=lambda x: (x.container, x.member or "", x.class_descriptor or "")),
|
|
267
|
+
source_artifacts=sorted({x.member or x.container for x in matched_locs}),
|
|
268
|
+
limitations=(
|
|
269
|
+
([] if version else ["release version cannot be distinguished from structural fingerprints alone"])
|
|
270
|
+
+ (["Exact reference bytes are assigned to multiple library families; identity/version remain ambiguous"] if ambiguous_exact else [])
|
|
271
|
+
+ (["partial library match; not all reference classes were observed"] if partial_match else [])
|
|
272
|
+
+ (
|
|
273
|
+
["matched code is assigned by structural shape because reference descriptors were not observed"]
|
|
274
|
+
if cand["match_mode"] == "structural_shape"
|
|
275
|
+
else []
|
|
276
|
+
)
|
|
277
|
+
),
|
|
278
|
+
)
|
|
279
|
+
)
|
|
280
|
+
return components, hints, assigned
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _elf_matches(
|
|
284
|
+
conn: sqlite3.Connection,
|
|
285
|
+
elf_features: list[tuple[ElfFeature, object]],
|
|
286
|
+
*,
|
|
287
|
+
input_sha: str,
|
|
288
|
+
container: str,
|
|
289
|
+
options: IdentifyOptions,
|
|
290
|
+
) -> list[ComponentMatch]:
|
|
291
|
+
components: dict[str, ComponentMatch] = {}
|
|
292
|
+
for feature, artifact in elf_features:
|
|
293
|
+
rows = list(conn.execute(
|
|
294
|
+
"""
|
|
295
|
+
SELECT DISTINCT r.family_id, r.version, r.purl, f.name, f.ecosystem
|
|
296
|
+
FROM elf_identity e
|
|
297
|
+
JOIN library_release r USING(release_id)
|
|
298
|
+
JOIN library_family f USING(family_id)
|
|
299
|
+
WHERE e.sha256=? ORDER BY f.name, r.version, r.purl
|
|
300
|
+
""", (feature.sha256,),
|
|
301
|
+
))
|
|
302
|
+
# Identical bytes assigned to different families cannot establish identity.
|
|
303
|
+
if not rows or len({r[0] for r in rows}) != 1:
|
|
304
|
+
continue
|
|
305
|
+
fid, _, _, name, ecosystem = rows[0]
|
|
306
|
+
versions = sorted({r[1] for r in rows})
|
|
307
|
+
version = versions[0] if len(versions) == 1 else None
|
|
308
|
+
purls = {r[2] for r in rows}
|
|
309
|
+
purl = next(iter(purls)) if version and len(purls) == 1 else None
|
|
310
|
+
families = {family_purl(r[2]) for r in rows}
|
|
311
|
+
cid = _component_id(name, fid, input_sha + feature.sha256)
|
|
312
|
+
locator = CodeLocator(container=getattr(artifact, "container", None) or container,
|
|
313
|
+
member=getattr(artifact, "member", None), sha256=feature.sha256)
|
|
314
|
+
source = getattr(artifact, "member", None) or getattr(artifact, "path", None) or container
|
|
315
|
+
if cid in components:
|
|
316
|
+
components[cid].matched_code.append(locator)
|
|
317
|
+
if source not in components[cid].source_artifacts:
|
|
318
|
+
components[cid].source_artifacts.append(source)
|
|
319
|
+
continue
|
|
320
|
+
candidates = versions[:options.max_version_candidates]
|
|
321
|
+
components[cid] = ComponentMatch(
|
|
322
|
+
component_id=cid, name=name, ecosystem=ecosystem, purl=purl,
|
|
323
|
+
family_purl=next(iter(families)) if len(families) == 1 else None,
|
|
324
|
+
version=version, version_candidates=candidates,
|
|
325
|
+
version_candidates_total=len(versions), version_candidates_truncated=len(candidates) < len(versions),
|
|
326
|
+
identity_confidence="high", version_confidence="high" if version else "unknown",
|
|
327
|
+
identification_methods=["elf_identity_v1"], score=1.0,
|
|
328
|
+
score_details={"matched_classes": 0, "profile_classes": 0, "weighted_coverage": 1.0,
|
|
329
|
+
"unique_anchors": 1, "runner_up_delta": 1.0},
|
|
330
|
+
matched_code=[locator], source_artifacts=[source],
|
|
331
|
+
limitations=[] if version else ["Identical ELF bytes occur in multiple reference releases; exact version is unknown"],
|
|
332
|
+
)
|
|
333
|
+
for component in components.values():
|
|
334
|
+
component.matched_code.sort(key=lambda loc: (loc.container, loc.member or ""))
|
|
335
|
+
component.source_artifacts.sort()
|
|
336
|
+
return list(components.values())
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def match_inventory(conn: sqlite3.Connection, inventory, options: IdentifyOptions):
|
|
340
|
+
dex_components, hints, assigned = _dex_matches(conn, inventory.dex_classes, input_sha=inventory.input.sha256, options=options)
|
|
341
|
+
elf_components = _elf_matches(conn, inventory.elf_features, input_sha=inventory.input.sha256, container=inventory.input.path, options=options)
|
|
342
|
+
from dataclasses import replace
|
|
343
|
+
from .models import OwnershipAssertion
|
|
344
|
+
claims = defaultdict(set)
|
|
345
|
+
ambiguous = set()
|
|
346
|
+
for component in dex_components:
|
|
347
|
+
for loc in component.matched_code:
|
|
348
|
+
claims[(loc.container, loc.member, loc.class_descriptor)].add(component.component_id)
|
|
349
|
+
for raw in component.score_details.get('ambiguous_code', []):
|
|
350
|
+
ambiguous.add((raw['container'], raw['member'], raw['class_descriptor']))
|
|
351
|
+
descriptors = Counter(c.descriptor for c in inventory.dex_classes)
|
|
352
|
+
assigned = set()
|
|
353
|
+
assertions = []
|
|
354
|
+
for feature in inventory.dex_classes:
|
|
355
|
+
loc = feature.locator
|
|
356
|
+
key = (loc.container, loc.member, loc.class_descriptor)
|
|
357
|
+
owners = sorted(claims[key])
|
|
358
|
+
disputed = len(owners) > 1 or key in ambiguous or descriptors[feature.descriptor] > 1
|
|
359
|
+
status = 'ambiguous' if disputed else ('attributed' if owners else 'unattributed')
|
|
360
|
+
if status == 'attributed':
|
|
361
|
+
assigned.add(key)
|
|
362
|
+
assertions.append(OwnershipAssertion(loc, status, owners,
|
|
363
|
+
limitations=['Competing or duplicate code definitions; ownership unresolved'] if disputed else []))
|
|
364
|
+
inventory.ownership_assertions = assertions
|
|
365
|
+
cleaned = []
|
|
366
|
+
for component in dex_components:
|
|
367
|
+
locations = [loc for loc in component.matched_code if (loc.container, loc.member, loc.class_descriptor) in assigned]
|
|
368
|
+
cleaned.append(replace(component, matched_code=locations,
|
|
369
|
+
limitations=component.limitations + (['Some matching evidence has ambiguous ownership'] if len(locations) < len(component.matched_code) else [])))
|
|
370
|
+
components = sorted(cleaned + elf_components, key=lambda c: (c.name, c.component_id))
|
|
371
|
+
hints = [CodeOwnerHint(loc.class_descriptor or '', 'third_party', c.component_id, c.identity_confidence, 'structural_fingerprint')
|
|
372
|
+
for c in cleaned for loc in c.matched_code]
|
|
373
|
+
hints.sort(key=lambda h: (h.component_id, h.subject))
|
|
374
|
+
unknown = [
|
|
375
|
+
feature.locator
|
|
376
|
+
for feature in inventory.dex_classes
|
|
377
|
+
if (feature.locator.container, feature.locator.member, feature.locator.class_descriptor) not in assigned
|
|
378
|
+
]
|
|
379
|
+
unknown.sort(key=lambda x: (x.container, x.member or "", x.class_descriptor or ""))
|
|
380
|
+
return components, hints, unknown
|
|
381
|
+
|