libidentifier 0.2.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. libidentifier/__init__.py +36 -0
  2. libidentifier/api.py +90 -0
  3. libidentifier/archive.py +133 -0
  4. libidentifier/cli.py +340 -0
  5. libidentifier/cli_resources.py +36 -0
  6. libidentifier/cli_setup.py +177 -0
  7. libidentifier/config.py +54 -0
  8. libidentifier/detect.py +30 -0
  9. libidentifier/errors.py +30 -0
  10. libidentifier/extractors/__init__.py +2 -0
  11. libidentifier/extractors/dex.py +121 -0
  12. libidentifier/extractors/dex_tables.py +137 -0
  13. libidentifier/extractors/elf.py +66 -0
  14. libidentifier/extractors/instructions.py +77 -0
  15. libidentifier/extractors/metadata.py +99 -0
  16. libidentifier/extractors/version_metadata.py +72 -0
  17. libidentifier/fingerprint/__init__.py +2 -0
  18. libidentifier/fingerprint/dex_v2.py +97 -0
  19. libidentifier/fingerprint/elf_v1.py +16 -0
  20. libidentifier/grouping.py +74 -0
  21. libidentifier/inventory.py +260 -0
  22. libidentifier/matcher.py +381 -0
  23. libidentifier/models.py +423 -0
  24. libidentifier/profiles/__init__.py +2 -0
  25. libidentifier/profiles/builder.py +182 -0
  26. libidentifier/profiles/catalog.py +135 -0
  27. libidentifier/profiles/distribution.py +303 -0
  28. libidentifier/profiles/fetcher.py +141 -0
  29. libidentifier/profiles/pom.py +66 -0
  30. libidentifier/profiles/schema.py +137 -0
  31. libidentifier/profiles/store.py +212 -0
  32. libidentifier/purl.py +36 -0
  33. libidentifier/relationships.py +168 -0
  34. libidentifier/sbom.py +62 -0
  35. libidentifier/serialization.py +53 -0
  36. libidentifier/table.py +254 -0
  37. libidentifier/version.py +2 -0
  38. libidentifier/vulnerabilities/__init__.py +2 -0
  39. libidentifier/vulnerabilities/correlate.py +164 -0
  40. libidentifier/vulnerabilities/database.py +213 -0
  41. libidentifier/vulnerabilities/models.py +155 -0
  42. libidentifier/vulnerabilities/runner.py +148 -0
  43. libidentifier/vulnerabilities/setup.py +74 -0
  44. libidentifier-0.2.0rc1.dist-info/METADATA +156 -0
  45. libidentifier-0.2.0rc1.dist-info/RECORD +50 -0
  46. libidentifier-0.2.0rc1.dist-info/WHEEL +5 -0
  47. libidentifier-0.2.0rc1.dist-info/entry_points.txt +2 -0
  48. libidentifier-0.2.0rc1.dist-info/licenses/LICENSE +62 -0
  49. libidentifier-0.2.0rc1.dist-info/licenses/THIRD_PARTY_NOTICES.md +15 -0
  50. libidentifier-0.2.0rc1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,260 @@
1
+ """Input inspection and feature aggregation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import os
7
+ from dataclasses import dataclass, field, replace
8
+ from pathlib import Path
9
+
10
+ from .archive import read_selected_members
11
+ from .config import FINGERPRINT_ALGORITHM, SCHEMA_VERSION, IdentifyOptions
12
+ from .detect import detect_bytes, detect_file
13
+ from .errors import InputError
14
+ from .extractors.dex import ClassFeature, parse_dex
15
+ from .extractors.elf import ElfFeature, parse_elf
16
+ from .extractors.metadata import detect_frameworks
17
+ from .models import (
18
+ AnalysisError,
19
+ ArtifactRecord,
20
+ CoverageRecord,
21
+ FrameworkHint,
22
+ InputRecord,
23
+ ProfileDatabaseRecord,
24
+ ScanResult,
25
+ )
26
+
27
+
28
+ @dataclass
29
+ class Inventory:
30
+ input: InputRecord
31
+ analyzed_files: list[ArtifactRecord] = field(default_factory=list)
32
+ dex_classes: list[ClassFeature] = field(default_factory=list)
33
+ elf_features: list[tuple[ElfFeature, ArtifactRecord]] = field(default_factory=list)
34
+ frameworks_detected: list[FrameworkHint] = field(default_factory=list)
35
+ errors: list[AnalysisError] = field(default_factory=list)
36
+ coverage: CoverageRecord = field(default_factory=CoverageRecord)
37
+ selected_bytes: int = 0
38
+ ownership_assertions: list = field(default_factory=list)
39
+ dex_tables: dict = field(default_factory=dict)
40
+ version_metadata: list = field(default_factory=list)
41
+ metadata_issues: list = field(default_factory=list)
42
+
43
+
44
+ def sha256_bytes(data: bytes) -> str:
45
+ return hashlib.sha256(data).hexdigest()
46
+
47
+
48
+ def sha256_file(path: Path) -> str:
49
+ h = hashlib.sha256()
50
+ with path.open("rb") as fh:
51
+ for chunk in iter(lambda: fh.read(1024 * 1024), b""):
52
+ h.update(chunk)
53
+ return h.hexdigest()
54
+
55
+
56
+ def _coverage(inv: Inventory) -> None:
57
+ dex_seen = any(a.format == "dex" for a in inv.analyzed_files)
58
+ elf_seen = any(a.format == "elf" for a in inv.analyzed_files)
59
+ incomplete = any(e.code in {
60
+ "member_too_large", "archive_budget", "encrypted_member", "unsupported_compression",
61
+ "compression_ratio", "unsafe_member_path", "unsupported_member", "file_read_error",
62
+ "member_read_error", "duplicate_member", "unsupported_file", "directory_depth_limit",
63
+ } for e in inv.errors)
64
+ dex_errors = incomplete or any(e.code.startswith("dex_") or e.code == "parse_error" for e in inv.errors)
65
+ elf_errors = incomplete or any(e.code.startswith("elf_") for e in inv.errors)
66
+ analyzed_count = {
67
+ "dex": sum(1 for a in inv.analyzed_files if a.format == "dex" and a.status == "analyzed"),
68
+ "elf": sum(1 for a in inv.analyzed_files if a.format == "elf" and a.status == "analyzed"),
69
+ "classes": len(inv.dex_classes),
70
+ "native_binaries": sum(1 for a in inv.analyzed_files if a.format == "elf" and a.status == "analyzed"),
71
+ }
72
+ skipped = dict(inv.coverage.skipped_count)
73
+ for e in inv.errors:
74
+ if e.code in {"member_too_large", "archive_budget"}:
75
+ skipped["size_limit"] += 1
76
+ elif e.code == "encrypted_member":
77
+ skipped["encrypted"] += 1
78
+ elif "unsupported" in e.code:
79
+ skipped["unsupported"] += 1
80
+ elif "parse" in e.code:
81
+ skipped["parse_error"] += 1
82
+ inv.coverage = CoverageRecord(
83
+ dex="partial" if dex_errors else ("analyzed" if dex_seen else "absent"),
84
+ elf="partial" if elf_errors else ("analyzed" if elf_seen else "absent"),
85
+ metadata="partial" if inv.frameworks_detected or inv.version_metadata or inv.metadata_issues else "absent",
86
+ analyzed_count=analyzed_count,
87
+ skipped_count=skipped,
88
+ limitations=(["Some input members were skipped; absence of detections is inconclusive"] if incomplete else [])
89
+ + ["max_scan_seconds is not enforced in-process; use a host-managed subprocess for a hard timeout"],
90
+ )
91
+
92
+
93
+ def _analyze_blob(inv: Inventory, data: bytes, *, fmt: str, container: str, member: str | None, path: str | None) -> None:
94
+ digest = sha256_bytes(data)
95
+ inv.selected_bytes += len(data)
96
+ record = ArtifactRecord(member=member, format=fmt, sha256=digest, status="analyzed", path=path, container=container)
97
+ if fmt == "dex":
98
+ try:
99
+ from .extractors.dex_tables import DexTables
100
+ tables = DexTables(data)
101
+ classes = parse_dex(data, container=container, member=member, sha256=digest, tables=tables)
102
+ inv.dex_classes.extend(classes)
103
+ inv.dex_tables[(container, member)] = tables
104
+ inv.analyzed_files.append(record)
105
+ except Exception as exc:
106
+ inv.analyzed_files.append(replace(record, status="failed"))
107
+ inv.errors.append(AnalysisError("dex_parse_error", member, str(exc), "error"))
108
+ elif fmt == "elf":
109
+ try:
110
+ elf = parse_elf(data)
111
+ inv.elf_features.append((elf, record))
112
+ inv.analyzed_files.append(record)
113
+ except Exception as exc:
114
+ inv.analyzed_files.append(replace(record, status="failed"))
115
+ inv.errors.append(AnalysisError("elf_parse_error", member, str(exc), "error"))
116
+
117
+
118
+ def _scan_file(path: Path, options: IdentifyOptions, *, top_input: Path | None = None, member_prefix: str | None = None) -> Inventory:
119
+ fmt = detect_file(path)
120
+ if fmt == "unknown":
121
+ raise InputError(f"unsupported input format: {path}")
122
+ size = path.stat().st_size
123
+ if fmt in {"dex", "elf"} and size > min(options.max_member_uncompressed_bytes, options.max_total_selected_uncompressed_bytes):
124
+ raise InputError("standalone binary exceeds scan byte limit")
125
+ input_path = top_input or path
126
+ inv = Inventory(InputRecord(path=str(input_path), format=fmt, sha256=sha256_file(path), size=size))
127
+ if fmt in {"dex", "elf"}:
128
+ limit = min(options.max_member_uncompressed_bytes, options.max_total_selected_uncompressed_bytes)
129
+ if path.stat().st_size > limit:
130
+ raise InputError("standalone binary exceeds scan byte limit")
131
+ with path.open("rb") as stream:
132
+ data = stream.read(limit + 1)
133
+ if len(data) > limit:
134
+ raise InputError("standalone binary exceeds scan byte limit")
135
+ _analyze_blob(inv, data, fmt=fmt, container=str(input_path), member=member_prefix, path=str(path))
136
+ inv.frameworks_detected = detect_frameworks([], inv.dex_classes, inv.elf_features, container=str(input_path))
137
+ elif fmt in {"apk", "zip", "jar", "aar"}:
138
+ try:
139
+ members, errors = read_selected_members(path, options)
140
+ inv.errors.extend(errors)
141
+ except Exception as exc:
142
+ raise InputError(f"cannot inspect ZIP/APK: {exc}") from exc
143
+ for member in members:
144
+ if member.format in {"dex", "elf"}:
145
+ _analyze_blob(inv, member.data, fmt=member.format, container=str(path), member=member.name, path=None)
146
+ elif member.format == 'metadata':
147
+ from .extractors.version_metadata import read_properties
148
+ from .models import CodeLocator
149
+ props = read_properties(member.data, member.name)
150
+ if props:
151
+ inv.version_metadata.append((CodeLocator(str(path), member.name, sha256=member.sha256), props))
152
+ else:
153
+ inv.metadata_issues.append(AnalysisError('version_metadata_unsupported', member.name,
154
+ 'Unsupported or inconsistent Maven properties; no version inferred'))
155
+ inv.frameworks_detected = detect_frameworks(members, inv.dex_classes, inv.elf_features, container=str(path))
156
+ inv.selected_bytes = sum(len(member.data) for member in members)
157
+ _coverage(inv)
158
+ return inv
159
+
160
+
161
+ def _merge(base: Inventory, child: Inventory) -> None:
162
+ base.analyzed_files.extend(child.analyzed_files)
163
+ base.dex_classes.extend(child.dex_classes)
164
+ base.elf_features.extend(child.elf_features)
165
+ base.frameworks_detected.extend(child.frameworks_detected)
166
+ base.errors.extend(child.errors)
167
+ base.selected_bytes += child.selected_bytes
168
+ base.dex_tables.update(child.dex_tables)
169
+ base.version_metadata.extend(child.version_metadata)
170
+ base.metadata_issues.extend(child.metadata_issues)
171
+
172
+
173
+ def inspect_path(path, options: IdentifyOptions) -> Inventory:
174
+ try:
175
+ return _inspect_path(path, options)
176
+ except OSError as exc:
177
+ raise InputError(f"cannot read input {path}: {exc}") from exc
178
+
179
+
180
+ def _inspect_path(path, options: IdentifyOptions) -> Inventory:
181
+ p = Path(path)
182
+ if not p.exists():
183
+ raise InputError(f"input does not exist: {p}")
184
+ if p.is_symlink():
185
+ raise InputError("top-level symlink inputs are not supported")
186
+ if p.is_file():
187
+ return _scan_file(p, options)
188
+ if not p.is_dir():
189
+ raise InputError("input must be a file or directory")
190
+
191
+ files = []
192
+ traversal_errors = []
193
+ root_depth = len(p.resolve().parts)
194
+ total = 0
195
+ def walk_error(exc):
196
+ traversal_errors.append(AnalysisError("file_read_error", str(exc.filename), str(exc)))
197
+
198
+ for folder, directories, names in os.walk(p, followlinks=False, onerror=walk_error):
199
+ directories[:] = sorted(d for d in directories if not (Path(folder) / d).is_symlink())
200
+ depth = len(Path(folder).resolve().parts) - root_depth
201
+ if depth + 1 >= options.max_directory_depth and directories:
202
+ traversal_errors.append(AnalysisError("directory_depth_limit", str(Path(folder).relative_to(p)), "Deeper directories skipped"))
203
+ directories[:] = []
204
+ for name in sorted(names):
205
+ child = Path(folder) / name
206
+ if child.is_symlink() or not child.is_file():
207
+ continue
208
+ files.append(child)
209
+ total += child.stat().st_size
210
+ if len(files) > options.max_directory_files:
211
+ raise InputError("directory contains too many files")
212
+ if total > options.max_total_selected_uncompressed_bytes:
213
+ raise InputError("directory exceeds scan byte budget")
214
+ files.sort()
215
+
216
+ root_hash = hashlib.sha256()
217
+ for child in files:
218
+ root_hash.update(str(child.relative_to(p)).replace("\\", "/").encode())
219
+ root_hash.update(sha256_file(child).encode())
220
+ inv = Inventory(InputRecord(path=str(p), format="directory", sha256=root_hash.hexdigest(), size=total))
221
+ inv.errors.extend(traversal_errors)
222
+ for child in files:
223
+ try:
224
+ fmt = detect_file(child)
225
+ except OSError as exc:
226
+ inv.errors.append(AnalysisError("file_read_error", str(child.relative_to(p)), str(exc), "warning"))
227
+ continue
228
+ if fmt in {"unknown", "zip", "jar", "aar"}:
229
+ continue
230
+ try:
231
+ remaining = options.max_total_selected_uncompressed_bytes - inv.selected_bytes
232
+ if remaining <= 0:
233
+ inv.errors.append(AnalysisError("archive_budget", str(child.relative_to(p)), "Directory selected-byte budget exhausted"))
234
+ break
235
+ child_inv = _scan_file(child, replace(options, max_total_selected_uncompressed_bytes=remaining), top_input=p, member_prefix=str(child.relative_to(p)).replace("\\", "/"))
236
+ child_inv.errors = [replace(e, member=f"{child.relative_to(p).as_posix()}!/{e.member}" if e.member else child.relative_to(p).as_posix()) for e in child_inv.errors]
237
+ _merge(inv, child_inv)
238
+ except InputError as exc:
239
+ inv.errors.append(AnalysisError("unsupported_file", str(child.relative_to(p)), str(exc), "warning"))
240
+ inv.frameworks_detected = sorted(inv.frameworks_detected, key=lambda h: (h.name, h.evidence))
241
+ _coverage(inv)
242
+ return inv
243
+
244
+
245
+ def empty_scan_result(inv: Inventory, tool_version: str) -> ScanResult:
246
+ return ScanResult(
247
+ schema_version=SCHEMA_VERSION,
248
+ tool_name="libidentifier",
249
+ tool_version=tool_version,
250
+ fingerprint_algorithm=FINGERPRINT_ALGORITHM,
251
+ profile_database=ProfileDatabaseRecord(),
252
+ input=inv.input,
253
+ analyzed_files=sorted(inv.analyzed_files, key=lambda a: (a.member or "", a.sha256)),
254
+ frameworks_detected=inv.frameworks_detected,
255
+ components=[],
256
+ class_ownership_hints=[],
257
+ unknown_classes=sorted((c.locator for c in inv.dex_classes), key=lambda x: (x.member or "", x.class_descriptor or "")),
258
+ coverage=inv.coverage,
259
+ errors=inv.errors,
260
+ )
@@ -0,0 +1,381 @@
1
+ """Generic profile matching engine."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import math
7
+ import sqlite3
8
+ from collections import Counter, defaultdict
9
+
10
+ from .config import IdentifyOptions
11
+ from .extractors.dex import ClassFeature
12
+ from .extractors.elf import ElfFeature
13
+ from .fingerprint.dex_v2 import digest_class, is_trivial_shape
14
+ from .models import CodeLocator, CodeOwnerHint, ComponentMatch
15
+ from .profiles.store import dex_class_rows, release_rows
16
+ from .purl import family_purl
17
+
18
+
19
+ def _component_id(name: str, family_id: str, input_sha: str) -> str:
20
+ return "cmp-" + hashlib.sha256(f"{name}:{family_id}:{input_sha}".encode()).hexdigest()[:16]
21
+
22
+
23
+ def _dex_matches(
24
+ conn: sqlite3.Connection,
25
+ classes: list[ClassFeature],
26
+ *,
27
+ input_sha: str,
28
+ options: IdentifyOptions,
29
+ ) -> tuple[list[ComponentMatch], list[CodeOwnerHint], set[tuple[str, str | None, str | None]]]:
30
+ observed = Counter()
31
+ locators: dict[str, list[CodeLocator]] = defaultdict(list)
32
+ observed_descriptors: dict[str, Counter[str]] = defaultdict(Counter)
33
+ discriminative_observed: set[str] = set()
34
+ for feature in classes:
35
+ digest = digest_class(feature)
36
+ observed[digest] += 1
37
+ locators[digest].append(feature.locator)
38
+ observed_descriptors[digest][feature.descriptor] += 1
39
+ if not is_trivial_shape(feature):
40
+ discriminative_observed.add(digest)
41
+ if not observed:
42
+ return [], [], set()
43
+
44
+ placeholders = ",".join("?" for _ in observed)
45
+ rows = list(
46
+ conn.execute(
47
+ f"SELECT digest, release_id, multiplicity FROM dex_shape WHERE digest IN ({placeholders})",
48
+ tuple(sorted(observed)),
49
+ )
50
+ )
51
+ release_ids = {row[1] for row in rows}
52
+ releases = release_rows(conn, release_ids)
53
+ if not rows:
54
+ return [], [], set()
55
+
56
+ digest_family_counts = {
57
+ digest: count
58
+ for digest, count in conn.execute(
59
+ f"""
60
+ SELECT ds.digest, COUNT(DISTINCT lr.family_id)
61
+ FROM dex_shape ds JOIN library_release lr USING(release_id)
62
+ WHERE ds.digest IN ({placeholders})
63
+ GROUP BY ds.digest
64
+ """,
65
+ tuple(sorted(observed)),
66
+ )
67
+ }
68
+ family_count = conn.execute("SELECT COUNT(*) FROM library_family").fetchone()[0] or 1
69
+ weights = {d: 1.0 + math.log((1.0 + family_count) / (1.0 + digest_family_counts.get(d, family_count))) for d in observed}
70
+
71
+ reference_counts: dict[str, Counter[str]] = defaultdict(Counter)
72
+ for digest, rid, multiplicity in rows:
73
+ reference_counts[rid][digest] = int(multiplicity)
74
+ totals: dict[str, float] = {}
75
+ for rid in reference_counts:
76
+ totals[rid] = sum(
77
+ weights.get(d, 1.0) * count
78
+ for d, count in conn.execute("SELECT digest, multiplicity FROM dex_shape WHERE release_id=?", (rid,))
79
+ )
80
+ profile_class_totals = {
81
+ rid: sum(
82
+ count
83
+ for _digest, count in conn.execute("SELECT digest, multiplicity FROM dex_shape WHERE release_id=?", (rid,))
84
+ )
85
+ for rid in reference_counts
86
+ }
87
+
88
+ candidates = []
89
+ for rid, counts in reference_counts.items():
90
+ structural_matched = sum(min(observed[d], c) for d, c in counts.items())
91
+ structural_weighted = sum(weights.get(d, 1.0) * min(observed[d], c) for d, c in counts.items())
92
+ reference_descriptors = dex_class_rows(conn, rid, set(counts))
93
+ exact_matched = 0
94
+ exact_weighted = 0.0
95
+ for digest, descriptor_counts in reference_descriptors.items():
96
+ for descriptor, ref_count in descriptor_counts.items():
97
+ count = min(observed_descriptors[digest][descriptor], ref_count)
98
+ exact_matched += count
99
+ exact_weighted += weights.get(digest, 1.0) * count
100
+ matched = exact_matched or structural_matched
101
+ weighted = exact_weighted or structural_weighted
102
+ match_mode = "exact_descriptor" if exact_matched else "structural_shape"
103
+ total = totals.get(rid) or 1.0
104
+ discriminative = sum(min(observed[d], c) for d, c in counts.items() if d in discriminative_observed)
105
+ unique_anchors = sum(1 for d in counts if d in discriminative_observed and digest_family_counts.get(d, 99) <= 1)
106
+ candidates.append(
107
+ {
108
+ "release_id": rid,
109
+ "family_id": releases[rid]["family_id"],
110
+ "coverage": weighted / total,
111
+ "matched": matched,
112
+ "structural_matched": structural_matched,
113
+ "exact_descriptor_matches": exact_matched,
114
+ "profile_classes": profile_class_totals.get(rid, sum(counts.values())),
115
+ "discriminative": discriminative,
116
+ "unique_anchors": unique_anchors,
117
+ "digests": set(counts),
118
+ "counts": counts,
119
+ "reference_descriptors": reference_descriptors,
120
+ "match_mode": match_mode,
121
+ "weighted_total": total,
122
+ }
123
+ )
124
+
125
+ by_family: dict[str, list[dict]] = defaultdict(list)
126
+ for cand in candidates:
127
+ by_family[cand["family_id"]].append(cand)
128
+ family_best = []
129
+ for fid, cands in by_family.items():
130
+ best = sorted(cands, key=lambda c: (-c["coverage"], -c["matched"], c["release_id"]))[0]
131
+ family_best.append(best)
132
+ family_best.sort(key=lambda c: (-c["coverage"], -c["matched"], c["family_id"]))
133
+
134
+ components: list[ComponentMatch] = []
135
+ hints: list[CodeOwnerHint] = []
136
+ assigned: set[tuple[str, str | None, str | None]] = set()
137
+ for index, cand in enumerate(family_best):
138
+ runner = 0.0
139
+ for j, other in enumerate(family_best):
140
+ if j == index:
141
+ continue
142
+ overlap = cand["digests"].intersection(other["digests"])
143
+ if not overlap:
144
+ continue
145
+ overlap_weight = sum(
146
+ weights.get(d, 1.0) * min(cand["counts"].get(d, 0), other["counts"].get(d, 0))
147
+ for d in overlap
148
+ )
149
+ material_overlap = overlap_weight / max(min(cand["weighted_total"], other["weighted_total"]), 1.0)
150
+ if material_overlap >= options.min_material_runner_overlap:
151
+ runner = max(runner, other["coverage"])
152
+ runner_delta = cand["coverage"] - runner
153
+ if cand["discriminative"] >= options.high_min_discriminative_classes and cand["coverage"] >= options.high_min_weighted_coverage and cand["unique_anchors"] >= options.high_min_unique_anchors and runner_delta >= options.high_min_runner_up_delta:
154
+ confidence = "high"
155
+ partial_match = False
156
+ elif cand["discriminative"] >= options.medium_min_discriminative_classes and cand["coverage"] >= options.medium_min_weighted_coverage and runner_delta >= 0:
157
+ confidence = "medium"
158
+ partial_match = False
159
+ elif (
160
+ cand["match_mode"] == "exact_descriptor"
161
+ and cand["exact_descriptor_matches"] >= options.medium_partial_min_exact_classes
162
+ and cand["coverage"] >= options.medium_partial_min_weighted_coverage
163
+ and cand["unique_anchors"] >= options.medium_partial_min_unique_anchors
164
+ and runner_delta >= 0
165
+ ):
166
+ confidence = "medium"
167
+ partial_match = True
168
+ else:
169
+ continue
170
+ family_releases = sorted(by_family[cand["family_id"]], key=lambda c: (-c["coverage"], c["release_id"]))
171
+ supported_releases = [c for c in family_releases if c is cand or (
172
+ c["coverage"] >= options.medium_min_weighted_coverage
173
+ and c["discriminative"] >= options.medium_min_discriminative_classes
174
+ ) or (
175
+ c["match_mode"] == "exact_descriptor"
176
+ and c["coverage"] >= options.medium_partial_min_weighted_coverage
177
+ and c["exact_descriptor_matches"] >= options.medium_partial_min_exact_classes
178
+ and c["unique_anchors"] >= options.medium_partial_min_unique_anchors
179
+ )]
180
+ all_versions = sorted({releases[c["release_id"]]["version"] for c in supported_releases})
181
+ candidate_versions = all_versions[: options.max_version_candidates]
182
+ exact = [
183
+ releases[c["release_id"]]
184
+ for c in family_releases
185
+ if input_sha in releases[c["release_id"]]['source_hashes']
186
+ ]
187
+ ambiguous_exact = False
188
+ if exact:
189
+ families = conn.execute('SELECT COUNT(DISTINCT r.family_id) FROM reference_artifact a JOIN library_release r USING(release_id) WHERE a.sha256=?', (input_sha,)).fetchone()[0]
190
+ if families != 1:
191
+ ambiguous_exact = True
192
+ exact = []
193
+ version = exact[0]["version"] if len(exact) == 1 else None
194
+ purl = exact[0]["purl"] if len(exact) == 1 else None
195
+ version_confidence = "high" if version else "unknown"
196
+ matched_locs = []
197
+ ambiguous_locs = []
198
+ for digest in sorted(cand["digests"]):
199
+ remaining = cand["counts"][digest]
200
+ descriptor_counts = Counter(cand["reference_descriptors"].get(digest, {}))
201
+ exact_locators = [
202
+ locator
203
+ for locator in locators[digest]
204
+ if locator.class_descriptor in descriptor_counts and descriptor_counts[locator.class_descriptor] > 0
205
+ ]
206
+ if cand["exact_descriptor_matches"]:
207
+ locator_pool = exact_locators
208
+ else:
209
+ locator_pool = locators[digest]
210
+ if len(locator_pool) > remaining:
211
+ # A shape/count cannot choose which physical definition belongs.
212
+ ambiguous_locs.extend(locator_pool)
213
+ matched_locs.extend(locator_pool)
214
+ continue
215
+ for locator in locator_pool:
216
+ if remaining <= 0:
217
+ break
218
+ matched_locs.append(locator)
219
+ if locator.class_descriptor in descriptor_counts:
220
+ descriptor_counts[locator.class_descriptor] -= 1
221
+ remaining -= 1
222
+ if not matched_locs:
223
+ continue
224
+ release = releases[cand["release_id"]]
225
+ cid = _component_id(release["name"], cand["family_id"], input_sha)
226
+ for locator in matched_locs:
227
+ key = (locator.container, locator.member, locator.class_descriptor)
228
+ assigned.add(key)
229
+ hints.append(
230
+ CodeOwnerHint(
231
+ subject=locator.class_descriptor or "",
232
+ origin="third_party",
233
+ component_id=cid,
234
+ confidence=confidence,
235
+ method="structural_fingerprint",
236
+ )
237
+ )
238
+ components.append(
239
+ ComponentMatch(
240
+ component_id=cid,
241
+ name=release["name"],
242
+ ecosystem=release["ecosystem"],
243
+ purl=purl,
244
+ family_purl=family_purl(release["purl"]),
245
+ version_candidates_total=len(all_versions),
246
+ version_candidates_truncated=len(candidate_versions) < len(all_versions),
247
+ version=version,
248
+ version_candidates=candidate_versions,
249
+ identity_confidence=confidence,
250
+ version_confidence=version_confidence,
251
+ identification_methods=["dex_shape_v1"],
252
+ score=round(cand["coverage"], 6),
253
+ score_details={
254
+ "version_candidate_scores": {releases[c["release_id"]]["version"]: {"weighted_coverage": round(c["coverage"], 6), "exact_descriptor_matches": c["exact_descriptor_matches"], "supported": c in supported_releases} for c in family_releases},
255
+ "matched_classes": cand["matched"],
256
+ "profile_classes": cand["profile_classes"],
257
+ "exact_descriptor_matches": cand["exact_descriptor_matches"],
258
+ "structural_matched_classes": cand["structural_matched"],
259
+ "match_mode": cand["match_mode"],
260
+ "partial_match": partial_match,
261
+ "weighted_coverage": round(cand["coverage"], 6),
262
+ "unique_anchors": cand["unique_anchors"],
263
+ "runner_up_delta": round(runner_delta, 6),
264
+ "ambiguous_code": [x.to_dict() for x in ambiguous_locs],
265
+ },
266
+ matched_code=sorted(matched_locs, key=lambda x: (x.container, x.member or "", x.class_descriptor or "")),
267
+ source_artifacts=sorted({x.member or x.container for x in matched_locs}),
268
+ limitations=(
269
+ ([] if version else ["release version cannot be distinguished from structural fingerprints alone"])
270
+ + (["Exact reference bytes are assigned to multiple library families; identity/version remain ambiguous"] if ambiguous_exact else [])
271
+ + (["partial library match; not all reference classes were observed"] if partial_match else [])
272
+ + (
273
+ ["matched code is assigned by structural shape because reference descriptors were not observed"]
274
+ if cand["match_mode"] == "structural_shape"
275
+ else []
276
+ )
277
+ ),
278
+ )
279
+ )
280
+ return components, hints, assigned
281
+
282
+
283
+ def _elf_matches(
284
+ conn: sqlite3.Connection,
285
+ elf_features: list[tuple[ElfFeature, object]],
286
+ *,
287
+ input_sha: str,
288
+ container: str,
289
+ options: IdentifyOptions,
290
+ ) -> list[ComponentMatch]:
291
+ components: dict[str, ComponentMatch] = {}
292
+ for feature, artifact in elf_features:
293
+ rows = list(conn.execute(
294
+ """
295
+ SELECT DISTINCT r.family_id, r.version, r.purl, f.name, f.ecosystem
296
+ FROM elf_identity e
297
+ JOIN library_release r USING(release_id)
298
+ JOIN library_family f USING(family_id)
299
+ WHERE e.sha256=? ORDER BY f.name, r.version, r.purl
300
+ """, (feature.sha256,),
301
+ ))
302
+ # Identical bytes assigned to different families cannot establish identity.
303
+ if not rows or len({r[0] for r in rows}) != 1:
304
+ continue
305
+ fid, _, _, name, ecosystem = rows[0]
306
+ versions = sorted({r[1] for r in rows})
307
+ version = versions[0] if len(versions) == 1 else None
308
+ purls = {r[2] for r in rows}
309
+ purl = next(iter(purls)) if version and len(purls) == 1 else None
310
+ families = {family_purl(r[2]) for r in rows}
311
+ cid = _component_id(name, fid, input_sha + feature.sha256)
312
+ locator = CodeLocator(container=getattr(artifact, "container", None) or container,
313
+ member=getattr(artifact, "member", None), sha256=feature.sha256)
314
+ source = getattr(artifact, "member", None) or getattr(artifact, "path", None) or container
315
+ if cid in components:
316
+ components[cid].matched_code.append(locator)
317
+ if source not in components[cid].source_artifacts:
318
+ components[cid].source_artifacts.append(source)
319
+ continue
320
+ candidates = versions[:options.max_version_candidates]
321
+ components[cid] = ComponentMatch(
322
+ component_id=cid, name=name, ecosystem=ecosystem, purl=purl,
323
+ family_purl=next(iter(families)) if len(families) == 1 else None,
324
+ version=version, version_candidates=candidates,
325
+ version_candidates_total=len(versions), version_candidates_truncated=len(candidates) < len(versions),
326
+ identity_confidence="high", version_confidence="high" if version else "unknown",
327
+ identification_methods=["elf_identity_v1"], score=1.0,
328
+ score_details={"matched_classes": 0, "profile_classes": 0, "weighted_coverage": 1.0,
329
+ "unique_anchors": 1, "runner_up_delta": 1.0},
330
+ matched_code=[locator], source_artifacts=[source],
331
+ limitations=[] if version else ["Identical ELF bytes occur in multiple reference releases; exact version is unknown"],
332
+ )
333
+ for component in components.values():
334
+ component.matched_code.sort(key=lambda loc: (loc.container, loc.member or ""))
335
+ component.source_artifacts.sort()
336
+ return list(components.values())
337
+
338
+
339
+ def match_inventory(conn: sqlite3.Connection, inventory, options: IdentifyOptions):
340
+ dex_components, hints, assigned = _dex_matches(conn, inventory.dex_classes, input_sha=inventory.input.sha256, options=options)
341
+ elf_components = _elf_matches(conn, inventory.elf_features, input_sha=inventory.input.sha256, container=inventory.input.path, options=options)
342
+ from dataclasses import replace
343
+ from .models import OwnershipAssertion
344
+ claims = defaultdict(set)
345
+ ambiguous = set()
346
+ for component in dex_components:
347
+ for loc in component.matched_code:
348
+ claims[(loc.container, loc.member, loc.class_descriptor)].add(component.component_id)
349
+ for raw in component.score_details.get('ambiguous_code', []):
350
+ ambiguous.add((raw['container'], raw['member'], raw['class_descriptor']))
351
+ descriptors = Counter(c.descriptor for c in inventory.dex_classes)
352
+ assigned = set()
353
+ assertions = []
354
+ for feature in inventory.dex_classes:
355
+ loc = feature.locator
356
+ key = (loc.container, loc.member, loc.class_descriptor)
357
+ owners = sorted(claims[key])
358
+ disputed = len(owners) > 1 or key in ambiguous or descriptors[feature.descriptor] > 1
359
+ status = 'ambiguous' if disputed else ('attributed' if owners else 'unattributed')
360
+ if status == 'attributed':
361
+ assigned.add(key)
362
+ assertions.append(OwnershipAssertion(loc, status, owners,
363
+ limitations=['Competing or duplicate code definitions; ownership unresolved'] if disputed else []))
364
+ inventory.ownership_assertions = assertions
365
+ cleaned = []
366
+ for component in dex_components:
367
+ locations = [loc for loc in component.matched_code if (loc.container, loc.member, loc.class_descriptor) in assigned]
368
+ cleaned.append(replace(component, matched_code=locations,
369
+ limitations=component.limitations + (['Some matching evidence has ambiguous ownership'] if len(locations) < len(component.matched_code) else [])))
370
+ components = sorted(cleaned + elf_components, key=lambda c: (c.name, c.component_id))
371
+ hints = [CodeOwnerHint(loc.class_descriptor or '', 'third_party', c.component_id, c.identity_confidence, 'structural_fingerprint')
372
+ for c in cleaned for loc in c.matched_code]
373
+ hints.sort(key=lambda h: (h.component_id, h.subject))
374
+ unknown = [
375
+ feature.locator
376
+ for feature in inventory.dex_classes
377
+ if (feature.locator.container, feature.locator.member, feature.locator.class_descriptor) not in assigned
378
+ ]
379
+ unknown.sort(key=lambda x: (x.container, x.member or "", x.class_descriptor or ""))
380
+ return components, hints, unknown
381
+