libidentifier 0.2.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- libidentifier/__init__.py +36 -0
- libidentifier/api.py +90 -0
- libidentifier/archive.py +133 -0
- libidentifier/cli.py +340 -0
- libidentifier/cli_resources.py +36 -0
- libidentifier/cli_setup.py +177 -0
- libidentifier/config.py +54 -0
- libidentifier/detect.py +30 -0
- libidentifier/errors.py +30 -0
- libidentifier/extractors/__init__.py +2 -0
- libidentifier/extractors/dex.py +121 -0
- libidentifier/extractors/dex_tables.py +137 -0
- libidentifier/extractors/elf.py +66 -0
- libidentifier/extractors/instructions.py +77 -0
- libidentifier/extractors/metadata.py +99 -0
- libidentifier/extractors/version_metadata.py +72 -0
- libidentifier/fingerprint/__init__.py +2 -0
- libidentifier/fingerprint/dex_v2.py +97 -0
- libidentifier/fingerprint/elf_v1.py +16 -0
- libidentifier/grouping.py +74 -0
- libidentifier/inventory.py +260 -0
- libidentifier/matcher.py +381 -0
- libidentifier/models.py +423 -0
- libidentifier/profiles/__init__.py +2 -0
- libidentifier/profiles/builder.py +182 -0
- libidentifier/profiles/catalog.py +135 -0
- libidentifier/profiles/distribution.py +303 -0
- libidentifier/profiles/fetcher.py +141 -0
- libidentifier/profiles/pom.py +66 -0
- libidentifier/profiles/schema.py +137 -0
- libidentifier/profiles/store.py +212 -0
- libidentifier/purl.py +36 -0
- libidentifier/relationships.py +168 -0
- libidentifier/sbom.py +62 -0
- libidentifier/serialization.py +53 -0
- libidentifier/table.py +254 -0
- libidentifier/version.py +2 -0
- libidentifier/vulnerabilities/__init__.py +2 -0
- libidentifier/vulnerabilities/correlate.py +164 -0
- libidentifier/vulnerabilities/database.py +213 -0
- libidentifier/vulnerabilities/models.py +155 -0
- libidentifier/vulnerabilities/runner.py +148 -0
- libidentifier/vulnerabilities/setup.py +74 -0
- libidentifier-0.2.0rc1.dist-info/METADATA +156 -0
- libidentifier-0.2.0rc1.dist-info/RECORD +50 -0
- libidentifier-0.2.0rc1.dist-info/WHEEL +5 -0
- libidentifier-0.2.0rc1.dist-info/entry_points.txt +2 -0
- libidentifier-0.2.0rc1.dist-info/licenses/LICENSE +62 -0
- libidentifier-0.2.0rc1.dist-info/licenses/THIRD_PARTY_NOTICES.md +15 -0
- libidentifier-0.2.0rc1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Public API for LibIdentifier."""
|
|
2
|
+
|
|
3
|
+
from .api import build_profile, export_cyclonedx, identify, inspect, correlate, scan
|
|
4
|
+
from .vulnerabilities.models import AssessmentResult, VulnerabilityOptions, VulnerabilityReport
|
|
5
|
+
from .vulnerabilities.database import provision_advisory_database, update_advisory_database, validate_advisory_database
|
|
6
|
+
from .config import IdentifyOptions
|
|
7
|
+
from .models import ProfileBuildResult, ScanResult
|
|
8
|
+
from .models import (Producer, AnalysisRun, EvidenceRecord, EntityRef, OwnershipAssertion,
|
|
9
|
+
Relationship, UnattributedCluster)
|
|
10
|
+
from .version import __version__
|
|
11
|
+
from .profiles.distribution import install_profiles, package_profiles, validate_profiles
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"install_profiles",
|
|
15
|
+
"package_profiles",
|
|
16
|
+
"validate_profiles",
|
|
17
|
+
"IdentifyOptions",
|
|
18
|
+
"ProfileBuildResult",
|
|
19
|
+
"ScanResult",
|
|
20
|
+
"build_profile",
|
|
21
|
+
"export_cyclonedx",
|
|
22
|
+
"identify",
|
|
23
|
+
"inspect",
|
|
24
|
+
"correlate",
|
|
25
|
+
"scan",
|
|
26
|
+
"AssessmentResult",
|
|
27
|
+
"VulnerabilityOptions",
|
|
28
|
+
"VulnerabilityReport",
|
|
29
|
+
"provision_advisory_database",
|
|
30
|
+
"update_advisory_database",
|
|
31
|
+
"validate_advisory_database",
|
|
32
|
+
"Producer", "AnalysisRun", "EvidenceRecord", "EntityRef", "OwnershipAssertion",
|
|
33
|
+
"Relationship", "UnattributedCluster",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
|
libidentifier/api.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Python API entry points."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sqlite3
|
|
6
|
+
from dataclasses import replace
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .config import FINGERPRINT_ALGORITHM, SCHEMA_VERSION, IdentifyOptions
|
|
10
|
+
from .errors import ProfileError
|
|
11
|
+
from .inventory import empty_scan_result, inspect_path
|
|
12
|
+
from .matcher import match_inventory
|
|
13
|
+
from .models import ScanResult
|
|
14
|
+
from .profiles.builder import build_profile
|
|
15
|
+
from .profiles.store import connect_readonly, database_record
|
|
16
|
+
from .sbom import export_cyclonedx
|
|
17
|
+
from .version import __version__
|
|
18
|
+
from .vulnerabilities.models import AssessmentResult, VulnerabilityOptions, VulnerabilityReport
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _result_coverage(coverage, *, total: int, emitted: int, identification: str):
|
|
22
|
+
limitations = list(coverage.limitations)
|
|
23
|
+
limitations.append("Identification is limited to supplied reference profiles; no matches does not mean no third-party code")
|
|
24
|
+
if identification == "not_performed":
|
|
25
|
+
limitations.append("Library matching was not performed (inspect has no reference database)")
|
|
26
|
+
elif identification == "no_profiles":
|
|
27
|
+
limitations.append("The reference database contains no profiles")
|
|
28
|
+
if emitted < total:
|
|
29
|
+
limitations.append(f"Unknown class output truncated: emitted {emitted} of {total} unmatched definitions")
|
|
30
|
+
return replace(coverage, library_identification=identification,
|
|
31
|
+
unknown_classes_total=total, unknown_classes_emitted=emitted,
|
|
32
|
+
unknown_classes_truncated=emitted < total, limitations=limitations)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def inspect(path, *, options: IdentifyOptions | None = None) -> ScanResult:
|
|
36
|
+
opts = options or IdentifyOptions()
|
|
37
|
+
inv = inspect_path(Path(path), opts)
|
|
38
|
+
result = empty_scan_result(inv, __version__)
|
|
39
|
+
from .relationships import enrich
|
|
40
|
+
from .extractors.version_metadata import apply_version_metadata
|
|
41
|
+
result = enrich(apply_version_metadata(result, inv), inv, opts)
|
|
42
|
+
unknown = result.unknown_classes if opts.max_emitted_unknown_classes is None else result.unknown_classes[:opts.max_emitted_unknown_classes]
|
|
43
|
+
return replace(result, unknown_classes=unknown,
|
|
44
|
+
coverage=_result_coverage(result.coverage, total=len(inv.dex_classes), emitted=len(unknown), identification="not_performed"))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def identify(path, *, profiles, options: IdentifyOptions | None = None) -> ScanResult:
|
|
48
|
+
opts = options or IdentifyOptions()
|
|
49
|
+
inv = inspect_path(Path(path), opts)
|
|
50
|
+
conn = connect_readonly(profiles)
|
|
51
|
+
try:
|
|
52
|
+
components, hints, unknown = match_inventory(conn, inv, opts)
|
|
53
|
+
database = database_record(profiles)
|
|
54
|
+
result = ScanResult(
|
|
55
|
+
schema_version=SCHEMA_VERSION, tool_name="libidentifier", tool_version=__version__,
|
|
56
|
+
fingerprint_algorithm=FINGERPRINT_ALGORITHM, profile_database=database,
|
|
57
|
+
input=inv.input,
|
|
58
|
+
analyzed_files=sorted(inv.analyzed_files, key=lambda a: (a.member or "", a.sha256)),
|
|
59
|
+
frameworks_detected=inv.frameworks_detected, components=components,
|
|
60
|
+
class_ownership_hints=hints, unknown_classes=unknown, coverage=inv.coverage, errors=inv.errors)
|
|
61
|
+
from .extractors.version_metadata import apply_version_metadata
|
|
62
|
+
from .relationships import enrich
|
|
63
|
+
result = enrich(apply_version_metadata(result, inv), inv, opts, conn)
|
|
64
|
+
except sqlite3.Error as exc:
|
|
65
|
+
raise ProfileError(f"cannot query profile database: {exc}") from exc
|
|
66
|
+
finally:
|
|
67
|
+
conn.close()
|
|
68
|
+
emitted = unknown if opts.max_emitted_unknown_classes is None else unknown[:opts.max_emitted_unknown_classes]
|
|
69
|
+
return replace(result, unknown_classes=emitted,
|
|
70
|
+
coverage=_result_coverage(inv.coverage, total=len(unknown), emitted=len(emitted),
|
|
71
|
+
identification="limited" if database.profile_count else "no_profiles"))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def correlate(result: ScanResult, *, options: VulnerabilityOptions) -> VulnerabilityReport:
|
|
76
|
+
"""Assess an existing inventory using explicitly configured local OSV data."""
|
|
77
|
+
from .vulnerabilities.correlate import correlate as assess
|
|
78
|
+
return assess(result, options=options)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def scan(path, *, profiles, options: IdentifyOptions | None = None,
|
|
82
|
+
vulnerabilities: VulnerabilityOptions | None = None) -> AssessmentResult:
|
|
83
|
+
"""Identify, then optionally assess; never provision or download dependencies."""
|
|
84
|
+
from .vulnerabilities.models import AssessmentResult, VulnerabilityReport
|
|
85
|
+
result = identify(path, profiles=profiles, options=options)
|
|
86
|
+
report = correlate(result, options=vulnerabilities) if vulnerabilities is not None else VulnerabilityReport(
|
|
87
|
+
"not_requested", result.input, result.tool_version, result.profile_database,
|
|
88
|
+
limitations=["Vulnerability assessment was not requested"],
|
|
89
|
+
identification_coverage=result.coverage, identification_errors=result.errors)
|
|
90
|
+
return AssessmentResult(result, report)
|
libidentifier/archive.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Bounded ZIP/APK traversal for untrusted archives."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import posixpath
|
|
7
|
+
import stat
|
|
8
|
+
import zipfile
|
|
9
|
+
import zlib
|
|
10
|
+
from collections import Counter
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import PurePosixPath
|
|
13
|
+
|
|
14
|
+
from .config import IdentifyOptions
|
|
15
|
+
from .detect import detect_bytes
|
|
16
|
+
from .models import AnalysisError
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class ArchiveMember:
|
|
21
|
+
name: str
|
|
22
|
+
format: str
|
|
23
|
+
data: bytes
|
|
24
|
+
sha256: str
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _safe_member_name(name: str) -> bool:
|
|
28
|
+
if "\x00" in name or "\\" in name:
|
|
29
|
+
return False
|
|
30
|
+
if name.startswith("/") or name.startswith("//"):
|
|
31
|
+
return False
|
|
32
|
+
if len(name) >= 2 and name[1] == ":":
|
|
33
|
+
return False
|
|
34
|
+
if ".." in PurePosixPath(name).parts:
|
|
35
|
+
return False
|
|
36
|
+
normalized = posixpath.normpath(name)
|
|
37
|
+
if normalized == "." or normalized.startswith("../") or normalized == "..":
|
|
38
|
+
return False
|
|
39
|
+
parts = PurePosixPath(normalized).parts
|
|
40
|
+
return ".." not in parts
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _is_symlink(info: zipfile.ZipInfo) -> bool:
|
|
44
|
+
mode = (info.external_attr >> 16) & 0o170000
|
|
45
|
+
return mode == stat.S_IFLNK
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _interesting_by_path(name: str) -> bool:
|
|
49
|
+
from .extractors.metadata import ASSETS
|
|
50
|
+
|
|
51
|
+
lowered = name.lower()
|
|
52
|
+
base = PurePosixPath(lowered).name
|
|
53
|
+
return (
|
|
54
|
+
(base.startswith("classes") and base.endswith(".dex"))
|
|
55
|
+
or (lowered.startswith("lib/") and lowered.endswith(".so"))
|
|
56
|
+
or lowered.endswith("/pom.properties")
|
|
57
|
+
or lowered == "androidmanifest.xml"
|
|
58
|
+
or base in {"libflutter.so", "libapp.so"}
|
|
59
|
+
or any(name in paths for paths in ASSETS.values())
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def read_selected_members(path, options: IdentifyOptions) -> tuple[list[ArchiveMember], list[AnalysisError]]:
|
|
64
|
+
errors: list[AnalysisError] = []
|
|
65
|
+
members: list[ArchiveMember] = []
|
|
66
|
+
total = 0
|
|
67
|
+
with zipfile.ZipFile(path) as zf:
|
|
68
|
+
infos = sorted(zf.infolist(), key=lambda i: i.filename)
|
|
69
|
+
if len(infos) > options.max_archive_members:
|
|
70
|
+
raise ValueError("archive has too many members")
|
|
71
|
+
occurrences = Counter(info.filename for info in infos)
|
|
72
|
+
duplicates_reported = set()
|
|
73
|
+
for info in infos:
|
|
74
|
+
name = info.filename
|
|
75
|
+
if info.is_dir():
|
|
76
|
+
continue
|
|
77
|
+
if occurrences[name] > 1:
|
|
78
|
+
if name not in duplicates_reported:
|
|
79
|
+
errors.append(AnalysisError("duplicate_member", name, "Duplicate ZIP member names cannot be located unambiguously; skipped"))
|
|
80
|
+
duplicates_reported.add(name)
|
|
81
|
+
continue
|
|
82
|
+
if not _safe_member_name(name):
|
|
83
|
+
errors.append(AnalysisError("unsafe_member_path", name, "ZIP member path is unsafe", "warning"))
|
|
84
|
+
continue
|
|
85
|
+
if _is_symlink(info):
|
|
86
|
+
errors.append(AnalysisError("unsupported_member", name, "ZIP symlink member skipped", "warning"))
|
|
87
|
+
continue
|
|
88
|
+
if info.flag_bits & 0x1:
|
|
89
|
+
errors.append(AnalysisError("encrypted_member", name, "Encrypted ZIP member skipped", "warning"))
|
|
90
|
+
continue
|
|
91
|
+
if info.compress_type not in {zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED}:
|
|
92
|
+
errors.append(AnalysisError("unsupported_compression", name, "Unsupported ZIP compression skipped", "warning"))
|
|
93
|
+
continue
|
|
94
|
+
if info.file_size > options.max_member_uncompressed_bytes:
|
|
95
|
+
errors.append(AnalysisError("member_too_large", name, "ZIP member exceeds size limit", "warning"))
|
|
96
|
+
continue
|
|
97
|
+
if info.compress_size and info.file_size / max(info.compress_size, 1) > options.max_compression_ratio:
|
|
98
|
+
errors.append(AnalysisError("compression_ratio", name, "ZIP member compression ratio exceeds limit", "warning"))
|
|
99
|
+
continue
|
|
100
|
+
|
|
101
|
+
selected = _interesting_by_path(name)
|
|
102
|
+
remaining = options.max_total_selected_uncompressed_bytes - total
|
|
103
|
+
try:
|
|
104
|
+
with zf.open(info) as fh:
|
|
105
|
+
head = fh.read(8)
|
|
106
|
+
fmt = detect_bytes(head, name=name)
|
|
107
|
+
selected = selected or fmt in {"dex", "elf"}
|
|
108
|
+
if not selected:
|
|
109
|
+
continue
|
|
110
|
+
if info.file_size > remaining:
|
|
111
|
+
errors.append(AnalysisError("archive_budget", name, "Selected archive members exceed total byte limit"))
|
|
112
|
+
break
|
|
113
|
+
limit = min(options.max_member_uncompressed_bytes, remaining)
|
|
114
|
+
rest = fh.read(max(0, limit + 1 - len(head)))
|
|
115
|
+
data = head + rest
|
|
116
|
+
except (OSError, RuntimeError, zipfile.BadZipFile, zlib.error) as exc:
|
|
117
|
+
errors.append(AnalysisError("member_read_error", name, str(exc), "error"))
|
|
118
|
+
continue
|
|
119
|
+
if len(data) > options.max_member_uncompressed_bytes:
|
|
120
|
+
errors.append(AnalysisError("member_too_large", name, "ZIP member exceeds size limit", "warning"))
|
|
121
|
+
continue
|
|
122
|
+
total += len(data)
|
|
123
|
+
if total > options.max_total_selected_uncompressed_bytes:
|
|
124
|
+
errors.append(AnalysisError("archive_budget", name, "Selected archive members exceed total byte limit", "warning"))
|
|
125
|
+
break
|
|
126
|
+
fmt = detect_bytes(data, name=name)
|
|
127
|
+
if fmt == "unknown" and (name.lower().endswith(".dex") or name.lower().endswith(".so")):
|
|
128
|
+
errors.append(AnalysisError("member_read_error", name, "Selected binary has invalid magic", "error"))
|
|
129
|
+
continue
|
|
130
|
+
if fmt in {"unknown", "zip"} and name.lower().endswith("/pom.properties"):
|
|
131
|
+
fmt = "metadata"
|
|
132
|
+
members.append(ArchiveMember(name=name, format=fmt, data=data, sha256=hashlib.sha256(data).hexdigest()))
|
|
133
|
+
return members, errors
|
libidentifier/cli.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
"""Command-line interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from .api import identify, inspect, scan, correlate
|
|
12
|
+
from .vulnerabilities.models import AssessmentResult
|
|
13
|
+
from .vulnerabilities.setup import prepare_options
|
|
14
|
+
from .vulnerabilities.correlate import unavailable_report
|
|
15
|
+
from .vulnerabilities.database import provision_advisory_database, update_advisory_database, validate_advisory_database
|
|
16
|
+
from .errors import AdvisoryError, InputError, LibIdentifierError, ProfileError
|
|
17
|
+
from .models import ScanResult
|
|
18
|
+
from .config import IdentifyOptions
|
|
19
|
+
from .serialization import write_stream
|
|
20
|
+
from .table import write_table
|
|
21
|
+
from .profiles.builder import build_profile, validate_database
|
|
22
|
+
from .profiles.catalog import load_catalog
|
|
23
|
+
from .profiles.fetcher import fetch_catalog, rebuild_database
|
|
24
|
+
from .profiles.distribution import DEFAULT_ENGINE_REQUIREMENT, DEFAULT_PROFILE_MANIFEST, install_profiles, package_profiles
|
|
25
|
+
from .sbom import export_cyclonedx
|
|
26
|
+
from .version import __version__
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _write_result(result: ScanResult, args) -> None:
|
|
30
|
+
if args.format == "json":
|
|
31
|
+
if args.output:
|
|
32
|
+
result.write_json(args.output)
|
|
33
|
+
else:
|
|
34
|
+
write_stream(result, sys.stdout)
|
|
35
|
+
else:
|
|
36
|
+
write_table(result)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _identify_options(args):
|
|
40
|
+
return IdentifyOptions(max_emitted_unknown_classes=args.max_unknown_classes,
|
|
41
|
+
extract_relationships=not args.no_relationships, extract_calls=not args.no_calls,
|
|
42
|
+
group_unknown_classes=not args.no_grouping)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _cmd_inspect(args) -> int:
|
|
46
|
+
result = inspect(args.path, options=_identify_options(args))
|
|
47
|
+
_write_result(result, args)
|
|
48
|
+
return 4 if any(e.severity == "error" for e in result.errors) else 0
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _cmd_identify(args) -> int:
|
|
52
|
+
profiles = args.profiles or os.environ.get("LIBIDENTIFIER_PROFILES")
|
|
53
|
+
if not profiles:
|
|
54
|
+
raise ProfileError(
|
|
55
|
+
"no profile database specified; pass --profiles, set LIBIDENTIFIER_PROFILES, "
|
|
56
|
+
"or build one with 'libidentifier profiles rebuild'"
|
|
57
|
+
)
|
|
58
|
+
result = identify(args.path, profiles=profiles, options=_identify_options(args))
|
|
59
|
+
if args.vulnerabilities:
|
|
60
|
+
assessment = AssessmentResult(result, _assess(result, args))
|
|
61
|
+
_write_assessment(assessment, args)
|
|
62
|
+
return _assessment_exit(assessment)
|
|
63
|
+
_write_result(result, args)
|
|
64
|
+
return 4 if any(e.severity == "error" for e in result.errors) else 0
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _cmd_profiles_build(args) -> int:
|
|
68
|
+
built = build_profile(
|
|
69
|
+
args.path,
|
|
70
|
+
purl=args.purl,
|
|
71
|
+
database=args.database,
|
|
72
|
+
d8=args.d8,
|
|
73
|
+
android_jar=args.android_jar,
|
|
74
|
+
reference_sha256=args.sha256,
|
|
75
|
+
)
|
|
76
|
+
sys.stdout.write(json.dumps(built.to_dict(), sort_keys=True, separators=(",", ":")) + "\n")
|
|
77
|
+
return 0
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _cmd_profiles_package(args) -> int:
|
|
81
|
+
manifest = package_profiles(args.database, output_dir=args.output_dir,
|
|
82
|
+
snapshot_version=args.snapshot_version, source_revision=args.source_revision,
|
|
83
|
+
requires_engine=args.requires_engine)
|
|
84
|
+
sys.stdout.write(json.dumps(manifest, indent=2) + "\n")
|
|
85
|
+
return 0
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _cmd_profiles_install(args) -> int:
|
|
89
|
+
manifest = install_profiles(args.manifest, database=args.database, offline=args.offline,
|
|
90
|
+
replace=args.replace, manifest_sha256=args.manifest_sha256,
|
|
91
|
+
timeout_seconds=args.timeout_seconds)
|
|
92
|
+
sys.stdout.write(json.dumps(manifest, indent=2) + "\n")
|
|
93
|
+
return 0
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _cmd_profiles_validate(args) -> int:
|
|
97
|
+
validate_database(args.database)
|
|
98
|
+
sys.stdout.write("profile database is valid\n")
|
|
99
|
+
return 0
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _cmd_profiles_fetch(args) -> int:
|
|
103
|
+
catalog = load_catalog(args.catalog)
|
|
104
|
+
rows = fetch_catalog(catalog, args.cache, offline=args.offline)
|
|
105
|
+
sys.stdout.write(json.dumps(rows, sort_keys=True, separators=(",", ":")) + "\n")
|
|
106
|
+
return 0
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _cmd_profiles_rebuild(args) -> int:
|
|
110
|
+
rows = rebuild_database(
|
|
111
|
+
args.catalog,
|
|
112
|
+
cache_dir=args.cache,
|
|
113
|
+
database=args.database,
|
|
114
|
+
d8=args.d8,
|
|
115
|
+
android_jar=args.android_jar,
|
|
116
|
+
offline=args.offline,
|
|
117
|
+
)
|
|
118
|
+
sys.stdout.write(json.dumps(rows, sort_keys=True, separators=(",", ":")) + "\n")
|
|
119
|
+
return 0
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _cmd_sbom(args) -> int:
|
|
123
|
+
data = json.loads(Path(args.input).read_text(encoding="utf-8"))
|
|
124
|
+
export_cyclonedx(ScanResult.from_dict(data), args.output)
|
|
125
|
+
return 0
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _assess(result, args):
|
|
129
|
+
try:
|
|
130
|
+
if error := getattr(args, "_setup_error", None):
|
|
131
|
+
raise error
|
|
132
|
+
report = correlate(result, options=prepare_options(args))
|
|
133
|
+
except (AdvisoryError, OSError) as exc:
|
|
134
|
+
report = unavailable_report(result, exc)
|
|
135
|
+
if report.errors:
|
|
136
|
+
issue = report.errors[0]
|
|
137
|
+
download = (' Download OSV-Scanner v2 from https://github.com/google/osv-scanner/releases.'
|
|
138
|
+
if issue.code == 'scanner_missing' else '')
|
|
139
|
+
print(f"libidentifier: warning: advisories not fully assessed ({issue.code}): {issue.message}{download}", file=sys.stderr)
|
|
140
|
+
return report
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _write_assessment(result, args):
|
|
144
|
+
if args.format == "json":
|
|
145
|
+
if args.output:
|
|
146
|
+
result.write_json(args.output)
|
|
147
|
+
else:
|
|
148
|
+
write_stream(result, sys.stdout)
|
|
149
|
+
else:
|
|
150
|
+
inventory = result.identification if hasattr(result, 'identification') else None
|
|
151
|
+
report = result.vulnerabilities if inventory is not None else result
|
|
152
|
+
if args.output:
|
|
153
|
+
with Path(args.output).open("w", encoding="utf-8") as stream:
|
|
154
|
+
write_table(inventory, report, stream=stream)
|
|
155
|
+
else:
|
|
156
|
+
write_table(inventory, report)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _assessment_exit(result):
|
|
160
|
+
report = result.vulnerabilities if hasattr(result, "identification") else result
|
|
161
|
+
if any(e.severity == "error" for e in report.identification_errors):
|
|
162
|
+
return 4
|
|
163
|
+
if report.errors and all(e.code == 'scanner_missing' for e in report.errors):
|
|
164
|
+
return 0 # Identification completed; assessment is explicitly unavailable.
|
|
165
|
+
if report.status in {"blocked", "failed"}:
|
|
166
|
+
return 3
|
|
167
|
+
if report.errors or any(e.severity == "error" for e in report.identification_errors):
|
|
168
|
+
return 4
|
|
169
|
+
return 0
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _cmd_scan(args):
|
|
173
|
+
profiles = args.profiles or os.environ.get("LIBIDENTIFIER_PROFILES")
|
|
174
|
+
managed = not profiles
|
|
175
|
+
if args.format is None:
|
|
176
|
+
args.format = "table" if managed and not args.output else "json"
|
|
177
|
+
if not profiles:
|
|
178
|
+
from .cli_setup import prepare_managed_scan
|
|
179
|
+
profiles = prepare_managed_scan(args)
|
|
180
|
+
if args.vulnerabilities:
|
|
181
|
+
identification = identify(args.path, profiles=profiles,
|
|
182
|
+
options=_identify_options(args))
|
|
183
|
+
result = AssessmentResult(identification, _assess(identification, args))
|
|
184
|
+
else:
|
|
185
|
+
result = scan(args.path, profiles=profiles,
|
|
186
|
+
options=_identify_options(args))
|
|
187
|
+
_write_assessment(result, args)
|
|
188
|
+
return _assessment_exit(result)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _cmd_correlate(args):
|
|
192
|
+
try:
|
|
193
|
+
result = ScanResult.from_dict(json.loads(Path(args.input).read_text(encoding="utf-8")))
|
|
194
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
195
|
+
raise InputError(f"invalid identification JSON: {exc}") from exc
|
|
196
|
+
report = _assess(result, args)
|
|
197
|
+
_write_assessment(report, args)
|
|
198
|
+
return _assessment_exit(report)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _cmd_advisories(args):
|
|
202
|
+
if args.advisory_command == "provision":
|
|
203
|
+
record = provision_advisory_database(args.database, archives={args.ecosystem: args.archive}, fetched_at=args.fetched_at)
|
|
204
|
+
elif args.advisory_command == "update":
|
|
205
|
+
record = update_advisory_database(args.database, ecosystems=[args.ecosystem])
|
|
206
|
+
else:
|
|
207
|
+
record = validate_advisory_database(args.database, max_age_days=args.max_database_age_days)
|
|
208
|
+
sys.stdout.write(record.to_json() + "\n")
|
|
209
|
+
return 0
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
213
|
+
parser = argparse.ArgumentParser(prog="libidentifier")
|
|
214
|
+
parser.add_argument("--version", action="version", version=f"libidentifier {__version__}")
|
|
215
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
216
|
+
|
|
217
|
+
inspect_p = sub.add_parser("inspect")
|
|
218
|
+
inspect_p.add_argument("path")
|
|
219
|
+
inspect_p.add_argument("--format", choices=["json", "table"], default="json")
|
|
220
|
+
inspect_p.add_argument("--output")
|
|
221
|
+
inspect_p.add_argument("--max-unknown-classes", type=int, default=None)
|
|
222
|
+
inspect_p.set_defaults(func=_cmd_inspect)
|
|
223
|
+
|
|
224
|
+
identify_p = sub.add_parser("identify")
|
|
225
|
+
identify_p.add_argument("path")
|
|
226
|
+
identify_p.add_argument("--profiles")
|
|
227
|
+
identify_p.add_argument("--format", choices=["json", "table"], default="json")
|
|
228
|
+
identify_p.add_argument("--output")
|
|
229
|
+
identify_p.add_argument("--max-unknown-classes", type=int, default=None)
|
|
230
|
+
identify_p.add_argument("--vulnerabilities", "--vulns", action="store_true")
|
|
231
|
+
identify_p.set_defaults(func=_cmd_identify)
|
|
232
|
+
|
|
233
|
+
profiles_p = sub.add_parser("profiles")
|
|
234
|
+
profiles_sub = profiles_p.add_subparsers(dest="profiles_command", required=True)
|
|
235
|
+
package_p = profiles_sub.add_parser("package", help="prepare portable profile release assets")
|
|
236
|
+
package_p.add_argument("--database", required=True)
|
|
237
|
+
package_p.add_argument("--output-dir", required=True)
|
|
238
|
+
package_p.add_argument("--snapshot-version", required=True)
|
|
239
|
+
package_p.add_argument("--source-revision", required=True)
|
|
240
|
+
package_p.add_argument("--requires-engine", default=DEFAULT_ENGINE_REQUIREMENT)
|
|
241
|
+
package_p.set_defaults(func=_cmd_profiles_package)
|
|
242
|
+
install_p = profiles_sub.add_parser("install", help="explicitly install a local or HTTPS profile release")
|
|
243
|
+
install_p.add_argument("--manifest", default=DEFAULT_PROFILE_MANIFEST)
|
|
244
|
+
install_p.add_argument("--database", required=True)
|
|
245
|
+
install_p.add_argument("--offline", action="store_true")
|
|
246
|
+
install_p.add_argument("--replace", action="store_true")
|
|
247
|
+
install_p.add_argument("--manifest-sha256")
|
|
248
|
+
install_p.add_argument("--timeout-seconds", type=float, default=60)
|
|
249
|
+
install_p.set_defaults(func=_cmd_profiles_install)
|
|
250
|
+
build_p = profiles_sub.add_parser("build")
|
|
251
|
+
build_p.add_argument("path")
|
|
252
|
+
build_p.add_argument("--purl", required=True)
|
|
253
|
+
build_p.add_argument("--database", required=True)
|
|
254
|
+
build_p.add_argument("--d8")
|
|
255
|
+
build_p.add_argument("--android-jar")
|
|
256
|
+
build_p.add_argument("--sha256")
|
|
257
|
+
build_p.set_defaults(func=_cmd_profiles_build)
|
|
258
|
+
validate_p = profiles_sub.add_parser("validate")
|
|
259
|
+
validate_p.add_argument("database")
|
|
260
|
+
validate_p.set_defaults(func=_cmd_profiles_validate)
|
|
261
|
+
fetch_p = profiles_sub.add_parser("fetch")
|
|
262
|
+
fetch_p.add_argument("--catalog", default="catalog/references.json")
|
|
263
|
+
fetch_p.add_argument("--cache", default="reference-artifacts")
|
|
264
|
+
fetch_p.add_argument("--offline", action="store_true")
|
|
265
|
+
fetch_p.set_defaults(func=_cmd_profiles_fetch)
|
|
266
|
+
rebuild_p = profiles_sub.add_parser("rebuild")
|
|
267
|
+
rebuild_p.add_argument("--catalog", default="catalog/references.json")
|
|
268
|
+
rebuild_p.add_argument("--cache", default="reference-artifacts")
|
|
269
|
+
rebuild_p.add_argument("--database", required=True)
|
|
270
|
+
rebuild_p.add_argument("--d8")
|
|
271
|
+
rebuild_p.add_argument("--android-jar")
|
|
272
|
+
rebuild_p.add_argument("--offline", action="store_true")
|
|
273
|
+
rebuild_p.set_defaults(func=_cmd_profiles_rebuild)
|
|
274
|
+
|
|
275
|
+
sbom_p = sub.add_parser("sbom")
|
|
276
|
+
sbom_p.add_argument("input")
|
|
277
|
+
sbom_p.add_argument("--output", required=True)
|
|
278
|
+
sbom_p.set_defaults(func=_cmd_sbom)
|
|
279
|
+
|
|
280
|
+
scan_p = sub.add_parser("scan")
|
|
281
|
+
scan_p.add_argument("path")
|
|
282
|
+
scan_p.add_argument("--profiles")
|
|
283
|
+
scan_p.add_argument("--vulnerabilities", "--vulns", action="store_true")
|
|
284
|
+
scan_p.add_argument("--yes", action="store_true", help="authorize first-run managed resource downloads without prompting")
|
|
285
|
+
scan_p.add_argument("--max-unknown-classes", type=int, default=None)
|
|
286
|
+
scan_p.set_defaults(func=_cmd_scan)
|
|
287
|
+
correlate_p = sub.add_parser("correlate")
|
|
288
|
+
correlate_p.add_argument("input")
|
|
289
|
+
correlate_p.set_defaults(func=_cmd_correlate)
|
|
290
|
+
for command in (inspect_p, identify_p, scan_p):
|
|
291
|
+
command.add_argument('--no-relationships', action='store_true', help='skip structural and published relationships')
|
|
292
|
+
command.add_argument('--no-calls', action='store_true', help='skip explicit invocation extraction')
|
|
293
|
+
command.add_argument('--no-grouping', action='store_true', help='skip unattributed clustering')
|
|
294
|
+
for command in (scan_p, correlate_p, identify_p):
|
|
295
|
+
if command is not identify_p:
|
|
296
|
+
command.add_argument("--format", choices=["json", "table"], default=None if command is scan_p else "json")
|
|
297
|
+
command.add_argument("--output")
|
|
298
|
+
command.add_argument("--osv-scanner")
|
|
299
|
+
command.add_argument("--osv-scanner-sha256", help="optional SHA-256 pin for the selected executable")
|
|
300
|
+
command.add_argument("--osv-db")
|
|
301
|
+
command.add_argument("--offline", action="store_true", help="prohibit all setup downloads (assessment always runs offline)")
|
|
302
|
+
command.add_argument("--osv-timeout", type=float, default=60)
|
|
303
|
+
command.add_argument("--max-database-age-days", type=float, default=30)
|
|
304
|
+
command.add_argument("--no-potential", action="store_true")
|
|
305
|
+
command.add_argument("--max-potential", type=int, default=100)
|
|
306
|
+
command.add_argument("--min-identity-score", type=float, default=0.75,
|
|
307
|
+
help="minimum component score for advisory attribution (0 to 1; default: 0.75)")
|
|
308
|
+
advisories = sub.add_parser("vulnerabilities")
|
|
309
|
+
actions = advisories.add_subparsers(dest="advisory_command", required=True)
|
|
310
|
+
for name in ("provision", "update", "validate"):
|
|
311
|
+
action = actions.add_parser(name)
|
|
312
|
+
action.add_argument("--database", required=True)
|
|
313
|
+
if name != "validate":
|
|
314
|
+
action.add_argument("--ecosystem", choices=["Maven"], default="Maven")
|
|
315
|
+
if name == "provision":
|
|
316
|
+
action.add_argument("--archive", required=True)
|
|
317
|
+
action.add_argument("--fetched-at")
|
|
318
|
+
if name == "validate":
|
|
319
|
+
action.add_argument("--max-database-age-days", type=float, default=30)
|
|
320
|
+
action.set_defaults(func=_cmd_advisories)
|
|
321
|
+
return parser
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def main(argv: list[str] | None = None) -> int:
|
|
325
|
+
# JSON and table output must preserve non-ASCII descriptors on Windows,
|
|
326
|
+
# including when stdout is a pipe rather than an interactive terminal.
|
|
327
|
+
for stream in (sys.stdout, sys.stderr):
|
|
328
|
+
if hasattr(stream, "reconfigure"):
|
|
329
|
+
stream.reconfigure(encoding="utf-8")
|
|
330
|
+
parser = build_parser()
|
|
331
|
+
args = parser.parse_args(argv)
|
|
332
|
+
try:
|
|
333
|
+
return int(args.func(args))
|
|
334
|
+
except (InputError, ProfileError, LibIdentifierError, OSError, ValueError) as exc:
|
|
335
|
+
print(f"libidentifier: {exc}", file=sys.stderr)
|
|
336
|
+
return 2 if isinstance(exc, InputError) else 3
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
if __name__ == "__main__":
|
|
340
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Project-qualified managed scanner builds, independent of GitHub's latest tag."""
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
import platform
|
|
4
|
+
|
|
5
|
+
from .errors import AdvisoryError
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True)
|
|
9
|
+
class ScannerRelease:
|
|
10
|
+
version: str
|
|
11
|
+
filename: str
|
|
12
|
+
sha256: str
|
|
13
|
+
|
|
14
|
+
@property
|
|
15
|
+
def url(self):
|
|
16
|
+
return f"https://github.com/google/osv-scanner/releases/download/v{self.version}/{self.filename}"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# Publisher release SHA-256 values qualified by the project's native CI matrix.
|
|
20
|
+
SUPPORTED_SCANNERS = {
|
|
21
|
+
("Windows", "amd64"): ScannerRelease("2.7.0", "osv-scanner_windows_amd64.exe",
|
|
22
|
+
"9964ed64ec9a387391eb4cc006ad5c531154821aef29b793e02c448de4686438"),
|
|
23
|
+
("Linux", "amd64"): ScannerRelease("2.7.0", "osv-scanner_linux_amd64",
|
|
24
|
+
"160c6253e2922e9e0a6a9771c5c4c40a4b3c6090b34a3cdd5e80c96cc2e4218b"),
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def supported_scanner(system=None, machine=None):
|
|
29
|
+
system, machine = system or platform.system(), (machine or platform.machine()).lower()
|
|
30
|
+
architecture = "amd64" if machine in {"amd64", "x86_64"} else machine
|
|
31
|
+
try:
|
|
32
|
+
return SUPPORTED_SCANNERS[(system, architecture)]
|
|
33
|
+
except KeyError as error:
|
|
34
|
+
raise AdvisoryError(f"No project-qualified automatic OSV installation for {system}/{machine}; "
|
|
35
|
+
"use an explicitly configured compatible native scanner",
|
|
36
|
+
code="scanner_platform_unsupported") from error
|