libidentifier 0.2.0rc1__py3-none-any.whl

libidentifier/cli_setup.py

@@ -0,0 +1,177 @@
+"""Consent-based setup for the managed CLI scan path; analysis APIs never call it."""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import sys
+import tempfile
+
+from .cli_resources import supported_scanner
+from .errors import AdvisoryError, InputError, ProfileError
+from .profiles.builder import validate_database
+from .profiles.distribution import install_profiles, _read_url
+from .profiles.store import sha256_file
+from .vulnerabilities.database import update_advisory_database, validate_advisory_database
+from .vulnerabilities.models import VulnerabilityOptions
+from .vulnerabilities.runner import verify_scanner
+from .vulnerabilities.setup import discover_scanner
+
+
+def resource_home():
+    if override := os.environ.get("LIBIDENTIFIER_HOME"):
+        return Path(override).expanduser().resolve()
+    if os.name == "nt":
+        base = Path(os.environ.get("LOCALAPPDATA") or Path.home() / "AppData/Local")
+        return base / "LibIdentifier"
+    return Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / "libidentifier"
+
+
+def _step(label, operation):
+    print(label + "...", end=" ", file=sys.stderr, flush=True)
+    try:
+        value = operation()
+    except Exception:
+        print("failed", file=sys.stderr)
+        raise
+    print("done", file=sys.stderr)
+    return value
+
+
+def _consent(args, missing, root):
+    print("\nFirst-time setup\n\nThe following resources are missing:", file=sys.stderr)
+    for name in missing:
+        print("  - " + name, file=sys.stderr)
+    print(f"\nDownloads will be stored in:\n  {root}\n\n"
+          "Your APK will stay on this computer.\n"
+          "Installed resources will be reused for future scans.\n", file=sys.stderr)
+    if args.vulnerabilities and args.osv_db and not Path(args.osv_db).is_relative_to(root):
+        print(f"Advisory database override:\n  {args.osv_db}\n", file=sys.stderr)
+    if args.offline:
+        print("Offline mode: downloads are prohibited.", file=sys.stderr)
+        return False
+    if args.yes:
+        return True
+    if not sys.stdin.isatty():
+        print("Setup requires an interactive terminal or --yes; no resources downloaded.", file=sys.stderr)
+        return False
+    print("Download and set up these resources? [y/N]: ", end="", file=sys.stderr, flush=True)
+    try:
+        answer = sys.stdin.readline().strip().lower()
+    except (EOFError, OSError):
+        return False
+    return answer in {"y", "yes"}
+
+
+def _options(args, scanner):
+    return VulnerabilityOptions(scanner, args.osv_db, scanner_sha256=args.osv_scanner_sha256,
+                                timeout_seconds=args.osv_timeout)
+
+
+def _verify(args, scanner):
+    return verify_scanner(_options(args, scanner))
+
+
+def _install_scanner(args, release, target):
+    target.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        with tempfile.TemporaryDirectory(prefix=".download-", dir=target.parent) as temporary:
+            candidate = Path(temporary) / release.filename
+            def download():
+                with candidate.open("wb") as output:
+                    _read_url(release.url, output, limit=128 * 1024 * 1024, timeout=args.osv_timeout)
+            _step("Downloading OSV-Scanner", download)
+            def checksum():
+                if sha256_file(candidate) != release.sha256:
+                    raise AdvisoryError("Downloaded OSV-Scanner checksum does not match the project pin",
+                                        code="scanner_unapproved")
+            _step("Verifying scanner checksum", checksum)
+            candidate.chmod(0o755)
+            _step("Checking scanner compatibility", lambda: _verify(args, candidate))
+            if os.name == "nt":
+                os.rename(candidate, target)  # Refuses a concurrently created destination.
+            else:
+                os.link(candidate, target)
+    except ProfileError as error:
+        raise AdvisoryError(str(error), code="scanner_download_failed") from error
+
+
+def prepare_managed_scan(args):
+    """Resolve resources for scan without --profiles/environment; return profile path.
+
+    Consent applies only to missing resources. Existing invalid resources require
+    explicit maintenance. Optional setup failures preserve library identification.
+    """
+    if not Path(args.path).exists():
+        raise InputError(f"input does not exist: {args.path}")
+    root = resource_home()
+    profiles = root / "profiles/profiles.sqlite"
+    profile_missing = not profiles.exists()
+    if not profile_missing:
+        validate_database(profiles)
+    missing = ["Library profiles: official preview snapshot"] if profile_missing else []
+    scanner_missing = database_missing = False
+    release = None
+    args._setup_error = None
+    if args.vulnerabilities:
+        args.osv_db = str(Path(args.osv_db).resolve() if args.osv_db else root / "advisory-db")
+        try:
+            if args.osv_scanner:
+                args.osv_scanner = str(Path(args.osv_scanner).resolve())
+                _verify(args, args.osv_scanner)
+            else:
+                release = supported_scanner()
+                if args.osv_scanner_sha256 and args.osv_scanner_sha256.lower() != release.sha256:
+                    raise AdvisoryError("Requested hash disagrees with the managed scanner pin; "
+                                        "use --osv-scanner for an explicit override", code="scanner_unapproved")
+                args.osv_scanner_sha256 = release.sha256
+                target = root / "tools/osv-scanner" / release.version / release.filename
+                # Reuse a discovered executable only if its bytes match the project pin.
+                if not target.exists():
+                    try:
+                        discovered = discover_scanner()
+                        if sha256_file(discovered) == release.sha256:
+                            target = discovered
+                    except (AdvisoryError, OSError):
+                        pass
+                args.osv_scanner = str(target)
+                scanner_missing = not target.exists()
+                if scanner_missing:
+                    missing.append(f"OSV-Scanner: project-supported {release.filename} ({release.version})")
+                else:
+                    _verify(args, target)  # Includes hash validation before execution.
+            database_missing = not Path(args.osv_db).exists()
+            if database_missing:
+                missing.append("Maven vulnerability database: current snapshot")
+        except (AdvisoryError, OSError, ValueError) as error:
+            args._setup_error = error if isinstance(error, AdvisoryError) else AdvisoryError(str(error), code="setup_failed")
+
+    accepted = _consent(args, missing, root) if missing else False
+    if profile_missing:
+        if not accepted:
+            raise ProfileError("Library profiles are required. Run scan interactively or with --yes, "
+                               "or supply local --profiles. --offline prohibits setup downloads.")
+        _step("Downloading and validating library profiles", lambda: install_profiles(database=profiles))
+    if args.vulnerabilities and args._setup_error is None:
+        optional_missing = scanner_missing or database_missing
+        if optional_missing and not accepted:
+            args._setup_error = AdvisoryError("Vulnerability setup was not authorized or resources are missing "
+                                             "in offline mode; identification is retained", code="setup_not_performed")
+        elif optional_missing:
+            try:
+                if scanner_missing:
+                    _install_scanner(args, release, Path(args.osv_scanner))
+                # Never refresh any existing directory, even an empty/corrupt one.
+                if database_missing and not Path(args.osv_db).exists():
+                    _step("Downloading vulnerability data", lambda: update_advisory_database(
+                        args.osv_db, timeout_seconds=args.osv_timeout))
+                    _step("Validating vulnerability data", lambda: validate_advisory_database(
+                        args.osv_db, max_age_days=args.max_database_age_days))
+            except (AdvisoryError, ProfileError, OSError, ValueError) as error:
+                args._setup_error = error if isinstance(error, AdvisoryError) else AdvisoryError(str(error), code="setup_failed")
+    # The single consent decision is final. No later assessment path may provision.
+    args.offline = True
+    if accepted:
+        print("\nSetup complete." if args._setup_error is None else
+              "\nLibrary profiles ready; vulnerability setup incomplete. Identification will continue.",
+              file=sys.stderr)
+    return profiles

libidentifier/config.py

@@ -0,0 +1,54 @@
+"""Configuration knobs for scanning and matching."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class IdentifyOptions:
+    max_archive_members: int = 20_000
+    max_member_uncompressed_bytes: int = 128 * 1024 * 1024
+    max_total_selected_uncompressed_bytes: int = 512 * 1024 * 1024
+    max_compression_ratio: int = 200
+    max_directory_files: int = 20_000
+    max_directory_depth: int = 8
+    max_scan_seconds: int = 180  # Compatibility-only: not enforced in-process; use a host subprocess timeout.
+    max_emitted_unknown_classes: int | None = None
+    medium_min_discriminative_classes: int = 3
+    medium_min_weighted_coverage: float = 0.75
+    medium_partial_min_exact_classes: int = 25
+    medium_partial_min_weighted_coverage: float = 0.65
+    medium_partial_min_unique_anchors: int = 10
+    high_min_discriminative_classes: int = 5
+    high_min_weighted_coverage: float = 0.90
+    high_min_unique_anchors: int = 2
+    high_min_runner_up_delta: float = 0.15
+    min_material_runner_overlap: float = 0.25
+    max_version_candidates: int = 12
+    extract_relationships: bool = True
+    extract_calls: bool = True
+    group_unknown_classes: bool = True
+
+
+    def __post_init__(self):
+        for name in ('extract_relationships', 'extract_calls', 'group_unknown_classes'):
+            if not isinstance(getattr(self, name), bool):
+                raise ValueError(f'{name} must be boolean')
+        for name in ("max_archive_members", "max_member_uncompressed_bytes",
+                     "max_total_selected_uncompressed_bytes", "max_compression_ratio",
+                     "max_directory_files", "max_directory_depth"):
+            value = getattr(self, name)
+            if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
+                raise ValueError(f"{name} must be a positive integer")
+        for name in ("max_emitted_unknown_classes", "max_version_candidates"):
+            value = getattr(self, name)
+            if name == "max_emitted_unknown_classes" and value is None:
+                continue
+            if not isinstance(value, int) or isinstance(value, bool) or value < 0:
+                raise ValueError(f"{name} must be a non-negative integer")
+
+
+FINGERPRINT_ALGORITHM = "dex-shape-v2-canonical+elf-identity-v1"
+SCHEMA_VERSION = 1
+DATABASE_SCHEMA_VERSION = 2

libidentifier/detect.py

@@ -0,0 +1,30 @@
+"""Magic-byte format detection."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def detect_bytes(data: bytes, *, name: str | None = None) -> str:
+    if data.startswith(b"dex\n") and len(data) >= 8:
+        return "dex"
+    if data.startswith(b"\x7fELF"):
+        return "elf"
+    if data.startswith(b"PK\x03\x04") or data.startswith(b"PK\x05\x06") or data.startswith(b"PK\x07\x08"):
+        lowered = (name or "").lower()
+        if lowered.endswith(".apk"):
+            return "apk"
+        if lowered.endswith(".aar"):
+            return "aar"
+        if lowered.endswith(".jar"):
+            return "jar"
+        return "zip"
+    return "unknown"
+
+
+def detect_file(path: str | Path, read_size: int = 8) -> str:
+    p = Path(path)
+    with p.open("rb") as fh:
+        head = fh.read(read_size)
+    return detect_bytes(head, name=p.name)
+

libidentifier/errors.py

@@ -0,0 +1,30 @@
+"""Structured exceptions used by LibIdentifier."""
+
+
+class LibIdentifierError(Exception):
+    """Base class for package errors."""
+
+
+class InputError(LibIdentifierError):
+    """The top-level input cannot be inspected safely."""
+
+
+class ProfileError(LibIdentifierError):
+    """The profile database is missing, incompatible, or corrupt."""
+
+
+class ProfileBuildError(LibIdentifierError):
+    """A trusted reference profile could not be built."""
+
+
+class ParserError(LibIdentifierError):
+    """A binary parser failed for a selected artifact."""
+
+
+class AdvisoryError(LibIdentifierError):
+    """An advisory dependency or snapshot could not be used or provisioned."""
+    def __init__(self, message, *, code="advisory_error", database=None):
+        super().__init__(message)
+        self.code = code
+        self.database = database
+

libidentifier/extractors/__init__.py

@@ -0,0 +1,2 @@
+"""Binary feature extractors."""
+

libidentifier/extractors/dex.py

@@ -0,0 +1,121 @@
+"""LIEF DEX feature extraction."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from ..models import CodeLocator
+
+
+@dataclass(frozen=True)
+class FieldFeature:
+    name: str
+    type_descriptor: str
+    access_flags: list[str]
+
+
+@dataclass(frozen=True)
+class MethodFeature:
+    name: str
+    return_type: str
+    parameter_types: tuple[str, ...]
+    access_flags: list[str]
+    descriptor: str | None = None
+    method_index: int | None = None
+    code_offset: int | None = None
+    bytecode: bytes = b''
+
+
+@dataclass(frozen=True)
+class ClassFeature:
+    descriptor: str
+    access_flags: list[str]
+    superclass: str | None
+    interfaces: tuple[str, ...]
+    fields: tuple[FieldFeature, ...]
+    methods: tuple[MethodFeature, ...]
+    source_filename: str | None
+    locator: CodeLocator
+
+
+def _flags(value: Any) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, int):
+        return [str(value)] if value else []
+    try:
+        iterable = list(value)
+    except TypeError:
+        return [str(value)]
+    return sorted(str(x).split(".")[-1].lower() for x in iterable)
+
+
+def parse_dex(data: bytes, *, container: str, member: str | None, sha256: str, tables=None) -> list[ClassFeature]:
+    import lief
+    from .dex_tables import DexTables
+
+    tables = tables or DexTables(data)
+    defined_descriptors = set(tables.classes)
+    try:
+        parsed = lief.DEX.parse(list(data), name=member or container)
+    except TypeError:
+        parsed = lief.DEX.parse(list(data))
+    if parsed is None:
+        raise ValueError("LIEF returned no DEX object")
+    classes = []
+    for cls in getattr(parsed, "classes", []) or []:
+        descriptor = cls.fullname
+        if descriptor not in defined_descriptors:
+            continue
+        superclass, interfaces = tables.classes[descriptor]
+        interfaces = tuple(sorted(interfaces))
+        fields = []
+        for field in getattr(cls, "fields", []) or []:
+            if field.index not in tables.defined_fields:
+                continue
+            fields.append(
+                FieldFeature(
+                    name=str(getattr(field, "name", "")),
+                    type_descriptor=tables.at(tables.fields, field.index)[1],
+                    access_flags=_flags(getattr(field, "access_flags", None)),
+                )
+            )
+        methods = []
+        for method in getattr(cls, "methods", []) or []:
+            index = method.index
+            if index not in tables.defined_methods:
+                continue
+            _, _, (ret, params) = tables.at(tables.methods, index)
+            code_offset, bytecode = tables.code.get(index, (None, b''))
+            methods.append(
+                MethodFeature(
+                    name=str(getattr(method, "name", "")),
+                    return_type=ret,
+                    parameter_types=params,
+                    access_flags=_flags(getattr(method, "access_flags", None)),
+                    descriptor=tables.method_descriptor(index),
+                    method_index=index,
+                    code_offset=code_offset,
+                    bytecode=bytecode,
+                )
+            )
+        access_flags = _flags(getattr(cls, "access_flags", None))
+        # The raw class_defs table already proves this is an embedded definition.
+        # Package-private empty classes are valid and must remain in the inventory.
+        classes.append(
+            ClassFeature(
+                descriptor=descriptor,
+                access_flags=access_flags,
+                superclass=superclass,
+                interfaces=interfaces,
+                fields=tuple(sorted(fields, key=lambda f: (f.type_descriptor, f.name))),
+                methods=tuple(sorted(methods, key=lambda m: (m.name, m.return_type, m.parameter_types))),
+                source_filename=getattr(cls, "source_filename", None),
+                locator=CodeLocator(container=container, member=member, class_descriptor=descriptor, sha256=sha256),
+            )
+        )
+    if not classes:
+        raise ValueError("DEX contains no class definitions")
+    return sorted(classes, key=lambda c: c.descriptor)
+

libidentifier/extractors/dex_tables.py

@@ -0,0 +1,137 @@
+"""Bounded standard DEX tables supplementing LIEF's missing descriptor/interfaces API.
+
+This reader never executes code. Offsets are absolute bytes; instructions are
+retained as bytes and decoded separately. Unsupported container versions fail.
+"""
+from __future__ import annotations
+
+import struct
+
+
+class DexTables:
+    def __init__(self, data: bytes):
+        self.data = data
+        self.require(0, 0x70)
+        if data[:8] not in {b'dex\n035\0', b'dex\n037\0', b'dex\n038\0', b'dex\n039\0', b'dex\n040\0'}:
+            raise ValueError('Unsupported standard DEX version')
+        if self.u32(0x28) != 0x12345678 or self.u32(0x24) != 0x70:
+            raise ValueError('Unsupported DEX header/endian format')
+        if self.u32(0x20) != len(data):
+            raise ValueError('DEX file size disagrees with available data')
+        self.strings = [self.string(self.u32(p)) for p in self.table(0x38, 4)]
+        self.types = [self.at(self.strings, self.u32(p)) for p in self.table(0x40, 4)]
+        self.protos = [(self.at(self.types, self.u32(p + 4)), self.type_list(self.u32(p + 8)))
+                       for p in self.table(0x48, 12)]
+        self.fields = [(self.at(self.types, self.u16(p)), self.at(self.types, self.u16(p + 2)),
+                        self.at(self.strings, self.u32(p + 4))) for p in self.table(0x50, 8)]
+        self.methods = [(self.at(self.types, self.u16(p)), self.at(self.strings, self.u32(p + 4)),
+                         self.at(self.protos, self.u16(p + 2))) for p in self.table(0x58, 8)]
+        self.classes = {}
+        self.code = {}
+        self.defined_fields = set()
+        self.defined_methods = set()
+        for p in self.table(0x60, 32):
+            desc = self.at(self.types, self.u32(p))
+            if desc in self.classes:
+                raise ValueError('Duplicate DEX class definition')
+            parent = self.u32(p + 8)
+            self.classes[desc] = (None if parent == 0xffffffff else self.at(self.types, parent),
+                                  self.type_list(self.u32(p + 12)))
+            offset = self.u32(p + 24)
+            if offset:
+                self.class_data(offset, desc)
+
+    def require(self, offset, size):
+        if offset < 0 or size < 0 or offset + size > len(self.data):
+            raise ValueError('DEX table/data out of bounds')
+
+    def u16(self, offset):
+        self.require(offset, 2)
+        return struct.unpack_from('<H', self.data, offset)[0]
+
+    def u32(self, offset):
+        self.require(offset, 4)
+        return struct.unpack_from('<I', self.data, offset)[0]
+
+    @staticmethod
+    def at(values, index):
+        if index < 0 or index >= len(values):
+            raise ValueError('DEX reference index out of bounds')
+        return values[index]
+
+    def table(self, header, width):
+        size, offset = self.u32(header), self.u32(header + 4)
+        if size and not offset:
+            raise ValueError('DEX table offset missing')
+        self.require(offset, size * width)
+        return range(offset, offset + size * width, width)
+
+    def uleb(self, offset):
+        value = 0
+        for shift in range(0, 35, 7):
+            self.require(offset, 1)
+            byte = self.data[offset]
+            offset += 1
+            if shift == 28 and byte > 15:
+                raise ValueError('DEX ULEB128 overflow')
+            value |= (byte & 127) << shift
+            if not byte & 128:
+                return value, offset
+        raise ValueError('DEX ULEB128 overflow')
+
+    def string(self, offset):
+        length, pos = self.uleb(offset)
+        end = self.data.find(b'\0', pos)
+        if end < 0:
+            raise ValueError('Unterminated DEX string')
+        # DEX MUTF-8 encodes NUL and UTF-16 surrogate pairs specially.
+        raw = self.data[pos:end].replace(b'\xc0\x80', b'\0')
+        value = raw.decode('utf-8', errors='surrogatepass')
+        utf16 = value.encode('utf-16-le', errors='surrogatepass')
+        if len(utf16) // 2 != length:
+            raise ValueError('DEX string UTF-16 length mismatch')
+        return utf16.decode('utf-16-le')
+
+    def type_list(self, offset):
+        if not offset:
+            return ()
+        size = self.u32(offset)
+        self.require(offset + 4, size * 2)
+        return tuple(self.at(self.types, self.u16(offset + 4 + 2 * i)) for i in range(size))
+
+    def class_data(self, pos, owner):
+        counts = []
+        for _ in range(4):
+            count, pos = self.uleb(pos)
+            counts.append(count)
+        for count in counts[:2]:
+            index = 0
+            for _ in range(count):
+                delta, pos = self.uleb(pos)
+                flags, pos = self.uleb(pos)
+                index += delta
+                if self.at(self.fields, index)[0] != owner:
+                    raise ValueError('DEX field definition owner mismatch')
+                if index in self.defined_fields:
+                    raise ValueError('Duplicate DEX field definition')
+                self.defined_fields.add(index)
+        for count in counts[2:]:
+            index = 0
+            for _ in range(count):
+                delta, pos = self.uleb(pos)
+                flags, pos = self.uleb(pos)
+                offset, pos = self.uleb(pos)
+                index += delta
+                if self.at(self.methods, index)[0] != owner:
+                    raise ValueError('DEX method definition owner mismatch')
+                if index in self.defined_methods:
+                    raise ValueError('Duplicate DEX method definition')
+                self.defined_methods.add(index)
+                if offset:
+                    size = self.u32(offset + 12)
+                    self.require(offset, 16 + size * 2)
+                    self.code[index] = (offset + 16, self.data[offset + 16:offset + 16 + size * 2])
+
+    def method_descriptor(self, index):
+        owner, name, (ret, params) = self.at(self.methods, index)
+        return owner + '->' + name + '(' + ''.join(params) + ')' + ret

libidentifier/extractors/elf.py

@@ -0,0 +1,66 @@
+"""LIEF ELF identity extraction."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ElfFeature:
+    sha256: str
+    abi: str
+    soname: str | None
+    build_id: str | None
+    imported_symbols: tuple[str, ...]
+    exported_symbols: tuple[str, ...]
+    symbol_digest: str | None
+
+
+def _name(value) -> str | None:
+    if value is None:
+        return None
+    n = getattr(value, "name", None)
+    return str(n) if n else None
+
+
+def parse_elf(data: bytes) -> ElfFeature:
+    import lief
+
+    parsed = lief.ELF.parse(list(data))
+    if parsed is None:
+        raise ValueError("LIEF returned no ELF object")
+    header = getattr(parsed, "header", None)
+    machine = str(getattr(header, "machine_type", "unknown")).split(".")[-1].lower()
+    cls = str(getattr(header, "identity_class", "")).split(".")[-1].lower()
+    abi = ":".join(x for x in (machine, cls) if x)
+    soname = None
+    try:
+        soname = parsed.soname
+    except Exception:
+        soname = None
+    build_id = None
+    for note in getattr(parsed, "notes", []) or []:
+        desc = getattr(note, "description", None)
+        if desc and "build" in str(getattr(note, "type", "")).lower():
+            try:
+                build_id = bytes(desc).hex()
+            except Exception:
+                build_id = str(desc)
+            break
+    imported = tuple(sorted(n for n in (_name(s) for s in getattr(parsed, "imported_symbols", []) or []) if n))
+    exported = tuple(sorted(n for n in (_name(s) for s in getattr(parsed, "exported_symbols", []) or []) if n))
+    symbol_digest = None
+    if imported or exported:
+        payload = "\n".join(["I:" + x for x in imported] + ["E:" + x for x in exported]).encode()
+        symbol_digest = hashlib.sha256(b"elf-identity-v1\0" + payload).hexdigest()
+    return ElfFeature(
+        sha256=hashlib.sha256(data).hexdigest(),
+        abi=abi or "unknown",
+        soname=soname,
+        build_id=build_id,
+        imported_symbols=imported,
+        exported_symbols=exported,
+        symbol_digest=symbol_digest,
+    )
+