libidentifier 0.2.0rc1__py3-none-any.whl

libidentifier/extractors/instructions.py

@@ -0,0 +1,77 @@
+"""Instruction-boundary traversal for standard DEX explicit invocations.
+
+Only declared references are returned. All supported instruction widths are
+accounted for, including switch/array payloads. Invalid widths/targets stop the
+method and retain preceding observations with a diagnostic.
+"""
+from __future__ import annotations
+
+import struct
+
+WIDTHS = {}
+for start, end, width in [(0x00,0x01,1),(0x02,0x02,2),(0x03,0x03,3),
+                         (0x04,0x04,1),(0x05,0x05,2),(0x06,0x06,3),
+                         (0x07,0x07,1),(0x08,0x08,2),(0x09,0x09,3),
+                         (0x0a,0x12,1),(0x13,0x13,2),(0x14,0x14,3),
+                         (0x15,0x16,2),(0x17,0x17,3),(0x18,0x18,5),
+                         (0x19,0x1a,2),(0x1b,0x1b,3),(0x1c,0x1c,2),
+                         (0x1d,0x1e,1),(0x1f,0x20,2),(0x21,0x21,1),
+                         (0x22,0x23,2),(0x24,0x26,3),(0x27,0x28,1),
+                         (0x29,0x29,2),(0x2a,0x2c,3),(0x2d,0x3d,2),
+                         (0x44,0x6d,2),(0x6e,0x72,3),(0x74,0x78,3),
+                         (0x7b,0x8f,1),(0x90,0xaf,2),(0xb0,0xcf,1),
+                         (0xd0,0xe2,2),(0xfa,0xfb,4),(0xfc,0xfd,3),(0xfe,0xff,2)]:
+    WIDTHS.update({op: width for op in range(start, end + 1)})
+INVOKES = dict(zip(range(0x6e,0x73), ('virtual','super','direct','static','interface')))
+INVOKES.update({op+6: kind+'/range' for op,kind in list(INVOKES.items())})
+
+
+def decode_invocations(code: bytes, *, method_count: int, dex_version: int = 35):
+    calls, issues = [], []
+    units = len(code) // 2
+    def u16(i):
+        if i >= units:
+            raise ValueError('Truncated instruction or payload')
+        return struct.unpack_from('<H', code, i * 2)[0]
+    pc = 0
+    if len(code) % 2:
+        return [], ['Odd byte length in instruction stream']
+    try:
+        while pc < units:
+            first = u16(pc)
+            op = first & 255
+            width = WIDTHS.get(op)
+            if width is None:
+                raise ValueError(f'Unsupported opcode 0x{op:02x} at code unit {pc}')
+            if op >= 0xfa and dex_version < (39 if op >= 0xfe else 38):
+                raise ValueError('Opcode is unsupported for this DEX version')
+            if op == 0 and first:
+                if pc % 2:
+                    raise ValueError('Misaligned instruction payload')
+                count = u16(pc + 1)
+                if first == 0x0100:
+                    width = 4 + 2 * count
+                elif first == 0x0200:
+                    width = 2 + 4 * count
+                elif first == 0x0300:
+                    size = u16(pc + 2) | (u16(pc + 3) << 16)
+                    if count not in (1,2,4,8):
+                        raise ValueError('Invalid array payload element width')
+                    width = 4 + (count * size + 1) // 2
+                else:
+                    raise ValueError('Unknown instruction payload')
+            if pc + width > units:
+                raise ValueError(f'Truncated instruction/payload at code unit {pc}')
+            if op in INVOKES:
+                if op <= 0x72 and first >> 12 > 5:
+                    raise ValueError('Invalid invoke register count')
+                index = u16(pc + 1)
+                if index >= method_count:
+                    raise ValueError('Invocation method index out of bounds')
+                calls.append((pc * 2, INVOKES[op], index))
+            elif op in (0xfa,0xfb,0xfc,0xfd):
+                issues.append(f'Special invocation at code unit {pc}; dynamic target resolution unsupported')
+            pc += width
+    except ValueError as exc:
+        issues.append(str(exc))
+    return calls, issues

libidentifier/extractors/metadata.py

@@ -0,0 +1,99 @@
+"""Conservative static framework indicators; no framework payload execution."""
+from __future__ import annotations
+from pathlib import PurePosixPath
+from ..models import CodeLocator, FrameworkEvidence, FrameworkHint
+
+# Exact runtime entry points, not generic AndroidX/Kotlin namespaces.
+RULES = {
+    "flutter": ("Flutter", {
+        "Lio/flutter/embedding/engine/FlutterEngine;", "Lio/flutter/embedding/android/FlutterActivity;",
+        "Lio/flutter/view/FlutterNativeView;",
+    }, {"libflutter.so"}, "Dart application logic and package dependencies are not analyzed"),
+    "react_native": ("React Native", {
+        "Lcom/facebook/react/ReactInstanceManager;", "Lcom/facebook/react/ReactNativeHost;",
+        "Lcom/facebook/react/bridge/CatalystInstanceImpl;",
+    }, {"libreactnative.so", "libreactnativejni.so"}, "JavaScript and Hermes application logic and dependencies are not analyzed"),
+    "jetpack_compose": ("Jetpack Compose", {
+        "Landroidx/compose/runtime/Composer;", "Landroidx/compose/runtime/ComposerImpl;",
+        "Landroidx/compose/ui/platform/AndroidComposeView;",
+    }, set(), "DEX structure is analyzed; Compose usage and UI behavior are not established"),
+    "cordova": ("Cordova", {
+        "Lorg/apache/cordova/CordovaActivity;", "Lorg/apache/cordova/CordovaWebViewImpl;",
+        "Lorg/apache/cordova/PluginManager;",
+    }, set(), "JavaScript application logic and dependencies are not analyzed"),
+    "unity": ("Unity", {
+        "Lcom/unity3d/player/UnityPlayer;", "Lcom/unity3d/player/UnityPlayerActivity;",
+    }, {"libunity.so"}, "Unity managed code and IL2CPP application logic are not analyzed"),
+    "dotnet_android": ("Xamarin / .NET MAUI", {
+        "Lmono/android/Runtime;", "Lmono/MonoPackageManager;", "Lmono/MonoPackageManager_Resources;",
+    }, {"libmonodroid.so", "libmonosgen-2.0.so"}, ".NET assemblies and MAUI application logic are not analyzed; Xamarin versus MAUI is not always distinguishable"),
+}
+ASSETS = {
+    "flutter": {"assets/flutter_assets/" + x for x in (
+        "AssetManifest.json", "AssetManifest.bin", "NOTICES.Z", "kernel_blob.bin",
+        "isolate_snapshot_data", "vm_snapshot_data")},
+    "react_native": {"assets/index.android.bundle"},
+    "cordova": {"assets/www/cordova.js"},
+    "unity": {"assets/bin/Data/globalgamemanagers", "assets/bin/Data/Managed/Metadata/global-metadata.dat"},
+    "dotnet_android": {"assemblies/" + x for x in (
+        "assemblies.blob", "assemblies.manifest", "mscorlib.dll", "Mono.Android.dll",
+        "Microsoft.Maui.dll", "Microsoft.Maui.Controls.dll")},
+}
+
+
+def detect_frameworks(members, classes, elf_features, *, container: str) -> list[FrameworkHint]:
+    """Combine independent evidence kinds; multiple ABIs do not increase confidence.
+
+    A single exact class definition is a low-confidence presence hint. Filenames
+    alone never suffice; parsed native artifacts need corroboration.
+    """
+    results = []
+    for framework_id, (name, descriptors, native_names, limitation) in RULES.items():
+        evidence, kinds, found_classes = [], set(), set()
+
+        def add(kind, indicator, location):
+            kinds.add(kind)
+            evidence.append(FrameworkEvidence(indicator, location))
+
+        for cls in classes:
+            if cls.descriptor in descriptors:
+                found_classes.add(cls.descriptor)
+                add("dex", "defined framework runtime class", cls.locator)
+        for elf, artifact in elf_features:
+            basename = PurePosixPath(artifact.member or artifact.path or "").name
+            loc = CodeLocator(container, artifact.member, sha256=elf.sha256)
+            if basename in native_names or elf.soname in native_names:
+                add("native", "parsed ELF runtime: " + (elf.soname or basename), loc)
+            if framework_id == "flutter":
+                for symbol in elf.exported_symbols:
+                    if symbol in {"FlutterEngineRun", "FlutterEngineInitialize"}:
+                        add("symbol", "ELF exported symbol: " + symbol, loc)
+        for m in members:
+            if m.name not in ASSETS.get(framework_id, set()):
+                continue
+            if framework_id == "cordova" and not any(marker in m.data for marker in (
+                b"cordova.define", b'define("cordova', b"define('cordova")):
+                continue
+            add("asset", name + " asset layout" + (" with bootstrap marker" if framework_id == "cordova" else " (content not analyzed)"),
+                CodeLocator(container, m.name, sha256=m.sha256))
+        if not found_classes and not ({"native", "asset"} <= kinds or "symbol" in kinds):
+            continue
+        confidence = "low"
+        if len(found_classes) >= 2 or len(kinds) >= 2:
+            confidence = "medium"
+        if len(kinds) >= 3 or ({"dex", "native"} <= kinds and len(found_classes) >= 2):
+            confidence = "high"
+        evidence.sort(key=lambda e: (e.location.member or "", e.location.class_descriptor or "", e.indicator))
+        results.append(FrameworkHint(
+            name=name, evidence="; ".join(sorted({e.indicator for e in evidence})),
+            member=evidence[0].location.member, confidence=confidence,
+            limitations=[limitation, "Presence does not prove active use; no framework version is inferred"],
+            framework_id=framework_id, evidence_records=evidence,
+            application_logic="dex_structure_only" if framework_id == "jetpack_compose" else "not_supported",
+        ))
+    return sorted(results, key=lambda h: h.name)
+
+
+def framework_hints_from_names(member_names, classes):
+    """Legacy internal helper: filenames alone no longer establish presence."""
+    return detect_frameworks([], classes, [], container="")

libidentifier/extractors/version_metadata.py

@@ -0,0 +1,72 @@
+"""A deliberately small pom.properties subset, never a global version-string search."""
+from __future__ import annotations
+
+import re
+from dataclasses import replace
+from ..models import CodeLocator, EvidenceRecord, Producer, AnalysisRun
+from ..grouping import stable_id
+from ..purl import reference_coordinates
+
+
+def read_properties(data, member):
+    match = re.fullmatch(r'META-INF/maven/([^/]+)/([^/]+)/pom\.properties', member)
+    if not match or len(data) > 65536:
+        return None
+    values = {}
+    try:
+        for line in data.decode('utf-8-sig').splitlines():
+            line = line.strip()
+            if not line or line.startswith(('#','!')):
+                continue
+            key, value = re.split(r'\s*[=:]\s*', line, maxsplit=1)
+            if key in values or '\\' in line:
+                return None  # Unsupported Java-properties escapes/continuations.
+            values[key] = value.strip()
+    except (ValueError, UnicodeError):
+        return None
+    if (values.get('groupId'), values.get('artifactId')) != match.groups() or not values.get('version'):
+        return None
+    return values
+
+
+def apply_version_metadata(result, inv):
+    components, evidence = [], list(result.evidence_records)
+    conflicts = 0
+    for component in result.components:
+        records = []
+        containers = {loc.container for loc in component.matched_code}
+        if component.family_purl:
+            try:
+                base = re.split(r'[?#]', component.family_purl, maxsplit=1)[0]
+                ecosystem, group, name, _, _ = reference_coordinates(base + '@__unknown__')
+            except ValueError:
+                ecosystem, group, name = None, None, None
+            for loc, props in inv.version_metadata:
+                if loc.container in containers and ecosystem == 'maven' and (group, name) == (props['groupId'], props['artifactId']):
+                    eid = stable_id('evidence', ['pom.properties', loc.to_dict(), props])
+                    evidence.append(EvidenceRecord(eid, 'version_metadata', Producer('maven-properties','1'), loc,
+                        {'component_id':component.component_id, 'declared':props},
+                        limitations=['Embedded metadata is a declaration and can be modified; structural corroboration is required']))
+                    records.append(props)
+        versions = {r['version'] for r in records}
+        if len(versions) > 1 or (component.version and versions and versions != {component.version}):
+            conflicts += 1
+            component = replace(component, version=None, purl=None, version_confidence='unknown',
+                limitations=component.limitations + ['Conflicting embedded version metadata'])
+        elif (len(versions) == 1 and not component.version and len(containers) == 1 and len(component.matched_code) >= 3
+              and component.score_details.get('exact_descriptor_matches',0) >= 3):
+            version = next(iter(versions))
+            # Full supported candidates remain conservative; never use mere best release.
+            supported = {v for v,details in component.score_details.get('version_candidate_scores',{}).items() if details.get('supported')}
+            if version in supported or version in component.version_candidates:
+                from urllib.parse import quote
+                base = re.split(r'[?#]', component.family_purl, maxsplit=1)[0]
+                suffix = component.family_purl[len(base):]
+                component = replace(component, version=version, purl=base+'@'+quote(version,safe='')+suffix,
+                    version_confidence='high', identification_methods=component.identification_methods+['maven_properties'],
+                    limitations=component.limitations+['Version uses attributable embedded Maven declaration plus matching code; not cryptographic provenance'])
+        components.append(component)
+    return replace(result, components=components, evidence_records=evidence,
+        analysis_runs=result.analysis_runs+[AnalysisRun('version_metadata', 'partial' if conflicts or inv.metadata_issues or inv.coverage.dex=='partial' else 'complete',
+            Producer('maven-properties','1'), {'records':len(inv.version_metadata), 'conflicting_components':conflicts},
+            ['Only simple canonical pom.properties associated with identified code is supported'], inv.metadata_issues)])

libidentifier/fingerprint/__init__.py

@@ -0,0 +1,2 @@
+"""Fingerprint algorithms."""
+

libidentifier/fingerprint/dex_v2.py

@@ -0,0 +1,97 @@
+"""dex-shape-v2 canonical class fingerprints."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections import Counter
+
+from ..extractors.dex import ClassFeature
+
+PLATFORM_PREFIXES = (
+    "Ljava/",
+    "Ljavax/",
+    "Landroid/",
+    "Landroidx/",
+    "Ldalvik/",
+    "Lkotlin/",
+)
+
+
+def normalize_type(descriptor: str | None) -> str:
+    if not descriptor:
+        return "UNKNOWN"
+    primitive_names = {
+        "void": "V",
+        "boolean": "Z",
+        "byte": "B",
+        "short": "S",
+        "char": "C",
+        "int": "I",
+        "long": "J",
+        "float": "F",
+        "double": "D",
+    }
+    descriptor = primitive_names.get(descriptor, descriptor)
+    depth = 0
+    while descriptor.startswith("["):
+        depth += 1
+        descriptor = descriptor[1:]
+    if descriptor in {"V", "Z", "B", "S", "C", "I", "J", "F", "D"}:
+        base = descriptor
+    elif descriptor.startswith(PLATFORM_PREFIXES):
+        base = descriptor
+    elif descriptor.startswith("L"):
+        base = "OBJECT"
+    else:
+        base = "UNKNOWN"
+    return ("ARRAY:" * depth) + base
+
+
+def _method_kind(name: str) -> str:
+    if name == "<init>":
+        return "constructor"
+    if name == "<clinit>":
+        return "class_initializer"
+    return "method"
+
+
+def class_shape_map(feature: ClassFeature) -> dict:
+    fields = [
+        {"type": normalize_type(f.type_descriptor), "flags": sorted(f.access_flags)} for f in feature.fields
+    ]
+    methods = [
+        {
+            "kind": _method_kind(m.name),
+            "return": normalize_type(m.return_type),
+            "params": [normalize_type(p) for p in m.parameter_types],
+            "flags": sorted(m.access_flags),
+        }
+        for m in feature.methods
+    ]
+    return {
+        "class_flags": sorted(feature.access_flags),
+        "field_count": len(feature.fields),
+        "method_count": len(feature.methods),
+        "interface_count": len(feature.interfaces),
+        "super": normalize_type(feature.superclass),
+        "interfaces": sorted(normalize_type(i) for i in feature.interfaces),
+        "fields": sorted(fields, key=lambda x: json.dumps(x, sort_keys=True, separators=(",", ":"))),
+        "methods": sorted(methods, key=lambda x: json.dumps(x, sort_keys=True, separators=(",", ":"))),
+    }
+
+
+def digest_class(feature: ClassFeature) -> str:
+    payload = json.dumps(class_shape_map(feature), sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode(
+        "utf-8"
+    )
+    return hashlib.sha256(b"dex-shape-v2\0" + payload).hexdigest()
+
+
+def digest_counts(features: list[ClassFeature]) -> Counter[str]:
+    return Counter(digest_class(f) for f in features)
+
+
+def is_trivial_shape(feature: ClassFeature) -> bool:
+    return len(feature.fields) == 0 and len(feature.methods) <= 1 and len(feature.interfaces) == 0
+

libidentifier/fingerprint/elf_v1.py

@@ -0,0 +1,16 @@
+"""elf-identity-v1 helper functions."""
+
+from __future__ import annotations
+
+from ..extractors.elf import ElfFeature
+
+
+def identity_record(feature: ElfFeature) -> dict[str, str | None]:
+    return {
+        "abi": feature.abi,
+        "sha256": feature.sha256,
+        "build_id": feature.build_id,
+        "soname": feature.soname,
+        "symbol_digest": feature.symbol_digest,
+    }
+

libidentifier/grouping.py

@@ -0,0 +1,74 @@
+"""Deterministic inexpensive clustering; clusters make no library-origin claim."""
+from __future__ import annotations
+
+import hashlib
+import json
+from collections import defaultdict
+from .models import UnattributedCluster
+
+
+def locator_key(loc):
+    return (loc.container, loc.member or '', loc.class_descriptor or '')
+
+
+def stable_id(prefix, value):
+    return prefix + '-' + hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=True).encode()).hexdigest()[:24]
+
+
+def group_unknown(classes, unknown, relationships):
+    locators = {locator_key(loc): loc for loc in unknown}
+    parent = {key: key for key in locators}
+    def root(key):
+        while parent[key] != key:
+            parent[key] = parent[parent[key]]
+            key = parent[key]
+        return key
+    def join(a, b):
+        a, b = root(a), root(b)
+        parent[max(a,b)] = min(a,b)
+    namespaces = defaultdict(list)
+    for key in locators:
+        desc = key[2]
+        package = desc[1:].rsplit('/', 1)[0] if '/' in desc else None
+        # Root/default and broad namespace prefixes are not useful identities.
+        if package and '/' in package and package not in {'com/google', 'com/android', 'org/apache'}:
+            namespaces[package].append(key)
+    for members in namespaces.values():
+        for key in members[1:]:
+            join(members[0], key)
+    structural_pairs = []
+    for rel in relationships:
+        if rel.source.location is None or rel.target.location is None:
+            continue
+        a, b = locator_key(rel.source.location), locator_key(rel.target.location)
+        if a in parent and b in parent and rel.type in {'inheritance', 'interface_implementation'}:
+            # Type-signature/call connectivity alone easily bridges separate code.
+            join(a, b)
+            structural_pairs.append((a,b,rel.type))
+    groups = defaultdict(list)
+    for key in sorted(locators):
+        groups[root(key)].append(key)
+    supports = defaultdict(list)
+    used_methods = defaultdict(set)
+    for members in namespaces.values():
+        if len(members) > 1:
+            used_methods[root(members[0])].add('namespace')
+    for a,b,kind in structural_pairs:
+        used_methods[root(a)].add(kind)
+    for rel in relationships:
+        if rel.type not in {'inheritance','interface_implementation'} or not rel.source.location or not rel.target.location:
+            continue
+        a,b=locator_key(rel.source.location),locator_key(rel.target.location)
+        if a in parent and b in parent and root(a)==root(b):
+            supports[root(a)].append(rel.id)
+    results = []
+    for group,members in groups.items():
+        methods = used_methods[group] or {'singleton'}
+        packages = {k[2][1:].rsplit('/',1)[0] for k in members if '/' in k[2]}
+        hint = next(iter(packages)).replace('/', '.') if len(packages) == 1 else None
+        results.append(UnattributedCluster(stable_id('cluster', members), len(members),
+            [locators[k] for k in members], hint, sorted(methods),
+            quality={'namespace_count': len(packages), 'singleton': len(members) == 1},
+            limitations=['Grouping does not establish common library origin; namespace reuse and obfuscation may mix code'],
+            supporting_relationship_ids=sorted(supports[group])))
+    return sorted(results, key=lambda c:c.id)