narvy-cli 1.0.0__py3-none-any.whl

narvy/scope_config.py

@@ -0,0 +1,195 @@
+"""Loads and validates `.narvy-scope.yml`, the first-party/third-party override.
+
+Parses the `own`/`vendor` lists. Never raises: a bad or missing file gives EMPTY.
+"""
+from __future__ import annotations
+
+import os
+import sys
+from typing import Dict, FrozenSet, NamedTuple, Optional
+
+try:
+    import yaml
+except ImportError:  # pragma: no cover
+    yaml = None
+
+SCOPE_CONFIG_FILENAME = ".narvy-scope.yml"
+SCOPE_CONFIG_FILENAMES = (SCOPE_CONFIG_FILENAME,)
+
+MAX_ENTRIES_PER_LIST = 500
+
+
+class ScopeConfig(NamedTuple):
+    """Validated config: lower-cased patterns, exact unless they end in `*`."""
+    own: FrozenSet[str] = frozenset()
+    vendor: FrozenSet[str] = frozenset()
+    reasons: Dict[str, str] = {}
+    source_path: Optional[str] = None
+
+    def __bool__(self) -> bool:
+        return bool(self.own) or bool(self.vendor)
+
+
+EMPTY = ScopeConfig()
+
+
+def _warn(msg: str) -> None:
+    print(f"[narvy] WARNING: {msg}", file=sys.stderr)
+
+
+def entry_matches(entry: str, candidate: str) -> bool:
+    """Match an undelimited token: exact, or prefix if entry ends in '*'. Args lower-cased."""
+    if entry.endswith("*"):
+        return candidate.startswith(entry[:-1])
+    return candidate == entry
+
+
+def segment_prefix_match(entry: str, candidate: str, delimiter: str = ".") -> bool:
+    """Whole-segment prefix match for delimiter-separated identifiers. Args lower-cased."""
+    e = entry[:-1] if entry.endswith("*") else entry
+    e = e.rstrip(delimiter)
+    if not e:
+        return False
+    entry_segs = e.split(delimiter)
+    cand_segs = candidate.split(delimiter)
+    return cand_segs[:len(entry_segs)] == entry_segs
+
+
+def find_scope_config_path(target_path: str) -> Optional[str]:
+    """First `.narvy-scope.yml` in the cwd, then next to target_path."""
+    target_dir = target_path if os.path.isdir(target_path) else os.path.dirname(
+        os.path.abspath(target_path))
+    target_dir = target_dir or "."
+
+    # cwd is checked before the target dir.
+    for base in (os.getcwd(), target_dir):
+        for name in SCOPE_CONFIG_FILENAMES:
+            candidate = os.path.join(base, name)
+            if os.path.isfile(candidate):
+                return candidate
+    return None
+
+
+def _parse_entry(item, key: str, path: str):
+    """Return (name, reason) for one list item, or None if it is malformed."""
+    if isinstance(item, str):
+        name = item.strip()
+        if not name:
+            _warn(f"{path}: '{key}' contains an empty string entry - skipping it.")
+            return None
+        return name.lower(), None
+    if isinstance(item, dict):
+        name = item.get("name")
+        if not isinstance(name, str) or not name.strip():
+            _warn(f"{path}: '{key}' has a mapping entry with no valid 'name' "
+                  f"({item!r}) - skipping it.")
+            return None
+        reason = item.get("reason")
+        reason = reason.strip() if isinstance(reason, str) and reason.strip() else None
+        return name.strip().lower(), reason
+    _warn(f"{path}: '{key}' contains an entry that is neither a string nor a "
+          f"{{name, reason}} mapping ({item!r}) - skipping it.")
+    return None
+
+
+def _as_pattern_list(data: dict, key: str, path: str):
+    """(name, reason) tuples in file order, or None if data[key] is malformed."""
+    val = data.get(key)
+    if val is None:
+        return []
+    if not isinstance(val, list):
+        _warn(f"{path}: '{key}' should be a YAML list, got "
+              f"{type(val).__name__} - ignoring this config file, continuing "
+              f"with no overrides.")
+        return None
+    if len(val) > MAX_ENTRIES_PER_LIST:
+        _warn(f"{path}: '{key}' has {len(val)} entries, more than the "
+              f"{MAX_ENTRIES_PER_LIST} cap - using only the first "
+              f"{MAX_ENTRIES_PER_LIST} (in file order).")
+        val = val[:MAX_ENTRIES_PER_LIST]
+    out = []
+    for item in val:
+        parsed = _parse_entry(item, key, path)
+        if parsed is not None:
+            out.append(parsed)
+    return out
+
+
+def _strip_star(pattern: str) -> str:
+    return pattern[:-1] if pattern.endswith("*") else pattern
+
+
+def _patterns_overlap(a: str, b: str) -> bool:
+    """Load-time heuristic: could an own and a vendor pattern both match?"""
+    sa, sb = _strip_star(a), _strip_star(b)
+    return sa == sb or sa.startswith(sb) or sb.startswith(sa)
+
+
+def load_scope_config(target_path: str) -> ScopeConfig:
+    """Auto-detect and parse the scope config. Never raises: degrades to EMPTY."""
+    path = find_scope_config_path(target_path)
+    if path is None:
+        return EMPTY
+
+    if yaml is None:  # pragma: no cover
+        _warn(f"found {path} but PyYAML is not installed - overrides ignored.")
+        return EMPTY
+
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+    except Exception as e:
+        _warn(f"could not read/parse {path} ({e.__class__.__name__}: {e}) - "
+              f"continuing with no overrides.")
+        return EMPTY
+
+    if data is None:
+        return EMPTY
+    if not isinstance(data, dict):
+        _warn(f"{path} does not look like a valid scope config (expected a "
+              f"YAML mapping with 'own'/'vendor' keys, got "
+              f"{type(data).__name__}) - continuing with no overrides.")
+        return EMPTY
+
+    own_entries = _as_pattern_list(data, "own", path)
+    vendor_entries = _as_pattern_list(data, "vendor", path)
+    if own_entries is None or vendor_entries is None:
+        return EMPTY
+
+    own_names = {name for name, _ in own_entries}
+    vendor_names = {name for name, _ in vendor_entries}
+
+    # Identical entry in both lists: ambiguous, drop from both.
+    conflicts = own_names & vendor_names
+    if conflicts:
+        plural = len(conflicts) != 1
+        _warn(
+            f"{path}: entr{'ies' if plural else 'y'} "
+            f"{', '.join(sorted(conflicts))!r} listed in BOTH 'own' and "
+            f"'vendor' - ambiguous, falling back to automatic classification "
+            f"for {'them' if plural else 'it'}."
+        )
+        own_names -= conflicts
+        vendor_names -= conflicts
+
+    # Two patterns that could both match: warn, drop neither ('own' wins at match time).
+    overlap_pairs = sorted({
+        (o, v) for o in own_names for v in vendor_names if _patterns_overlap(o, v)
+    })
+    for o, v in overlap_pairs:
+        _warn(
+            f"{path}: 'own' pattern {o!r} and 'vendor' pattern {v!r} could "
+            f"both match the same name - 'own' wins for anything matched by "
+            f"both, per policy."
+        )
+
+    if not own_names and not vendor_names:
+        return EMPTY
+
+    reasons = {name: reason for name, reason in (own_entries + vendor_entries) if reason}
+    return ScopeConfig(
+        own=frozenset(own_names),
+        vendor=frozenset(vendor_names),
+        reasons=reasons,
+        source_path=path,
+    )

narvy/semgrep_engine.py

@@ -0,0 +1,219 @@
+"""Semgrep-backed detection engine for the Android rule pack: runs Semgrep CE as a
+subprocess over the jadx-decompiled tree. rule_engine.py stays the fallback for manifest/XML rules."""
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+from typing import Any, Dict, List, Optional
+
+import yaml
+
+from .third_party_filter import THIRD_PARTY_PREFIXES
+from .scope_config import ScopeConfig
+from . import weak_prng_context
+
+RULES_PATH = os.path.join(os.path.dirname(__file__), 'rules', 'android.yml')
+# Third-party prefixes as --exclude globs; leading '**/' de-anchors from the scan root.
+_EXCLUDE_ARGS = []
+for _p in THIRD_PARTY_PREFIXES:
+    _EXCLUDE_ARGS += ['--exclude', f"**/{_p.replace('.', '/').rstrip('/')}/**"]
+
+SEVERITY_MAP = {
+    'ERROR': 'CRITICAL',
+    'WARNING': 'HIGH',
+    'INFO': 'MEDIUM',
+}
+
+# Whole-run timeout scales with tree size; a timeout is recorded in LAST_RUN.
+SEMGREP_BASE_TIMEOUT_SEC = 600
+SEMGREP_MAX_TIMEOUT_SEC = 1800
+SEMGREP_TIMEOUT_PER_1K_FILES = 120
+
+# Callers read this to tell "0 findings" apart from "never finished".
+LAST_RUN: Dict[str, Any] = {
+    'status': 'not_run',      # ok | timeout | error | unavailable | not_run
+    'timeout_s': None,
+    'file_count': None,
+    'degraded': False,
+    'message': None,
+}
+
+
+def count_source_files(source_dir: str) -> int:
+    """Cheap file count used to size the semgrep timeout."""
+    skip = {'.git', 'node_modules', 'build', '.gradle', '.idea', 'Pods',
+            'Carthage', '__pycache__', '.venv', 'venv', 'DerivedData'}
+    n = 0
+    try:
+        for dirpath, dirnames, filenames in os.walk(source_dir):
+            dirnames[:] = [d for d in dirnames if d not in skip]
+            n += len(filenames)
+    except OSError:
+        return 0
+    return n
+
+
+def compute_semgrep_timeout(source_dir: str,
+                            base: int = SEMGREP_BASE_TIMEOUT_SEC) -> int:
+    """Scale the whole-run timeout with tree size. Never below `base`."""
+    files = count_source_files(source_dir)
+    if files < 2000:
+        return base
+    extra = ((files - 2000) // 1000) * SEMGREP_TIMEOUT_PER_1K_FILES
+    return min(base + extra, SEMGREP_MAX_TIMEOUT_SEC)
+
+
+def _record_run(status: str, timeout_s: Optional[int] = None,
+                file_count: Optional[int] = None,
+                message: Optional[str] = None) -> None:
+    LAST_RUN.update({
+        'status': status,
+        'timeout_s': timeout_s,
+        'file_count': file_count,
+        'degraded': status in ('timeout', 'error'),
+        'message': message,
+    })
+
+
+def degraded_coverage_note() -> Optional[str]:
+    """User-facing sentence when the last pass did not complete, else None."""
+    if not LAST_RUN.get('degraded'):
+        return None
+    if LAST_RUN.get('status') == 'timeout':
+        return (
+            "PARTIAL COVERAGE: the deep structural (Semgrep) pass did NOT "
+            f"finish within its {LAST_RUN.get('timeout_s')}s budget on this "
+            f"target ({LAST_RUN.get('file_count')} files). Zero findings from "
+            "that pass means 'not fully scanned', NOT 'clean'. The regex, "
+            "taint and SCA passes still ran to completion and their findings "
+            "are complete. Re-run with NARVY_SEMGREP_TIMEOUT set higher "
+            "(seconds), or scan a narrower path."
+        )
+    return (
+        "PARTIAL COVERAGE: the deep structural (Semgrep) pass failed "
+        f"({LAST_RUN.get('message')}). Zero findings from that pass means "
+        "'not fully scanned', NOT 'clean'."
+    )
+
+
+def is_available() -> bool:
+    return shutil.which('semgrep') is not None
+
+
+def load_rule_defs(rules_path: str = RULES_PATH) -> List[Dict[str, Any]]:
+    """Synthesize a rule-def dict per rule id in the YAML pack, for SARIF."""
+    try:
+        with open(rules_path, 'r') as f:
+            data = yaml.safe_load(f)
+    except OSError:
+        return []
+    defs = []
+    for rule in (data or {}).get('rules', []):
+        meta = rule.get('metadata', {}) or {}
+        defs.append({
+            'id': rule['id'],
+            'name': rule.get('message', rule['id'])[:120],
+            'severity': SEVERITY_MAP.get(rule.get('severity', 'INFO'), 'MEDIUM'),
+            'details': {
+                'cwe': meta.get('cwe', ''),
+                'masvs': meta.get('masvs', ''),
+                'description': rule.get('message', ''),
+                'recommendation': rule.get('message', ''),
+            },
+        })
+    return defs
+
+
+def run_semgrep(source_dir: str, timeout: Optional[int] = None,
+                 own_roots: Optional[set] = None,
+                 override_config: Optional[ScopeConfig] = None,
+                 include_path_prefixes: Optional[List[str]] = None) -> Optional[List[Dict[str, Any]]]:
+    """Run the Semgrep rule pack against source_dir; findings, or None if semgrep is unavailable. own_roots scopes --include; include_path_prefixes replaces the 'sources/' anchor."""
+    if not is_available():
+        _record_run('unavailable')
+        return None
+
+    file_count = count_source_files(source_dir)
+    if timeout is None:
+        env_to = os.environ.get('NARVY_SEMGREP_TIMEOUT', '').strip()
+        timeout = (int(env_to) if env_to.isdigit()
+                   else compute_semgrep_timeout(source_dir))
+
+    override_own = {e for e in (override_config.own if override_config else ())
+                     if not e.endswith('*')}
+    override_vendor = {e for e in (override_config.vendor if override_config else ())
+                        if not e.endswith('*')}
+
+    if own_roots or override_own:
+        scope_args = []
+        prefixes = include_path_prefixes if include_path_prefixes is not None else ["sources/"]
+        anchor = "**/" if include_path_prefixes is not None else ""
+        for root in (set(own_roots or ()) | override_own):
+            root_path = root.replace('.', '/')
+            for prefix in prefixes:
+                scope_args += ['--include', f"{anchor}{prefix}{root_path}/**"]
+    else:
+        scope_args = list(_EXCLUDE_ARGS)
+
+    for entry in override_vendor:
+        # --exclude wins over --include in semgrep's path filtering, so this
+        # suppresses a vendor entry nested inside an included own_root.
+        scope_args += ['--exclude', f"**/{entry.replace('.', '/')}/**"]
+
+    cmd = (
+        ['semgrep', '--config', RULES_PATH, source_dir,
+         '--json', '--quiet', '--no-git-ignore',
+         '--timeout', '60']  # per-file timeout, not the whole run
+        + scope_args
+    )
+    try:
+        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+    except subprocess.TimeoutExpired:
+        # LAST_RUN stops the caller presenting the empty list as a clean pass.
+        _record_run('timeout', timeout_s=timeout, file_count=file_count)
+        return []
+
+    if not proc.stdout:
+        if proc.returncode not in (0, 1):
+            _record_run('error', timeout_s=timeout, file_count=file_count,
+                        message=f"semgrep exited rc={proc.returncode}")
+        else:
+            _record_run('ok', timeout_s=timeout, file_count=file_count)
+        return []
+
+    try:
+        data = json.loads(proc.stdout)
+    except json.JSONDecodeError:
+        _record_run('error', timeout_s=timeout, file_count=file_count,
+                    message="semgrep output was not valid JSON")
+        return []
+
+    _record_run('ok', timeout_s=timeout, file_count=file_count)
+    findings = []
+    for r in data.get('results', []):
+        rel_path = os.path.relpath(r.get('path', ''), source_dir)
+        extra = r.get('extra', {})
+        meta = extra.get('metadata', {})
+        # Strip semgrep's dotted config namespace; our own ids never contain dots.
+        check_id = r.get('check_id', 'semgrep-finding').rsplit('.', 1)[-1]
+        findings.append({
+            'rule_id': check_id,
+            'file_path': rel_path,
+            'name': extra.get('message', r.get('check_id', ''))[:120],
+            'severity': SEVERITY_MAP.get(extra.get('severity', 'INFO'), 'MEDIUM'),
+            'line': r.get('start', {}).get('line', 1),
+            'details': {
+                'cwe': meta.get('cwe', ''),
+                'masvs': meta.get('masvs', ''),
+                'description': extra.get('message', ''),
+                'recommendation': extra.get('message', ''),
+            },
+            'engine': 'semgrep',
+        })
+
+    # Dedupe before grading: one source line can hold several Random calls.
+    findings = weak_prng_context.dedupe_by_location(findings)
+    weak_prng_context.apply(findings, source_dir)
+    return findings

narvy/stack_protector_evidence.py

@@ -0,0 +1,96 @@
+"""Decide whether a missing `__stack_chk_fail` symbol really means the stack
+protector was off, by detecting whether the ELF holds any function that
+`-fstack-protector-strong` would have instrumented.
+"""
+from __future__ import annotations
+
+import logging
+import struct
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+try:
+    import lief
+    LIEF_AVAILABLE = True
+except ImportError:  # pragma: no cover
+    LIEF_AVAILABLE = False
+
+# Cap pure-Python decoding on very large .text sections.
+_MAX_TEXT_SCAN_BYTES = 8 * 1024 * 1024
+
+
+def _aarch64_takes_stack_address(code: bytes) -> bool:
+    """AArch64 is fixed-width, so every 4-byte aligned word is an instruction."""
+    for off in range(0, min(len(code), _MAX_TEXT_SCAN_BYTES) // 4 * 4, 4):
+        w = struct.unpack_from("<I", code, off)[0]
+        # ADD (immediate), 64-bit; mask covers both shift forms.
+        if (w & 0xFF800000) == 0x91000000:
+            rd, rn = w & 0x1F, (w >> 5) & 0x1F
+            # Rd 31/29 are sp/frame-pointer setup, not an escaping address.
+            if rd not in (29, 31) and rn in (29, 31):
+                return True
+        # SUB (immediate), 64-bit: a slot below the frame pointer.
+        elif (w & 0xFF800000) == 0xD1000000:
+            rd, rn = w & 0x1F, (w >> 5) & 0x1F
+            if rd not in (29, 31) and rn == 29:
+                return True
+    return False
+
+
+def _arm32_takes_stack_address(code: bytes) -> bool:
+    """ARM32 mixes A32 and Thumb-2 in one .text, so both decodings are scanned."""
+    limit = min(len(code), _MAX_TEXT_SCAN_BYTES)
+    # A32
+    for off in range(0, limit // 4 * 4, 4):
+        w = struct.unpack_from("<I", code, off)[0]
+        rd = (w >> 12) & 0xF
+        rn = (w >> 16) & 0xF
+        if rd == 13:
+            continue
+        if (w & 0x0FE00000) == 0x02800000 and rn == 13:  # ADD Rd, sp, #imm
+            return True
+        if (w & 0x0FE00000) == 0x02400000 and rn == 11:  # SUB Rd, r11(fp), #imm
+            return True
+        if (w & 0x0FEF0FFF) == 0x01A0000D:               # MOV Rd, sp
+            return True
+    # Thumb-2
+    for off in range(0, limit - 1, 2):
+        h = struct.unpack_from("<H", code, off)[0]
+        if 0xA800 <= h <= 0xAFFF:                        # ADD Rd, SP, #imm8*4
+            return True
+        if (h & 0xFF78) == 0x4668:                       # MOV Rd, SP
+            return True
+        # ADD.W Rd, SP, #const (T3) and ADDW Rd, SP, #imm12 (T4).
+        if h in (0xF10D, 0xF50D, 0xF20D, 0xF60D) and off + 3 < limit:
+            h2 = struct.unpack_from("<H", code, off + 2)[0]
+            if ((h2 >> 8) & 0xF) != 13:
+                return True
+    return False
+
+
+def has_instrumentable_stack_code(binary) -> Optional[bool]:
+    """Does this ELF hold a function `-fstack-protector-strong` would instrument? True: absent `__stack_chk_fail` means protector off; False: proves nothing; None: no analysis. Never raises."""
+    if not LIEF_AVAILABLE or binary is None:
+        return None
+    try:
+        machine = binary.header.machine_type
+        if machine == lief.ELF.ARCH.AARCH64:
+            decode = _aarch64_takes_stack_address
+        elif machine == lief.ELF.ARCH.ARM:
+            decode = _arm32_takes_stack_address
+        else:
+            return None
+
+        code = None
+        for section in binary.sections:
+            if section.name == ".text":
+                code = bytes(section.content)
+                break
+        if not code:
+            # A missing .text is "no analysis"; an empty one is a real verdict.
+            return None if code is None else False
+        return decode(code)
+    except Exception as e:  # pragma: no cover
+        logger.debug(f"[stack-protector-evidence] analysis failed: {e}")
+        return None