narvy-cli 1.1.2__tar.gz → 1.1.3__tar.gz

{narvy_cli-1.1.2 → narvy_cli-1.1.3}/CHANGELOG.md

@@ -5,6 +5,36 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.1.3]
+
+### Fixed
+
+- APK scans were refused before decompiling on machines with less free
+  memory than `--max-mem` needs (about 1.3x the heap plus 1 GB), even for
+  small apps. On an 8 GB laptop at the default `--max-mem 4g` that was
+  nearly every app, and on macOS the free-memory reading was low by
+  several GB. `--max-mem` is a ceiling: when it does not fit but a smaller
+  heap still covers the app's estimated need by 1.5x, jadx now runs with
+  that smaller heap and the scan says so. A machine that cannot hold even
+  that is still refused, with the same advice as before.
+- macOS: available memory comes from the kernel's own figure
+  (`kern.memorystatus_level`, what `memory_pressure` reports), which counts
+  memory the compressor can reclaim. The `vm_stat` page count is the
+  fallback and now includes speculative pages.
+- APK scans: a structural (Semgrep) pass that failed was reported as
+  complete with 0 findings when semgrep exited with an error but still
+  printed JSON. It is now reported as incomplete with the error, after one
+  retry with a single worker. A run whose path filters matched none of the
+  app's own source files is reported the same way instead of as a clean
+  pass. Rule ids no longer depend on where the package is installed.
+
+### Changed
+
+- Platform CI: the check removes the user Scripts/bin directory from PATH
+  before running scans (hosted runners ship with it on PATH, a new user's
+  shell does not), and prints the APK scan output on success so the
+  structural-pass status is in the log.
+
 ## [1.1.2]
 
 ### Added

{narvy_cli-1.1.2/narvy_cli.egg-info → narvy_cli-1.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: narvy-cli
-Version: 1.1.2
+Version: 1.1.3
 Summary: Free, local SAST scanner for Android, iOS, and polyglot web/backend source - plus passive web, host-audit, and cloud-posture scanning.
 Author: Narvy
 License: Apache-2.0

{narvy_cli-1.1.2 → narvy_cli-1.1.3}/narvy/__init__.py

@@ -1,3 +1,3 @@
 """Narvy CLI - free, local SAST for Android, iOS and polyglot source."""
 
-__version__ = "1.1.2"
+__version__ = "1.1.3"

{narvy_cli-1.1.2 → narvy_cli-1.1.3}/narvy/apk_memory_preflight.py

@@ -24,6 +24,13 @@ JVM_RSS_OVERHEAD = 1.30
 
 RAM_SAFETY_RESERVE_MB = 1024
 
+# When --max-mem does not fit in free RAM but a smaller heap still leaves this
+# much headroom over the estimate, jadx runs with that smaller -Xmx instead of
+# the scan being refused. -Xmx bounds the JVM, so the smaller heap is what keeps
+# memory in check. Measured: four real apps (6.5k to 60k classes) decompiled
+# fully at 1.5x the estimate, peak RSS about 1.2x that heap.
+AUTO_LOWER_MARGIN = 1.5
+
 
 class DexStats(NamedTuple):
     methods: int
@@ -41,6 +48,8 @@ class PreflightVerdict(NamedTuple):
     max_mem_mb: int
     available_mb: Optional[int]
     stats: DexStats
+    # -Xmx value to run jadx with when it differs from --max-mem, else None.
+    lowered_max_mem: Optional[str] = None
 
 
 def parse_mem_arg(value: str) -> Optional[int]:
@@ -71,19 +80,7 @@ def available_ram_mb() -> Optional[int]:
                     if line.startswith("MemAvailable:"):
                         return int(line.split()[1]) // 1024
         elif system == "Darwin":
-            # vm_stat exposes reclaimable inactive+purgeable pools that SC_AVPHYS_PAGES omits.
-            import subprocess
-            out = subprocess.run(["vm_stat"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10).stdout
-            page = 4096
-            pm = re.search(r"page size of (\d+) bytes", out)
-            if pm:
-                page = int(pm.group(1))
-            counts = dict(re.findall(r"^(.+?):\s+(\d+)\.", out, re.M))
-            free = int(counts.get("Pages free", 0))
-            inactive = int(counts.get("Pages inactive", 0))
-            purgeable = int(counts.get("Pages purgeable", 0))
-            if free or inactive:
-                return ((free + inactive + purgeable) * page) // (1024 * 1024)
+            return _darwin_available_mb()
         elif system == "Windows":
             class _MemStatusEx(ctypes.Structure):
                 _fields_ = [
@@ -109,6 +106,58 @@ def available_ram_mb() -> Optional[int]:
     return None
 
 
+def _sysctl_int(name: str) -> Optional[int]:
+    import subprocess
+    try:
+        out = subprocess.run(["sysctl", "-n", name], capture_output=True, text=True,
+                             encoding="utf-8", errors="replace", timeout=10).stdout.strip()
+    except (OSError, subprocess.SubprocessError):
+        return None
+    return int(out) if out.isdigit() else None
+
+
+def _vm_stat_available_mb(out: str) -> Optional[int]:
+    """free + inactive + speculative + purgeable pages from `vm_stat` output, in MB."""
+    page = 4096
+    pm = re.search(r"page size of (\d+) bytes", out)
+    if pm:
+        page = int(pm.group(1))
+    counts = dict(re.findall(r"^(.+?):\s+(\d+)\.", out, re.M))
+    pages = sum(int(counts.get(k, 0)) for k in
+                ("Pages free", "Pages inactive", "Pages speculative", "Pages purgeable"))
+    if not pages:
+        return None
+    return (pages * page) // (1024 * 1024)
+
+
+def _darwin_available_mb(memsize: Optional[int] = None, level: Optional[int] = None,
+                         vm_stat_out: Optional[str] = None) -> Optional[int]:
+    """Available memory on macOS.
+
+    The kernel's own figure is kern.memorystatus_level (the "free percentage"
+    `memory_pressure` prints), which counts memory the compressor can reclaim.
+    Page counts from vm_stat miss that and read a few GB low on a machine that
+    is not under pressure at all, so they are only the fallback.
+    """
+    import subprocess
+    if memsize is None:
+        memsize = _sysctl_int("hw.memsize")
+    if level is None:
+        level = _sysctl_int("kern.memorystatus_level")
+    from_level = None
+    if memsize and level is not None and 0 < level <= 100:
+        from_level = (memsize * level // 100) // (1024 * 1024)
+    if vm_stat_out is None:
+        try:
+            vm_stat_out = subprocess.run(["vm_stat"], capture_output=True, text=True,
+                                         encoding="utf-8", errors="replace", timeout=10).stdout
+        except (OSError, subprocess.SubprocessError):
+            vm_stat_out = ""
+    from_pages = _vm_stat_available_mb(vm_stat_out or "")
+    known = [v for v in (from_level, from_pages) if v is not None]
+    return max(known) if known else None
+
+
 def _is_app_dex(name: str) -> bool:
     """True for app-code DEX (APK `classes*.dex` or App Bundle `<module>/dex/classes*.dex`), not asset .dex."""
     base = name.rsplit("/", 1)[-1]
@@ -225,8 +274,20 @@ def check_memory_preflight(apk_path: str, max_mem: str,
 
     if available_mb is not None:
         needed_rss = int(max_mem_mb * JVM_RSS_OVERHEAD)
-        if needed_rss > (available_mb - RAM_SAFETY_RESERVE_MB):
-            safe_gb = max(1, int((available_mb - RAM_SAFETY_RESERVE_MB) / JVM_RSS_OVERHEAD / 1024))
+        room_mb = available_mb - RAM_SAFETY_RESERVE_MB
+        if needed_rss > room_mb:
+            # -Xmx is a ceiling the JVM only grows to under pressure: when a smaller
+            # heap still fits this app with room to spare, use that instead of refusing.
+            fit_mb = (max(0, int(room_mb / JVM_RSS_OVERHEAD)) // 256) * 256
+            if fit_mb >= max(MIN_ESTIMATE_MB, need_mb * AUTO_LOWER_MARGIN):
+                note = (
+                    f"Only {_fmt_mb(available_mb)} of memory is available right now, so JADX runs "
+                    f"with a {_fmt_mb(fit_mb)} Java heap instead of --max-mem {max_mem} "
+                    f"(this app needs about {_fmt_mb(need_mb)})."
+                )
+                return PreflightVerdict(False, "lowered", note, need_mb, max_mem_mb,
+                                        available_mb, stats, f"{fit_mb}m")
+            safe_gb = max(1, int(room_mb / JVM_RSS_OVERHEAD / 1024))
             lower_line = f"  - lower to --max-mem {safe_gb}g, which fits in what's free"
             if safe_gb * 1024 < need_mb:
                 lower_line += (f" (though this app looks like it needs about {_fmt_mb(need_mb)}, "
@@ -234,7 +295,8 @@ def check_memory_preflight(apk_path: str, max_mem: str,
             msg = (
                 f"--max-mem {max_mem} asks JADX for {_fmt_mb(max_mem_mb)} of Java heap, which needs "
                 f"about {_fmt_mb(needed_rss)} of real memory once JVM overhead is counted. This "
-                f"machine only has {_fmt_mb(available_mb)} available right now.\n\n"
+                f"machine only has {_fmt_mb(available_mb)} available right now, not enough for "
+                f"this app ({scale}, about {_fmt_mb(need_mb)} of heap).\n\n"
                 f"Running this would push the machine into swap and most likely get JADX killed by "
                 f"the OS out-of-memory killer (and may take other apps down with it). Options:\n"
                 f"  - free up memory and try again\n"

{narvy_cli-1.1.2 → narvy_cli-1.1.3}/narvy/decompiler.py

@@ -222,6 +222,10 @@ def _as_limit_preexec(max_mem_mb):
     return _set_limits
 
 
+# Set by decompile_apk when it ran jadx with a smaller heap than --max-mem; the caller prints it.
+LAST_HEAP_NOTE = None
+
+
 def decompile_apk(apk_path: str, output_dir: str, max_mem: str = "4g", force: bool = False,
                   extra_inputs=None):
     """Decompile an APK with jadx; returns (success, error_detail). jadx keeps the FIRST input's manifest, so apk_path stays first; `extra_inputs` merge into one tree."""
@@ -229,10 +233,16 @@ def decompile_apk(apk_path: str, output_dir: str, max_mem: str = "4g", force: bo
     if validation_error:
         return False, validation_error
 
+    global LAST_HEAP_NOTE
+    LAST_HEAP_NOTE = None
     if not force:
         verdict = check_memory_preflight(apk_path, max_mem)
         if verdict.should_block:
             return False, verdict.message
+        lowered = getattr(verdict, "lowered_max_mem", None)
+        if lowered:
+            max_mem = lowered
+            LAST_HEAP_NOTE = verdict.message
 
     try:
         jadx_bin = resolve_jadx_binary()

{narvy_cli-1.1.2 → narvy_cli-1.1.3}/narvy/main.py

@@ -37,6 +37,7 @@ except Exception:
 from narvy import __version__
 from narvy.reporter import generate_sarif_report, sarif_level, sarif_security_severity
 from .decompiler import decompile_apk
+from . import decompiler as _decompiler
 from .rule_engine import load_rules_from_dir, run_rules_on_file
 from . import auth, own_reports, uploader, telemetry
 from .third_party_filter import get_own_package_roots, resolve_file_scope, check_android_override
@@ -866,6 +867,8 @@ def _run_local_scan(apk_path, max_mem, override_config=None, force=False,
                 console.print(f"\n[bold red]{decompile_error}[/bold red]\n")
                 return None
             progress.update(task1, completed=1, description="[green]Decompilation Complete.")
+            if _decompiler.LAST_HEAP_NOTE:
+                console.print(f"[dim]{_decompiler.LAST_HEAP_NOTE}[/dim]")
 
             task2 = progress.add_task("[cyan]Analyzing files...", total=None)
             rules_path = os.path.join(os.path.dirname(__file__), 'rules', 'android')

{narvy_cli-1.1.2 → narvy_cli-1.1.3}/narvy/semgrep_engine.py

@@ -231,6 +231,50 @@ def load_rule_defs(rules_path: str = RULES_PATH) -> List[Dict[str, Any]]:
     return defs
 
 
+def _attempt(cmd: List[str], timeout: int):
+    """One semgrep run: (status, data, message). Raises TimeoutExpired."""
+    proc = run_tree(cmd, timeout=timeout, env=semgrep_env())
+    if not proc.stdout:
+        if proc.returncode not in (0, 1):
+            return 'error', None, _rc_message(proc)
+        return 'ok', {'results': []}, None
+    try:
+        data = json.loads(proc.stdout)
+    except json.JSONDecodeError:
+        return 'error', None, "semgrep output was not valid JSON"
+    if not isinstance(data, dict):
+        return 'error', None, "semgrep output was not valid JSON"
+    if proc.returncode not in (0, 1) and not data.get('results'):
+        # Exit 2+ with a JSON body is a fatal error (bad config, crash), not a clean pass.
+        errs = [e.get('message') or e.get('type') for e in data.get('errors') or []
+                if isinstance(e, dict)]
+        detail = next((str(e) for e in errs if e), None)
+        return 'error', None, _rc_message(proc, detail)
+    return 'ok', data, None
+
+
+def _rc_message(proc, detail: Optional[str] = None) -> str:
+    text = detail or (proc.stderr or '').strip()
+    text = ' '.join(str(text).split())[:200]
+    return f"semgrep exited rc={proc.returncode}" + (f": {text}" if text else "")
+
+
+def _scanned_nothing(data: Optional[Dict[str, Any]]) -> bool:
+    paths = (data or {}).get('paths')
+    return isinstance(paths, dict) and 'scanned' in paths and not paths['scanned']
+
+
+def _count_own_code_files(source_dir: str, own_roots) -> int:
+    from .third_party_filter import is_own_package_path
+    n = 0
+    for dirpath, _dirs, files in os.walk(source_dir):
+        for f in files:
+            if f.endswith(('.java', '.kt')) and is_own_package_path(
+                    os.path.relpath(os.path.join(dirpath, f), source_dir), own_roots):
+                n += 1
+    return n
+
+
 def run_semgrep(source_dir: str, timeout: Optional[int] = None,
                  own_roots: Optional[set] = None,
                  override_config: Optional[ScopeConfig] = None,
@@ -267,33 +311,37 @@ def run_semgrep(source_dir: str, timeout: Optional[int] = None,
         # suppresses a vendor entry nested inside an included own_root.
         scope_args += ['--exclude', f"**/{entry.replace('.', '/')}/**"]
 
-    cmd = (
-        [semgrep_bin() or 'semgrep', '--config', RULES_PATH, source_dir,
-         '--json', '--quiet', '--no-git-ignore',
-         '--timeout', '60']  # per-file timeout, not the whole run
-        + resource_args(semgrep_jobs())
-        + scope_args
-    )
+    def _cmd(jobs: int, scope: List[str]) -> List[str]:
+        return (
+            [semgrep_bin() or 'semgrep', '--config', RULES_PATH, source_dir,
+             '--json', '--quiet', '--no-git-ignore', '--no-rewrite-rule-ids',
+             '--timeout', '60']  # per-file timeout, not the whole run
+            + resource_args(jobs)
+            + scope
+        )
+
+    jobs = semgrep_jobs()
     try:
-        proc = run_tree(cmd, timeout=timeout, env=semgrep_env())
+        status, data, message = _attempt(_cmd(jobs, scope_args), timeout)
+        if status == 'error' and jobs > 1:
+            # A crashed multi-worker run is retried once with one worker before
+            # the pass is reported as incomplete.
+            status, data, message = _attempt(_cmd(1, scope_args), timeout)
     except subprocess.TimeoutExpired:
         # LAST_RUN stops the caller presenting the empty list as a clean pass.
         _record_run('timeout', timeout_s=timeout, file_count=file_count)
         return []
 
-    if not proc.stdout:
-        if proc.returncode not in (0, 1):
-            _record_run('error', timeout_s=timeout, file_count=file_count,
-                        message=f"semgrep exited rc={proc.returncode}")
-        else:
-            _record_run('ok', timeout_s=timeout, file_count=file_count)
-        return []
+    if status == 'ok' and own_roots and _scanned_nothing(data):
+        expected = _count_own_code_files(source_dir, own_roots)
+        if expected:
+            # The --include globs matched none of the app's own files: say so
+            # instead of reporting a clean pass over nothing.
+            status, message = 'error', (
+                f"the structural pass matched none of the {expected} app source files")
 
-    try:
-        data = json.loads(proc.stdout)
-    except json.JSONDecodeError:
-        _record_run('error', timeout_s=timeout, file_count=file_count,
-                    message="semgrep output was not valid JSON")
+    if status != 'ok':
+        _record_run('error', timeout_s=timeout, file_count=file_count, message=message)
         return []
 
     _record_run('ok', timeout_s=timeout, file_count=file_count)

{narvy_cli-1.1.2 → narvy_cli-1.1.3/narvy_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: narvy-cli
-Version: 1.1.2
+Version: 1.1.3
 Summary: Free, local SAST scanner for Android, iOS, and polyglot web/backend source - plus passive web, host-audit, and cloud-posture scanning.
 Author: Narvy
 License: Apache-2.0