jscope 0.4.2__py3-none-any.whl

jscope/config.py

@@ -0,0 +1,72 @@
+"""Configuration. Supports a jscope.yml/json in cwd or --config path.
+Users can enable/disable analyzers, set confidence threshold, output format,
+concurrency, exclusions. """
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+DEFAULTS: dict[str, Any] = {
+    "analyzers": {
+        "enabled": ["framework_detector", "endpoint_analyzer", "secret_analyzer", "router_analyzer",
+                    "auth_analyzer", "authorization_analyzer", "storage_analyzer", "graphql_analyzer",
+                    "service_worker_analyzer", "source_map_analyzer",
+                    "html_analyzer", "openapi_analyzer", "graphql_schema_analyzer", "wasm_analyzer",
+                    "dataflow_analyzer"],
+    },
+    "artifacts": {"enabled": True},                       # Phase 4: HTML, Vue SFC, OpenAPI, GraphQL schema, WASM
+    "incremental": {"cache_dir": None, "enabled": True},
+    "plugins": {"paths": [], "verified": [], "community_confidence_cap": "MEDIUM", "entry_points": True,
+                "isolation": "auto", "timeout_seconds": 30, "memory_mb": 1024, "grant_read": []},
+    "source_maps": {"expand": False},
+    "min_confidence": "LOW",
+    "output": {"format": "human", "verbose": False},
+    "concurrency": 4,
+    "exclude": ["**/node_modules/**"],
+    "max_file_bytes": 5 * 1024 * 1024,
+    "runtime": {"rate_per_second": 2.0, "max_requests": 100, "timeout_seconds": 8},
+    "ai": {"enabled": False, "provider": "anthropic", "model": "claude-sonnet-5-5", "api_key_env": "ANTHROPIC_API_KEY",
+           "top_n": 8, "redact_paths": True, "max_tokens": 1500},
+}
+
+
+@dataclass
+class Config:
+    data: dict[str, Any] = field(default_factory=lambda: json.loads(json.dumps(DEFAULTS)))
+
+    @classmethod
+    def load(cls, path: str | None) -> "Config":
+        cfg = cls()
+        if path and Path(path).exists():
+            text = Path(path).read_text()
+            if path.endswith((".yml", ".yaml")):
+                try:
+                    import yaml  # optional dependency
+                    user_cfg = yaml.safe_load(text) or {}
+                except ImportError:
+                    raise RuntimeError("PyYAML not installed; use a .json config or `pip install pyyaml`")
+            else:
+                user_cfg = json.loads(text)
+            cfg.data = _deep_merge(cfg.data, user_cfg)
+        return cfg
+
+    def get(self, dotted_key: str, default: Any = None) -> Any:
+        node: Any = self.data
+        for part in dotted_key.split("."):
+            if not isinstance(node, dict) or part not in node:
+                return default
+            node = node[part]
+        return node
+
+
+def _deep_merge(base: dict, override: dict) -> dict:
+    out = dict(base)
+    for k, v in override.items():
+        if isinstance(v, dict) and isinstance(out.get(k), dict):
+            out[k] = _deep_merge(out[k], v)
+        else:
+            out[k] = v
+    return out

jscope/core/artifacts.py

@@ -0,0 +1,139 @@
+"""Non-JavaScript artifacts (Phase 4).
+
+JScope started as a JavaScript tool; the PRD's final phase widens it to the other client-side and
+API-description artifacts that sit next to a web app's JS: HTML pages, Vue single-file components,
+OpenAPI/Swagger specs, GraphQL schemas (SDL or introspection JSON) and WebAssembly modules.
+
+Collection is purely offline and content-sniffed (a `.json` file is only treated as an OpenAPI spec if it
+parses as one), so dropping a whole downloaded site into a directory just works and unrelated JSON is ignored.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from jscope.parsing.ast_engine import ASTEngine
+
+KIND_HTML = "html"
+KIND_VUE = "vue"
+KIND_OPENAPI = "openapi"
+KIND_GRAPHQL_SCHEMA = "graphql_schema"
+KIND_WASM = "wasm"
+ALL_KINDS = (KIND_HTML, KIND_VUE, KIND_OPENAPI, KIND_GRAPHQL_SCHEMA, KIND_WASM)
+
+_HTML_EXT = (".html", ".htm")
+_DATA_EXT = (".json", ".yaml", ".yml")
+_GQL_EXT = (".graphql", ".gql", ".graphqls")
+_MAX_ARTIFACT_BYTES = 20 * 1024 * 1024
+_WASM_MAGIC = b"\x00asm"
+
+
+@dataclass
+class Artifact:
+    kind: str
+    path: str
+    data: bytes
+    content_hash: str
+    origin: str = "disk"                                   # "disk" | "html-inline" ...
+    memo: dict[str, Any] = field(default_factory=dict, repr=False)   # parse-once cache shared by pipeline + analyzers
+
+    @property
+    def text(self) -> str:
+        return self.data.decode("utf-8", errors="replace")
+
+
+def _load_structured(text: str, path: str) -> Any | None:
+    """JSON or (if PyYAML is installed) YAML. Returns None when it cannot be parsed."""
+    if path.lower().endswith(".json") or text.lstrip().startswith(("{", "[")):
+        try:
+            return json.loads(text)
+        except ValueError:
+            return None
+    try:
+        import yaml
+    except ImportError:
+        return None
+    try:
+        return yaml.safe_load(text)
+    except Exception:       # noqa: BLE001 -- any YAML error just means "not a spec we can read"
+        return None
+
+
+def structured(art: Artifact) -> Any | None:
+    if "structured" not in art.memo:
+        art.memo["structured"] = _load_structured(art.text, art.path)
+    return art.memo["structured"]
+
+
+def _is_openapi(doc: Any) -> bool:
+    return (isinstance(doc, dict) and isinstance(doc.get("paths"), dict)
+            and (str(doc.get("openapi", "")).startswith("3") or str(doc.get("swagger", "")).startswith("2")))
+
+
+def _is_introspection(doc: Any) -> bool:
+    if not isinstance(doc, dict):
+        return False
+    schema = (doc.get("data") or doc).get("__schema") if isinstance(doc.get("data") or doc, dict) else None
+    return isinstance(schema, dict) and isinstance(schema.get("types"), list)
+
+
+def classify_artifact(path: str, data: bytes) -> str | None:
+    """Content-sniffing classifier. Returns an artifact kind or None (not an artifact we handle)."""
+    low = path.lower()
+    if low.endswith(".wasm") or data[:4] == _WASM_MAGIC:
+        return KIND_WASM if data[:4] == _WASM_MAGIC else None
+    if low.endswith(_HTML_EXT):
+        return KIND_HTML
+    if low.endswith(".vue"):
+        return KIND_VUE
+    if low.endswith(_GQL_EXT):
+        text = data[:200_000].decode("utf-8", errors="replace")
+        return KIND_GRAPHQL_SCHEMA if _looks_like_sdl(text) else None
+    if low.endswith(_DATA_EXT):
+        head = data[:4096].decode("utf-8", errors="replace").lower()
+        if "openapi" in head or "swagger" in head:
+            doc = _load_structured(data.decode("utf-8", errors="replace"), path)
+            if _is_openapi(doc):
+                return KIND_OPENAPI
+        if "__schema" in data[:2_000_000].decode("utf-8", errors="replace"):
+            doc = _load_structured(data.decode("utf-8", errors="replace"), path)
+            if _is_introspection(doc):
+                return KIND_GRAPHQL_SCHEMA
+    return None
+
+
+def _looks_like_sdl(text: str) -> bool:
+    import re
+    return bool(re.search(r"^\s*(extend\s+)?(type|input|interface|enum|schema|union|scalar)\s+\w*", text, re.M)
+                and re.search(r"\btype\s+(Query|Mutation|Subscription)\b|\bschema\s*\{", text))
+
+
+def collect_artifacts(paths: list[str], exclude: list[str], kinds: tuple[str, ...] = ALL_KINDS) -> list[Artifact]:
+    out: dict[str, Artifact] = {}
+
+    def consider(f: Path):
+        if not f.is_file() or f.stat().st_size > _MAX_ARTIFACT_BYTES:
+            return
+        rel = str(f)
+        if any(fnmatch.fnmatch(rel, pat) for pat in exclude):
+            return
+        suffix = f.suffix.lower()
+        if suffix not in _HTML_EXT + _DATA_EXT + _GQL_EXT + (".wasm", ".vue"):
+            return
+        data = f.read_bytes()
+        kind = classify_artifact(rel, data)
+        if kind and kind in kinds:
+            out[rel] = Artifact(kind, rel, data, ASTEngine.hash_bytes(data))
+
+    for p in paths:
+        path = Path(p)
+        if path.is_file():
+            consider(path)
+        elif path.is_dir():
+            for f in path.rglob("*"):
+                consider(f)
+    return [out[k] for k in sorted(out)]

jscope/core/cache.py

@@ -0,0 +1,198 @@
+"""Dependency-aware incremental analysis (FR-07).
+
+What is cached: the data-flow engine's per-root results. A *root* is a file; its result (flows, limitations, unresolved
+imports) is a pure function of the file's content, the content of every file it transitively imports, the resolver
+configuration (tsconfig/aliases), the Vue templates and the engine version. That function's input is hashed into a
+`closure hash`; an entry is reused only when its closure hash is unchanged, so:
+
+  * unchanged project            -> every root reused, no analysis
+  * one file edited              -> that file and everything that (transitively) imports it is re-analysed
+  * a file removed               -> its entry is dropped; importers' closure hashes change so they re-analyse
+  * corrupt / foreign cache file -> ignored, full scan (never an error)
+
+Parsed ASTs are NOT persisted (tree-sitter trees are not serialisable and parsing is cheap relative to analysis).
+`--no-cache` / clean scans bypass the cache entirely, and a cold incremental run uses the same per-root isolation as a
+warm one, so incremental output equals clean output for the same inputs (verified by tests/test_incremental.py).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from jscope.dataflow.engine import DataFlowEngine, Flow, Limitation
+from jscope.dataflow.rules import Mitigation
+from jscope.parsing.ast_engine import ParsedFile
+
+CACHE_VERSION = 3
+ENGINE_VERSION = "dataflow-0.5.0"
+CACHE_FILE = "dataflow-cache.json"
+
+
+def _h(*parts: str) -> str:
+    m = hashlib.sha256()
+    for p in parts:
+        m.update(p.encode("utf-8", "replace"))
+        m.update(b"\x00")
+    return m.hexdigest()
+
+
+def flow_to_json(f: Flow) -> dict:
+    return {"source_kind": f.source_kind, "source_file": f.source_file, "source_line": f.source_line, "source_snippet": f.source_snippet,
+            "sink_class": f.sink_class, "sink_name": f.sink_name, "sink_file": f.sink_file, "sink_line": f.sink_line,
+            "sink_snippet": f.sink_snippet, "steps": [list(s) for s in f.steps], "status": f.status,
+            "mitigations": [{"name": m.name, "effective": sorted(m.effective), "uncertain": sorted(m.uncertain), "note": m.note}
+                            for m in f.mitigations],
+            "assumptions": list(f.assumptions), "files": list(f.files), "root": f.root}
+
+
+def flow_from_json(d: dict) -> Flow:
+    return Flow(d["source_kind"], d["source_file"], d["source_line"], d["source_snippet"], d["sink_class"], d["sink_name"],
+                d["sink_file"], d["sink_line"], d["sink_snippet"], tuple(tuple(s) for s in d["steps"]), d["status"],
+                tuple(Mitigation(m["name"], frozenset(m["effective"]), frozenset(m["uncertain"]), m.get("note", "")) for m in d["mitigations"]),
+                tuple(d["assumptions"]), tuple(d["files"]), d.get("root", ""))
+
+
+@dataclass
+class CacheStats:
+    reused: list[str] = field(default_factory=list)
+    reanalyzed: list[str] = field(default_factory=list)
+    dropped: list[str] = field(default_factory=list)
+    note: str = ""
+
+    def summary(self) -> str:
+        return (f"[incremental] reused {len(self.reused)} file(s), re-analyzed {len(self.reanalyzed)}, "
+                f"dropped {len(self.dropped)} removed" + (f" ({self.note})" if self.note else ""))
+
+
+def static_import_specs(pf: ParsedFile) -> list[str]:
+    """Every literal module specifier in the file, without executing anything (import/export-from/require/import())."""
+    out: list[str] = []
+    stack = [pf.tree.root_node]
+    while stack:
+        n = stack.pop()
+        t = n.type
+        if t in ("import_statement", "export_statement"):
+            src = n.child_by_field_name("source")
+            if src is not None:
+                out.append(pf.node_text(src).strip("'\"`"))
+        elif t == "call_expression":
+            fn = n.child_by_field_name("function")
+            args = n.child_by_field_name("arguments")
+            if fn is not None and args is not None and (fn.type == "import" or (fn.type == "identifier" and pf.node_text(fn) == "require")):
+                ch = [c for c in args.children if c.is_named]
+                if ch and ch[0].type == "string":
+                    out.append(pf.node_text(ch[0]).strip("'\"`"))
+        stack.extend(n.children)
+    return out
+
+
+class AnalysisCache:
+    def __init__(self, directory: str | None, enabled: bool = True):
+        self.dir = Path(directory) if directory else None
+        self.enabled = enabled and self.dir is not None
+        self.stats = CacheStats()
+
+    # -- persistence -------------------------------------------------------------------------------
+    def _load(self) -> dict:
+        if not self.enabled:
+            return {}
+        p = self.dir / CACHE_FILE
+        try:
+            data = json.loads(p.read_text(encoding="utf-8"))
+            if data.get("cache_version") != CACHE_VERSION or data.get("engine") != ENGINE_VERSION or not isinstance(data.get("entries"), dict):
+                self.stats.note = "cache from a different JScope version ignored"
+                return {}
+            return data["entries"]
+        except FileNotFoundError:
+            return {}
+        except (OSError, ValueError, KeyError, TypeError):
+            self.stats.note = "cache unreadable; performing a full scan"
+            return {}
+
+    def _save(self, entries: dict) -> None:
+        if not self.enabled:
+            return
+        try:
+            self.dir.mkdir(parents=True, exist_ok=True)
+            tmp = self.dir / (CACHE_FILE + ".tmp")
+            tmp.write_text(json.dumps({"cache_version": CACHE_VERSION, "engine": ENGINE_VERSION, "entries": entries}), encoding="utf-8")
+            os.replace(tmp, self.dir / CACHE_FILE)
+        except OSError:
+            self.stats.note = (self.stats.note + "; " if self.stats.note else "") + "cache could not be written"
+
+    # -- closure hashing -----------------------------------------------------------------------------
+    def closure_hashes(self, engine: DataFlowEngine, files: list[ParsedFile]) -> tuple[dict[str, str], dict[str, set[str]]]:
+        resolver = engine.resolver
+        edges: dict[str, set[str]] = {}
+        for pf in files:
+            deps: set[str] = set()
+            for spec in static_import_specs(pf):
+                r = resolver.resolve(spec, pf.path)
+                if r.status == "resolved" and r.path:
+                    real = engine.real_path(r.path)
+                    if real and real != pf.path:
+                        deps.add(real)
+                elif r.status == "unresolved":
+                    deps.add("unresolved:" + spec)           # a spec that starts resolving later must invalidate the entry
+            edges[pf.path] = deps
+        ts_digest = _h(*[Path(p).read_text(errors="replace") for p in sorted(set(resolver.tsconfig_files)) if os.path.exists(p)],
+                       json.dumps(sorted(resolver.aliases.items())))
+        tpl_digest = _h(repr(sorted(map(repr, engine.vue_templates))))
+        content = {pf.path: pf.content_hash for pf in files}
+        out: dict[str, str] = {}
+        for pf in files:
+            seen, stack = set(), [pf.path]
+            while stack:
+                cur = stack.pop()
+                if cur in seen:
+                    continue
+                seen.add(cur)
+                stack.extend(d for d in edges.get(cur, ()) if not d.startswith("unresolved:"))
+            parts = [f"{p}={content.get(p, '')}" for p in sorted(seen)]
+            parts += sorted(d for p in seen for d in edges.get(p, ()) if d.startswith("unresolved:"))
+            out[pf.path] = _h(ENGINE_VERSION, ts_digest, tpl_digest, *parts)
+        return out, edges
+
+    # -- the one entry point used by DataFlowAnalyzer ------------------------------------------------------
+    def run(self, engine: DataFlowEngine, files: list[ParsedFile]) -> CacheStats:
+        engine.isolate_roots = True
+        entries = self._load()
+        hashes, edges = self.closure_hashes(engine, files)
+        engine.import_edges = {k: {d for d in v if not d.startswith("unresolved:")} for k, v in edges.items()}
+        current = set(hashes)
+        reuse = {p for p, h in hashes.items() if entries.get(p, {}).get("closure") == h}
+        self.stats.reused = sorted(reuse)
+        self.stats.reanalyzed = sorted(current - reuse)
+        self.stats.dropped = sorted(set(entries) - current)
+        engine.suppressed = set(reuse)
+        engine.run(only_files=current - reuse)
+        for p in reuse:                                          # replay cached roots
+            e = entries[p]
+            for fd in e["flows"]:
+                engine.keep(flow_from_json(fd))
+            for ld in e["limitations"]:
+                engine.lim(ld["file"], ld["line"], ld["kind"], ld["message"])
+            for u in e["unresolved_imports"]:
+                engine.unresolved_imports.append(tuple(u))
+        engine.unresolved_imports = sorted(set(engine.unresolved_imports))
+        by_root: dict[str, list[Flow]] = {}
+        for f in engine.flows.values():
+            by_root.setdefault(f.root, []).append(f)
+        new_entries = {}
+        for p in current:
+            if p in reuse:
+                new_entries[p] = entries[p]
+                continue
+            new_entries[p] = {
+                "closure": hashes[p],
+                "flows": [flow_to_json(f) for f in sorted(by_root.get(p, []), key=lambda f: f.key)],
+                "limitations": [{"file": l.file, "line": l.line, "kind": l.kind, "message": l.message}
+                                for l in engine.limitations if l.file == p],
+                "unresolved_imports": [list(u) for u in engine.unresolved_imports if u[0] == p],
+            }
+        self._save(new_entries)
+        return self.stats

jscope/core/classification.py

@@ -0,0 +1,56 @@
+"""Finding classification (FR-08): what KIND of statement is a finding, and how far has it been verified?
+
+Two orthogonal axes, deliberately separate from `confidence`:
+
+  evidence_class        observed_fact | static_inference | security_hypothesis | verified_behavior | confirmed_vulnerability
+  verification_status   unverified | probable | confirmed | inconclusive | rejected
+
+`confidence` says how sure JScope is that a code pattern EXISTS and means what it looks like. It never says the
+weakness is exploitable. Only verification can move a finding to confirmed_vulnerability, and only by the documented
+procedure for its class (docs/VERIFICATION.md).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from jscope import __version__
+
+EVIDENCE_CLASSES = ("observed_fact", "static_inference", "security_hypothesis", "verified_behavior", "confirmed_vulnerability")
+VERIFICATION_STATUSES = ("unverified", "probable", "confirmed", "inconclusive", "rejected")
+SCHEMA_VERSION = "0.5"
+
+HYPOTHESIS_CATEGORIES = {"dataflow", "authorization", "idor", "csrf", "xss", "redirect", "cors", "session"}
+
+
+def classify(category: str, metadata: dict[str, Any], evidence_types: set[str], confidence: str) -> dict[str, Any]:
+    ver = metadata.get("verification") or {}
+    status = ver.get("status") or metadata.get("verification_status") or "unverified"
+    if status not in VERIFICATION_STATUSES:
+        status = "unverified"
+    explicit = metadata.get("evidence_class")
+    if status == "confirmed" and ver.get("procedure"):
+        cls = "confirmed_vulnerability"
+    elif status == "probable":
+        cls = "verified_behavior" if ver.get("behavior_reproduced") else "security_hypothesis"
+    elif status == "rejected" and ver.get("procedure"):
+        cls = "verified_behavior"
+    elif explicit in EVIDENCE_CLASSES and explicit not in ("verified_behavior", "confirmed_vulnerability"):
+        cls = explicit                                  # an analyzer cannot claim verification by labelling itself
+    elif "RUNTIME_CONTEXT" in evidence_types or "RUNTIME_NOT_FOUND" in evidence_types:
+        cls = "observed_fact"
+    elif category in HYPOTHESIS_CATEGORIES or metadata.get("attack_surface_hypothesis"):
+        cls = "security_hypothesis"
+    else:
+        cls = "static_inference"
+    out: dict[str, Any] = {
+        "evidence_class": cls,
+        "verification_status": status,
+        "analysis_version": __version__,
+        "confidence_note": "confidence measures how well the evidence supports that this pattern exists; it is not proof of exploitability",
+    }
+    if metadata.get("limitations"):
+        out["limitations"] = list(metadata["limitations"])
+    if ver:
+        out["verification"] = ver
+    return out