jscope 0.4.2__py3-none-any.whl

jscope/dataflow/modules.py

@@ -0,0 +1,205 @@
+"""Module resolution (FR-02): maps an import specifier to a file that is part of the analyzed project.
+
+Supported, in order: relative paths (extension probing incl. the TypeScript ESM habit of writing `./x.js` for `x.ts`,
+and `dir/index.*`), tsconfig/jsconfig `baseUrl` + `paths` (wildcards, `extends` chain), explicit `module_aliases` from
+the JScope config (bundler aliases such as `@` -> `src`). Bare package specifiers resolve to nothing and are reported as
+EXTERNAL; anything else that fails is reported as UNRESOLVED with a reason. Resolution never touches the network and
+never leaves the set of files given to the analyzer.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+PROBE_EXTS = (".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts", ".vue")
+_JS_TO_TS = {".js": (".ts", ".tsx"), ".jsx": (".tsx",), ".mjs": (".mts",), ".cjs": (".cts",)}
+
+
+@dataclass
+class Resolution:
+    path: str | None
+    status: str            # "resolved" | "external" | "unresolved"
+    reason: str = ""
+    via: str = "relative"  # relative | tsconfig-paths | tsconfig-baseUrl | config-alias
+
+
+def _strip_jsonc(text: str) -> str:
+    """tsconfig.json is JSON-with-comments-and-trailing-commas. Strip both without touching string contents."""
+    out, i, n, in_str, esc = [], 0, len(text), False, False
+    while i < n:
+        c = text[i]
+        if in_str:
+            out.append(c)
+            if esc:
+                esc = False
+            elif c == "\\":
+                esc = True
+            elif c == '"':
+                in_str = False
+            i += 1
+            continue
+        if c == '"':
+            in_str = True
+            out.append(c)
+        elif text.startswith("//", i):
+            while i < n and text[i] != "\n":
+                i += 1
+            continue
+        elif text.startswith("/*", i):
+            j = text.find("*/", i + 2)
+            i = n if j < 0 else j + 2
+            continue
+        else:
+            out.append(c)
+        i += 1
+    return re.sub(r",(\s*[}\]])", r"\1", "".join(out))
+
+
+@dataclass
+class TsConfig:
+    directory: str
+    base_url: str | None = None                       # absolute directory
+    paths: dict[str, list[str]] = field(default_factory=dict)
+    paths_base: str | None = None                     # directory `paths` targets are relative to
+    source: str = ""
+
+
+def load_tsconfig(path: Path, _seen: set[str] | None = None) -> TsConfig | None:
+    _seen = _seen or set()
+    key = str(path.resolve())
+    if key in _seen:
+        return None
+    _seen.add(key)
+    try:
+        raw = json.loads(_strip_jsonc(path.read_text(encoding="utf-8", errors="replace")))
+    except (OSError, ValueError):
+        return None
+    cfg = TsConfig(directory=str(path.parent), source=str(path))
+    ext = raw.get("extends")
+    if isinstance(ext, str) and ext.startswith("."):
+        parent = (path.parent / ext)
+        for cand in (parent, parent.with_suffix(".json")):
+            if cand.is_file():
+                base = load_tsconfig(cand, _seen)
+                if base:
+                    cfg.base_url, cfg.paths, cfg.paths_base = base.base_url, dict(base.paths), base.paths_base
+                break
+    co = raw.get("compilerOptions") or {}
+    if isinstance(co.get("baseUrl"), str):
+        cfg.base_url = str((path.parent / co["baseUrl"]).resolve())
+    if isinstance(co.get("paths"), dict):
+        cfg.paths = {k: [x for x in v if isinstance(x, str)] for k, v in co["paths"].items() if isinstance(v, list)}
+        cfg.paths_base = cfg.base_url or str(path.parent.resolve())
+    return cfg
+
+
+class ModuleResolver:
+    def __init__(self, files: list[str], aliases: dict[str, str] | None = None, roots: list[str] | None = None):
+        self.files = {os.path.normpath(os.path.abspath(f)): f for f in files}
+        self.aliases = dict(aliases or {})
+        self._tscache: dict[str, TsConfig | None] = {}
+        dirs = [os.path.dirname(a) for a in self.files] or ["."]
+        self.root = os.path.commonpath(dirs) if dirs else "."
+        self.search_roots = [os.path.abspath(r) for r in (roots or [])]
+        self.tsconfig_files: list[str] = []
+
+    # -- tsconfig discovery ----------------------------------------------------------------------
+    def tsconfig_for(self, file: str) -> TsConfig | None:
+        d = os.path.dirname(os.path.abspath(file))
+        ceiling = self.root
+        for _ in range(4):                       # allow the config to sit a few levels above the analyzed folder
+            ceiling = os.path.dirname(ceiling) or ceiling
+        return self.tsconfig_for_dir(d, ceiling)
+
+    def tsconfig_for_dir(self, d: str, ceiling: str) -> TsConfig | None:
+        while True:
+            if d in self._tscache:
+                return self._tscache[d]
+            for name in ("tsconfig.json", "jsconfig.json"):
+                p = Path(d) / name
+                if p.is_file():
+                    cfg = load_tsconfig(p)
+                    if cfg:
+                        self.tsconfig_files.append(str(p))
+                        self._tscache[d] = cfg
+                        return cfg
+            if d == ceiling or d == os.path.dirname(d):
+                self._tscache[d] = None
+                return None
+            d = os.path.dirname(d)
+
+    # -- probing ---------------------------------------------------------------------------------
+    def _probe(self, base: str) -> str | None:
+        base = os.path.normpath(base)
+        if base in self.files:
+            return self.files[base]
+        stem, ext = os.path.splitext(base)
+        for alt in _JS_TO_TS.get(ext, ()):                     # ./x.js written in TS source really means ./x.ts
+            if stem + alt in self.files:
+                return self.files[stem + alt]
+        for e in PROBE_EXTS:
+            if base + e in self.files:
+                return self.files[base + e]
+        for e in PROBE_EXTS:
+            idx = os.path.join(base, "index" + e)
+            if idx in self.files:
+                return self.files[idx]
+        return None
+
+    def resolve(self, spec: str, from_file: str) -> Resolution:
+        if not spec:
+            return Resolution(None, "unresolved", "empty specifier")
+        spec = spec.split("?")[0]
+        fdir = os.path.dirname(os.path.abspath(from_file))
+        if spec.startswith((".", "/")):
+            base = spec if spec.startswith("/") else os.path.join(fdir, spec)
+            hit = self._probe(base)
+            if hit:
+                return Resolution(hit, "resolved", via="relative")
+            return Resolution(None, "unresolved", f"no file matches '{spec}' relative to {os.path.basename(fdir) or '.'}/")
+        tried: list[str] = []
+        cfg = self.tsconfig_for(from_file)
+        if cfg and cfg.paths:
+            for pattern, targets in cfg.paths.items():
+                m = self._match_pattern(pattern, spec)
+                if m is None:
+                    continue
+                for t in targets:
+                    cand = os.path.join(cfg.paths_base or cfg.directory, t.replace("*", m))
+                    tried.append(cand)
+                    hit = self._probe(cand)
+                    if hit:
+                        return Resolution(hit, "resolved", via="tsconfig-paths")
+        for alias, target in self.aliases.items():
+            if spec == alias or spec.startswith(alias.rstrip("/") + "/"):
+                rest = spec[len(alias):].lstrip("/")
+                cand = os.path.join(target if os.path.isabs(target) else os.path.join(self.root, target), rest) \
+                    if rest else (target if os.path.isabs(target) else os.path.join(self.root, target))
+                tried.append(cand)
+                hit = self._probe(cand)
+                if hit:
+                    return Resolution(hit, "resolved", via="config-alias")
+        if cfg and cfg.base_url:
+            cand = os.path.join(cfg.base_url, spec)
+            hit = self._probe(cand)
+            if hit:
+                return Resolution(hit, "resolved", via="tsconfig-baseUrl")
+        if tried:
+            return Resolution(None, "unresolved", f"alias matched but no file found (tried {len(tried)} location(s))")
+        if spec.startswith(("@/", "~/", "#")):
+            return Resolution(None, "unresolved", f"'{spec}' looks like a bundler alias but none is configured "
+                                                  "(add tsconfig `paths` or config `module_aliases`)")
+        return Resolution(None, "external", "package import (not part of the analyzed files)")
+
+    @staticmethod
+    def _match_pattern(pattern: str, spec: str) -> str | None:
+        if "*" not in pattern:
+            return "" if pattern == spec else None
+        pre, post = pattern.split("*", 1)
+        if spec.startswith(pre) and spec.endswith(post) and len(spec) >= len(pre) + len(post):
+            return spec[len(pre):len(spec) - len(post) if post else len(spec)]
+        return None

jscope/dataflow/rules.py

@@ -0,0 +1,118 @@
+"""Sources, sinks, sanitizers and validators for the data-flow engine (FR-01).
+
+Everything the engine believes about the web platform lives here as plain data so it can be reviewed and extended
+without touching the engine. Sink *classes* group sinks that are neutralised by the same kind of sanitizer:
+    html        DOM HTML insertion / client-side HTML rendering
+    code        dynamic code evaluation / script loading
+    navigation  location changes, window.open
+    sql         database query text
+    command     OS command execution
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+SINK_CLASSES = ("html", "code", "navigation", "sql", "command")
+ALL = frozenset(SINK_CLASSES)
+
+
+@dataclass(frozen=True)
+class Mitigation:
+    """A sanitizer / validator applied to a value. `effective` classes are neutralised, `uncertain` classes MAY be,
+    anything else is untouched (an ineffective mitigation is recorded for the report but changes nothing)."""
+    name: str
+    effective: frozenset = frozenset()
+    uncertain: frozenset = frozenset()
+    note: str = ""
+
+
+# ---------------------------------------------------------------------------------------------------- sources
+URL_MEMBER = re.compile(r"^(?:(?:window|self|globalThis|top|parent)\.)?(?:document\.)?location\.(hash|search|href|pathname)$")
+SOURCE_DOTTED = {
+    "document.URL": ("url", "document.URL"), "document.documentURI": ("url", "document.documentURI"),
+    "document.referrer": ("referrer", "document.referrer"), "document.cookie": ("cookie", "document.cookie"),
+    "window.name": ("window_name", "window.name"), "name": None,
+}
+URL_KIND = {"hash": "url_hash", "search": "url_query", "href": "url", "pathname": "url_path"}
+STORAGE_OBJECTS = {"localStorage", "sessionStorage"}
+FORM_CALL_RE = re.compile(r"(getElementById|querySelector|querySelectorAll|getElementsBy\w+|forms)\b")
+NETWORK_CALLEES = {"fetch", "axios", "$http.get", "$http.post", "$.get", "$.post", "$.getJSON", "ky", "got"}
+NETWORK_PREFIX = ("axios.",)
+HUMAN_SOURCE = {
+    "url_hash": "URL fragment (location.hash)", "url_query": "URL query string (location.search)",
+    "url": "page URL", "url_path": "URL path", "referrer": "document.referrer", "cookie": "document.cookie",
+    "window_name": "window.name", "postmessage": "postMessage event data", "storage": "web storage",
+    "form_input": "user-controlled form input", "network_response": "network response body",
+}
+
+# ---------------------------------------------------------------------------------------------------- sinks
+HTML_ASSIGN_PROPS = {"innerHTML": "innerHTML", "outerHTML": "outerHTML", "srcdoc": "iframe.srcdoc"}
+CODE_ASSIGN_PROPS = {"src": "script.src (only when the element is a <script>)", "text": None}
+NAV_ASSIGN_DOTTED = re.compile(r"^(?:(?:window|self|top)\.)?(?:document\.)?location(?:\.href)?$")
+JQUERY_HTML_METHODS = {"html", "append", "prepend", "after", "before", "replaceWith", "wrap", "wrapAll", "appendTo", "prependTo"}
+JQUERY_RECEIVER = re.compile(r"^(?:\$|jQuery)\(")
+DB_PACKAGES = {"mysql", "mysql2", "pg", "sqlite3", "better-sqlite3", "mssql", "knex", "sequelize", "typeorm", "oracledb",
+               "pg-promise", "slonik", "@prisma/client", "mysql2/promise", "sql.js", "alasql"}
+DB_METHODS = {"query", "execute", "raw", "queryRaw", "$queryRawUnsafe", "$executeRawUnsafe", "exec", "run", "all", "get"}
+CMD_PACKAGES = {"child_process", "node:child_process", "execa", "shelljs", "cross-spawn"}
+CMD_FUNCS = {"exec", "execSync", "spawn", "spawnSync", "execFile", "execFileSync", "fork", "shell.exec"}
+
+# ---------------------------------------------------------------------------------------------------- sanitizers
+_HTML = frozenset({"html"})
+SANITIZERS: list[tuple[re.Pattern, Mitigation]] = [
+    (re.compile(r"(^|\.)(dompurify\.)?sanitize$|^sanitizeHtml$|^xss$|(^|\.)filterXSS$", re.I),
+     Mitigation("HTML sanitizer (DOMPurify/sanitize-html/xss)", effective=_HTML)),
+    (re.compile(r"(^|\.)(escapeHtml|escapeHTML|escape)$|^he\.(encode|escape)$|validator\.escape$"),
+     Mitigation("HTML escaping", effective=_HTML, note="does not make a value safe as a URL, SQL or shell string")),
+    (re.compile(r"^(encodeURIComponent)$"),
+     Mitigation("encodeURIComponent", effective=_HTML, uncertain=frozenset({"navigation"}),
+                note="neutralises markup but is not a URL validator")),
+    (re.compile(r"^(encodeURI|escape|encodeURL)$"),
+     Mitigation("encodeURI", uncertain=_HTML, note="leaves < > \" ' ( ) intact")),
+    (re.compile(r"^(Number|parseInt|parseFloat|Boolean|BigInt)$|^Math\.(floor|ceil|round|abs|trunc)$|^Number\.parse(Int|Float)$"),
+     Mitigation("numeric coercion", effective=ALL, note="value can only be a number/boolean afterwards")),
+    (re.compile(r"(^|\.)quote$|^shellescape$|^escapeShellArg$"),
+     Mitigation("shell quoting", effective=frozenset({"command"}))),
+]
+# A distinct, narrow table for SQL escapers so `escape` on a random object does not count as SQL escaping.
+DB_ESCAPERS = re.compile(r"(^|\.)(mysql|connection|pool|db|client|sqlstring|SqlString|pg)\.(escape|escapeId|escapeLiteral|format)$")
+PARTIAL_REPLACE = Mitigation("string replace()", uncertain=frozenset({"html", "navigation", "code"}),
+                             note="partial filtering by replace() is rarely complete")
+# Methods that transform a string but do NOT neutralise it. Taint passes straight through.
+PASS_METHODS = {"trim", "trimStart", "trimEnd", "toLowerCase", "toUpperCase", "slice", "substring", "substr", "split", "join",
+                "concat", "toString", "padStart", "padEnd", "repeat", "normalize", "at", "charAt", "json", "text", "blob",
+                "get", "getAll", "toJSON", "valueOf", "flat", "reverse", "sort", "pop", "shift", "values", "entries", "keys",
+                "decodeURIComponent", "decodeURI", "atob", "btoa", "stringify", "parse", "from", "assign", "freeze",
+                "clone", "cloneDeep", "map", "filter", "find", "reduce", "forEach", "some", "every", "flatMap", "entries"}
+PASSTHROUGH_FUNCS = {"String", "decodeURIComponent", "decodeURI", "atob", "btoa", "JSON.parse", "JSON.stringify", "Object.assign",
+                     "Object.values", "Object.entries", "Object.keys", "Array.from", "Array.of", "structuredClone",
+                     "Object.freeze", "unescape", "Symbol", "Promise.resolve", "Promise.all", "Promise.race", "Promise.allSettled",
+                     "ref", "reactive", "computed", "shallowRef", "toRef", "toRefs", "useState", "useMemo", "useRef", "useCallback",
+                     "useReducer", "Array", "Object", "Buffer.from", "Reflect.get"}
+PASSTHROUGH_CTORS = {"URLSearchParams", "URL", "String", "Array", "Object", "Set", "Map", "Buffer", "Blob", "Uint8Array", "FormData"}
+CLEAN_FUNCS = {"Array.isArray", "isNaN", "Number.isInteger", "Number.isFinite", "Number.isNaN", "console.log", "console.error",
+               "console.warn", "console.info", "console.debug", "Date.now", "Math.random", "Math.max", "Math.min", "setState"}
+
+# ---------------------------------------------------------------------------------------------------- validators
+_SAFE_ATOM = r"(?:\[[A-Za-z0-9_\-\\wd.\s]+\]|\\w|\\d|[A-Za-z0-9_\-])"
+_STRICT_RE = re.compile(r"^\^\(?(?:" + _SAFE_ATOM + r"(?:[+*]|\{\d+(?:,\d*)?\})?)+\)?\$$")
+
+
+def regex_is_strict_allowlist(src: str) -> bool:
+    """True for an anchored pattern built only from alphanumeric classes (`^[a-z0-9_-]+$`, `^\\d{1,6}$`).
+    No `.`, no negated class, no alternation: those can admit markup, `javascript:` or quotes. Anything we cannot
+    positively show is strict is reported as UNCERTAIN, never as effective."""
+    src = src.strip()
+    if "[^" in src or "|" in src or ".*" in src or ".+" in src:
+        return False
+    body = src.replace("\\.", "")                 # escaped dot is allowed only inside a class below
+    if "." in re.sub(r"\[[^\]]*\]", "", body):
+        return False
+    return bool(_STRICT_RE.match(src))
+
+
+def sink_label(kind: str) -> str:
+    return {"html": "DOM HTML insertion", "code": "dynamic code execution", "navigation": "URL navigation",
+            "sql": "database query", "command": "OS command execution"}.get(kind, kind)

jscope/downloader/downloader.py

@@ -0,0 +1,116 @@
+"""Download Mode: text file of URLs -> project directory (JS, HTML, specs, schemas, WASM).
+Dedupes by content hash, preserves metadata (source URL, fetch time, status).
+No analysis happens here (PRD §6: crawling/downloading produce no findings)."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import urlparse
+
+import requests
+from jscope.security_transport import GuardedSession, TransportPolicy, PolicyError, normalize_host
+
+DEFAULT_TIMEOUT = 15
+DEFAULT_UA = "JScope/0.4 (+https://github.com/jscope)"
+
+
+class DownloadError(Exception):
+    """A user-facing failure (bad input file) -- never a traceback."""
+
+
+def extension_for(url: str, content: bytes, content_type: str = "") -> str:
+    """Pick the saved file's extension from what the bytes ARE, so the analyzers' content sniffing sees the right kind."""
+    path = urlparse(url).path.lower()
+    ctype = content_type.lower()
+    if content[:4] == b"\x00asm":
+        return ".wasm"
+    if path.endswith((".js", ".mjs", ".cjs", ".jsx")) or "javascript" in ctype:
+        return ".js"
+    head = content[:512].lstrip().lower()
+    if "html" in ctype or head.startswith((b"<!doctype html", b"<html", b"<head", b"<body")):
+        return ".html"
+    if path.endswith((".yaml", ".yml")) or "yaml" in ctype:
+        return ".yaml"
+    if path.endswith((".graphql", ".gql", ".graphqls")):
+        return ".graphql"
+    if "json" in ctype or head.startswith((b"{", b"[")):
+        return ".json"
+    return ".js"
+
+
+@dataclass
+class DownloadResult:
+    url: str
+    ok: bool
+    path: str | None = None
+    error: str | None = None
+    status_code: int | None = None
+    duplicate_of: str | None = None
+
+
+def download_urls(url_file: str, out_dir: str, timeout: int = DEFAULT_TIMEOUT,
+                   user_agent: str = DEFAULT_UA, policy: TransportPolicy | None = None) -> list[DownloadResult]:
+    if policy is None:
+        raise DownloadError("network downloads require an explicit assessment TransportPolicy; refusing unguarded URL-list downloads")
+    try:
+        text = Path(url_file).read_text()
+    except OSError as exc:
+        raise DownloadError(f"cannot read URL list {url_file!r}: {exc.strerror or exc}. "
+                            f"Create it first with `jscope crawl <url> --out {url_file}`.") from exc
+    urls = [
+        line.strip() for line in text.splitlines()
+        if line.strip() and not line.strip().startswith("#")
+    ]
+    urls = list(dict.fromkeys(urls))  # de-dupe identical URL strings, preserve order
+
+    out_path = Path(out_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+    meta_path = out_path / "_jscope_meta.json"
+    meta: dict[str, dict] = {}
+    seen_hashes: dict[str, str] = {}  # content_hash -> first filename
+    results: list[DownloadResult] = []
+
+    session = GuardedSession(policy)
+    session.session.headers.update({"User-Agent": user_agent})
+
+    for i, url in enumerate(urls):
+        try:
+            resp = session.get(url, timeout=timeout)
+        except (requests.RequestException, PolicyError) as e:
+            results.append(DownloadResult(url=url, ok=False, error=str(e)))
+            continue
+
+        if resp.status_code != 200:
+            results.append(DownloadResult(url=url, ok=False, status_code=resp.status_code,
+                                           error=f"HTTP {resp.status_code}"))
+            resp.close()
+            continue
+
+        content = resp.content
+        resp.close()
+        content_hash = hashlib.sha256(content).hexdigest()[:16]
+
+        if content_hash in seen_hashes:
+            results.append(DownloadResult(url=url, ok=True, duplicate_of=seen_hashes[content_hash]))
+            continue
+
+        safe_name = f"{i:05d}_{content_hash}{extension_for(url, content, resp.headers.get('Content-Type', ''))}"
+        (out_path / safe_name).write_bytes(content)
+        seen_hashes[content_hash] = safe_name
+        meta[safe_name] = {
+            "source_url": url,
+            "fetched_at": time.time(),
+            "status_code": resp.status_code,
+            "content_hash": content_hash,
+            "bytes": len(content),
+        }
+        results.append(DownloadResult(url=url, ok=True, path=str(out_path / safe_name),
+                                       status_code=resp.status_code))
+
+    meta_path.write_text(json.dumps(meta, indent=2))
+    session.close()
+    return results