jscope 0.4.2__py3-none-any.whl

jscope/workspace/assessment.py

@@ -0,0 +1,133 @@
+"""Domain-centric assessment scope and audit records.
+
+This module deliberately separates discovered hosts from hosts approved for the
+current assessment. Workspace history is knowledge, never standing permission.
+No network activity is performed here.
+"""
+from __future__ import annotations
+
+import hashlib
+import ipaddress
+import json
+import re
+import secrets
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import urlparse
+from typing import Any
+
+from jscope.workspace.common import WorkspaceError, _atomic_write
+
+_LABEL = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
+
+
+def normalize_domain(value: str) -> str:
+    """Normalize a DNS name or URL to a hostname; reject ambiguous/non-domain inputs."""
+    raw = value.strip()
+    if not raw:
+        raise WorkspaceError("target domain cannot be empty")
+    if "://" in raw:
+        parsed = urlparse(raw)
+        if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password:
+            raise WorkspaceError("target must be a DNS domain or an http(s) URL without credentials")
+        if parsed.path not in ("", "/") or parsed.query or parsed.fragment:
+            raise WorkspaceError("provide a domain/origin only, not a path, query, or fragment")
+        host = parsed.hostname
+    else:
+        if any(ch in raw for ch in "/?#@"):
+            raise WorkspaceError(f"invalid domain {value!r}; provide a hostname only")
+        host = raw.rstrip(".")
+        if ":" in host:
+            raise WorkspaceError("ports and IP literals are not supported as domain workspace roots")
+    try:
+        host = host.encode("idna").decode("ascii").lower().rstrip(".")
+    except UnicodeError as exc:
+        raise WorkspaceError(f"invalid internationalized domain: {value!r}") from exc
+    if len(host) > 253 or "." not in host or host.startswith("."):
+        raise WorkspaceError(f"{value!r} is not a fully qualified DNS domain")
+    try:
+        ipaddress.ip_address(host)
+    except ValueError:
+        pass
+    else:
+        raise WorkspaceError("domain-centric workspaces require a DNS name, not an IP address")
+    labels = host.split(".")
+    if any(not _LABEL.fullmatch(label) for label in labels):
+        raise WorkspaceError(f"invalid DNS hostname: {value!r}")
+    return host
+
+
+def validate_scope_file(path: str | Path, root_domain: str) -> dict[str, Any]:
+    """Parse a host-per-line authorization file; no DNS/network calls are made."""
+    root = normalize_domain(root_domain)
+    file_path = Path(path)
+    try:
+        lines = file_path.read_text(encoding="utf-8").splitlines()
+    except OSError as exc:
+        raise WorkspaceError(f"cannot read scope file {str(path)!r}: {exc}") from exc
+    approved: list[str] = []
+    rejected: list[dict[str, str | int]] = []
+    for number, line in enumerate(lines, 1):
+        item = line.strip()
+        if not item or item.startswith("#"):
+            continue
+        try:
+            host = normalize_domain(item)
+            if host != root and not host.endswith("." + root):
+                raise WorkspaceError(f"host is outside root domain {root}")
+            approved.append(host)
+        except WorkspaceError as exc:
+            rejected.append({"line": number, "value": item, "reason": str(exc)})
+    # Preserve order while removing duplicates.
+    approved = list(dict.fromkeys(approved))
+    return {
+        "path": str(file_path.resolve()),
+        "sha256": hashlib.sha256(file_path.read_bytes()).hexdigest(),
+        "approved_hosts": approved,
+        "rejected_entries": rejected,
+        "valid": not rejected and bool(approved),
+    }
+
+
+def create_assessment_record(root: str, known_hosts: list[str] | None = None,
+                             approved_hosts: list[str] | None = None,
+                             limits: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Create a new assessment manifest; approvals are explicit and per-run."""
+    domain = normalize_domain(root)
+    known = list(dict.fromkeys(normalize_domain(h) for h in (known_hosts or [domain])))
+    approved = list(dict.fromkeys(normalize_domain(h) for h in (approved_hosts or [])))
+    invalid = [h for h in known + approved if h != domain and not h.endswith("." + domain)]
+    if invalid:
+        raise WorkspaceError(f"hosts outside root domain {domain}: {', '.join(invalid)}")
+    if not set(approved).issubset(set(known)):
+        raise WorkspaceError("every approved host must first be present in the known-host list")
+    now = datetime.now(timezone.utc).isoformat(timespec="seconds")
+    return {
+        "schema": "jscope-assessment/1",
+        "assessment_id": datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + secrets.token_hex(3),
+        "root_domain": domain,
+        "created_at": now,
+        "status": "approved" if approved else "planned",
+        "authorization": {"fresh_for_this_run": True, "approved_at": now if approved else None, "approved_hosts": approved},
+        "hosts": {h: {"state": "approved" if h in approved else "known", "discovery_source": "user-supplied" if h == domain else "scope-input"} for h in known},
+        "limits": limits or {"max_requests": 500, "rate_per_second": 2.0, "depth": "deep"},
+        "stages": [],
+        "findings": [],
+        "evidence": [],
+        "coverage": {"tested_hosts": [], "untested_hosts": known, "limitations": []},
+    }
+
+
+def record_stage(manifest: dict[str, Any], stage: str, outcome: str, detail: str = "") -> None:
+    manifest.setdefault("stages", []).append({
+        "stage": stage, "outcome": outcome,
+        "at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "detail": detail,
+    })
+
+
+def write_assessment(root: str | Path, manifest: dict[str, Any]) -> Path:
+    """Atomically persist a manifest under .jscope-ws/assessments."""
+    root = Path(root)
+    out = root / ".jscope-ws" / "assessments" / f"{manifest['assessment_id']}.json"
+    _atomic_write(out, manifest)
+    return out

jscope/workspace/common.py

@@ -0,0 +1,25 @@
+"""Small workspace primitives kept independent from the analysis engine."""
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from pathlib import Path
+
+
+class WorkspaceError(Exception):
+    """A user-correctable workspace or assessment error."""
+
+
+def _atomic_write(path: Path, data: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=path.parent, suffix=".tmp")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=2, sort_keys=True)
+            f.write("\n")
+        os.replace(tmp, path)
+    except BaseException:
+        if os.path.exists(tmp):
+            os.unlink(tmp)
+        raise

jscope/workspace/diff.py

@@ -0,0 +1,143 @@
+"""Scan-to-scan diffing (Phase 3).
+
+Compares two saved scan records at three levels: files (by content hash), findings (by stable
+`category|subject` key, NOT the order-based JS-#### id) and the knowledge graph's route map.
+
+The graph-level diff is the interesting one: "this route lost its guard" or "this route now reaches a
+new endpoint" is a change in application *behavior*, which a file diff cannot express. These changes
+are inherently worth re-testing, so they feed the deterministic priority ranking (see priority.py).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from jscope.reporting.priority import PriorityDelta, finding_key
+
+_ORDER = {"SUPPRESSED": 0, "INFORMATIONAL": 1, "LOW": 2, "MEDIUM": 3, "HIGH": 4, "CONFIRMED": 5}
+
+
+def _protection(chain: dict) -> set[str]:
+    return {f"guard:{g}" for g in chain.get("guards", [])} | {f"role:{r}" for r in chain.get("roles", [])}
+
+
+@dataclass
+class ScanDiff:
+    old_id: str
+    new_id: str
+    files_added: list[str] = field(default_factory=list)
+    files_removed: list[str] = field(default_factory=list)
+    files_modified: list[str] = field(default_factory=list)
+    findings_new: list[dict] = field(default_factory=list)
+    findings_resolved: list[dict] = field(default_factory=list)
+    findings_changed: list[dict] = field(default_factory=list)
+    routes_added: list[dict] = field(default_factory=list)
+    routes_removed: list[dict] = field(default_factory=list)
+    routes_changed: list[dict] = field(default_factory=list)
+
+    def is_empty(self) -> bool:
+        return not any([self.files_added, self.files_removed, self.files_modified, self.findings_new,
+                        self.findings_resolved, self.findings_changed, self.routes_added,
+                        self.routes_removed, self.routes_changed])
+
+    def to_dict(self) -> dict[str, Any]:
+        return {k: v for k, v in self.__dict__.items()}
+
+    def priority_delta(self) -> PriorityDelta:
+        return PriorityDelta(
+            new_keys={f["key"] for f in self.findings_new},
+            raised_keys={f["key"] for f in self.findings_changed if f["direction"] == "up"},
+            weakened_routes={r["id"] for r in self.routes_changed if r["change"] == "weakened"})
+
+    def regressions(self, min_confidence: str = "HIGH") -> list[str]:
+        """Human-readable reasons a CI/cron run should alert. Deliberately narrow: new findings at/above
+        `min_confidence`, confidence rising to it, and routes that lost protection."""
+        bar = _ORDER[min_confidence]
+        out = [f"new {f['confidence']} finding: {f['category']} {f['subject']}"
+               for f in self.findings_new if _ORDER[f["confidence"]] >= bar]
+        out += [f"{f['category']} {f['subject']} rose {f['old']} -> {f['new']}"
+                for f in self.findings_changed if f["direction"] == "up" and _ORDER[f["new"]] >= bar]
+        out += [f"route {r['route']} lost client-side protection ({', '.join(sorted(r['lost']))})"
+                for r in self.routes_changed if r["change"] == "weakened"]
+        return out
+
+
+def _brief(f: dict) -> dict:
+    return {"key": finding_key(f["category"], f["subject"]), "id": f["id"], "category": f["category"],
+            "subject": f["subject"], "confidence": f["confidence"], "attack_surface": f.get("attack_surface")}
+
+
+def diff_scans(old: dict, new: dict) -> ScanDiff:
+    """`old`/`new` are scan records as written by Workspace (they carry `scan_id`, `files`, `payload`)."""
+    d = ScanDiff(old_id=old["scan_id"], new_id=new["scan_id"])
+
+    of, nf = old.get("files", {}), new.get("files", {})
+    d.files_added = sorted(set(nf) - set(of))
+    d.files_removed = sorted(set(of) - set(nf))
+    d.files_modified = sorted(p for p in set(of) & set(nf) if of[p] != nf[p])
+
+    ofi = {finding_key(f["category"], f["subject"]): f for f in old["payload"]["findings"]}
+    nfi = {finding_key(f["category"], f["subject"]): f for f in new["payload"]["findings"]}
+    d.findings_new = [_brief(nfi[k]) for k in sorted(set(nfi) - set(ofi))]
+    d.findings_resolved = [_brief(ofi[k]) for k in sorted(set(ofi) - set(nfi))]
+    for k in sorted(set(ofi) & set(nfi)):
+        a, b = ofi[k]["confidence"], nfi[k]["confidence"]
+        if a != b:
+            d.findings_changed.append({**_brief(nfi[k]), "old": a, "new": b,
+                                       "direction": "up" if _ORDER[b] > _ORDER[a] else "down"})
+
+    orm = {c["id"]: c for c in old["payload"].get("route_map", [])}
+    nrm = {c["id"]: c for c in new["payload"].get("route_map", [])}
+    d.routes_added = [{"route": nrm[i]["route"], "id": i, "unguarded": nrm[i]["unguarded"],
+                       "endpoints": nrm[i]["endpoints"]} for i in sorted(set(nrm) - set(orm))]
+    d.routes_removed = [{"route": orm[i]["route"], "id": i} for i in sorted(set(orm) - set(nrm))]
+    for i in sorted(set(orm) & set(nrm)):
+        o, n = orm[i], nrm[i]
+        po, pn = _protection(o), _protection(n)
+        if not o["unguarded"] and n["unguarded"] or pn < po:
+            change = "weakened"
+        elif pn > po:
+            change = "strengthened"
+        elif pn != po:
+            change = "changed"
+        else:
+            change = None
+        ep_add = sorted(set(n["endpoints"]) - set(o["endpoints"]))
+        ep_rem = sorted(set(o["endpoints"]) - set(n["endpoints"]))
+        if change or ep_add or ep_rem:
+            d.routes_changed.append({
+                "route": n["route"], "id": i, "change": change or "endpoints_only",
+                "lost": sorted(po - pn), "gained": sorted(pn - po),
+                "endpoints_added": ep_add, "endpoints_removed": ep_rem,
+                "now_unguarded": n["unguarded"]})
+    return d
+
+
+def render_diff(d: ScanDiff) -> str:
+    if d.is_empty():
+        return f"[changes since scan {d.old_id}] none"
+    L = [f"[changes since scan {d.old_id}]  (-> {d.new_id})"]
+    L.append(f"  files    : +{len(d.files_added)} -{len(d.files_removed)} ~{len(d.files_modified)}")
+    for r in d.routes_changed:
+        if r["change"] == "weakened":
+            L.append(f"  !! route {r['route']} LOST protection: {', '.join(r['lost']) or 'all guards'}")
+    for r in d.routes_changed:
+        if r["change"] == "strengthened":
+            L.append(f"  +  route {r['route']} gained protection: {', '.join(r['gained'])}")
+        for e in r["endpoints_added"]:
+            L.append(f"  +  route {r['route']} now reaches {e}")
+        for e in r["endpoints_removed"]:
+            L.append(f"  -  route {r['route']} no longer reaches {e}")
+    for r in d.routes_added:
+        L.append(f"  +  new route {r['route']}" + ("  (UNPROTECTED)" if r["unguarded"] else ""))
+    for r in d.routes_removed:
+        L.append(f"  -  route removed {r['route']}")
+    for f in d.findings_new:
+        L.append(f"  +  new [{f['confidence']}] {f['category']}: {f['subject']}")
+    for f in d.findings_changed:
+        arrow = "^" if f["direction"] == "up" else "v"
+        L.append(f"  {arrow}  {f['category']}: {f['subject']}  {f['old']} -> {f['new']}")
+    for f in d.findings_resolved:
+        L.append(f"  -  gone [{f['confidence']}] {f['category']}: {f['subject']}")
+    return "\n".join(L)

jscope/workspace/store.py

@@ -0,0 +1,206 @@
+"""Workspaces: named targets, immutable scan history, incremental rescans (Phase 3).
+
+Layout (all JSON, human-inspectable, safe to commit or rsync):
+
+    .jscope-ws/
+      workspace.json                   {"version": 1, "targets": {name: {"paths": [...]}}}
+      targets/<name>/index.json        {"scans": [{id, timestamp, fingerprint, finding_count}]}
+      targets/<name>/scans/0001.json   {scan_id, timestamp, fingerprint, files{path:hash}, payload}
+
+Incremental rescans, stated precisely: a scan is skipped when its *fingerprint* (every input file's content
+hash + the analysis-relevant config + plugin source hashes + the JScope version) equals the previous scan's.
+When ANY file changed the whole target is re-analyzed, because route -> component -> API correlation is
+cross-file: re-analyzing only the changed file would silently produce a wrong graph. Skipping is therefore
+all-or-nothing, which is the only form that is both fast in the common cron case ("nothing changed") and
+never wrong. (tree-sitter trees are not serializable, so there is no per-file parse cache on disk.)
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import shutil
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from jscope import __version__
+from jscope.config import Config
+from jscope.pipeline import collect_manifest, run_analysis
+from jscope.reporting import json_report
+from jscope.workspace.diff import ScanDiff, diff_scans
+
+_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$")
+_SCAN_RE = re.compile(r"^\d{4,}$")
+DIRNAME = ".jscope-ws"
+
+
+from jscope.workspace.common import WorkspaceError, _atomic_write
+
+
+@dataclass
+class ScanOutcome:
+    status: str                       # "scanned" | "unchanged"
+    record: dict[str, Any]            # the scan record now considered latest
+    previous: dict[str, Any] | None
+    diff: ScanDiff | None             # None on a first scan (a baseline has no diff)
+    result: Any = None                # AnalysisResult when status == "scanned" (graph, findings objects)
+
+
+def config_fingerprint(config: Config) -> str:
+    """Hash of every setting that can change analysis OUTPUT. Output-format/verbosity are excluded on
+    purpose, and so is concurrency. Plugin *source* is hashed too: editing a plugin must invalidate."""
+    relevant = {k: config.get(k) for k in ("analyzers.enabled", "min_confidence", "exclude", "max_file_bytes",
+                                            "source_maps.expand", "artifacts.enabled", "plugins.verified",
+                                            "plugins.community_confidence_cap")}
+    plugin_hashes = {}
+    for p in config.get("plugins.paths", []) or []:
+        pp = Path(p)
+        for f in ([pp] if pp.is_file() else sorted(pp.rglob("*.py")) if pp.is_dir() else []):
+            plugin_hashes[str(f)] = hashlib.sha256(f.read_bytes()).hexdigest()[:16]
+    blob = json.dumps({"cfg": relevant, "plugins": plugin_hashes, "v": __version__}, sort_keys=True)
+    return hashlib.sha256(blob.encode()).hexdigest()[:16]
+
+
+def scan_fingerprint(manifest: dict[str, str], config: Config) -> str:
+    blob = json.dumps({"files": manifest, "cfg": config_fingerprint(config)}, sort_keys=True)
+    return hashlib.sha256(blob.encode()).hexdigest()[:16]
+
+
+class Workspace:
+    def __init__(self, root: Path):
+        self.root = Path(root)
+        self.dir = self.root / DIRNAME
+
+    # -- lifecycle --------------------------------------------------------
+    @classmethod
+    def init(cls, root: str | Path) -> "Workspace":
+        ws = cls(Path(root))
+        if (ws.dir / "workspace.json").exists():
+            raise WorkspaceError(f"workspace already exists at {ws.dir}")
+        _atomic_write(ws.dir / "workspace.json", {"version": 1, "targets": {}})
+        return ws
+
+    @classmethod
+    def open(cls, root: str | Path) -> "Workspace":
+        ws = cls(Path(root))
+        if not (ws.dir / "workspace.json").exists():
+            raise WorkspaceError(f"no workspace at {ws.dir} (run `jscope ws init`)")
+        return ws
+
+    def _meta(self) -> dict:
+        return json.loads((self.dir / "workspace.json").read_text())
+
+    @staticmethod
+    def _check_name(name: str) -> str:
+        if not _NAME_RE.match(name):
+            raise WorkspaceError(f"invalid target name {name!r} (letters, digits, '_', '.', '-'; max 64)")
+        return name
+
+    def _tdir(self, name: str) -> Path:
+        return self.dir / "targets" / self._check_name(name)
+
+    # -- targets ----------------------------------------------------------
+    def add_target(self, name: str, paths: list[str]) -> None:
+        self._check_name(name)
+        meta = self._meta()
+        if name in meta["targets"]:
+            raise WorkspaceError(f"target {name!r} already exists")
+        missing = [p for p in paths if not Path(p).exists()]
+        if missing:
+            raise WorkspaceError(f"path(s) not found: {', '.join(missing)}")
+        meta["targets"][name] = {"paths": [str(Path(p).resolve()) for p in paths]}
+        _atomic_write(self.dir / "workspace.json", meta)
+
+    def remove_target(self, name: str) -> None:
+        meta = self._meta()
+        if name not in meta["targets"]:
+            raise WorkspaceError(f"no such target {name!r}")
+        del meta["targets"][name]
+        _atomic_write(self.dir / "workspace.json", meta)
+        shutil.rmtree(self._tdir(name), ignore_errors=True)      # name validated by _tdir
+
+    def targets(self) -> dict[str, dict]:
+        return self._meta()["targets"]
+
+    # -- history ----------------------------------------------------------
+    def history(self, name: str) -> list[dict]:
+        idx = self._tdir(name) / "index.json"
+        return json.loads(idx.read_text())["scans"] if idx.exists() else []
+
+    def load_scan(self, name: str, ref: str = "latest") -> dict:
+        hist = self.history(name)
+        if not hist:
+            raise WorkspaceError(f"target {name!r} has no scans yet")
+        if ref == "latest":
+            sid = hist[-1]["id"]
+        elif ref == "previous":
+            if len(hist) < 2:
+                raise WorkspaceError(f"target {name!r} has only one scan")
+            sid = hist[-2]["id"]
+        else:
+            sid = ref.zfill(4)
+            if not _SCAN_RE.match(sid):
+                raise WorkspaceError(f"invalid scan id {ref!r}")
+        path = self._tdir(name) / "scans" / f"{sid}.json"
+        if not path.exists():
+            raise WorkspaceError(f"no scan {sid} for target {name!r}")
+        return json.loads(path.read_text())
+
+    # -- scanning ---------------------------------------------------------
+    def scan(self, name: str, config: Config, force: bool = False, now: datetime | None = None) -> ScanOutcome:
+        targets = self.targets()
+        if name not in targets:
+            raise WorkspaceError(f"no such target {name!r}")
+        paths = targets[name]["paths"]
+        hist = self.history(name)
+        prev = self.load_scan(name, "latest") if hist else None
+
+        manifest = collect_manifest(paths, config)
+        fp = scan_fingerprint(manifest, config)
+        if prev is not None and prev["fingerprint"] == fp and not force:
+            return ScanOutcome("unchanged", prev, prev, None)
+
+        res = run_analysis(paths, config)
+        payload = json_report.payload(res.findings, res.frameworks, res.file_count, res.graph, res.diagnostics,
+                                   dataflow=res.dataflow, file_diagnostics=res.file_diagnostics)
+        sid = f"{(int(hist[-1]['id']) + 1) if hist else 1:04d}"
+        ts = (now or datetime.now(timezone.utc)).strftime("%Y-%m-%dT%H:%M:%SZ")
+        record = {"scan_id": sid, "target": name, "timestamp": ts, "fingerprint": fp,
+                  "jscope_version": __version__, "files": manifest, "payload": payload}
+        diff = diff_scans(prev, record) if prev is not None else None
+        # re-render priority with the delta so the stored ranking already reflects "what changed"
+        if diff is not None:
+            payload = json_report.payload(res.findings, res.frameworks, res.file_count, res.graph,
+                                          res.diagnostics, diff.priority_delta(), dataflow=res.dataflow,
+                                          file_diagnostics=res.file_diagnostics)
+            record["payload"] = payload
+        _atomic_write(self._tdir(name) / "scans" / f"{sid}.json", record)
+        _atomic_write(self._tdir(name) / "index.json", {"scans": hist + [{
+            "id": sid, "timestamp": ts, "fingerprint": fp, "finding_count": len(res.findings),
+            "file_count": len(manifest)}]})
+        return ScanOutcome("scanned", record, prev, diff, res)
+
+    # -- multi-target overview -------------------------------------------
+    def summary(self) -> list[dict]:
+        rows = []
+        for name in sorted(self.targets()):
+            hist = self.history(name)
+            if not hist:
+                rows.append({"target": name, "scans": 0})
+                continue
+            rec = self.load_scan(name, "latest")
+            fs = rec["payload"]["findings"]
+            conf: dict[str, int] = {}
+            for f in fs:
+                conf[f["confidence"]] = conf.get(f["confidence"], 0) + 1
+            top = rec["payload"]["priority"][0] if rec["payload"]["priority"] else None
+            top_f = next((f for f in fs if top and f["id"] == top["id"]), None)
+            rows.append({
+                "target": name, "scans": len(hist), "last_scan": rec["timestamp"], "files": len(rec["files"]),
+                "findings": len(fs), "by_confidence": conf, "frameworks": rec["payload"]["summary"]["frameworks"],
+                "unprotected_routes": sum(1 for c in rec["payload"].get("route_map", []) if c["unguarded"]),
+                "top": {"id": top_f["id"], "subject": top_f["subject"], "priority": top["priority"]} if top_f else None})
+        return rows