cg-code-graph 0.20.0__tar.gz → 0.21.0__tar.gz

{cg_code_graph-0.20.0/cg_code_graph.egg-info → cg_code_graph-0.21.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cg-code-graph
-Version: 0.20.0
+Version: 0.21.0
 Summary: cg: a code graph of routes, calls, data access and platform boundaries for agents and humans (CLI + MCP server)
 Author: cyberchronos00
 License-Expression: MIT
@@ -90,6 +90,7 @@ Go and others: import a SCIP index with `cg index --scip FILE`. Run `cg doctor`
 - `cg affected --base main`: the tests and entry points a change reaches (`--quiet` feeds your test runner).
 - Across repos: `cg link` matches client HTTP calls (frontends and Laravel / Guzzle backends) to server routes, and checks request keys (Laravel FormRequest, Elysia schemas, Nuxt / TypeScript bodies).
 - Route guards: `cg routes --writes --unguarded` lists routes that write with no auth guard. Laravel checks inside the action count as inline guards unless `--strict`.
+- Security views: `cg surface` lists the inbound / outbound attack surface and findings (hardcoded credentials, plaintext protocols, unverified webhooks, unguarded handlers, listeners on all interfaces, disabled TLS / SSH verification, plaintext gRPC, exposed IPC, outbound URLs built from request input) as text, JSON or SARIF, with `--fail-on` for CI.
 - MCP server (`cg-mcp`) so agents query the graph instead of grepping.
 - Visual view (`cg serve`) and plan checks (`cg plan check`) for planned changes.
 

{cg_code_graph-0.20.0 → cg_code_graph-0.21.0}/README.md

@@ -63,6 +63,7 @@ Go and others: import a SCIP index with `cg index --scip FILE`. Run `cg doctor`
 - `cg affected --base main`: the tests and entry points a change reaches (`--quiet` feeds your test runner).
 - Across repos: `cg link` matches client HTTP calls (frontends and Laravel / Guzzle backends) to server routes, and checks request keys (Laravel FormRequest, Elysia schemas, Nuxt / TypeScript bodies).
 - Route guards: `cg routes --writes --unguarded` lists routes that write with no auth guard. Laravel checks inside the action count as inline guards unless `--strict`.
+- Security views: `cg surface` lists the inbound / outbound attack surface and findings (hardcoded credentials, plaintext protocols, unverified webhooks, unguarded handlers, listeners on all interfaces, disabled TLS / SSH verification, plaintext gRPC, exposed IPC, outbound URLs built from request input) as text, JSON or SARIF, with `--fail-on` for CI.
 - MCP server (`cg-mcp`) so agents query the graph instead of grepping.
 - Visual view (`cg serve`) and plan checks (`cg plan check`) for planned changes.
 

{cg_code_graph-0.20.0 → cg_code_graph-0.21.0}/cg_code_graph/__init__.py

@@ -1,2 +1,2 @@
 """code-graph: deterministic code knowledge graph (beta)."""
-__version__ = "0.20.0"
+__version__ = "0.21.0"

{cg_code_graph-0.20.0 → cg_code_graph-0.21.0}/cg_code_graph/blindspots.py

@@ -35,6 +35,8 @@ KINDS = {
                                      "Nuxt imports.dirs entry that is not a literal path or glob"),
     "vue_unresolved_navigation": ("route", "typescript",
                                   "Nuxt / Vue / Astro navigation target that matches no page or several pages"),
+    "ts_unrecognised_http_client": ("endpoint", "typescript",
+                                    "outbound HTTP call through an imported client (http, https, undici, got, node-fetch, superagent, ...) that made no endpoint"),
 }
 
 
@@ -415,6 +417,110 @@ def nuxt_unevaluable_import_dirs(root: Path) -> dict | None:
     return _finding("nuxt_unevaluable_import_dirs", hits)
 
 
+# ------------------------------------------------------------------------------------------- TS HTTP clients
+
+HTTP_CLIENT_MODS = ("http", "https", "undici", "got", "node-fetch", "superagent", "needle", "phin", "request")
+_HTTP_VERBS = {"request", "get", "post", "put", "patch", "delete", "del", "head", "options", "fetch", "stream"}
+_HTTP_ONLY = {"request", "get"}      # node:http / node:https: createServer, Agent, ... are not client calls
+_HTTP_MOD_ALT = "|".join(re.escape(m) for m in HTTP_CLIENT_MODS)
+_IMPORT_DEFAULT = re.compile(r"""import\s+(?:type\s+)?(\w+)\s*(?:,\s*\{([^}]*)\}\s*)?from\s*['"](?:node:)?(%s)['"]""" % _HTTP_MOD_ALT)
+_IMPORT_NS = re.compile(r"""import\s+\*\s+as\s+(\w+)\s+from\s*['"](?:node:)?(%s)['"]""" % _HTTP_MOD_ALT)
+_IMPORT_NAMED = re.compile(r"""import\s+(?:type\s+)?\{([^}]*)\}\s*from\s*['"](?:node:)?(%s)['"]""" % _HTTP_MOD_ALT)
+_REQ_BIND = re.compile(r"""(?:const|let|var)\s+(\w+)\s*=\s*(?:await\s+)?require\(\s*['"](?:node:)?(%s)['"]\s*\)""" % _HTTP_MOD_ALT)
+_REQ_DESTR = re.compile(r"""(?:const|let|var)\s+\{([^}]*)\}\s*=\s*require\(\s*['"](?:node:)?(%s)['"]\s*\)""" % _HTTP_MOD_ALT)
+_TEST_FILE = re.compile(r"(^|/)(tests?|__tests__|e2e|spec|specs|fixtures?)/|\.(test|spec)\.[cm]?[jt]sx?$")
+
+
+def _named(spec: str) -> list[tuple[str, str]]:
+    out = []
+    for part in spec.split(","):
+        bits = [b for b in re.split(r"\s+as\s+|\s+", part.replace("type ", "").strip()) if b]
+        if bits:
+            out.append((bits[0], bits[-1]))
+    return out
+
+
+def _http_client_calls(src: str) -> int | None:
+    """Offset of the first call through an imported HTTP client module in `src`, else None."""
+    binds: dict[str, str] = {}      # name -> module (the module object itself)
+    named: dict[str, str] = {}      # local name -> module (an imported client function)
+    for m in _IMPORT_DEFAULT.finditer(src):
+        binds[m.group(1)] = m.group(3)
+        for imp, loc in _named(m.group(2) or ""):
+            named[loc] = f"{m.group(3)}:{imp}"
+    for m in _IMPORT_NS.finditer(src):
+        binds[m.group(1)] = m.group(2)
+    for m in _IMPORT_NAMED.finditer(src):
+        for imp, loc in _named(m.group(1)):
+            named[loc] = f"{m.group(2)}:{imp}"
+    for m in _REQ_BIND.finditer(src):
+        binds[m.group(1)] = m.group(2)
+    for m in _REQ_DESTR.finditer(src):
+        for imp, loc in _named(m.group(1).replace(":", " as ")):
+            named[loc] = f"{m.group(2)}:{imp}"
+    best = None
+
+    def hit(pos):
+        nonlocal best
+        if best is None or pos < best:
+            best = pos
+
+    for name, mod in binds.items():
+        verbs = _HTTP_ONLY if mod in ("http", "https") else _HTTP_VERBS
+        if mod not in ("http", "https"):
+            m = re.search(r"(?<![\w.$])%s\s*\(" % re.escape(name), src)       # got(url), request(opts), needle(...)
+            if m:
+                hit(m.start())
+        m = re.search(r"(?<![\w.$])%s\s*\.\s*(%s)\s*\(" % (re.escape(name), "|".join(sorted(verbs))), src)
+        if m:
+            hit(m.start())
+    for name, spec in named.items():
+        mod, _, imp = spec.partition(":")
+        ok = imp in (_HTTP_ONLY if mod in ("http", "https") else _HTTP_VERBS | {"Client", "Pool"})
+        if not ok:
+            continue
+        m = re.search(r"(?<![\w.$])(?:new\s+)?%s\s*\(" % re.escape(name), src)
+        if m:
+            hit(m.start())
+    return best
+
+
+def _opaque_origin(origin: str | None) -> bool:
+    """An origin no host can be read from: a bare placeholder that is not a configured / env value, or a templated host."""
+    o = origin or ""
+    if not o:
+        return False
+    if o.startswith("{"):
+        return not o.startswith(("{env.", "{runtimeConfig."))
+    return "://" in o and "{" in o.split("://", 1)[1].split("/", 1)[0]
+
+
+def ts_unrecognised_http_client(root: Path, files: list[str], builder) -> dict | None:
+    """Files that import an HTTP client module (http, https, undici, got, node-fetch, superagent, needle, phin, request)
+    and call it, with no client endpoint made there (a call cg could not follow, or only an origin it could not read)."""
+    if builder is None:
+        return None
+    made: dict[str, bool] = {}
+    for e in builder.edges.values():
+        if e.kind != "HTTP_CALLS" or not e.file:
+            continue
+        n = builder.nodes.get(e.dst)
+        if n is not None and not _opaque_origin((n.attrs or {}).get("origin")):
+            made[e.file] = True
+    hits = []
+    for f in files:
+        if made.get(f) or _TEST_FILE.search(f):
+            continue
+        src = _read(root, f)
+        if not src or not re.search(r"(?:from|require\s*\()\s*['\"](?:node:)?(?:%s)['\"]" % _HTTP_MOD_ALT, src):
+            continue
+        src = _strip_comments(src)
+        pos = _http_client_calls(src)
+        if pos is not None:
+            hits.append((f, _line(src, pos)))
+    return _finding("ts_unrecognised_http_client", hits)
+
+
 # ------------------------------------------------------------------------------------------- driver
 
 def detect(root: str | Path, files_by_ext: dict[str, list[str]], builder=None, programs: dict | None = None) -> list[dict]:
@@ -442,6 +548,7 @@ def detect(root: str | Path, files_by_ext: dict[str, list[str]], builder=None, p
     run(express_loop_routes, root, js)
     run(nuxt_unevaluable_import_dirs, root)
     run(vue_unresolved_navigation, builder)
+    run(ts_unrecognised_http_client, root, js, builder)
     run(laravel_loop_routes, root, php)
     py = programs.get("python")
     if py is not None:

{cg_code_graph-0.20.0 → cg_code_graph-0.21.0}/cg_code_graph/cli.py

@@ -207,6 +207,23 @@ def main(argv=None):
     p.add_argument("--source", help="address or credential source: literal, env, env-example, compose, config")
     p.add_argument("--tls-off", action="store_true", help="only systems known to be reached without TLS")
     p.add_argument("--max-items", type=int, default=60)
+    p = sub.add_parser("surface", help="security views over recorded facts: inbound / outbound attack surface and findings "
+                                        "(hardcoded credentials, plaintext protocols, disabled TLS / SSH verification, unverified "
+                                        "webhooks, unguarded handlers, listeners on all interfaces, exposed IPC); text, JSON or "
+                                        "SARIF; --fail-on for CI gates")
+    p.add_argument("--db", required=True)
+    p.add_argument("--inbound", action="store_true", help="list the inbound surface (routes, webhooks, sockets, consumers, IPC)")
+    p.add_argument("--outbound", action="store_true", help="list the outbound surface (every external system)")
+    p.add_argument("--protocol", help="only this protocol (http, ws, grpc, tcp, redis, postgres, saas, ...)")
+    from .surface import finding_names
+    p.add_argument("--finding", help="only this finding: " + ", ".join(finding_names()))
+    p.add_argument("--min-confidence", choices=["heuristic", "resolved", "exact"], default="heuristic",
+                   help="drop findings and items below this confidence (default: heuristic, keep all)")
+    p.add_argument("--format", choices=["text", "json", "sarif"], default="text")
+    p.add_argument("--fail-on", help="comma-separated finding types; exit 1 when any remains after ignores")
+    p.add_argument("--max-items", type=int, default=200)
+    p.add_argument("--strict", action="store_true", help="unguarded: count route guards only (an inline check does not protect)")
+    p.add_argument("--show-ignored", action="store_true", help="also list findings accepted by .cg.yaml surface.ignore")
     p = sub.add_parser("tools", help="LLM tools and MCP tools / resources / prompts: handler, tables it reaches, agents "
                                       "offering it, callers, checks; agents, dynamic dispatch, model calls")
     p.add_argument("pattern", nargs="?", help="tool name, substring or glob")
@@ -569,7 +586,14 @@ def main(argv=None):
         res = external(st, a.pattern, protocol=a.protocol, source=a.source, tls_off=a.tls_off,
                        max_items=max(a.max_items, 200) if a.json else a.max_items)
         print(json.dumps(res, indent=1, default=str) if a.json else render_external(res, max_items=a.max_items))
+        if not a.json:
+            from .coverage import endpoint_note
+            n = endpoint_note(st)
+            if n:
+                print(n)
         return
+    if a.cmd == "surface":
+        return surface_cmd(a, st)
     if a.cmd == "tools":
         from .aitools import render_tools, tools
         res = tools(st, a.pattern, framework=a.framework, unmatched=a.unmatched, agent=a.agent,
@@ -721,6 +745,11 @@ def main(argv=None):
     elif a.cmd == "api-calls":
         rows = Q.api_calls(st, a.spec)
         print(json.dumps(rows, indent=1, default=str) if a.json else Q.render_api_calls(rows))
+        if not a.json:
+            from .coverage import endpoint_note
+            n = endpoint_note(st)
+            if n:
+                print(n)
     elif a.cmd == "stats":
         doc = stats_doc(st)
         print(json.dumps(doc, indent=1, default=str) if a.json else render_stats(st, doc))
@@ -824,6 +853,26 @@ def _note(comp: dict, as_json: bool) -> None:
             print(n)
 
 
+def surface_cmd(a, st) -> int:
+    """`cg surface`: exit 0, or 1 when a --fail-on finding remains after ignores; 2 for a bad finding name."""
+    from . import surface as S
+    fail_on = [x.strip() for x in a.fail_on.split(",") if x.strip()] if a.fail_on else []
+    try:
+        res = S.surface(st, inbound=a.inbound, outbound=a.outbound, protocol=a.protocol, finding=a.finding,
+                        min_confidence=a.min_confidence, fail_on=fail_on, max_items=a.max_items, strict=a.strict,
+                        show_ignored=a.show_ignored)
+    except ValueError as ex:
+        print(f"cg surface: {ex}", file=sys.stderr)
+        return 2
+    if a.format == "json":
+        print(S.to_json(res))
+    elif a.format == "sarif":
+        print(json.dumps(S.to_sarif(res), indent=1))
+    else:
+        print(S.render_surface(res, max_items=a.max_items))
+    return 1 if res["failed"] else 0
+
+
 def config_cmd(a) -> int:
     from .config import ConfigError, effective, find, load_file, render_effective, unknown_key_warnings
     if a.action == "validate":

{cg_code_graph-0.20.0 → cg_code_graph-0.21.0}/cg_code_graph/config.py

@@ -26,6 +26,11 @@ Records project-specific knowledge once, for the CLI, the MCP server and the vis
     protocols:
       external: ["kafka:audit.*", "http:GET /status"]   # <protocol>:<name glob> handled outside the analysed repos
                                              # (not reported as no_receiver / no_sender by cg protocols)
+    surface:
+      ignore:                                # accepted risks of `cg surface` (reason is required)
+        - {finding: unguarded, path: "app/Http/Controllers/HealthController.php", reason: public health probe}
+        - {finding: plaintext, id: "external:redis:cache:6379", reason: compose-internal network}
+        - {finding: hardcoded, fingerprint: 3f9a1c0b7d2e4a55, reason: rotated test key}
     gates: config/gates.json                 # gate scenarios file (cg index --gates)
     plans:
       dir: docs/plans                        # plans directory (--plans-dir)
@@ -53,7 +58,7 @@ SCHEMA: dict[str, set | None] = {     # top-level key -> allowed sub-keys (None:
     "frameworks": {"add", "remove"}, "auth": {"extra_patterns"}, "secret": {"extra_patterns"}, "gates": None,
     "plans": {"dir", "text_mention_dirs"}, "viz": {"presets"}, "generated": {"paths", "vendored", "keep", "include"},
     "platforms": {"targets", "paths", "file_suffixes", "path_conventions"}, "include": None, "apps": None,
-    "rust": {"targets"}, "protocols": {"external"}, "lossy": None,
+    "rust": {"targets"}, "protocols": {"external"}, "lossy": None, "surface": {"ignore"},
 }
 APP_KEYS = ("name", "root", "role", "links")
 APP_ROLES = ("backend", "frontend")
@@ -298,6 +303,9 @@ def parse(data: Any, fname: str = ".cg.yaml") -> dict:
         if bad:
             raise ConfigError(f"{fname}: protocols.external: {bad[0]!r} is not <protocol>:<name glob> (e.g. kafka:audit.*)")
         out["protocols"] = {"external": ext}
+    sf = _section(data, "surface", fname)
+    if sf is not None and sf.get("ignore") is not None:
+        out["surface"] = {"ignore": _surface_ignores(sf["ignore"], f"{fname}: surface.ignore")}
     if data.get("gates") is not None:
         out["gates"] = norm_root(data["gates"], f"{fname}: gates")
     pl = _section(data, "plans", fname)
@@ -319,6 +327,38 @@ def parse(data: Any, fname: str = ".cg.yaml") -> dict:
     return out
 
 
+SURFACE_IGNORE_KEYS = ("finding", "path", "id", "fingerprint", "reason")
+
+
+def _surface_ignores(v: Any, where: str) -> list[dict]:
+    """`surface.ignore`: [{finding, path | id | fingerprint, reason}]; the reason is required."""
+    from .surface import FINDINGS
+    if not isinstance(v, list):
+        raise ConfigError(f"{where}: expected a list of {{finding, path | id | fingerprint, reason}} entries")
+    out = []
+    for i, e in enumerate(v):
+        w = f"{where}[{i}]"
+        if not isinstance(e, dict):
+            raise ConfigError(f"{w}: expected a mapping with finding, path | id | fingerprint and reason")
+        bad = sorted(str(k) for k in set(e) - set(SURFACE_IGNORE_KEYS))
+        if bad:
+            raise ConfigError(f"{w}: unknown key {bad[0]!r} (allowed: {', '.join(SURFACE_IGNORE_KEYS)})")
+        f = e.get("finding")
+        if f not in FINDINGS:
+            raise ConfigError(f"{w}: finding {f!r} is not one of {', '.join(FINDINGS)}")
+        reason = e.get("reason")
+        if not isinstance(reason, str) or not reason.strip():
+            raise ConfigError(f"{w}: reason is required (say why the risk is accepted)")
+        sel = {k: e[k] for k in ("path", "id", "fingerprint") if e.get(k) is not None}
+        if not sel:
+            raise ConfigError(f"{w}: give at least one of path, id or fingerprint")
+        for k, x in sel.items():
+            if not isinstance(x, str) or not x.strip():
+                raise ConfigError(f"{w}: {k} must be a non-empty string")
+        out.append({"finding": f, **{k: x.strip() for k, x in sel.items()}, "reason": reason.strip()})
+    return out
+
+
 def _apps(v: Any, fname: str) -> list[dict]:
     """apps as a list of {name, root, role, links} (or a mapping name -> {root, role, links}), validated: unique names,
     role backend / frontend, links naming backend apps. `root` may be relative, `../other-repo` or absolute.
@@ -516,7 +556,7 @@ def effective(root: str | Path, python_roots: list[str] | None = None, gates: st
         row(f"platforms.{k}", pc.get(k, True), f"{fname} platforms.{k}" if k in pc else f"built-in ({what})")
     row("auth.token_pattern", PR.values("common", "auth", "token_pattern"), "preset common")
     for section in ("auth", "secret"):
-        for key in ("guards", "not_auth") if section == "auth" else ("guards",):
+        for key in ("guards", "not_auth", "session_only") if section == "auth" else ("guards",):
             for p in plan["presets"]:
                 g = PR.values(p, section, key, default=None)
                 if g:

{cg_code_graph-0.20.0 → cg_code_graph-0.21.0}/cg_code_graph/core/model.py

@@ -72,6 +72,7 @@ EDGE_KINDS: dict[str, tuple[bool, str]] = {
     "READS_PROP": (True, "code reads a stored property (field node: Swift `var x`, `@State`, `@Published`)"),
     "WRITES_PROP": (True, "code writes a stored property (assignment, compound assignment)"),
     "USES_VALUE": (True, "code refers to an enum case, a constant, a static/global variable or an object-like macro"),
+    "REGISTERS_CALLBACK": (False, "function builds a callback URL for one of the app's own routes and registers it with an external party (attrs.body_key, endpoint)"),
     "REFERENCES_FN": (True, "code/data takes a function as a value (callback, dispatch table, handler registration)"),
     "USES_UNSAFE": (True, "code contains an unsafe block or is an unsafe fn (sink node unsafe:<crate>)"),
     "GATED_BY": (True, "code compiled only under a condition: Cargo feature, cfg predicate or preprocessor macro"),

{cg_code_graph-0.20.0 → cg_code_graph-0.21.0}/cg_code_graph/core/plugin.py

@@ -59,6 +59,9 @@ class GraphBuilder:
         # Pigeon API definitions found by the Dart plugin ({Api: {kind: host | flutter, methods, file}}): bridges.py
         # links their native implementations / native callers
         self.pigeon_apis: dict = {}
+        # keyed-hash salt for the markers that stand in for secret values (cg_code_graph/core/redact.py); the indexer
+        # sets it from the graph DB before any plugin runs
+        self.redact_salt: str | None = None
 
     def add_node(self, kind: str, key: str, name: str | None = None, **kw) -> str:
         nid = node_id(kind, key)

cg_code_graph-0.21.0/cg_code_graph/core/redact.py

@@ -0,0 +1,158 @@
+"""Secret values never reach the graph database (#47).
+
+A literal that belongs to a credential-looking key (password, secret, token, key, api_key, auth, private, credential,
+passphrase, dsn, ...) is stored as `redacted:hmac:<first 8 hex of HMAC-SHA-256(graph salt, value)>`: enough to see that
+a literal exists and to tell two values apart within one graph, not enough to recover one, and not recognisable by
+hashing a common password. The salt is `secrets.token_hex(16)`, created on the first index of a graph DB, kept in its
+`meta` table (`redact_salt`) and carried over by a re-index or refresh of that DB; a new DB gets a new salt, so markers
+of different graphs never compare. Graphs written before part 2d hold `redacted:sha256:<8 hex>` markers, which
+`is_redacted` still accepts. A password inside a URL (`scheme://user:pw@host`) gets the same marker in place of the
+password so the host stays readable. Placeholders (`${X}`, `%(X)s`, `<x>`, `null`) and
+values that are already markers are left alone. Applied at index time by the plugins that keep literal config values
+(Laravel config/*.php `value` and `env()` defaults) and as a last pass over every node / edge attr before the write.
+"""
+from __future__ import annotations
+
+import hashlib
+import hmac
+import re
+import secrets
+
+MARK = "redacted:hmac:"
+LEGACY_MARK = "redacted:sha256:"
+SECRET_WORDS = ("PASSWORD", "PASS", "PASSWD", "SECRET", "TOKEN", "SECRET_KEY", "SECRET_ACCESS_KEY", "KEY", "AUTH", "PWD")
+_EXTRA_WORDS = {"PASSWORD", "PASSWD", "PWD", "PASSPHRASE", "SECRET", "SECRETS", "TOKEN", "TOKENS", "AUTH", "AUTHORIZATION",
+                "CREDENTIAL", "CREDENTIALS", "PRIVATE", "APIKEY", "DSN", "BEARER", "SALT", "SIGNATURE"}
+_NOT_SECRET_TAIL = ("_KEY_ID", "_PUBLIC_KEY", "_KEY_PATH")
+_URL_PW = re.compile(r"(?P<head>\b[A-Za-z][\w+.-]*://[^/\s:@'\"]*:)(?P<pw>[^@/\s'\"]+)(?P<tail>@)")
+_PLACEHOLDER = ("${", "{{", "%(", "<", "$")
+
+
+def secret_key(key: str) -> bool:
+    """The `cg surface` rule for a config key / env name that holds a credential."""
+    k = re.sub(r"[^A-Za-z0-9]+", "_", key).upper().strip("_")
+    return any(k.endswith("_" + s) or k == s for s in SECRET_WORDS) and not k.endswith(_NOT_SECRET_TAIL)
+
+
+def looks_secret(key: str) -> bool:
+    """Wider than `secret_key`: any word of the name says credential (private_key_pem, db_dsn, apiKey, auth_header)."""
+    if secret_key(key):
+        return True
+    spaced = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", str(key))
+    words = [w for w in re.split(r"[^A-Za-z0-9]+", spaced.upper()) if w]
+    if any(w in _EXTRA_WORDS for w in words):
+        return True
+    return "KEY" in words and not set(words) & {"PUBLIC", "ID", "PATH", "FILE", "NAME", "PREFIX", "TYPE", "SIZE", "LENGTH"}
+
+
+def is_redacted(v) -> bool:
+    return isinstance(v, str) and v.startswith((MARK, LEGACY_MARK))
+
+
+def new_salt() -> str:
+    return secrets.token_hex(16)
+
+
+def marker(value: str, salt: str) -> str:
+    """The stored stand-in for `value` in the graph that owns `salt`. There is no default salt on purpose: an unsalted
+    marker would be a plain hash of the value."""
+    if not salt:
+        raise ValueError("marker() needs the graph's redact salt")
+    return MARK + hmac.new(salt.encode(), value.encode("utf-8", "replace"), hashlib.sha256).hexdigest()[:8]
+
+
+def placeholder(v) -> bool:
+    if not isinstance(v, str):
+        return True
+    t = v.strip()
+    return not t or t.startswith(_PLACEHOLDER) or t.lower() in ("null", "none", "false", "true", "changeme?") or is_redacted(t)
+
+
+def redact_url(v: str, salt: str) -> str:
+    if "://" not in v or "@" not in v:
+        return v
+    return _URL_PW.sub(lambda m: m["head"] + (m["pw"] if placeholder(m["pw"]) else marker(m["pw"], salt)) + m["tail"], v)
+
+
+def redact(value, key: str | None, salt: str):
+    """`value` as it may be stored: a marker for a literal under a credential-looking `key`, a URL password replaced
+    by one, anything else unchanged. Non-strings and placeholders pass through."""
+    if not isinstance(value, str):
+        return value
+    if key is not None and looks_secret(key) and not placeholder(value):
+        return marker(value, salt)
+    return redact_url(value, salt)
+
+
+_VALUE_KEYS = {"value", "literal", "default", "env_default", "default_value"}
+_NAME_KEYS = ("key", "name", "env", "env_key", "setting")
+# Names that only ever carry a literal credential. `token`, `secret`, `credential(s)` are left out on purpose: plugins
+# use them for identifiers (the Nest dependency-injection token on INJECTS edges, a secret's name, a credential source).
+_DIRECT = {"password", "passwd", "pwd", "passphrase", "api_key", "apikey", "access_key", "secret_key", "client_secret",
+           "private_key", "auth_token"}
+_REFERENCE = re.compile(r"^(?:env|config|file|const|literal)[: ]|^\$\{|^[\w./-]+:\d+$")
+STRUCTURAL_EDGES = {"INJECTS", "BOUND_TO", "CREDENTIAL_FROM"}
+# Attr maps keyed by source identifiers: an exported class called `ApiKey` is a name, not a literal under a credential key.
+_IDENTIFIER_MAPS = {"reexports"}
+
+
+def _holds_secret(d: dict, k: str) -> bool:
+    if k.lower() in _DIRECT:
+        return not _REFERENCE.match(str(d[k]))
+    if k in _VALUE_KEYS:
+        return any(isinstance(d.get(nk), str) and looks_secret(d[nk]) for nk in _NAME_KEYS)
+    return False
+
+
+def sweep(builder, salt: str | None = None) -> int:
+    """Last pass over node / edge attrs; returns the count of replaced values. Replaces the password of a
+    `scheme://user:password@host` string, a literal under a credential-named attr (`password`, `token`, ...), and the
+    `value` / `default` of a `{key|name: <credential-looking>, value: ...}` record (TypeScript / Python config readers,
+    Laravel config). Booleans such as `credential_literal`, hosts and every other value stay as they are. The salt is
+    `salt`, else `builder.redact_salt` (set by the indexer from the graph DB)."""
+    salt = salt or getattr(builder, "redact_salt", None)
+    if not salt:
+        raise ValueError("sweep() needs the graph's redact salt")
+    n = 0
+
+    def walk(x, name=None):
+        nonlocal n
+        if name in _IDENTIFIER_MAPS:
+            return x
+        if isinstance(x, str):
+            if "@" in x and "://" in x:
+                y = redact_url(x, salt)
+                if y != x:
+                    n += 1
+                return y
+            return x
+        if isinstance(x, dict):
+            for k, v in x.items():
+                if isinstance(v, str) and _holds_secret(x, k) and not placeholder(v):
+                    x[k] = marker(v, salt)
+                    n += 1
+                    continue
+                nv = walk(v, k)
+                if nv is not v:
+                    x[k] = nv
+            return x
+        if isinstance(x, list):
+            for i, v in enumerate(x):
+                nv = walk(v)
+                if nv is not v:
+                    x[i] = nv
+            return x
+        return x
+    for node in builder.nodes.values():
+        doc = getattr(node, "doc", None)
+        if isinstance(doc, str) and "@" in doc and "://" in doc:
+            y = redact_url(doc, salt)
+            if y != doc:
+                node.doc = y
+                n += 1
+        if node.attrs:
+            walk(node.attrs)
+    for e in builder.edges.values():
+        if e.attrs and e.kind not in STRUCTURAL_EDGES:
+            walk(e.attrs)
+    return n

{cg_code_graph-0.20.0 → cg_code_graph-0.21.0}/cg_code_graph/coverage.py

@@ -343,6 +343,11 @@ def compute(root: str | Path, plugins: dict, scip_imported: bool = False, report
             at = st.get("realtime_subscriptions_unresolved_receiver_at") or []
             if at:
                 e["realtime_subscriptions_unresolved_receiver_at"] = at[:8]
+        if lang == "typescript" and st and st.get("realtime_subscriptions_unresolved_wrapper"):
+            e["realtime_subscriptions_unresolved_wrapper"] = st["realtime_subscriptions_unresolved_wrapper"]
+            at = st.get("realtime_subscriptions_unresolved_wrapper_at") or []
+            if at:
+                e["realtime_subscriptions_unresolved_wrapper_at"] = at[:8]
         if lang == "python" and st:
             for k in ("roots_mode", "source_roots", "roots_warnings", "roots_ambiguous", "module_name_collisions"):
                 if st.get(k):
@@ -503,13 +508,22 @@ def _entry_text(e: dict) -> str:
 
 
 def realtime_unresolved_lines(e: dict, indent: str = "  ") -> list[str]:
-    """'.private(...)' on a receiver that is not an Echo client, with a channel literal: where to look."""
-    n = e.get("realtime_subscriptions_unresolved_receiver") or 0
-    if not n or e.get("language") != "typescript":
+    """'.private(...)' on a receiver that is not an Echo client, with a channel literal, and subscription wrapper calls whose
+    channel does not resolve: where to look."""
+    if e.get("language") != "typescript":
         return []
-    at = e.get("realtime_subscriptions_unresolved_receiver_at") or []
-    where = f" ({', '.join(at[:5])})" if at else ""
-    return [f"{indent}typescript: {n} .private(...) call{'s' if n != 1 else ''} on an unknown receiver{where}"]
+    out = []
+    n = e.get("realtime_subscriptions_unresolved_receiver") or 0
+    if n:
+        at = e.get("realtime_subscriptions_unresolved_receiver_at") or []
+        where = f" ({', '.join(at[:5])})" if at else ""
+        out.append(f"{indent}typescript: {n} .private(...) call{'s' if n != 1 else ''} on an unknown receiver{where}")
+    w = e.get("realtime_subscriptions_unresolved_wrapper") or 0
+    if w:
+        at = e.get("realtime_subscriptions_unresolved_wrapper_at") or []
+        where = f" ({', '.join(at[:5])})" if at else ""
+        out.append(f"{indent}typescript: {w} subscription wrapper call{'s' if w != 1 else ''} whose channel does not resolve{where}")
+    return out
 
 
 def syntax_error_lines(e: dict, all_files: bool = False, indent: str = "  ") -> list[str]:
@@ -553,9 +567,11 @@ def summary_line(cov: dict | None, repo: str | None = None) -> str:
 
 def _bs_count(bs: list[dict]) -> str:
     r = sum(b["count"] for b in bs if b["category"] == "route")
-    h = sum(b["count"] for b in bs if b["category"] != "route")
+    h = sum(b["count"] for b in bs if b["category"] == "handler")
+    x = sum(b["count"] for b in bs if b["category"] == "endpoint")
     parts = ([f"{r} unmodelled route registration{'s' if r != 1 else ''}"] if r else []) + \
-            ([f"{h} handler{'s' if h != 1 else ''} registered dynamically"] if h else [])
+            ([f"{h} handler{'s' if h != 1 else ''} registered dynamically"] if h else []) + \
+            ([f"{x} file{'s' if x != 1 else ''} with an outbound HTTP call that made no endpoint"] if x else [])
     return ", ".join(parts)
 
 
@@ -911,6 +927,21 @@ def completeness(covs: dict[str, dict | None], languages=None, dirs=None, repos=
     return out
 
 
+def endpoint_note(store) -> str:
+    """`coverage note:` for outbound HTTP calls cg could not turn into endpoints (the `endpoint` blind spots), shown after
+    `cg api-calls` / `cg external`; '' when there are none."""
+    try:
+        covs = for_graph(store)
+    except Exception:  # noqa: BLE001
+        return ""
+    found = [b for cov in covs.values() for b in blind_spots(cov) if b["category"] == "endpoint"]
+    if not found:
+        return ""
+    parts = [f"{b['count']} file{'s' if b['count'] != 1 else ''} call an HTTP client cg made no endpoint for ({', '.join(b['samples'][:3])})"
+             for b in found]
+    return "coverage note: " + "; ".join(parts) + ". Outbound calls from there may be missing; use your normal search and file reading."
+
+
 def completeness_for(store, node_ids=None, categories=("route", "handler"), whole: bool = False, **kw) -> dict:
     try:
         covs = for_graph(store)
@@ -931,23 +962,27 @@ def possibly_more(comp: dict) -> str:
         return ""
     parts = []
     r = sum(b["count"] for b in comp.get("blind_spots", []) if b["category"] == "route")
-    h = sum(b["count"] for b in comp.get("blind_spots", []) if b["category"] != "route")
+    h = sum(b["count"] for b in comp.get("blind_spots", []) if b["category"] == "handler")
+    x = sum(b["count"] for b in comp.get("blind_spots", []) if b["category"] == "endpoint")
     if r:
         parts.append(f"{r} unmodelled route registration{'s' if r != 1 else ''}")
     if h:
         parts.append(f"{h} dynamically registered handler{'s' if h != 1 else ''}")
+    if x:
+        parts.append(f"{x} file{'s' if x != 1 else ''} with an outbound HTTP call that made no endpoint")
     for k, v in comp.get("languages", {}).items():
         if v["complete"]:
             continue
-        lang = k.rsplit("/", 1)[-1]
+        repo, _, lang = k.rpartition("/")
         label = LANG_LABEL.get(lang, lang)
+        of = f" ({repo})" if repo else ""   # combined graph: keys are `<repo>/<language>`; two repos must not print identical fragments
         miss = sum(v.get(b, 0) for b in ("parse_failed", "skipped_oversize", "unmapped"))
         if v["mode"] in ("exact", "scip") and miss:
-            parts.append(f"{miss} {label} file{'s' if miss != 1 else ''} not indexed")
+            parts.append(f"{miss} {label} file{'s' if miss != 1 else ''} not indexed{of}")
         elif v["mode"] == "heuristic":
-            parts.append(f"{label} heuristic only")
+            parts.append(f"{label} heuristic only{of}")
         else:
-            parts.append(f"{label} {v['mode'].replace('_', ' ')}")
+            parts.append(f"{label} {v['mode'].replace('_', ' ')}{of}")
     se = comp.get("syntax_errors") or []
     if se:
         x = se[0]
@@ -968,7 +1003,7 @@ def answer_note(comp: dict) -> str:
         return ""
     parts = []
     for b in comp.get("blind_spots", []):
-        noun = "route registration" if b["category"] == "route" else "handler registration"
+        noun = {"route": "route registration", "endpoint": "outbound HTTP call"}.get(b["category"], "handler registration")
         parts.append(f"{b['count']} {noun}{'s' if b['count'] != 1 else ''} cg does not model ({b['what']}: {b['sample']})")
     rest = possibly_more({**comp, "blind_spots": []})
     if rest: