cg-code-graph 0.10.1__py3-none-any.whl

codegraph/plugins/python/external.py

@@ -0,0 +1,245 @@
+"""Python settings -> external-system facts for codegraph/external.py (#40).
+
+Module-level settings dicts (Django `DATABASES` / `CACHES`, NetBox-style `DATABASE` / `REDIS`, and nested aliases
+`{"default": {...}}`) with a HOST / LOCATION, and URL settings (`DATABASE_URL`, `CELERY_BROKER_URL`, `BROKER_URL`,
+`REDIS_URL`, `CACHE_URL`, `EMAIL_HOST`): literal values, `os.environ.get("K", default)` / `os.getenv` / `env("K")`
+reads and names of other module dicts. Client constructors with an address in their arguments (#77):
+`psycopg.connect(host=...)`, `redis.Redis(host=)` / `redis.from_url(url)`, `smtplib.SMTP(host, port)`,
+`pymongo.MongoClient(url)`, `ldap3.Server(host)`, `boto3.client("s3", endpoint_url=)` ... Only facts are recorded
+(builder.external_facts); passwords are kept as the env key or the location of a literal, never the value."""
+from __future__ import annotations
+
+import ast
+
+DICT_VARS = {"DATABASES": None, "DATABASE": None, "CACHES": None, "REDIS": "redis", "CACHE": None}
+URL_VARS = {"DATABASE_URL": None, "CELERY_BROKER_URL": None, "BROKER_URL": None, "REDIS_URL": "redis", "CACHE_URL": None,
+            "CELERY_RESULT_BACKEND": None, "EMAIL_HOST": "smtp", "MONGO_URL": "mongodb", "MONGODB_URI": "mongodb"}
+ENGINES = (("postgis", "postgres"), ("postgresql", "postgres"), ("psycopg", "postgres"), ("mysql", "mysql"),
+           ("oracle", "oracle"), ("mssql", "mssql"), ("sql_server", "mssql"), ("redis", "redis"), ("memcache", "memcached"),
+           ("sqlite", None), ("locmem", None), ("dummy", None), ("filebased", None), ("db.DatabaseCache", None))
+
+
+def _value(prog, m, e, depth=0):
+    """('lit', str) | ('env', key, default) | ('dict', ast.Dict, module) | None"""
+    if e is None or depth > 3:
+        return None
+    if isinstance(e, ast.Constant) and isinstance(e.value, (str, int)) and not isinstance(e.value, bool):
+        return ("lit", str(e.value))
+    if isinstance(e, ast.Dict):
+        return ("dict", e, m)
+    if isinstance(e, ast.Call):
+        fn = e.func
+        name = fn.attr if isinstance(fn, ast.Attribute) else fn.id if isinstance(fn, ast.Name) else ""
+        recv = fn.value if isinstance(fn, ast.Attribute) else None
+        rname = recv.attr if isinstance(recv, ast.Attribute) else recv.id if isinstance(recv, ast.Name) else None
+        envish = (isinstance(fn, ast.Name) and name in ("getenv", "env")) or (name == "get" and rname == "environ") \
+            or (name == "getenv" and rname == "os") or (rname in ("env", "environ") and name in ("str", "int", "url", "db", "cache_url", "db_url"))
+        if envish and e.args \
+                and isinstance(e.args[0], ast.Constant) and isinstance(e.args[0].value, str) and e.args[0].value.isupper():
+            d = e.args[1] if len(e.args) > 1 else next((k.value for k in e.keywords if k.arg == "default"), None)
+            dv = d.value if isinstance(d, ast.Constant) and isinstance(d.value, (str, int)) else None
+            return ("env", e.args[0].value, None if dv is None else str(dv))
+        if name in ("int", "str") and e.args:
+            return _value(prog, m, e.args[0], depth + 1)
+    if isinstance(e, ast.Subscript) and isinstance(e.slice, ast.Constant) and isinstance(e.slice.value, str) \
+            and (getattr(e.value, "attr", None) == "environ" or getattr(e.value, "id", None) == "environ"):
+        return ("env", e.slice.value, None)
+    if isinstance(e, ast.Name):
+        r = prog.resolve_name(m, e.id)
+        if r and r[0] == "var":
+            for v, _l, _a in r[1].vars.get(r[2], [])[-1:]:
+                return _value(prog, r[1], v, depth + 1)
+    return None
+
+
+def _proto(engine: str | None, default):
+    if engine:
+        for k, p in ENGINES:
+            if k in engine.lower():
+                return p, True
+    return default, False
+
+
+def index(prog, b, walk_body=None) -> int:
+    facts = []
+    for m in prog.modules.values():
+        if not any(s in m.name.rsplit(".", 1)[-1].lower() for s in ("settings", "config", "configuration")):
+            continue
+        for name, vals in m.vars.items():
+            if name not in DICT_VARS and name not in URL_VARS:
+                continue
+            v, line, _a = vals[-1]
+            if name in URL_VARS:
+                x = _value(prog, m, v)
+                if x and x[0] in ("lit", "env"):
+                    facts.append({"var": name, "module": m.name, "protocol": URL_VARS[name], "url": x, "src": f"module:{m.name}", "file": m.file, "line": line,
+                                  "host_only": name == "EMAIL_HOST"})
+                continue
+            x = _value(prog, m, v)
+            if not x or x[0] != "dict":
+                continue
+            d, dm = x[1], x[2]
+            entries = []
+            keys = {k.value: val for k, val in zip(d.keys, d.values) if isinstance(k, ast.Constant) and isinstance(k.value, str)}
+            if any(k in keys for k in ("HOST", "LOCATION", "ENGINE", "BACKEND", "URL")):
+                entries.append((None, keys, dm))
+            else:
+                for alias, val in keys.items():
+                    y = _value(prog, dm, val)
+                    if y and y[0] == "dict":
+                        sub = {k.value: vv for k, vv in zip(y[1].keys, y[1].values) if isinstance(k, ast.Constant) and isinstance(k.value, str)}
+                        entries.append((alias, sub, y[2]))
+            for alias, kv, em in entries:
+                eng = _value(prog, em, kv.get("ENGINE") or kv.get("BACKEND"))
+                proto, known = _proto(eng[1] if eng and eng[0] == "lit" else None, DICT_VARS[name])
+                if (eng and eng[0] == "lit") and known and proto is None:
+                    continue                                   # sqlite / local-memory cache: no network system
+                loc = _value(prog, em, kv.get("LOCATION") or kv.get("URL"))
+                nm = _value(prog, em, kv.get("NAME"))
+                f = {"var": name + (f"[{alias}]" if alias else ""), "module": m.name, "protocol": proto, "src": f"module:{m.name}", "file": m.file, "line": line,
+                     "resource": nm[1] if nm and nm[0] == "lit" and nm[1] else None}
+                if loc is not None and (loc[0] == "env" or (loc[0] == "lit" and "://" in loc[1])):
+                    f["url"] = loc
+                else:
+                    f["host"] = _value(prog, em, kv.get("HOST"))
+                    f["port"] = _value(prog, em, kv.get("PORT"))
+                    if f["host"] is None:
+                        continue
+                pw = kv.get("PASSWORD")
+                if pw is not None:
+                    pv = _value(prog, em, pw)
+                    f["password"] = ("env", pv[1]) if pv and pv[0] == "env" else ("literal", f"{em.file}:{pw.lineno}") \
+                        if pv and pv[0] == "lit" and pv[1] else None
+                facts.append(f)
+    if walk_body is not None:
+        facts += client_facts(prog, b, walk_body)
+    if facts:
+        b.external_facts = getattr(b, "external_facts", []) + facts
+    return len(facts)
+
+
+# top-level package -> (protocol, constructor / function names, default port override)
+CLIENTS = {
+    "psycopg": ("postgres", {"connect", "Connection", "AsyncConnection", "ConnectionPool", "AsyncConnectionPool"}),
+    "psycopg2": ("postgres", {"connect", "SimpleConnectionPool", "ThreadedConnectionPool"}),
+    "psycopg_pool": ("postgres", {"ConnectionPool", "AsyncConnectionPool"}),
+    "asyncpg": ("postgres", {"connect", "create_pool"}),
+    "pg8000": ("postgres", {"connect", "Connection"}),
+    "pymysql": ("mysql", {"connect", "Connection"}),
+    "MySQLdb": ("mysql", {"connect", "Connection"}),
+    "mysql": ("mysql", {"connect", "MySQLConnection"}),
+    "aiomysql": ("mysql", {"connect", "create_pool"}),
+    "redis": ("redis", {"Redis", "StrictRedis", "from_url", "ConnectionPool"}),
+    "aioredis": ("redis", {"Redis", "from_url", "create_redis_pool"}),
+    "pymongo": ("mongodb", {"MongoClient"}),
+    "motor": ("mongodb", {"AsyncIOMotorClient", "MotorClient"}),
+    "smtplib": ("smtp", {"SMTP", "SMTP_SSL"}),
+    "aiosmtplib": ("smtp", {"SMTP", "send"}),
+    "ftplib": ("ftp", {"FTP", "FTP_TLS"}),
+    "ldap3": ("ldap", {"Server"}),
+    "ldap": ("ldap", {"initialize"}),
+    "pika": ("amqp", {"URLParameters", "ConnectionParameters"}),
+    "aio_pika": ("amqp", {"connect", "connect_robust"}),
+    "kombu": ("amqp", {"Connection"}),
+    "elasticsearch": ("elasticsearch", {"Elasticsearch", "AsyncElasticsearch"}),
+    "pymemcache": ("memcached", {"Client", "PooledClient"}),
+    "boto3": ("s3", {"client", "resource"}),
+}
+TLS_CTORS = {"SMTP_SSL": "465", "FTP_TLS": None}
+URL_KW = ("dsn", "url", "conninfo", "host_url", "endpoint_url", "hosts")
+HOST_KW = ("host", "hostname", "server")
+
+
+def _dotted(m, e):
+    """`redis.asyncio.Redis` / `Redis` (from redis import Redis) -> 'redis.asyncio.Redis', through the imports."""
+    parts = []
+    while isinstance(e, ast.Attribute):
+        parts.append(e.attr)
+        e = e.value
+    if not isinstance(e, ast.Name):
+        return None
+    imp = m.imports.get(e.id)
+    if not isinstance(imp, tuple) or len(imp) < 2 or not isinstance(imp[1], str):
+        return None
+    base = imp[1] + (f".{imp[2]}" if len(imp) > 2 and imp[0] == "sym" and isinstance(imp[2], str) else "")
+    return ".".join([base, *reversed(parts)])
+
+
+def _kwarg(call, names):
+    for k in call.keywords:
+        if k.arg in names:
+            return k.value
+    return None
+
+
+def client_facts(prog, b, walk_body) -> list:
+    tops = set(CLIENTS)
+    facts = []
+    for m in prog.modules.values():
+        if not any(isinstance(v, tuple) and len(v) > 1 and isinstance(v[1], str) and v[1].split(".")[0] in tops
+                   for v in m.imports.values()):
+            continue
+        scopes = [(f.id, walk_body(f.node)) for f in m.funcs.values()]
+        scopes += [(f.id, walk_body(f.node)) for c in m.classes.values() for f in c.methods.values()]
+        scopes.append((f"module:{m.name}", (n for st in m.tree.body if not isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
+                                            for n in ast.walk(st))))
+        for src, nodes in scopes:
+            if src not in b.nodes:
+                continue
+            for c in nodes:
+                if not isinstance(c, ast.Call):
+                    continue
+                dn = _dotted(m, c.func)
+                if not dn:
+                    continue
+                top, last = dn.split(".")[0], dn.rsplit(".", 1)[-1]
+                if top not in CLIENTS or last not in CLIENTS[top][1]:
+                    continue
+                proto = CLIENTS[top][0]
+                if top == "boto3":
+                    if not (c.args and isinstance(c.args[0], ast.Constant) and c.args[0].value == "s3"):
+                        continue
+                    if _kwarg(c, ("endpoint_url",)) is None:
+                        continue
+                url = _kwarg(c, URL_KW)
+                host = _kwarg(c, HOST_KW)
+                pos = c.args[0] if c.args and top != "boto3" else None
+                if url is None and host is None and pos is not None:
+                    pv = _value(prog, m, pos)
+                    if pv and pv[0] == "lit" and "://" in pv[1] or last in ("from_url", "URLParameters") or top in ("pymongo", "motor", "asyncpg", "aio_pika") and pv and pv[0] == "env":
+                        url = pos
+                    elif proto in ("smtp", "ftp", "ldap", "elasticsearch", "memcached", "amqp") or (top == "redis" and last != "from_url"):
+                        host = pos
+                port = _kwarg(c, ("port",)) or (c.args[1] if len(c.args) > 1 and proto in ("smtp", "ftp", "redis") else None)
+                f = {"var": f"{dn}()", "module": m.name, "protocol": proto, "src": src, "file": m.file, "line": c.lineno,
+                     "client": top, "resource": None}
+                if url is not None:
+                    u = _value(prog, m, url)
+                    if not u or u[0] not in ("lit", "env") or (u[0] == "lit" and "://" not in u[1] and proto != "ldap"):
+                        continue
+                    if u[0] == "lit" and "://" not in u[1]:            # ldap3.Server("ldap.example.org")
+                        f["host"], f["port"] = u, None
+                    else:
+                        f["url"] = u
+                elif host is not None:
+                    h = _value(prog, m, host)
+                    if not h or h[0] not in ("lit", "env"):
+                        continue
+                    if h[0] == "lit" and "://" in h[1]:
+                        f["url"] = h
+                    else:
+                        f["host"] = h
+                        pv = _value(prog, m, port) if port is not None else None
+                        f["port"] = pv if pv and pv[0] in ("lit", "env") else (("lit", TLS_CTORS[last]) if TLS_CTORS.get(last) else None)
+                else:
+                    continue
+                db = _kwarg(c, ("dbname", "database", "db"))
+                dv = _value(prog, m, db) if db is not None else None
+                f["resource"] = dv[1] if dv and dv[0] == "lit" else None
+                pw = _kwarg(c, ("password", "passwd"))
+                if pw is not None:
+                    pv = _value(prog, m, pw)
+                    f["password"] = ("env", pv[1]) if pv and pv[0] == "env" else ("literal", f"{m.file}:{pw.lineno}") \
+                        if pv and pv[0] == "lit" and pv[1] else None
+                facts.append(f)
+    return facts

codegraph/plugins/python/fields.py

@@ -0,0 +1,107 @@
+"""Python stored attributes as field nodes (#88 phase 1).
+
+A class's instance attributes (`self.x = ...` in any of its methods) and its annotated class-level attributes
+(`x: int = 0` in a dataclass / pydantic model / attrs class; not `ClassVar[...]`) are `field:<Class>.<attr>` nodes,
+attrs `property: stored`, `declared: self | class`. Plain class-level assignments stay out: they are constants or
+framework descriptors (Django model fields are the Django plugin's own `field:` nodes), and methods, properties and
+nested classes are never fields.
+
+Every `self.x` / `v.x` attribute access whose receiver type the inference knows (`self`, a parameter annotation, a
+local `v = Cart()`, a typed attribute) and whose class (or a base) declares the field is a READS_PROP / WRITES_PROP
+edge (`resolved`, attr `receiver`). A write is an assignment / augmented assignment / `del` target, or the receiver
+of an in-place collection method (`self.items.append(x)`, `via: mutating`), or an item assignment / deletion
+(`self.cache[k] = v`, `via: item`). An unknown receiver binds nothing. Test
+code's accesses become TEST_USES (tests_index.py).
+"""
+from __future__ import annotations
+
+import ast
+
+from ...core.model import RESOLVED
+
+MUTATING = frozenset({
+    "append", "extend", "insert", "remove", "pop", "clear", "sort", "reverse", "add", "discard", "update",
+    "setdefault", "popitem", "difference_update", "intersection_update", "symmetric_difference_update", "appendleft",
+    "extendleft", "popleft", "rotate", "put", "put_nowait"})
+
+
+def _classvar(ann) -> bool:
+    t = ast.unparse(ann) if ann is not None else ""
+    return t.startswith(("ClassVar", "typing.ClassVar", "Final", "typing.Final", "InitVar", "dataclasses.InitVar"))
+
+
+def index(prog, b, Ctx, walk_body) -> dict:
+    fields: dict[str, dict[str, str]] = {}
+    names: set[str] = set()
+    n_nodes = 0
+    for c in prog.classes.values():
+        if c.id not in b.nodes:
+            continue
+        own: dict[str, tuple[int, str]] = {}
+        for nm, (val, line, ann) in c.attrs.items():
+            if ann is not None and not _classvar(ann):
+                own[nm] = (line, "class")
+        for nm, line in c.self_attr_lines.items():
+            own.setdefault(nm, (line, "self"))
+        inner = getattr(c, "inner_infos", {})
+        for nm, (line, how) in own.items():
+            if nm in c.methods or nm in inner:
+                continue
+            fid = b.add_node("field", f"{c.qual}.{nm}", name=nm, fqn=f"{c.qual}.{nm}", file=c.file, line=line,
+                             module=b.nodes[c.id].module, lang="python",
+                             attrs={"property": "stored", "declared": how})
+            b.add_edge(c.id, fid, "CONTAINS", c.file, line, RESOLVED)
+            fields.setdefault(c.qual, {})[nm] = fid
+            names.add(nm)
+            n_nodes += 1
+    if not names:
+        return {}
+
+    def lookup(cls, attr):
+        if cls.qual in fields and attr in fields[cls.qual]:
+            return fields[cls.qual][attr]
+        for bt in prog.mro(cls):
+            if bt[0] == "type" and bt[1].qual in fields and attr in fields[bt[1].qual]:
+                return fields[bt[1].qual][attr]
+        return None
+
+    reads = writes = mutating = unresolved = 0
+    for f in list(prog.funcs.values()):
+        ctx = Ctx(f.module, f, f.cls)
+        parent: dict[int, ast.AST] = {}
+        attrs = []
+        for n in walk_body(f.node):
+            for ch in ast.iter_child_nodes(n):
+                parent[id(ch)] = n
+            if isinstance(n, ast.Attribute) and n.attr in names:
+                attrs.append(n)
+        for a in attrs:
+            p = parent.get(id(a))
+            if isinstance(p, ast.Call) and p.func is a:
+                continue                                   # `self.save()`: a call, not an attribute access
+            try:
+                t = prog.infer(a.value, ctx)
+            except RecursionError:
+                t = None
+            fid = lookup(t[1], a.attr) if t is not None and t[0] == "inst" else None
+            if fid is None:
+                unresolved += 1
+                continue
+            via = None
+            write = isinstance(a.ctx, (ast.Store, ast.Del))
+            if not write and isinstance(p, ast.Subscript) and p.value is a and isinstance(p.ctx, (ast.Store, ast.Del)):
+                write, via = True, "item"                  # `self.cache[k] = v` / `del self.cache[k]`
+            elif not write and isinstance(p, ast.Attribute) and p.value is a and p.attr in MUTATING:
+                gp = parent.get(id(p))
+                if isinstance(gp, ast.Call) and gp.func is p:
+                    write, via = True, "mutating"
+            recv = ast.unparse(a.value)[:40] if not (isinstance(a.value, ast.Name) and a.value.id == "self") else "self"
+            b.add_edge(f.id, fid, "WRITES_PROP" if write else "READS_PROP", f.file, a.lineno, RESOLVED,
+                       receiver=recv, **({"via": via} if via else {}))
+            if write:
+                writes += 1
+                mutating += via == "mutating"
+            else:
+                reads += 1
+    return {"stored_attribute_nodes": n_nodes, "stored_attribute_reads": reads, "stored_attribute_writes": writes,
+            "stored_attribute_writes_mutating": mutating, "stored_attribute_refs_unresolved": unresolved}