code-node-graph 0.1.0__py3-none-any.whl

code_node/pages.py

@@ -0,0 +1,92 @@
+"""
+One page, made from scratch: read the folder, follow the code from the entry point, draw it. The command saves it
+once, and the server makes it again on every change.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from .analysis.graph import code_graph
+from .analysis.mapping import entry_file, map_code
+from .analysis.model import CodeMap
+from .deploys import find_deployment
+from .project import EntryPoint, Project, read_project
+from .report.build import page_blocks
+from .report.editor import editor_link
+from .report.style import render_page
+
+GRAPH_FUNCTIONS = 24  # functions the graph draws by default, the ones that lead to the most code first
+
+GRAPH_DEPTH = 5  # ...and how many calls deep
+
+_ALL = 10_000  # --all: no limit that a real codebase reaches
+
+_ALL_READ_DEPTH = 40  # --all: calls followed from the entry point
+
+
+@dataclass
+class PageOptions:
+    """
+    How much of the code the page draws, and which platform's checks it runs.
+    """
+
+    draw_all: bool = False
+    functions: int = GRAPH_FUNCTIONS
+    depth: int = GRAPH_DEPTH
+    platform: str = ""  # --platform: its checks run whatever the entry point is ('' leaves it to the entry point)
+    editor: str = "vscode"  # --editor: the editor its line links open ('vscode', 'cursor'; 'none' for no links)
+
+    @property
+    def limits(self) -> tuple[int, int]:
+        """
+        (functions, depth) the graph draws.
+        """
+        return (_ALL, _ALL) if self.draw_all else (self.functions, self.depth)
+
+
+def map_entry(root: Path, entry: EntryPoint, options: PageOptions) -> tuple[CodeMap, Project]:
+    """
+    The folder read as it is now, and the code followed from the entry point: the CodeMap, and the folder.
+    """
+    project = read_project(root)
+    file = entry.file or (entry_file(entry.name, project.files) or ("",))[0]
+    deployment = find_deployment(project.deploy_files, project.files, file, entry.name.rsplit(".", 1)[-1])
+    cmap = map_code(
+        project.files,
+        entry.name,
+        libraries=project.libraries(),
+        file=entry.file or None,
+        max_depth=_ALL_READ_DEPTH if options.draw_all else None,
+        platform=options.platform or entry.platform,
+        deployment=deployment,
+    )
+    return cmap, project
+
+
+def make_page(root: Path, entry: EntryPoint, options: PageOptions) -> tuple[str, CodeMap]:
+    """
+    The whole HTML page for an entry point, and the CodeMap it was drawn from.
+    """
+    cmap, project = map_entry(root, entry, options)
+    max_functions, max_depth = options.limits
+    blocks = page_blocks(
+        cmap,
+        project=project.root.name,
+        max_functions=max_functions,
+        max_depth=max_depth,
+        sources=project.files,
+        editor=editor_link(options.editor, str(project.root), os.environ.get("WSL_DISTRO_NAME", "")),
+    )
+    return render_page(blocks, f"How {entry.name} works"), cmap
+
+
+def make_mermaid(root: Path, entry: EntryPoint, options: PageOptions) -> str:
+    """
+    The graph as Mermaid.
+    """
+    cmap, _ = map_entry(root, entry, options)
+    max_functions, max_depth = options.limits
+    return code_graph(cmap, max_functions=max_functions, max_depth=max_depth).mermaid()

code_node/platforms/__init__.py

@@ -0,0 +1,203 @@
+"""
+Platforms: what a kind of entry point adds to reading plain Python. The reader, the findings and the graph are
+generic; a platform says what its entry points look like, what they're called with, what its code can't do there,
+and how to say so. AWS Lambda is one (aws_lambda.py), a web app (FastAPI, Flask, Django) another (web.py). Pure,
+like analysis/: no I/O, no rendering.
+
+Another package adds one by subclassing Platform and naming it under the `code_node.platforms` entry point group:
+
+    [project.entry-points."code_node.platforms"]
+    cloud_run = "my_package.cloud_run:CloudRun"
+
+or, from a script, with register_platform(CloudRun()).
+"""
+
+from __future__ import annotations
+
+import ast
+from typing import TYPE_CHECKING, Iterable
+
+if TYPE_CHECKING:
+    from ..analysis.model import CodeMap, EventField
+
+ENTRY_POINT_GROUP = "code_node.platforms"
+
+_BUILT_IN = ("code_node.platforms.aws_lambda:LambdaPlatform", "code_node.platforms.web:WebPlatform")
+
+
+class Platform:
+    """
+    Plain Python, and the base a platform plugin overrides: an entry point is a function called with nothing the
+    reader knows, code runs when Python imports it, and findings are worded for any code. A plugin sets `name` and
+    overrides what's different where it runs.
+    """
+
+    name = ""  # what --platform takes and CodeMap.platform holds: 'lambda'; '' is plain Python
+    title = "Python"  # 'AWS Lambda'
+    entry_kind = ""  # how its entry points are listed: 'a Lambda handler (event, context)'
+    inputs: tuple[str, ...] = ()  # what its entry function is called with: ('event', 'context'); the reader records
+    # the fields the code reads from the one named 'event'
+    input_name = "its input"  # 'the event'
+    input_word = "input"  # what a field of it is said to be read from, beside a step: 'event' ('event body.id')
+    draws_input = False  # the graph draws the input box even when the code reads no field of it
+    started = "When Python imports it"  # what runs at import is said to run: 'When Lambda starts it (...)'
+    web_apps: tuple[str, ...] = ()  # app classes that make an entry point of a module-level object, like Chalice
+    input_objects: tuple[str, ...] = ()  # names from a library that hold the input being handled: ('flask.request',)
+    input_aliases: dict[str, str] = {}  # its input's methods that give back a part of it: {'get_json': 'json'}
+
+    def find_entries(self, module: str, tree: ast.Module) -> list[tuple[str, int, str]]:
+        """
+        Its entry points in one module, as (name in the module, line, why): [('lambda_handler', 12, 'a Lambda
+        handler (event, context)')].
+        """
+        return []
+
+    def route_inputs(
+        self, function: ast.FunctionDef | ast.AsyncFunctionDef, path: str, imports: set[str], bound: bool = False
+    ) -> dict[str, tuple[str, ...]]:
+        """
+        What each parameter of a function a route runs holds from the input, as a path into it: {'order_id':
+        ('path', 'order_id'), 'request': ()}. path is the route's ('/orders/{order_id}'), imports the libraries its
+        module imports ({'fastapi'}), bound whether the first parameter is self. A parameter left out holds nothing
+        it knows.
+        """
+        return {}
+
+    def entry_icon(self, cmap: CodeMap) -> str:
+        """
+        The logo the entry point's box shows, by its name in report/logos/ ('fastapi-icon'); '' for its line icon.
+        """
+        return ""
+
+    def input_kind(self, fields: Iterable[EventField]) -> str | None:
+        """
+        What sends the input, from the fields the code reads: 'an SQS batch'. None when it can't tell.
+        """
+        return None
+
+    def python_version(self, cmap: CodeMap) -> tuple[int, int] | None:
+        """
+        The Python the code runs on, when it's known: (3, 12).
+        """
+        return None
+
+    def unset_env(self, cmap: CodeMap) -> set[str]:
+        """
+        The environment variables the code needs that are known not to be set where it runs.
+        """
+        return set()
+
+    def missing_entry(self, cmap: CodeMap, name: str, others: str) -> str:
+        """
+        The finding when the entry point isn't a function in its file.
+        """
+        return (
+            f"No function {name}() in {cmap.handler_file}: there's nothing to start from. Name one of its "
+            f"functions{others}."
+        )
+
+    def hang(self, cmap: CodeMap) -> str:
+        """
+        What an HTTP request without a timeout does when the server doesn't answer.
+        """
+        return "the call hangs for as long as the connection stays open."
+
+    def lost_error(self) -> str:
+        """
+        What a run whose errors are dropped looks like.
+        """
+        return "looks like it worked, so whoever called it never learns it didn't."
+
+    def import_error(self, module: str) -> str:
+        """
+        The error an import that can't be found raises: "ModuleNotFoundError: No module named 'x'".
+        """
+        return f"ModuleNotFoundError: No module named '{module}'"
+
+    def install(self, pip: str, them: str) -> str:
+        """
+        How to add libraries the code imports: 'install it (pip install x) and list it in the project's
+        dependencies'.
+        """
+        return f"install {them} (pip install {pip}) and list {them} in the project's dependencies"
+
+    def entry_findings(self, cmap: CodeMap) -> list[tuple[str, str]]:
+        """
+        Findings about the entry point itself, as (level, message): the graph marks its box with them.
+        """
+        return []
+
+    def findings(self, cmap: CodeMap) -> list[tuple[str, str]]:
+        """
+        Findings only this platform has, as (level, message): after the ones for any code.
+        """
+        return []
+
+    def denied(self, cmap: CodeMap) -> dict[str, str]:
+        """
+        The permissions its AWS calls need that where it runs is known not to allow, each with why, in a few words:
+        {'sns:Publish': 'not in its Policies in template.yaml'}. The graph marks those calls.
+        """
+        return {}
+
+    def marks(self, cmap: CodeMap) -> dict[str, list[str]]:
+        """
+        Why a function's box in the graph is amber, for this platform's findings: {'etl.save': ['writes data.csv
+        outside /tmp (line 9)']}.
+        """
+        return {}
+
+
+PYTHON = Platform()
+
+_platforms: dict[str, Platform] = {}
+
+_loaded = False
+
+
+def register_platform(platform: Platform) -> None:
+    """
+    Adds a platform (or replaces the one with its name): --platform takes its name, and its entry points are found.
+    """
+    if not platform.name:
+        raise ValueError("a platform needs a name")
+    _platforms[platform.name] = platform
+
+
+def _add(found: Platform | type[Platform]) -> None:
+    platform = found() if isinstance(found, type) else found
+    if platform.name:
+        _platforms.setdefault(platform.name, platform)  # one registered by hand first wins
+
+
+def platforms() -> list[Platform]:
+    """
+    Every platform: the built-in ones, then those installed packages add, loaded the first time it's asked.
+    """
+    global _loaded
+    if not _loaded:
+        _loaded = True
+        from importlib import import_module
+        from importlib.metadata import entry_points
+
+        for target in _BUILT_IN:
+            module, _, attr = target.partition(":")
+            _add(getattr(import_module(module), attr))
+        for point in entry_points(group=ENTRY_POINT_GROUP):
+            try:
+                _add(point.load())
+            except Exception:  # noqa: BLE001  a broken plugin leaves the others working
+                continue
+    return list(_platforms.values())
+
+
+def get_platform(name: str | None) -> Platform:
+    """
+    The platform with that name; plain Python for '' or a name no platform has.
+    """
+    if not name:
+        return PYTHON
+    return next((p for p in platforms() if p.name == name), PYTHON)
+
+
+__all__ = ["ENTRY_POINT_GROUP", "PYTHON", "Platform", "get_platform", "platforms", "register_platform"]

code_node/platforms/aws_lambda.py

@@ -0,0 +1,290 @@
+"""
+AWS Lambda as a platform: handlers are entry points, the handler is called with (event, context), code at import
+runs once per cold start, the disk is read-only outside /tmp, and the run is billed while it waits. Its findings
+check what a handler needs that plain Python doesn't.
+"""
+
+from __future__ import annotations
+
+import ast
+import posixpath
+from collections import defaultdict
+from typing import Iterable
+
+from ..analysis.aws_model import _event_kind, _service_name
+from ..analysis.findings import _listed, _where
+from ..analysis.mapping import _python_version
+from ..analysis.model import CodeMap, CodeSpot, EnvRead, EventField
+from ..deploys import allowed, get_deploy
+from ..kit.fmt import _plural
+from . import Platform
+
+_HANDLERS = ("lambda_handler", "handler")
+
+_WRITABLE = ("/tmp", "/mnt/")  # Lambda's disk is read-only but for /tmp (and a file system mounted under /mnt)
+
+_LAMBDA_ENV = frozenset(
+    {  # environment variables Lambda sets itself: reading one needs no setting
+        "AWS_REGION",
+        "AWS_DEFAULT_REGION",
+        "AWS_EXECUTION_ENV",
+        "AWS_LAMBDA_FUNCTION_NAME",
+        "AWS_LAMBDA_FUNCTION_MEMORY_SIZE",
+        "AWS_LAMBDA_FUNCTION_VERSION",
+        "AWS_LAMBDA_INITIALIZATION_TYPE",
+        "AWS_LAMBDA_LOG_GROUP_NAME",
+        "AWS_LAMBDA_LOG_STREAM_NAME",
+        "AWS_ACCESS_KEY_ID",
+        "AWS_SECRET_ACCESS_KEY",
+        "AWS_SESSION_TOKEN",
+        "AWS_LAMBDA_RUNTIME_API",
+        "LAMBDA_TASK_ROOT",
+        "LAMBDA_RUNTIME_DIR",
+        "_HANDLER",
+        "_X_AMZN_TRACE_ID",
+        "TZ",
+        "LANG",
+        "PATH",
+        "LD_LIBRARY_PATH",
+        "PYTHONPATH",
+        "AWS_XRAY_CONTEXT_MISSING",
+        "AWS_XRAY_DAEMON_ADDRESS",
+        "AWS_LAMBDA_LOG_FORMAT",
+        "AWS_LAMBDA_LOG_LEVEL",
+        "AWS_LAMBDA_MAX_CONCURRENCY",
+        "HOME",
+        "PWD",
+        "_AWS_XRAY_DAEMON_ADDRESS",
+        "_AWS_XRAY_DAEMON_PORT",
+        "AWS_LAMBDA_EXEC_WRAPPER",
+    }
+)
+
+
+class LambdaPlatform(Platform):
+    """
+    An AWS Lambda handler: def handler(event, context) in its deployment package.
+    """
+
+    name = "lambda"
+    title = "AWS Lambda"
+    entry_kind = "a Lambda handler (event, context)"
+    inputs = ("event", "context")
+    input_name = "the event"
+    input_word = "event"
+    draws_input = True
+    started = "When Lambda starts it (once per cold start)"
+    web_apps = (
+        "Chalice",
+        "APIGatewayRestResolver",
+        "APIGatewayHttpResolver",
+        "ALBResolver",
+        "LambdaFunctionUrlResolver",
+    )
+
+    def find_entries(self, module: str, tree: ast.Module) -> list[tuple[str, int, str]]:
+        found = []
+        for node in tree.body:
+            is_handler = (
+                isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
+                and node.name in _HANDLERS
+                and len(node.args.posonlyargs + node.args.args) == 2
+            )
+            if is_handler:
+                found.append((node.name, node.lineno, self.entry_kind))
+        return sorted(found, key=lambda entry: _HANDLERS.index(entry[0]))
+
+    def input_kind(self, fields: Iterable[EventField]) -> str | None:
+        return _event_kind(fields)
+
+    def python_version(self, cmap: CodeMap) -> tuple[int, int] | None:
+        return _python_version(cmap.function.runtime) if cmap.function is not None else None
+
+    def unset_env(self, cmap: CodeMap) -> set[str]:
+        fn = cmap.function
+        if fn is None or fn.env_error is not None:
+            return set()
+        return {e.name for e in cmap.env if e.required and e.name not in fn.env_names and e.name not in _LAMBDA_ENV}
+
+    def denied(self, cmap: CodeMap) -> dict[str, str]:
+        fn = cmap.function
+        if fn is None or getattr(fn, "permissions", None) is None:
+            return {}
+        return {
+            c.permission: f"{fn.name}'s role in {fn.source} doesn't allow {c.permission}"
+            for c in cmap.aws
+            if c.permission and not c.guessed and not allowed(c.permission, fn.permissions)
+        }
+
+    def missing_entry(self, cmap: CodeMap, name: str, others: str) -> str:
+        return (
+            f"No function {name}() in {cmap.handler_file}: every call fails with Runtime.HandlerNotFound before "
+            f"any of its code runs. Point the handler setting at one of its functions{others}, or add it."
+        )
+
+    def hang(self, cmap: CodeMap) -> str:
+        fn = cmap.function
+        timeout = f"its {fn.timeout:,} s timeout" if fn is not None else "its timeout"
+        return f"the run waits until Lambda stops it at {timeout}, and you pay for the wait."
+
+    def lost_error(self) -> str:
+        return "reports success, so Lambda doesn't retry it, CloudWatch doesn't count the error, and its event is lost."
+
+    def import_error(self, module: str) -> str:
+        return f"Runtime.ImportModuleError: No module named '{module}'"
+
+    def install(self, pip: str, them: str) -> str:
+        return f"add {them} to the package (pip install {pip} -t .) or a layer"
+
+    def entry_findings(self, cmap: CodeMap) -> list[tuple[str, str]]:
+        found: list[tuple[str, str]] = []
+        if cmap.note or cmap.entry is None:
+            return found
+        name = (cmap.handler or "").rsplit(".", 1)[-1]
+        if cmap.is_async:
+            found.append(
+                (
+                    "warn",
+                    (
+                        f"{name}() is async def: the Python runtime calls it without awaiting it, so its code never runs "
+                        "and the call returns a coroutine. Wrap it in a plain function that runs it: def handler(event, "
+                        "context): return asyncio.run(main(event, context))."
+                    ),
+                )
+            )
+        elif any(f.name == cmap.entry for f in cmap.functions):  # a function, not an app object
+            positional = [p for p in cmap.params if not p.startswith("*")]
+            required = [p for p in positional if "=" not in p]
+            star = any(p.startswith("*") and not p.startswith("**") for p in cmap.params)
+            if len(required) > 2 or (len(positional) < 2 and not star):
+                found.append(
+                    (
+                        "warn",
+                        (
+                            f"{name}() takes {_plural(len(required), 'argument')}, and Lambda passes two (event and "
+                            f"context): every call fails with TypeError. Make it def {name}(event, context)."
+                        ),
+                    )
+                )
+        unset = self.unset_env(cmap)
+        if unset:
+            missing: dict[str, list[EnvRead]] = defaultdict(list)
+            for read in cmap.env:
+                if read.name in unset:
+                    missing[read.name].append(read)
+            names = list(missing)
+            first = [reads[0] for reads in missing.values()]
+            startup = any(r.startup for r in first)
+            where = _listed([f"{r.file} line {r.line}" for r in first], 2)
+            them = "it" if len(names) == 1 else "them"
+            found.append(
+                (
+                    "warn",
+                    (
+                        f"{_listed(names)} {'isn' if len(names) == 1 else 'aren'}'t set on {cmap.function.name}, and its code "
+                        f"reads {them} with os.environ[...] ({where}): that line fails with KeyError: '{names[0]}'"
+                        + (", when Lambda loads the code, so every call fails" if startup else "")
+                        + f". Add {them} to its environment variables (the console's Configuration tab, Environment "
+                        "variables)."
+                    ),
+                )
+            )
+        return found
+
+    def findings(self, cmap: CodeMap) -> list[tuple[str, str]]:
+        found: list[tuple[str, str]] = []
+        if cmap.note:
+            return found
+        fn = cmap.function
+        layered = [i for i in cmap.imports if i.source == "layer" and not i.optional]
+        if layered and fn is not None:
+            tops = list(dict.fromkeys(i.top for i in layered))
+            found.append(
+                (
+                    "info",
+                    (
+                        f"{_listed(tops)} {'isn' if len(tops) == 1 else 'aren'}'t in the package, so "
+                        f"{'it has' if len(tops) == 1 else 'they have'} to come from its layers "
+                        f"({_listed([layer.rsplit(':', 2)[0].rsplit(':', 1)[-1] for layer in fn.layers], 2)}): if no layer has "
+                        f"{'it' if len(tops) == 1 else 'them'}, calls fail with Runtime.ImportModuleError."
+                    ),
+                )
+            )
+        for spot in self._outside_tmp(cmap):
+            name = posixpath.basename(spot.detail.rstrip("/")) or "out"
+            found.append(
+                (
+                    "warn",
+                    (
+                        f"It writes {spot.detail} at {_where(spot.file, spot.line)}, outside /tmp: Lambda's disk is read-only "
+                        f"there, so that line fails with OSError: [Errno 30] Read-only file system. Write under /tmp instead "
+                        f"(/tmp/{name})."
+                    ),
+                )
+            )
+        for spot in [s for s in cmap.spots if s.kind == "client" and not s.ok]:
+            service = _service_name(spot.detail) if spot.detail != "?" else "AWS"
+            found.append(
+                (
+                    "info",
+                    (
+                        f"It makes a new {service} client on every call, at {_where(spot.file, spot.line, spot.function)}"
+                        + (" inside a loop" if spot.loop else "")
+                        + ": making one takes time, more on a cold start. Make it once at module level (above the handler), "
+                        "so warm calls reuse it."
+                    ),
+                )
+            )
+        for spot in [s for s in cmap.spots if s.kind == "connect" and not s.ok and not s.startup]:
+            found.append(
+                (
+                    "info",
+                    (
+                        f"It opens a new {spot.detail} connection on every call, at "
+                        f"{_where(spot.file, spot.line, spot.function)}"
+                        + (" inside a loop" if spot.loop else "")
+                        + ": that's a round trip or more before any work, and many runs at once can use up the server's "
+                        "connections. Make it once at module level (above the handler), so warm calls reuse it."
+                    ),
+                )
+            )
+        denied = self.denied(cmap)
+        if denied and fn is not None:
+            calls = [c for c in cmap.aws if c.permission in denied]
+            missing = list(dict.fromkeys(c.permission for c in calls))
+            deploy = get_deploy(fn.reader)
+            fix = deploy.grant(fn, missing) if deploy is not None else f"allow {', '.join(missing)} in its role"
+            given = f" (it has {_listed(fn.policies, 3)})" if fn.policies else ""
+            found.append(
+                (
+                    "warn",
+                    (
+                        f"{fn.name}'s role in {fn.source}{given} doesn't allow {_listed(missing)}, which "
+                        + _listed([f"{c.label} at {_where(c.file, c.line)}" for c in calls], 2)
+                        + (" needs: that call fails" if len(calls) == 1 else " need: those calls fail")
+                        + f" with AccessDenied. {fix[0].upper()}{fix[1:]}."
+                    ),
+                )
+            )
+        for spot in [s for s in cmap.spots if s.kind == "sleep" and not s.startup]:
+            found.append(
+                (
+                    "info",
+                    (
+                        f"It sleeps at {_where(spot.file, spot.line, spot.function)} ({spot.code}): the function is billed "
+                        "for the time it waits. If it waits for something to finish, a Step Functions Wait state or an SQS "
+                        "delay costs nothing while it waits."
+                    ),
+                )
+            )
+        return found
+
+    def marks(self, cmap: CodeMap) -> dict[str, list[str]]:
+        marked: dict[str, list[str]] = defaultdict(list)
+        for spot in self._outside_tmp(cmap):
+            marked[spot.function].append(f"writes {spot.detail} outside /tmp (line {spot.line})")
+        return marked
+
+    @staticmethod
+    def _outside_tmp(cmap: CodeMap) -> list[CodeSpot]:
+        return [s for s in cmap.spots if s.kind == "write" and not s.detail.startswith(_WRITABLE)]