pymap-cli 0.3.0__py3-none-any.whl

pymap/mapper.py

@@ -0,0 +1,198 @@
+"""Orchestration: chain the analyses and write the map.
+
+This is also pymap's library API::
+
+    from pymap import Settings, map_codebase
+
+    report = map_codebase(Settings(target="src/mypkg"))
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable
+
+from pymap import render
+from pymap.analysis import cycles as cycle_analysis
+from pymap.analysis.calls import load_call_graph
+from pymap.analysis.symbols import Symbol, coverage, duplicate_modules, extract
+from pymap.settings import Settings, count_lines
+from pymap.tools import code2flow, pydeps, pyreverse, tach
+
+Log = Callable[[str], Any]
+
+#: Steps, in execution order. Adding a tool means adding one line here.
+STEPS = (
+    ("pydeps", "import graph", pydeps),
+    ("pyreverse", "classes and inheritance", pyreverse),
+    ("code2flow", "call graph", code2flow),
+    ("tach", "modules and cycles", tach),
+)
+
+
+@dataclass
+class Report:
+    """What pymap found, and where it wrote it."""
+
+    settings: Settings
+    files: list[Path] = field(default_factory=list)
+    lines: int = 0
+    symbols: list[Symbol] = field(default_factory=list)
+    documentable: int = 0
+    documented: int = 0
+    coverage: int = 0
+    cycles: list[list[str]] = field(default_factory=list)
+    duplicates: dict[str, int] = field(default_factory=dict)
+    tools: dict[str, dict[str, Any]] = field(default_factory=dict)
+    index: Path | None = None
+    explorer: Path | None = None
+
+    @property
+    def undocumented(self) -> int:
+        return self.documentable - self.documented
+
+
+def map_codebase(settings: Settings, log: Log = lambda _: None) -> Report:
+    """Analyse ``settings.target`` and write the map into ``settings.output``."""
+    if not settings.target.is_dir():
+        raise NotADirectoryError(f"directory not found: {settings.target}")
+    settings.output.mkdir(parents=True, exist_ok=True)
+
+    report = Report(settings=settings, files=settings.files())
+    if not report.files:
+        raise FileNotFoundError(f"no Python file under {settings.target}")
+    report.lines = count_lines(report.files)
+    log(f"Mapping {settings.name}: {len(report.files)} files, {report.lines} lines\n")
+
+    total = len(STEPS) + 1
+    for index, (name, role, module) in enumerate(STEPS, 1):
+        log(f"[{index}/{total}] {name:11} {role}...")
+        report.tools[name] = result = module.execute(settings)
+        log("      " + (result["log"] or _summary(name, result)))
+
+    log(f"[{total}/{total}] explorer    assembling...")
+    report.cycles = cycle_analysis.detect(report.tools["tach"]["deps"])
+    report.symbols = extract(settings.target, report.files)
+    report.duplicates = duplicate_modules(report.files)
+    report.documentable, report.documented, report.coverage = coverage(report.symbols)
+
+    relations = load_call_graph(report.tools["code2flow"]["json"] or "")
+    report.explorer = settings.output / "explorer.html"
+    report.explorer.write_text(
+        render.explorer_page(
+            settings.name,
+            settings.target,
+            report.symbols,
+            relations,
+            report.coverage,
+            settings.editor_uri,
+        ),
+        encoding="utf-8",
+    )
+    log(
+        f"      {len(report.symbols)} symbols, {report.coverage}% documented "
+        f"({report.undocumented} without a docstring)"
+    )
+    for stem, count in report.duplicates.items():
+        log(
+            f"      note: {count} files are named {stem}.py; "
+            f"their symbols share one namespace in the explorer"
+        )
+
+    report.index = settings.output / "index.html"
+    report.index.write_text(
+        render.index_page(settings.name, len(report.files), report.lines, _sections(report)),
+        encoding="utf-8",
+    )
+    return report
+
+
+def _summary(name: str, result: dict[str, Any]) -> str:
+    """One terminal line per tool that succeeded."""
+    if name == "pydeps":
+        return f"{result['modules']} modules, {result['imports']} imports"
+    if name == "pyreverse":
+        drawn = [kind for kind in ("classes", "packages") if result[kind]]
+        return ", ".join(drawn) + " rendered" if drawn else "nothing to draw"
+    if name == "code2flow":
+        return f"{result['functions']} functions, {result['calls']} calls"
+    if name == "tach":
+        cut = (
+            f" ({result['truncated']} modules beyond the limit)" if result["truncated"] else ""
+        )
+        return f"{len(result['deps'])} files linked{cut}"
+    return "done"
+
+
+def _sections(report: Report) -> list[str]:
+    """The landing page blocks, in the recommended reading order."""
+    tools = report.tools
+    sections = [
+        render.section(
+            title="Walkthrough tree",
+            what="Every module, class and function with its signature and docstring, "
+            "filterable from the keyboard. From any symbol you can jump to what "
+            "it calls and to what calls it. Open this one first.",
+            figures=[
+                (len(report.symbols), "symbols"),
+                (f"{report.coverage}%", "documented"),
+                (report.undocumented, "without a docstring"),
+            ],
+            buttons=[("explorer.html", "Open the explorer")],
+        )
+    ]
+
+    imports = tools["pydeps"]
+    sections.append(
+        render.section(
+            title="Imports between modules",
+            what="Who imports whom. The bird's-eye view: it shows the layers and "
+            "brings out the modules everyone pulls in."
+            if imports["svg"]
+            else "",
+            missing="" if imports["svg"] else imports["log"],
+            figures=[(imports["modules"], "modules"), (imports["imports"], "imports")]
+            if imports["svg"]
+            else [],
+            buttons=[("imports.svg", "Open the graph")] if imports["svg"] else [],
+        )
+    )
+
+    classes = tools["pyreverse"]
+    drawn = [
+        (path.name, label)
+        for path, label in (
+            (classes["classes"], "Classes and attributes"),
+            (classes["packages"], "Packages"),
+        )
+        if path
+    ]
+    sections.append(
+        render.section(
+            title="Classes and inheritance",
+            what="Classes, their attributes and their inheritance links. Useful when "
+            "the code is object-oriented; empty when it is not."
+            if drawn
+            else "",
+            missing="" if drawn else classes["log"],
+            buttons=drawn,
+        )
+    )
+
+    boundaries = tools["tach"]
+    known = bool(boundaries["deps"] or boundaries["mermaid"])
+    sections.append(
+        render.section(
+            title="Module boundaries",
+            what="The file-by-file dependency map, and the cycles. Paste the Mermaid "
+            "source into a README: it will still be readable in ten years."
+            if known
+            else "",
+            missing="" if known else boundaries["log"],
+            figures=[(len(boundaries["deps"]), "files linked")] if boundaries["deps"] else [],
+            detail=render.cycles_block(report.cycles) if boundaries["deps"] else "",
+            buttons=[("tach.mmd", "Mermaid source")] if boundaries["mermaid"] else [],
+        )
+    )
+    return sections

pymap/render.py

@@ -0,0 +1,170 @@
+"""Assembling the HTML pages from templates.
+
+Templates are real ``.html`` files (under ``pymap/templates/``), editable and
+readable as-is. Values are injected in place of ``__PYMAP_X__`` tokens: unlike
+``str.format``, no CSS or JavaScript brace has to be doubled.
+"""
+
+from __future__ import annotations
+
+import html
+import json
+import re
+from collections.abc import Iterable, Sequence
+from importlib import resources
+from pathlib import Path
+from typing import Any
+
+TOKEN = re.compile(r"__PYMAP_[A-Z_]+__")
+_CACHE: dict[str, str] = {}
+
+
+def template(filename: str) -> str:
+    """Contents of a template, read once and then kept in memory."""
+    if filename not in _CACHE:
+        _CACHE[filename] = (
+            resources.files("pymap.templates").joinpath(filename).read_text(encoding="utf-8")
+        )
+    return _CACHE[filename]
+
+
+def fill(filename: str, **values: Any) -> str:
+    """Substitute a template's tokens. A forgotten token is an error, not a hole.
+
+    Validation is done against the template, never against the result: injected
+    values legitimately contain ``__PYMAP_...__`` text (pymap can map itself).
+    Substitution is a single pass, so one value can never be re-read as the
+    token of the next.
+    """
+    text = template(filename)
+    table = {f"__PYMAP_{key.upper()}__": str(value) for key, value in values.items()}
+    expected = set(TOKEN.findall(text))
+    if table.keys() - expected:
+        raise KeyError(f"{filename}: unknown tokens {sorted(table.keys() - expected)}")
+    if expected - table.keys():
+        raise KeyError(f"{filename}: unfilled tokens {sorted(expected - table.keys())}")
+    return TOKEN.sub(lambda match: table[match.group(0)], text)
+
+
+def script_json(value: Any) -> str:
+    """Serialise ``value`` as JSON that is safe inside a ``<script>`` block.
+
+    A docstring containing ``</script>`` would otherwise close the tag early and
+    break the whole page, so the characters that can start HTML markup are
+    escaped. ``\\u2028`` and ``\\u2029`` are escaped too: they are valid JSON
+    but are line terminators to older JavaScript parsers.
+    """
+    text = json.dumps(value, ensure_ascii=False)
+    for char, escaped in (
+        ("<", "\\u003c"),
+        (">", "\\u003e"),
+        ("&", "\\u0026"),
+        ("\u2028", "\\u2028"),
+        ("\u2029", "\\u2029"),
+    ):
+        text = text.replace(char, escaped)
+    return text
+
+
+# --- fragments ---------------------------------------------------------------
+
+
+def stats(pairs: Sequence[tuple[Any, str]]) -> str:
+    """``[(value, label)]`` -> the row of headline figures."""
+    if not pairs:
+        return ""
+    return (
+        '<div class="stats">'
+        + "".join(
+            f'<div class="stat"><b>{html.escape(str(value))}</b>'
+            f"<span>{html.escape(str(label))}</span></div>"
+            for value, label in pairs
+        )
+        + "</div>"
+    )
+
+
+def links(pairs: Sequence[tuple[str, str]]) -> str:
+    """``[(href, label)]`` -> the open buttons."""
+    return "".join(
+        f'<a class="view" href="{html.escape(str(href))}">{html.escape(str(label))}</a>'
+        for href, label in pairs
+    )
+
+
+def note(text: str) -> str:
+    return f'<p class="what">{html.escape(text)}</p>' if text else ""
+
+
+def cycles_block(cycles: Iterable[Sequence[str]]) -> str:
+    """The red panel listing circular dependencies."""
+    cycles = list(cycles)
+    if not cycles:
+        return note("No circular dependency found.")
+    items = "".join(
+        f"<li><code>{html.escape(' -> '.join(Path(node).name for node in cycle))}</code></li>"
+        for cycle in cycles
+    )
+    plural = "y" if len(cycles) == 1 else "ies"
+    return (
+        f'<div class="cycles"><b>{len(cycles)} circular dependenc{plural}</b>'
+        f"<ul>{items}</ul></div>"
+    )
+
+
+def section(
+    title: str,
+    what: str = "",
+    missing: str = "",
+    figures: Sequence[tuple[Any, str]] = (),
+    detail: str = "",
+    buttons: Sequence[tuple[str, str]] = (),
+) -> str:
+    """One section of the landing page.
+
+    ``what`` is plain text (escaped here); ``missing`` replaces the description
+    when the matching tool did not run.
+    """
+    description = (
+        f'<span class="absent">{html.escape(missing)}</span>' if missing else html.escape(what)
+    )
+    return fill(
+        "section.html",
+        title=html.escape(title),
+        what=description,
+        stats=stats(figures),
+        detail=detail,
+        links=links(buttons),
+    )
+
+
+# --- pages -------------------------------------------------------------------
+
+
+def index_page(name: str, file_count: int, line_count: int, sections: Iterable[str]) -> str:
+    return fill(
+        "index.html",
+        name=html.escape(name),
+        files=file_count,
+        lines=f"{line_count:,}",
+        body="\n".join(sections),
+    )
+
+
+def explorer_page(
+    name: str,
+    root: str | Path,
+    symbols: list[dict[str, Any]],
+    relations: dict[str, Any],
+    coverage: int,
+    editor_uri: str = "",
+) -> str:
+    return fill(
+        "explorer.html",
+        name=html.escape(name),
+        coverage=coverage,
+        root=script_json(str(root)),
+        editor=script_json(editor_uri),
+        data=script_json(symbols),
+        relations=script_json(relations),
+    )

pymap/runner.py

@@ -0,0 +1,93 @@
+"""Running external tools, and detecting the ones that are missing.
+
+pymap needs none of these tools to start: every missing step is reported and
+then skipped. That is what makes it safe to install into a project without
+dragging a dependency chain along.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import shutil
+import subprocess
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+from typing import NamedTuple
+
+
+class Tool(NamedTuple):
+    """An external tool pymap knows how to drive."""
+
+    name: str
+    module: str | None
+    install: str
+    role: str
+
+
+#: ``module`` is the fallback: when the console script is not on PATH but the
+#: package is installed in the current interpreter, ``python -m`` finds it.
+TOOLS: dict[str, Tool] = {
+    "pydeps": Tool("pydeps", "pydeps", "pip install pydeps", "import graph"),
+    "pyreverse": Tool("pyreverse", None, "pip install pylint", "class diagrams"),
+    "code2flow": Tool("code2flow", "code2flow", "pip install code2flow", "call graph"),
+    "tach": Tool("tach", "tach", "pip install tach", "module boundaries"),
+    "dot": Tool(
+        "dot", None, "brew install graphviz (or apt install graphviz)", "graph rendering"
+    ),
+}
+
+
+def _has_main_module(module: str) -> bool:
+    """True when ``python -m module`` makes sense: not every package has a __main__."""
+    try:
+        return importlib.util.find_spec(f"{module}.__main__") is not None
+    except (ImportError, ValueError, AttributeError):
+        return False
+
+
+def resolve(name: str) -> list[str] | None:
+    """How to launch ``name`` here, or None when it cannot be found.
+
+    Look next to the running interpreter first: when pymap is installed in a
+    project's virtualenv and invoked by its full path, the venv's ``bin/`` is
+    not on PATH even though the tools live there. PATH takes over next, and
+    ``python -m`` is the last resort.
+    """
+    sibling = Path(sys.executable).parent
+    for path in (shutil.which(name, path=str(sibling)), shutil.which(name)):
+        if path:
+            return [path]
+    tool = TOOLS.get(name)
+    if tool and tool.module and _has_main_module(tool.module):
+        return [sys.executable, "-m", tool.module]
+    return None
+
+
+def available(name: str) -> bool:
+    return resolve(name) is not None
+
+
+def missing() -> list[Tool]:
+    """Tools that cannot be found, in declaration order."""
+    return [tool for name, tool in TOOLS.items() if not available(name)]
+
+
+def run(
+    cmd: Sequence[str], cwd: str | None = None, timeout: int = 300, full: bool = False
+) -> tuple[bool, str]:
+    """Run a command. Pass ``full=True`` when the output has to be parsed.
+
+    Returns ``(succeeded, output)``. Without ``full`` the output is trimmed to
+    its last 1500 characters: enough to diagnose, not enough to drown a log.
+    """
+    try:
+        completed = subprocess.run(
+            cmd, cwd=cwd, capture_output=True, text=True, timeout=timeout
+        )
+        output = completed.stdout + completed.stderr
+        return completed.returncode == 0, output if full else output[-1500:]
+    except subprocess.TimeoutExpired:
+        return False, f"timed out after {timeout}s"
+    except (FileNotFoundError, NotADirectoryError, PermissionError) as exc:
+        return False, f"command unreachable: {exc}"

pymap/settings.py

@@ -0,0 +1,176 @@
+"""Settings: what to analyse, where to write, what to skip.
+
+A single place decides which files enter the map. Without that filter, running
+pymap at a project root would swallow ``.venv/`` and ``node_modules/``.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import fnmatch
+import os
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+#: Directories and files never analysed: environments, caches, build artefacts.
+EXCLUDED: tuple[str, ...] = tuple(
+    """
+    .git .hg .svn __pycache__ .venv venv env .env node_modules site-packages
+    build dist *.egg-info .tox .nox .mypy_cache .pytest_cache .ruff_cache
+    .ipynb_checkpoints .eggs pymap-out
+    """.split()
+)
+
+#: URI templates to jump from a diagram to the line of code.
+#: ``{f}`` is the absolute file path, ``{l}`` the line number.
+EDITORS: dict[str, str] = {
+    "vscode": "vscode://file/{f}:{l}",
+    "vscodium": "vscodium://file/{f}:{l}",
+    "cursor": "cursor://file/{f}:{l}",
+    "windsurf": "windsurf://file/{f}:{l}",
+    "zed": "zed://file/{f}:{l}",
+    "pycharm": "pycharm://open?file={f}&line={l}",
+    "idea": "idea://open?file={f}&line={l}",
+    "sublime": "subl://open?url=file://{f}&line={l}",
+    "none": "",
+}
+
+#: Directory names that are never a project's main package.
+NOT_A_PACKAGE = set("tests test docs doc examples scripts tools benchmarks migrations".split())
+
+
+def is_excluded(name: str, patterns: Sequence[str] = EXCLUDED) -> bool:
+    """True when ``name`` matches one of the exclusion patterns."""
+    return any(fnmatch.fnmatch(name, pattern) for pattern in patterns)
+
+
+def python_files(root: Path, excluded: Sequence[str] = EXCLUDED) -> list[Path]:
+    """Sorted list of ``.py`` sources under ``root``, excluded branches pruned.
+
+    Pruning happens during the walk rather than after it, so a ``.venv`` holding
+    30,000 files is never traversed.
+    """
+    found: list[Path] = []
+    for directory, subdirs, filenames in os.walk(root):
+        subdirs[:] = sorted(d for d in subdirs if not is_excluded(d, excluded))
+        for filename in sorted(filenames):
+            if filename.endswith(".py") and not is_excluded(filename, excluded):
+                found.append(Path(directory) / filename)
+    return found
+
+
+def count_lines(files: Iterable[Path]) -> int:
+    """Total line count of the given files; unreadable ones count as zero."""
+    total = 0
+    for path in files:
+        with contextlib.suppress(OSError):
+            total += len(path.read_text(encoding="utf-8", errors="ignore").splitlines())
+    return total
+
+
+def read_pyproject(start: Path) -> tuple[dict[str, Any], Path | None]:
+    """Return the contents of the nearest ``pyproject.toml`` and its path.
+
+    Walks up from ``start``. Without a TOML reader (Python < 3.11 and no
+    ``tomli`` installed) it returns an empty mapping: configuration is optional
+    and pymap must still run.
+    """
+    try:
+        import tomllib
+    except ModuleNotFoundError:  # pragma: no cover - Python < 3.11
+        try:
+            import tomli as tomllib  # type: ignore[no-redef]
+        except ModuleNotFoundError:
+            return {}, None
+
+    for directory in [start, *start.parents]:
+        config = directory / "pyproject.toml"
+        if config.is_file():
+            try:
+                return tomllib.loads(config.read_text(encoding="utf-8")), config
+            except (OSError, ValueError):
+                return {}, config
+    return {}, None
+
+
+def detect_target(start: Path) -> Path | None:
+    """Guess which package to map when no target is given.
+
+    In order: ``[tool.pymap] target``, a ``src/`` layout, the package named
+    after the project, then the single top-level package.
+    """
+    data, config = read_pyproject(start)
+    root = config.parent if config else start
+
+    declared = data.get("tool", {}).get("pymap", {}).get("target")
+    if declared:
+        path = (root / declared).resolve()
+        return path if path.is_dir() else None
+
+    def packages(directory: Path) -> list[Path]:
+        if not directory.is_dir():
+            return []
+        return sorted(
+            child
+            for child in directory.iterdir()
+            if child.is_dir()
+            and (child / "__init__.py").is_file()
+            and not is_excluded(child.name)
+            and child.name not in NOT_A_PACKAGE
+        )
+
+    under_src = packages(root / "src")
+    if len(under_src) == 1:
+        return under_src[0]
+
+    name = data.get("project", {}).get("name", "")
+    if name:
+        expected = root / name.replace("-", "_")
+        if (expected / "__init__.py").is_file():
+            return expected
+
+    top_level = packages(root)
+    if len(top_level) == 1:
+        return top_level[0]
+    return None
+
+
+@dataclass
+class Settings:
+    """Everything a run needs, resolved once and for all.
+
+    ``target`` and ``output`` are declared as :class:`~pathlib.Path` but accept
+    plain strings too: both are coerced and made absolute on construction.
+    """
+
+    target: Path
+    output: Path = Path("pymap-out")
+    excluded: Sequence[str] = EXCLUDED
+    editor: str = "vscode"
+    timeout: int = 300
+    open_in_browser: bool = False
+    _files: list[Path] | None = field(default=None, repr=False, compare=False)
+
+    def __post_init__(self) -> None:
+        self.target = Path(self.target).resolve()
+        self.output = Path(self.output).resolve()
+        self.excluded = tuple(self.excluded)
+
+    @property
+    def name(self) -> str:
+        return self.target.name
+
+    @property
+    def editor_uri(self) -> str:
+        """URI template for the chosen editor; empty string when disabled."""
+        if self.editor in EDITORS:
+            return EDITORS[self.editor]
+        return self.editor if "{f}" in self.editor else ""
+
+    def files(self) -> list[Path]:
+        """Analysable sources, computed once."""
+        if self._files is None:
+            self._files = python_files(self.target, self.excluded)
+        return self._files

pymap/templates/__init__.py

@@ -0,0 +1,8 @@
+"""HTML templates, shipped as package data.
+
+This file is what makes ``templates`` a regular package rather than a namespace
+one. Without it, ``importlib.resources.files("pymap.templates")`` raises
+``TypeError`` on Python 3.9, where the fallback dereferences ``spec.origin`` --
+which is ``None`` for a namespace package. Python 3.10 and later handle the
+namespace case, so the failure only shows up on the oldest version we support.
+"""