untangled 1.0.0__py3-none-any.whl

untangled/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""Untangled for Python: dependency health your whole team can understand."""

untangled/__main__.py

@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0-only
+import sys
+
+from untangled.cli import main
+
+sys.exit(main())

untangled/analysis/__init__.py

@@ -0,0 +1 @@
+# SPDX-License-Identifier: GPL-2.0-only

untangled/analysis/analyzer.py

@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""Computes the analysis document of spec/metrics.md from a filtered file-level pairs document:
+per-unit coupling, Ca / Ce for clusters, cycles, and the project summary."""
+
+from typing import Any
+
+from untangled import version
+from untangled.analysis import cycles as cycle_finder
+from untangled.analysis.clustering import Clustering
+from untangled.graph import CLUSTER, EXTERNAL, INTERNAL, UNRESOLVED, PairsDocument, number
+
+
+def analyze(files: PairsDocument, clustering: Clustering | None = None) -> dict[str, Any]:
+    """Raises ValueError when the document is already clustered."""
+    if any(node["kind"] == CLUSTER for node in files.nodes()):
+        raise ValueError(
+            "the pairs document is already clustered; analyze the file-level document and choose a clustering instead"
+        )
+    graph = files
+    cluster_of = None
+    if clustering is not None:
+        clustered = clustering.apply(files)
+        graph, cluster_of = clustered.document, clustered.cluster_of
+
+    unit_kind = INTERNAL if clustering is None else CLUSTER
+    successors: dict[str, list[str]] = {}
+    predecessors: dict[str, list[str]] = {}
+    external: dict[str, set[str]] = {}
+    unresolved: dict[str, set[str]] = {}
+    members: dict[str, list[str]] = {}
+    for node in graph.nodes():
+        if node["kind"] == unit_kind:
+            successors[node["id"]] = []
+            predecessors[node["id"]] = []
+            external[node["id"]] = set()
+            unresolved[node["id"]] = set()
+            members[node["id"]] = node.get("members", [])
+
+    edges = 0
+    external_used: set[str] = set()
+    unresolved_used: set[str] = set()
+    for edge in graph.edges():
+        source, target = edge["from"], edge["to"]
+        kind = (graph.node(target) or {}).get("kind")
+        if kind == unit_kind:
+            successors[source].append(target)
+            predecessors[target].append(source)
+            edges += 1
+        elif kind == EXTERNAL:
+            external[source].add(target)
+            external_used.add(target)
+        elif kind == UNRESOLVED:
+            unresolved[source].add(target)
+            unresolved_used.add(target)
+
+    # Ca / Ce of a cluster count the distinct files outside it, on the file graph.
+    ca: dict[str, set[str]] = {}
+    ce: dict[str, set[str]] = {}
+    if cluster_of is not None:
+        for edge in files.edges():
+            source, target = cluster_of.get(edge["from"]), cluster_of.get(edge["to"])
+            if source is None or target is None or source == target:
+                continue
+            ce.setdefault(source, set()).add(edge["to"])
+            ca.setdefault(target, set()).add(edge["from"])
+
+    units = []
+    fan_out_total = 0
+    for unit_id, targets in successors.items():
+        fan_in, fan_out = len(predecessors[unit_id]), len(targets)
+        fan_out_total += fan_out
+        unit: dict[str, Any] = {"id": unit_id, "fan_in": fan_in, "fan_out": fan_out}
+        if cluster_of is not None:
+            unit["ca"], unit["ce"] = len(ca.get(unit_id, ())), len(ce.get(unit_id, ()))
+            afferent, efferent = unit["ca"], unit["ce"]
+        else:
+            afferent, efferent = fan_in, fan_out
+        unit["external"] = len(external[unit_id])
+        unit["unresolved"] = len(unresolved[unit_id])
+        unit["instability"] = None if afferent + efferent == 0 else number(efferent / (afferent + efferent))
+        if cluster_of is not None:
+            unit["members"] = members[unit_id]
+        units.append(unit)
+    units.sort(key=lambda unit: unit["id"])
+
+    cycles = cycle_finder.find(successors)
+    in_cycles = sum(cycle["size"] for cycle in cycles)
+    count = len(units)
+    return {
+        "format": "untangled-analysis",
+        "version": version.SPEC,
+        "language": files.language,
+        "generator": {"name": version.NAME, "version": version.VERSION},
+        "clustering": None if clustering is None else clustering.name(),
+        "summary": {
+            "units": count,
+            "edges": edges,
+            "cycles": len(cycles),
+            "largest_cycle": cycles[0]["size"] if cycles else 0,
+            "units_in_cycles": in_cycles,
+            "units_in_cycles_ratio": number(in_cycles / count) if count else 0,
+            "mean_fan_out": number(fan_out_total / count) if count else 0,
+            "external_packages": len(external_used),
+            "unresolved_references": len(unresolved_used),
+        },
+        "units": units,
+        "cycles": cycles,
+    }

untangled/analysis/clustering.py

@@ -0,0 +1,96 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""A clustering strategy from spec/clustering.md (`namespace:N`, `folder:N` or `rules`) and the
+step that applies it: one node per cluster, edges merged per (cluster, target), and edges
+inside a cluster dropped."""
+
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from untangled.graph import CLUSTER, INTERNAL, PairsDocument
+
+
+@dataclass
+class Clustered:
+    document: PairsDocument
+    cluster_of: dict[str, str]
+    """Each internal id's cluster."""
+    dropped: dict[str, int] = field(default_factory=dict)
+    """Per cluster, how many file edges fell inside it and were dropped."""
+
+
+class Clustering:
+    def __init__(self, strategy: str, depth: int = 0, rules: list[tuple[str, str]] | None = None) -> None:
+        if strategy in ("namespace", "folder") and depth < 1:
+            raise ValueError("clustering depth must be at least 1")
+        self.strategy = strategy
+        self.depth = depth
+        self.rules = rules or []
+
+    @classmethod
+    def from_string(cls, strategy: str) -> "Clustering":
+        """A strategy as written on the command line: `namespace:N` or `folder:N`.
+        (`rules` needs a rules file, so it is built with parse_rules().)"""
+        match = re.fullmatch(r"(namespace|folder):([1-9][0-9]*)", strategy)
+        if match is None:
+            raise ValueError(f'unknown clustering "{strategy}" (use namespace:N, folder:N or rules:<file>)')
+        return cls(match.group(1), int(match.group(2)))
+
+    @classmethod
+    def parse_rules(cls, text: str) -> "Clustering":
+        """One `prefix cluster` pair per line, `#` starts a comment."""
+        rules = []
+        for number, line in enumerate(text.splitlines(), start=1):
+            fields = line.split("#", 1)[0].split()
+            if not fields:
+                continue
+            if len(fields) != 2:
+                raise ValueError(f'rules line {number} must be "<prefix> <cluster>"')
+            rules.append((fields[0], fields[1]))
+        return cls("rules", rules=rules)
+
+    def name(self) -> str:
+        """The name written in the analysis document's `clustering` field."""
+        return "rules" if self.strategy == "rules" else f"{self.strategy}:{self.depth}"
+
+    def cluster_of(self, node: dict[str, Any]) -> str:
+        node_id = node["id"]
+        if self.strategy == "rules":
+            for prefix, cluster in self.rules:
+                if node_id.startswith(prefix):
+                    return cluster
+            return node_id
+        module = node.get("module")
+        if self.strategy == "namespace" and isinstance(module, str) and module:
+            # Segments are separated by `\\` (PHP namespaces) or `.` (Python modules); the
+            # separators are kept, so the cluster id reads like the module it shortens.
+            parts = re.split(r"([\\\\.])", module)
+            return "".join(parts[: 2 * self.depth - 1])
+        folders = node_id.split("/")[:-1]
+        return "/".join(folders[: self.depth]) if folders else "."
+
+    def apply(self, files: PairsDocument) -> Clustered:
+        cluster_of: dict[str, str] = {}
+        members: dict[str, list[str]] = {}
+        others = []
+        for node in files.nodes():
+            if node["kind"] == INTERNAL:
+                cluster = self.cluster_of(node)
+                cluster_of[node["id"]] = cluster
+                members.setdefault(cluster, []).append(node["id"])
+            else:
+                others.append(node)
+        document = PairsDocument(files.language)
+        for cluster, ids in members.items():
+            document.add_node(cluster, CLUSTER, members=sorted(ids))
+        for node in others:
+            document.add_node(node["id"], node["kind"], **{k: v for k, v in node.items() if k not in ("id", "kind")})
+        dropped: dict[str, int] = {}
+        for edge in files.edges():
+            source = cluster_of.get(edge["from"], edge["from"])
+            target = cluster_of.get(edge["to"], edge["to"])
+            if source == target:
+                dropped[source] = dropped.get(source, 0) + 1
+                continue
+            document.add_edge(source, target, edge["kinds"], edge["count"])
+        return Clustered(document, cluster_of, dict(sorted(dropped.items())))

untangled/analysis/cycles.py

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""Cycles as defined in spec/metrics.md: strongly connected components with two or more units,
+each with one concrete example loop."""
+
+from collections import deque
+from typing import Any
+
+
+def find(successors: dict[str, list[str]]) -> list[dict[str, Any]]:
+    """`successors` has every unit as a key, with the units it depends on."""
+    graph = {unit: sorted(set(targets)) for unit, targets in sorted(successors.items())}
+    cycles = []
+    for component in _components(graph):
+        if len(component) < 2:
+            continue
+        members = sorted(component)
+        cycles.append({"members": members, "size": len(members), "example": _example(members[0], set(members), graph)})
+    cycles.sort(key=lambda cycle: (-cycle["size"], cycle["members"][0]))
+    return cycles
+
+
+def _components(graph: dict[str, list[str]]) -> list[list[str]]:
+    """Tarjan's algorithm, written iteratively so deep dependency chains cannot overflow the stack."""
+    index: dict[str, int] = {}
+    low: dict[str, int] = {}
+    on_stack: set[str] = set()
+    stack: list[str] = []
+    components = []
+    for root in graph:
+        if root in index:
+            continue
+        # Each frame is [unit, position of the next successor to visit].
+        frames = [[root, 0]]
+        index[root] = low[root] = len(index)
+        stack.append(root)
+        on_stack.add(root)
+        while frames:
+            unit, position = frames[-1]
+            targets = graph[unit]
+            if position < len(targets):
+                frames[-1][1] += 1
+                target = targets[position]
+                if target not in index:
+                    index[target] = low[target] = len(index)
+                    stack.append(target)
+                    on_stack.add(target)
+                    frames.append([target, 0])
+                elif target in on_stack:
+                    low[unit] = min(low[unit], index[target])
+                continue
+            frames.pop()
+            if frames:
+                parent = frames[-1][0]
+                low[parent] = min(low[parent], low[unit])
+            if low[unit] == index[unit]:
+                component = []
+                while True:
+                    member = stack.pop()
+                    on_stack.discard(member)
+                    component.append(member)
+                    if member == unit:
+                        break
+                components.append(component)
+    return components
+
+
+def _example(start: str, members: set[str], graph: dict[str, list[str]]) -> list[str]:
+    """The shortest loop from start back to itself, by breadth-first search visiting
+    neighbours in byte order. Any such loop stays inside the cycle, so the search does too."""
+    parent: dict[str, str] = {}
+    queue = deque([start])
+    while queue:
+        unit = queue.popleft()
+        for target in graph[unit]:
+            if target not in members:
+                continue
+            if target == start:
+                path = [start]
+                step = unit
+                while step != start:
+                    path.append(step)
+                    step = parent[step]
+                path.append(start)
+                return path[::-1]
+            if target not in parent:
+                parent[target] = unit
+                queue.append(target)
+    raise AssertionError(f"no loop through {start}")

untangled/analysis/filtering.py

@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""The filtering step of spec/clustering.md: drops builtin nodes (unless kept), then applies
+ordered include / exclude globs to internal ids, where the last matching pattern wins."""
+
+import re
+
+from untangled.graph import BUILTIN, CLUSTER, INTERNAL, PairsDocument
+
+
+class Glob:
+    """`*` and `?` stay within one path segment, `**` matches across `/`, and `**` followed
+    by `/` also matches no directory at all."""
+
+    def __init__(self, pattern: str) -> None:
+        self.pattern = pattern
+        regex = ""
+        i = 0
+        while i < len(pattern):
+            char = pattern[i]
+            if pattern.startswith("**/", i):
+                regex += "(?:.*/)?"
+                i += 3
+                continue
+            if pattern.startswith("**", i):
+                regex += ".*"
+                i += 2
+                continue
+            regex += "[^/]*" if char == "*" else "[^/]" if char == "?" else re.escape(char)
+            i += 1
+        self._regex = re.compile(regex, re.DOTALL)
+
+    def matches(self, node_id: str) -> bool:
+        return self._regex.fullmatch(node_id) is not None
+
+
+class Filter:
+    def __init__(self, keep_builtins: bool = False) -> None:
+        self.keep_builtins = keep_builtins
+        self._patterns: list[tuple[bool, Glob]] = []
+
+    def include(self, pattern: str) -> "Filter":
+        self._patterns.append((True, Glob(pattern)))
+        return self
+
+    def exclude(self, pattern: str) -> "Filter":
+        self._patterns.append((False, Glob(pattern)))
+        return self
+
+    def keeps(self, internal_id: str) -> bool:
+        keep = True
+        for include, glob in self._patterns:
+            if glob.matches(internal_id):
+                keep = include
+        return keep
+
+    def apply(self, document: PairsDocument) -> PairsDocument:
+        """Returns the filtered document. Removing a node removes its edges, and a non-internal
+        node that no remaining edge points to is removed with them."""
+        kept = {}
+        for node in document.nodes():
+            if node["kind"] == BUILTIN and not self.keep_builtins:
+                continue
+            if node["kind"] == INTERNAL and not self.keeps(node["id"]):
+                continue
+            kept[node["id"]] = node
+        edges = [edge for edge in document.edges() if edge["from"] in kept and edge["to"] in kept]
+        targets = {edge["to"] for edge in edges}
+        filtered = PairsDocument(document.language)
+        for node_id, node in kept.items():
+            # Cluster nodes pass through so that the analyzer can refuse an already clustered document.
+            if node["kind"] in (INTERNAL, CLUSTER) or node_id in targets:
+                filtered.add_node(node_id, node["kind"], **{k: v for k, v in node.items() if k not in ("id", "kind")})
+        for edge in edges:
+            filtered.add_edge(edge["from"], edge["to"], edge["kinds"], edge["count"])
+        return filtered

untangled/cli.py

@@ -0,0 +1,223 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""`untangled <command> [options]`. The command and option names are shared by every
+Untangled library: `scan`, `analyze` and `report`."""
+
+import json
+import os
+import sys
+import tomllib
+from typing import TextIO
+
+from untangled import version
+from untangled.analysis.analyzer import analyze
+from untangled.analysis.clustering import Clustering
+from untangled.analysis.filtering import Filter
+from untangled.graph import DocumentError, PairsDocument, encode
+from untangled.report import dot, html
+from untangled.report.report import Report
+from untangled.scanner.project import ProjectError
+from untangled.scanner.scanner import Scanner
+
+USAGE = """\
+Usage: untangled scan [<path>] [--dev] [--source=<path>]... [--venv=<path>]...
+                      [--format=json|snakefood] [--output=<file>]
+       untangled analyze [<path>|<pairs.json>] [scan options] [--cluster=<strategy>]
+                         [--include=<glob>]... [--exclude=<glob>]... [--output=<file>]
+       untangled report [<path>|<pairs.json>] [--format=html|dot] [--style=classic]
+                         [--title=<text>] [analyze options]
+
+Commands:
+  scan       Write the dependency pairs of a Python project (default path: .)
+  analyze    Write cycles and coupling metrics for a project or a pairs document
+  report     Write a readable HTML report (or a Graphviz DOT graph) of the analysis
+
+Options:
+  --dev              Also scan tests (test/ and tests/ folders, test_*.py, *_test.py, conftest.py)
+  --source=PATH      Add PATH, a folder relative to the project, to the import roots (repeatable),
+                     for code that is importable from a folder the scanner does not find by itself
+  --venv=PATH        Read installed packages from this virtualenv instead of the ones found in the
+                     project (repeatable)
+  --format=FORMAT    scan: json (default, the Untangled pairs document) or snakefood
+                     report: html (default) or dot
+  --style=STYLE      report --format=dot only: default, or classic for snakefood's
+                     black and white sfood-graph look
+  --title=TEXT       report only: the report's title (default: the project name in pyproject.toml)
+  --cluster=STRATEGY analyze, report: namespace:N, folder:N or rules:<file>
+  --include=GLOB     analyze, report: keep files matching GLOB (repeatable, last match wins)
+  --exclude=GLOB     analyze, report: drop files matching GLOB (repeatable, last match wins)
+  --output=FILE      Write to FILE instead of standard output
+  -h, --help         Show this help
+  -V, --version      Show the version
+"""
+
+_SCAN = {"dev", "source", "venv"}
+_COMMANDS = {
+    "scan": _SCAN | {"format", "output"},
+    "analyze": _SCAN | {"cluster", "include", "exclude", "output"},
+    "report": _SCAN | {"cluster", "include", "exclude", "output", "format", "style", "title"},
+}
+_REPEATABLE = {"source", "venv"}
+
+
+class _Usage(Exception):
+    """A usage error: exit code 2."""
+
+
+class _Failure(Exception):
+    """An error reading or writing: exit code 1."""
+
+
+def main(argv: list[str] | None = None, stdout: TextIO | None = None, stderr: TextIO | None = None) -> int:
+    out = stdout or sys.stdout
+    err = stderr or sys.stderr
+    try:
+        return _run(sys.argv[1:] if argv is None else argv, out, err)
+    except _Usage as error:
+        err.write(f"{error}\n")
+        return 2
+    except _Failure as error:
+        err.write(f"error: {error}\n")
+        return 1
+
+
+def _run(args: list[str], out: TextIO, err: TextIO) -> int:
+    positional: list[str] = []
+    options: dict[str, str | bool] = {}
+    repeated: dict[str, list[str]] = {name: [] for name in _REPEATABLE}
+    patterns: list[tuple[str, str]] = []
+    for arg in args:
+        if arg in ("-h", "--help"):
+            out.write(USAGE)
+            return 0
+        if arg in ("-V", "--version"):
+            out.write(f"{version.NAME} {version.VERSION}\n")
+            return 0
+        if arg.startswith("--"):
+            name, has_value, value = arg[2:].partition("=")
+            options[name] = value if has_value else True
+            if has_value and name in _REPEATABLE:
+                repeated[name].append(value)
+            elif has_value and name in ("include", "exclude"):
+                # Order matters for these, so they are kept as a list rather than by name.
+                patterns.append((name, value))
+            continue
+        positional.append(arg)
+
+    command = positional.pop(0) if positional else None
+    if command not in _COMMANDS:
+        raise _Usage(USAGE.rstrip("\n") if command is None else f"Unknown command: {command}\n\n{USAGE.rstrip()}")
+    unknown = [name for name in options if name not in _COMMANDS[command]]
+    if unknown:
+        raise _Usage(f"Unknown option for {command}: --{unknown[0]}")
+    for name, value in options.items():
+        if name != "dev" and not isinstance(value, str):
+            raise _Usage(f"Option --{name} needs a value, as in --{name}=...")
+    if "dev" in options and options["dev"] is not True:
+        raise _Usage("Option --dev takes no value")
+
+    path = positional[0] if positional else "."
+    scanner = Scanner(dev="dev" in options, sources=repeated["source"], venvs=repeated["venv"] or None)
+    output = options.get("output")
+    output = output if isinstance(output, str) else None
+
+    if command == "scan":
+        output_format = options.get("format", "json")
+        if output_format not in ("json", "snakefood"):
+            raise _Usage(f"Unknown format: {output_format}")
+        pairs = _scan(scanner, path, err)
+        return _write(pairs.to_json() if output_format == "json" else pairs.to_snakefood(path), output, out)
+
+    output_format = options.get("format", "html")
+    if command == "report" and output_format not in ("html", "dot"):
+        raise _Usage(f"Unknown format: {output_format}")
+    style = options.get("style", "default")
+    if style not in dot.STYLES:
+        raise _Usage(f"Unknown style: {style}")
+    if "style" in options and output_format != "dot":
+        raise _Usage("--style only applies to --format=dot")
+    clustering = _clustering(options.get("cluster"))
+
+    if os.path.isfile(path):
+        warnings: list[str] = []
+        try:
+            with open(path, encoding="utf-8") as handle:
+                data = json.load(handle)
+            pairs = PairsDocument.from_dict(data, warnings)
+        except OSError as error:
+            raise _Failure(f"cannot read {path}") from error
+        except (json.JSONDecodeError, UnicodeDecodeError) as error:
+            raise _Failure(f"{path} is not valid JSON: {error}") from error
+        except DocumentError as error:
+            raise _Failure(str(error)) from error
+        for warning in warnings:
+            err.write(f"warning: {warning}\n")
+    else:
+        pairs = _scan(scanner, path, err)
+
+    selection = Filter()
+    for kind, glob in patterns:
+        selection.include(glob) if kind == "include" else selection.exclude(glob)
+    try:
+        if command == "analyze":
+            return _write(encode(analyze(selection.apply(pairs), clustering)), output, out)
+        title = options.get("title")
+        report = Report.build(title if isinstance(title, str) else _title(path), selection.apply(pairs), clustering)
+    except ValueError as error:
+        raise _Failure(str(error)) from error
+    return _write(dot.render(report, str(style)) if output_format == "dot" else html.render(report), output, out)
+
+
+def _scan(scanner: Scanner, path: str, err: TextIO) -> PairsDocument:
+    try:
+        result = scanner.scan(path)
+    except ProjectError as error:
+        raise _Failure(str(error)) from error
+    for warning in result.warnings:
+        err.write(f"warning: {warning}\n")
+    return result.pairs
+
+
+def _clustering(strategy: str | bool | None) -> Clustering | None:
+    if not isinstance(strategy, str):
+        return None
+    if strategy.startswith("rules:"):
+        rules_file = strategy[len("rules:") :]
+        try:
+            with open(rules_file, encoding="utf-8") as handle:
+                text = handle.read()
+        except (OSError, UnicodeDecodeError) as error:
+            raise _Failure(f"cannot read rules file {rules_file}") from error
+        try:
+            return Clustering.parse_rules(text)
+        except ValueError as error:
+            raise _Usage(f"error: {error}") from error
+    try:
+        return Clustering.from_string(strategy)
+    except ValueError as error:
+        raise _Usage(f"error: {error}") from error
+
+
+def _title(path: str) -> str:
+    """The default report title: the project name in pyproject.toml, else the folder or file name."""
+    if os.path.isfile(path):
+        return os.path.basename(path)
+    try:
+        with open(os.path.join(path, "pyproject.toml"), "rb") as handle:
+            name = tomllib.load(handle).get("project", {}).get("name")
+    except (OSError, tomllib.TOMLDecodeError, UnicodeDecodeError, AttributeError):
+        name = None
+    if isinstance(name, str) and name:
+        return name
+    return os.path.basename(os.path.abspath(path))
+
+
+def _write(text: str, output: str | None, out: TextIO) -> int:
+    if output is None:
+        out.write(text)
+        return 0
+    try:
+        with open(output, "w", encoding="utf-8", newline="\n") as handle:
+            handle.write(text)
+    except OSError as error:
+        raise _Failure(f"cannot write {output}") from error
+    return 0