codegraph-brain 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cgis/__init__.py +10 -0
- cgis/__main__.py +16 -0
- cgis/api/.gitkeep +0 -0
- cgis/api/__init__.py +1 -0
- cgis/api/mcp_server.py +550 -0
- cgis/cli.py +1516 -0
- cgis/core/.gitkeep +0 -0
- cgis/core/models.py +140 -0
- cgis/extractors/.gitkeep +0 -0
- cgis/extractors/_python_ast.py +194 -0
- cgis/extractors/_python_classes.py +126 -0
- cgis/extractors/_python_functions.py +312 -0
- cgis/extractors/_python_imports.py +188 -0
- cgis/extractors/_python_types.py +84 -0
- cgis/extractors/base.py +42 -0
- cgis/extractors/python_extractor.py +235 -0
- cgis/extractors/typescript_extractor.py +310 -0
- cgis/guardian/__init__.py +1 -0
- cgis/guardian/bench.py +199 -0
- cgis/guardian/chunked.py +240 -0
- cgis/guardian/chunker.py +148 -0
- cgis/guardian/collector.py +309 -0
- cgis/guardian/core.py +120 -0
- cgis/guardian/diff_index.py +151 -0
- cgis/guardian/findings.py +70 -0
- cgis/guardian/github_poster.py +133 -0
- cgis/guardian/metrics.py +108 -0
- cgis/guardian/prompts.py +217 -0
- cgis/guardian/providers/__init__.py +1 -0
- cgis/guardian/providers/base.py +112 -0
- cgis/guardian/providers/gemini.py +83 -0
- cgis/guardian/providers/mistral.py +83 -0
- cgis/guardian/providers/ollama.py +99 -0
- cgis/guardian/recording.py +82 -0
- cgis/guardian/render.py +107 -0
- cgis/guardian/runner.py +295 -0
- cgis/guardian/skeptic.py +208 -0
- cgis/pipeline.py +252 -0
- cgis/py.typed +0 -0
- cgis/query/analysis/__init__.py +0 -0
- cgis/query/analysis/analyzer.py +241 -0
- cgis/query/analysis/anomaly.py +34 -0
- cgis/query/analysis/cohesion.py +277 -0
- cgis/query/analysis/health.py +128 -0
- cgis/query/analysis/suggest_service.py +221 -0
- cgis/query/context/__init__.py +0 -0
- cgis/query/context/audit.py +134 -0
- cgis/query/context/context_service.py +129 -0
- cgis/query/context/prompt.py +184 -0
- cgis/query/context/snippet.py +96 -0
- cgis/query/drift/__init__.py +0 -0
- cgis/query/drift/_scc.py +90 -0
- cgis/query/drift/drift.py +867 -0
- cgis/query/drift/drift_service.py +217 -0
- cgis/query/drift/fingerprint.py +255 -0
- cgis/query/drift/fractal.py +292 -0
- cgis/query/drift/ontology_init.py +470 -0
- cgis/query/drift/quotient.py +75 -0
- cgis/query/drift/triads.py +234 -0
- cgis/query/engine.py +170 -0
- cgis/query/fqn.py +65 -0
- cgis/query/render/__init__.py +0 -0
- cgis/query/render/graph_json.py +42 -0
- cgis/query/render/mermaid.py +235 -0
- cgis/query/render/metrics.py +346 -0
- cgis/resolver/.gitkeep +0 -0
- cgis/resolver/__init__.py +1 -0
- cgis/resolver/engine.py +145 -0
- cgis/resolver/indices.py +184 -0
- cgis/resolver/symbols.py +179 -0
- cgis/resolver/uplift.py +263 -0
- cgis/storage/.gitkeep +0 -0
- cgis/storage/sqlite_store.py +589 -0
- codegraph_brain-0.6.0.dist-info/METADATA +240 -0
- codegraph_brain-0.6.0.dist-info/RECORD +78 -0
- codegraph_brain-0.6.0.dist-info/WHEEL +4 -0
- codegraph_brain-0.6.0.dist-info/entry_points.txt +3 -0
- codegraph_brain-0.6.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
"""Architectural anti-pattern detector powered by graph traversal algorithms."""
|
|
2
|
+
|
|
3
|
+
import structlog
|
|
4
|
+
|
|
5
|
+
from cgis.core.models import Edge, EdgeType, Node, NodeNamespace, NodeType
|
|
6
|
+
from cgis.query.analysis.anomaly import AnomalyType, ArchitecturalAnomaly, ArchitecturalReport
|
|
7
|
+
from cgis.query.drift._scc import build_adjacency, tarjan_scc
|
|
8
|
+
from cgis.storage.sqlite_store import SQLiteStore
|
|
9
|
+
|
|
10
|
+
log = structlog.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
# Thresholds -- tuned for typical Python codebases
|
|
13
|
+
_ZONE_OF_PAIN_DISTANCE = (
|
|
14
|
+
0.70 # D-distance threshold; Zone of Pain: D > 0.7 AND I <= 0.3 (stable + concrete)
|
|
15
|
+
)
|
|
16
|
+
_ZONE_OF_PAIN_MAX_INSTABILITY = 0.30 # I <= 0.3 means "stable" (more dependents than dependencies)
|
|
17
|
+
_ZONE_OF_PAIN_MIN_AFFERENT = 3 # require >= 3 callers to avoid flagging obscure low-usage classes
|
|
18
|
+
_GOD_OBJECT_MIN_METHODS = 10 # classes below this are too small to qualify regardless of coupling
|
|
19
|
+
_GOD_OBJECT_MIN_EFFERENT = 5 # low coupling means cohesion is still acceptable
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _build_method_to_class(nodes: list[Node], edges: list[Edge]) -> dict[str, str]:
|
|
23
|
+
"""Return a mapping of method FQN -> parent class FQN."""
|
|
24
|
+
method_ids = {n.id for n in nodes if n.type == NodeType.METHOD}
|
|
25
|
+
result: dict[str, str] = {}
|
|
26
|
+
for edge in edges:
|
|
27
|
+
if edge.type in (EdgeType.CONTAINS, EdgeType.DECLARES) and edge.target in method_ids:
|
|
28
|
+
result[edge.target] = edge.source
|
|
29
|
+
return result
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _compute_class_coupling(
|
|
33
|
+
internal_classes: set[str],
|
|
34
|
+
edges: list[Edge],
|
|
35
|
+
method_to_class: dict[str, str],
|
|
36
|
+
) -> tuple[dict[str, set[str]], dict[str, set[str]]]:
|
|
37
|
+
"""Compute afferent (Ca) and efferent (Ce) coupling per internal class."""
|
|
38
|
+
ca: dict[str, set[str]] = {fqn: set() for fqn in internal_classes}
|
|
39
|
+
ce: dict[str, set[str]] = {fqn: set() for fqn in internal_classes}
|
|
40
|
+
for edge in edges:
|
|
41
|
+
if edge.type != EdgeType.CALLS:
|
|
42
|
+
continue
|
|
43
|
+
src = method_to_class.get(edge.source, edge.source)
|
|
44
|
+
tgt = method_to_class.get(edge.target, edge.target)
|
|
45
|
+
if src == tgt:
|
|
46
|
+
continue
|
|
47
|
+
if tgt in internal_classes:
|
|
48
|
+
ca[tgt].add(src)
|
|
49
|
+
if src in internal_classes:
|
|
50
|
+
ce[src].add(tgt)
|
|
51
|
+
return ca, ce
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _get_abstract_fqns(internal_classes: set[str], nodes: list[Node]) -> set[str]:
|
|
55
|
+
"""Return FQNs of internal classes that are abstract (metadata only).
|
|
56
|
+
|
|
57
|
+
Only metadata-flagged classes qualify; using EXTENDS would misclassify
|
|
58
|
+
concrete subclasses (e.g. UserService(Base)) as abstract.
|
|
59
|
+
"""
|
|
60
|
+
return {n.id for n in nodes if n.id in internal_classes and n.metadata.get("is_abstract")}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _build_class_method_map(
|
|
64
|
+
internal_classes: set[str], nodes: list[Node], edges: list[Edge]
|
|
65
|
+
) -> dict[str, set[str]]:
|
|
66
|
+
"""Return a mapping of class FQN -> set of its method FQNs."""
|
|
67
|
+
method_ids = {n.id for n in nodes if n.type == NodeType.METHOD}
|
|
68
|
+
result: dict[str, set[str]] = {fqn: set() for fqn in internal_classes}
|
|
69
|
+
for edge in edges:
|
|
70
|
+
if (
|
|
71
|
+
edge.type in (EdgeType.CONTAINS, EdgeType.DECLARES)
|
|
72
|
+
and edge.source in internal_classes
|
|
73
|
+
and edge.target in method_ids
|
|
74
|
+
):
|
|
75
|
+
result[edge.source].add(edge.target)
|
|
76
|
+
return result
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class AnalyzerEngine:
|
|
80
|
+
"""Detects architectural anti-patterns from a fully-ingested code graph."""
|
|
81
|
+
|
|
82
|
+
def __init__(self, store: SQLiteStore) -> None:
|
|
83
|
+
"""Load all nodes and edges from the store for in-memory analysis."""
|
|
84
|
+
self._nodes: list[Node] = store.get_all_nodes()
|
|
85
|
+
self._edges: list[Edge] = store.get_all_edges()
|
|
86
|
+
|
|
87
|
+
def detect_cycles(self) -> list[ArchitecturalAnomaly]:
|
|
88
|
+
"""Find circular dependencies using Tarjan's SCC on IMPORTS edges.
|
|
89
|
+
|
|
90
|
+
Returns one anomaly per cycle, containing the full cycle as a list
|
|
91
|
+
in metrics["cycle_members"].
|
|
92
|
+
"""
|
|
93
|
+
internal_fqns = {
|
|
94
|
+
n.id
|
|
95
|
+
for n in self._nodes
|
|
96
|
+
if n.namespace == NodeNamespace.INTERNAL and n.type in (NodeType.FILE, NodeType.MODULE)
|
|
97
|
+
}
|
|
98
|
+
adj = build_adjacency(self._edges, frozenset({EdgeType.IMPORTS}))
|
|
99
|
+
adj = {
|
|
100
|
+
k: [v for v in vs if v in internal_fqns] for k, vs in adj.items() if k in internal_fqns
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
anomalies: list[ArchitecturalAnomaly] = []
|
|
104
|
+
for scc in tarjan_scc(adj):
|
|
105
|
+
if len(scc) < 2:
|
|
106
|
+
continue
|
|
107
|
+
sorted_members = sorted(scc)
|
|
108
|
+
severity = min(1.0, (len(scc) - 1) / 4)
|
|
109
|
+
anomalies.append(
|
|
110
|
+
ArchitecturalAnomaly(
|
|
111
|
+
id="cycle_" + "_".join(sorted_members[:3]),
|
|
112
|
+
type=AnomalyType.CIRCULAR_DEPENDENCY,
|
|
113
|
+
focal_fqn=sorted_members[0],
|
|
114
|
+
severity_score=round(severity, 2),
|
|
115
|
+
metrics={"cycle_length": len(scc), "cycle_members": sorted_members},
|
|
116
|
+
refactoring_hint=(
|
|
117
|
+
"Break the cycle with a Mediator, shared abstraction, or by "
|
|
118
|
+
"extracting the shared dependency into a third module."
|
|
119
|
+
),
|
|
120
|
+
)
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
log.info("Cycle detection complete.", cycles_found=len(anomalies))
|
|
124
|
+
return anomalies
|
|
125
|
+
|
|
126
|
+
def detect_zone_of_pain(self) -> list[ArchitecturalAnomaly]:
|
|
127
|
+
"""Detect classes in Uncle Bob's Zone of Pain (stable + concrete).
|
|
128
|
+
|
|
129
|
+
Uses afferent/efferent coupling from CALLS edges to compute instability I,
|
|
130
|
+
and node metadata to determine abstractness A.
|
|
131
|
+
High stability (I->0) + low abstractness (A=0) means D-distance approaches 1.0.
|
|
132
|
+
"""
|
|
133
|
+
internal_classes = {
|
|
134
|
+
n.id
|
|
135
|
+
for n in self._nodes
|
|
136
|
+
if n.type == NodeType.CLASS and n.namespace == NodeNamespace.INTERNAL
|
|
137
|
+
}
|
|
138
|
+
if not internal_classes:
|
|
139
|
+
return []
|
|
140
|
+
|
|
141
|
+
method_to_class = _build_method_to_class(self._nodes, self._edges)
|
|
142
|
+
ca, ce = _compute_class_coupling(internal_classes, self._edges, method_to_class)
|
|
143
|
+
abstract_fqns = _get_abstract_fqns(internal_classes, self._nodes)
|
|
144
|
+
|
|
145
|
+
anomalies: list[ArchitecturalAnomaly] = []
|
|
146
|
+
for fqn in internal_classes:
|
|
147
|
+
ca_count = len(ca[fqn])
|
|
148
|
+
ce_count = len(ce[fqn])
|
|
149
|
+
total = ca_count + ce_count
|
|
150
|
+
if total == 0 or ca_count < _ZONE_OF_PAIN_MIN_AFFERENT:
|
|
151
|
+
continue
|
|
152
|
+
instability = ce_count / total
|
|
153
|
+
abstractness = 1.0 if fqn in abstract_fqns else 0.0
|
|
154
|
+
d_distance = abs(abstractness + instability - 1.0)
|
|
155
|
+
if (
|
|
156
|
+
d_distance >= _ZONE_OF_PAIN_DISTANCE
|
|
157
|
+
and instability <= _ZONE_OF_PAIN_MAX_INSTABILITY
|
|
158
|
+
):
|
|
159
|
+
anomalies.append(
|
|
160
|
+
ArchitecturalAnomaly(
|
|
161
|
+
id=f"zop_{fqn}",
|
|
162
|
+
type=AnomalyType.ZONE_OF_PAIN,
|
|
163
|
+
focal_fqn=fqn,
|
|
164
|
+
severity_score=round(d_distance, 2),
|
|
165
|
+
metrics={
|
|
166
|
+
"afferent_coupling": ca_count,
|
|
167
|
+
"efferent_coupling": ce_count,
|
|
168
|
+
"instability": round(instability, 3),
|
|
169
|
+
"abstractness": abstractness,
|
|
170
|
+
"d_distance": round(d_distance, 3),
|
|
171
|
+
},
|
|
172
|
+
refactoring_hint=(
|
|
173
|
+
"Introduce an Abstract Base Class or Protocol to invert the "
|
|
174
|
+
"dependency, making this module stable-and-abstract rather than "
|
|
175
|
+
"stable-and-concrete."
|
|
176
|
+
),
|
|
177
|
+
)
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
log.info("Zone of Pain detection complete.", found=len(anomalies))
|
|
181
|
+
return anomalies
|
|
182
|
+
|
|
183
|
+
def detect_god_objects(self) -> list[ArchitecturalAnomaly]:
|
|
184
|
+
"""Detect God Objects: classes with too many methods and high efferent coupling."""
|
|
185
|
+
internal_classes = {
|
|
186
|
+
n.id
|
|
187
|
+
for n in self._nodes
|
|
188
|
+
if n.type == NodeType.CLASS and n.namespace == NodeNamespace.INTERNAL
|
|
189
|
+
}
|
|
190
|
+
if not internal_classes:
|
|
191
|
+
return []
|
|
192
|
+
|
|
193
|
+
class_methods = _build_class_method_map(internal_classes, self._nodes, self._edges)
|
|
194
|
+
method_to_class = {m: cls for cls, methods in class_methods.items() for m in methods}
|
|
195
|
+
efferent: dict[str, set[str]] = {fqn: set() for fqn in internal_classes}
|
|
196
|
+
for edge in self._edges:
|
|
197
|
+
if edge.type != EdgeType.CALLS:
|
|
198
|
+
continue
|
|
199
|
+
src_class = method_to_class.get(edge.source)
|
|
200
|
+
if src_class is None:
|
|
201
|
+
continue
|
|
202
|
+
tgt_class = method_to_class.get(edge.target, edge.target)
|
|
203
|
+
if tgt_class != src_class:
|
|
204
|
+
efferent[src_class].add(tgt_class)
|
|
205
|
+
|
|
206
|
+
anomalies: list[ArchitecturalAnomaly] = []
|
|
207
|
+
for fqn in internal_classes:
|
|
208
|
+
m_count = len(class_methods[fqn])
|
|
209
|
+
e_count = len(efferent[fqn])
|
|
210
|
+
if m_count < _GOD_OBJECT_MIN_METHODS or e_count < _GOD_OBJECT_MIN_EFFERENT:
|
|
211
|
+
continue
|
|
212
|
+
severity = min(
|
|
213
|
+
1.0,
|
|
214
|
+
(m_count / _GOD_OBJECT_MIN_METHODS) * (e_count / _GOD_OBJECT_MIN_EFFERENT) / 4,
|
|
215
|
+
)
|
|
216
|
+
anomalies.append(
|
|
217
|
+
ArchitecturalAnomaly(
|
|
218
|
+
id=f"god_{fqn}",
|
|
219
|
+
type=AnomalyType.GOD_OBJECT,
|
|
220
|
+
focal_fqn=fqn,
|
|
221
|
+
severity_score=round(severity, 2),
|
|
222
|
+
metrics={"method_count": m_count, "efferent_coupling": e_count},
|
|
223
|
+
refactoring_hint=(
|
|
224
|
+
"Apply Single Responsibility Principle: extract cohesive groups of "
|
|
225
|
+
"methods into focused collaborator classes."
|
|
226
|
+
),
|
|
227
|
+
)
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
log.info("God Object detection complete.", found=len(anomalies))
|
|
231
|
+
return anomalies
|
|
232
|
+
|
|
233
|
+
def run(self) -> ArchitecturalReport:
|
|
234
|
+
"""Run all detectors and return a consolidated health report."""
|
|
235
|
+
anomalies = self.detect_cycles() + self.detect_zone_of_pain() + self.detect_god_objects()
|
|
236
|
+
internal_count = sum(1 for n in self._nodes if n.namespace == NodeNamespace.INTERNAL)
|
|
237
|
+
return ArchitecturalReport(
|
|
238
|
+
total_nodes_analyzed=internal_count,
|
|
239
|
+
total_anomalies=len(anomalies),
|
|
240
|
+
anomalies=tuple(anomalies),
|
|
241
|
+
)
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Pydantic models for architectural anomaly reporting."""
|
|
2
|
+
|
|
3
|
+
from enum import StrEnum
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, Field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class AnomalyType(StrEnum):
|
|
9
|
+
"""Categories of architectural anti-patterns detectable from the code graph."""
|
|
10
|
+
|
|
11
|
+
CIRCULAR_DEPENDENCY = "CIRCULAR_DEPENDENCY"
|
|
12
|
+
ZONE_OF_PAIN = "ZONE_OF_PAIN"
|
|
13
|
+
GOD_OBJECT = "GOD_OBJECT"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ArchitecturalAnomaly(BaseModel, frozen=True):
|
|
17
|
+
"""A single detected architectural anti-pattern with supporting metrics."""
|
|
18
|
+
|
|
19
|
+
id: str = Field(..., description="Unique anomaly identifier (deterministic)")
|
|
20
|
+
type: AnomalyType
|
|
21
|
+
focal_fqn: str = Field(..., description="FQN of the primary node violating the pattern")
|
|
22
|
+
severity_score: float = Field(..., ge=0.0, le=1.0, description="Normalised severity 0-1")
|
|
23
|
+
metrics: dict[str, float | int | list[str]] = Field(
|
|
24
|
+
..., description="Raw computed metrics supporting this finding"
|
|
25
|
+
)
|
|
26
|
+
refactoring_hint: str = Field(..., description="Suggested untangling strategy")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ArchitecturalReport(BaseModel, frozen=True):
|
|
30
|
+
"""Full architectural health report for a codebase snapshot."""
|
|
31
|
+
|
|
32
|
+
total_nodes_analyzed: int
|
|
33
|
+
total_anomalies: int
|
|
34
|
+
anomalies: tuple[ArchitecturalAnomaly, ...]
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"""Package-cohesion analysis: community detection over a package's file graph (#242).
|
|
2
|
+
|
|
3
|
+
Pure logic, no I/O. The intra-package file graph is undirected and weighted; edge
|
|
4
|
+
reconciliation is done in-memory by dot-boundary FQN suffix so the result does not
|
|
5
|
+
depend on the ingest root (src/ vs src/cgis/ — the #242 load-bearing fix).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
from collections import Counter
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from typing import TYPE_CHECKING, Literal
|
|
14
|
+
|
|
15
|
+
from cgis.core.models import Edge, EdgeType, Node, NodeType
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from collections.abc import Hashable, Mapping
|
|
19
|
+
|
|
20
|
+
_FILE_TYPES = frozenset({NodeType.FILE, NodeType.MODULE})
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class FileGraph:
|
|
25
|
+
"""Undirected, weighted intra-package file graph.
|
|
26
|
+
|
|
27
|
+
``files`` lists every file under the package prefix (sorted, including
|
|
28
|
+
isolated leaves). ``adj`` is a symmetric weighted adjacency: ``adj[a][b]``
|
|
29
|
+
== ``adj[b][a]`` is the aggregated import (and optionally call) weight
|
|
30
|
+
between two files. Isolated files have no ``adj`` entry.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
files: tuple[str, ...]
|
|
34
|
+
adj: dict[str, dict[str, float]]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _under(fqn: str, prefix: str) -> bool:
|
|
38
|
+
"""Return True iff fqn is the prefix itself or a dot-boundary child of it.
|
|
39
|
+
|
|
40
|
+
An empty prefix matches nothing — the service normalizes a missing/blank
|
|
41
|
+
prefix to a no_signal report before this is ever reached.
|
|
42
|
+
"""
|
|
43
|
+
return bool(prefix) and (fqn == prefix or fqn.startswith(prefix + "."))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _build_suffix_index(file_ids: list[str]) -> dict[str, str]:
|
|
47
|
+
"""Map every dot-boundary suffix of each file id to that id.
|
|
48
|
+
|
|
49
|
+
A suffix shared by two files maps to the sentinel ``""`` (ambiguous — never
|
|
50
|
+
reconcile to it). Used to link an import target written with a different
|
|
51
|
+
root (``cgis.p.b``) back to the in-graph file (``p.b``).
|
|
52
|
+
"""
|
|
53
|
+
index: dict[str, str] = {}
|
|
54
|
+
for fid in file_ids:
|
|
55
|
+
parts = fid.split(".")
|
|
56
|
+
for i in range(len(parts)):
|
|
57
|
+
suffix = ".".join(parts[i:])
|
|
58
|
+
index[suffix] = "" if suffix in index and index[suffix] != fid else fid
|
|
59
|
+
return index
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _resolve_to_file(node_id: str, file_set: set[str], suffix_index: dict[str, str]) -> str | None:
|
|
63
|
+
"""Resolve an edge endpoint to a file under the prefix, or None.
|
|
64
|
+
|
|
65
|
+
Exact membership wins; otherwise the longest dot-boundary suffix of
|
|
66
|
+
``node_id`` that uniquely names a file. Ambiguous or absent → None.
|
|
67
|
+
"""
|
|
68
|
+
if node_id in file_set:
|
|
69
|
+
return node_id
|
|
70
|
+
parts = node_id.split(".")
|
|
71
|
+
for i in range(len(parts)):
|
|
72
|
+
hit = suffix_index.get(".".join(parts[i:]))
|
|
73
|
+
if hit:
|
|
74
|
+
return hit
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def build_file_graph(
|
|
79
|
+
nodes: list[Node], edges: list[Edge], prefix: str, with_calls: bool
|
|
80
|
+
) -> FileGraph:
|
|
81
|
+
"""Build the undirected weighted file graph for the package at ``prefix``.
|
|
82
|
+
|
|
83
|
+
Files = FILE/MODULE nodes under ``prefix``. Edges = IMPORTS (and CALLS when
|
|
84
|
+
``with_calls``) whose BOTH endpoints reconcile to distinct files under the
|
|
85
|
+
prefix; weights aggregate. Endpoints are reconciled by dot-boundary suffix
|
|
86
|
+
so a target written with a different root still links (root-agnostic, #242).
|
|
87
|
+
"""
|
|
88
|
+
file_ids = sorted(n.id for n in nodes if n.type in _FILE_TYPES and _under(n.id, prefix))
|
|
89
|
+
file_set = set(file_ids)
|
|
90
|
+
suffix_index = _build_suffix_index(file_ids)
|
|
91
|
+
wanted = {EdgeType.IMPORTS} | ({EdgeType.CALLS} if with_calls else set())
|
|
92
|
+
|
|
93
|
+
adj: dict[str, dict[str, float]] = {}
|
|
94
|
+
for e in edges:
|
|
95
|
+
if e.type not in wanted:
|
|
96
|
+
continue
|
|
97
|
+
a = _resolve_to_file(e.source, file_set, suffix_index)
|
|
98
|
+
b = _resolve_to_file(e.target, file_set, suffix_index)
|
|
99
|
+
if a is None or b is None or a == b:
|
|
100
|
+
continue
|
|
101
|
+
adj.setdefault(a, {})[b] = adj.setdefault(a, {}).get(b, 0.0) + e.weight
|
|
102
|
+
adj.setdefault(b, {})[a] = adj.setdefault(b, {}).get(a, 0.0) + e.weight
|
|
103
|
+
return FileGraph(files=tuple(file_ids), adj=adj)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
_MIN_GAIN = 1e-12 # ignore non-positive / floating-noise merges
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _modularity(
|
|
110
|
+
graph: FileGraph,
|
|
111
|
+
communities: list[list[str]],
|
|
112
|
+
deg: dict[str, float],
|
|
113
|
+
m2: float,
|
|
114
|
+
) -> float:
|
|
115
|
+
"""Newman modularity Q for a partition (m2 == 2m == sum of weighted degrees)."""
|
|
116
|
+
q = 0.0
|
|
117
|
+
for c in communities:
|
|
118
|
+
members = set(c)
|
|
119
|
+
l_c = sum(w for f in c for g, w in graph.adj.get(f, {}).items() if g in members)
|
|
120
|
+
d_c = sum(deg[f] for f in c)
|
|
121
|
+
q += l_c / m2 - (d_c / m2) ** 2
|
|
122
|
+
return q
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _best_merge(
|
|
126
|
+
graph: FileGraph,
|
|
127
|
+
members: dict[str, set[str]],
|
|
128
|
+
deg: dict[str, float],
|
|
129
|
+
m2: float,
|
|
130
|
+
) -> tuple[str, str] | None:
|
|
131
|
+
"""Return the (c1, c2) pair with the highest positive delta-Q, or None.
|
|
132
|
+
|
|
133
|
+
Iterates sorted label pairs for lexicographic tie-breaking (deterministic).
|
|
134
|
+
Returns None when no merge would improve modularity by more than _MIN_GAIN.
|
|
135
|
+
"""
|
|
136
|
+
best_gain: float = _MIN_GAIN
|
|
137
|
+
best_pair: tuple[str, str] | None = None
|
|
138
|
+
labels = sorted(members)
|
|
139
|
+
# Precompute each community's normalized degree once (O(V)) rather than
|
|
140
|
+
# re-summing it inside the O(K²) pair scan.
|
|
141
|
+
a_of = {c: sum(deg[f] for f in members[c]) / m2 for c in labels}
|
|
142
|
+
for i, c1 in enumerate(labels):
|
|
143
|
+
for c2 in labels[i + 1 :]:
|
|
144
|
+
# Sum edge weight between c1 and c2 over c1's actual neighbours
|
|
145
|
+
# (sparse, O(deg)) rather than a |c1|*|c2| Cartesian scan.
|
|
146
|
+
e_ij = (
|
|
147
|
+
sum(
|
|
148
|
+
w
|
|
149
|
+
for f in members[c1]
|
|
150
|
+
for g, w in graph.adj.get(f, {}).items()
|
|
151
|
+
if g in members[c2]
|
|
152
|
+
)
|
|
153
|
+
/ m2
|
|
154
|
+
)
|
|
155
|
+
if e_ij <= 0.0: # disconnected (weights are non-negative); merging can't raise Q
|
|
156
|
+
continue
|
|
157
|
+
dq = 2 * (e_ij - a_of[c1] * a_of[c2])
|
|
158
|
+
if dq > best_gain:
|
|
159
|
+
best_gain, best_pair = dq, (c1, c2)
|
|
160
|
+
return best_pair
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def greedy_modularity(graph: FileGraph) -> tuple[list[list[str]], float]:
|
|
164
|
+
"""Greedy (Clauset-Newman-Moore) community detection; returns (communities, Q).
|
|
165
|
+
|
|
166
|
+
Each file starts in its own community; the connected pair with the maximum
|
|
167
|
+
positive delta-Q = 2*(e_ij - a_i*a_j) is merged until no merge improves Q.
|
|
168
|
+
Ties break on the lexicographically smallest label pair (deterministic).
|
|
169
|
+
Isolated files stay singletons and contribute 0 to Q. Communities are
|
|
170
|
+
returned sorted (members sorted, then the list sorted).
|
|
171
|
+
"""
|
|
172
|
+
files = list(graph.files)
|
|
173
|
+
if not files:
|
|
174
|
+
return [], 0.0
|
|
175
|
+
deg = {f: sum(graph.adj.get(f, {}).values()) for f in files}
|
|
176
|
+
m2 = sum(deg.values())
|
|
177
|
+
if m2 <= 0.0: # all isolated (degree sum is non-negative)
|
|
178
|
+
return [[f] for f in files], 0.0
|
|
179
|
+
|
|
180
|
+
members: dict[str, set[str]] = {f: {f} for f in files}
|
|
181
|
+
while len(members) > 1:
|
|
182
|
+
best_pair = _best_merge(graph, members, deg, m2)
|
|
183
|
+
if best_pair is None:
|
|
184
|
+
break
|
|
185
|
+
c1, c2 = best_pair
|
|
186
|
+
members[c1] |= members.pop(c2)
|
|
187
|
+
|
|
188
|
+
communities = sorted(sorted(s) for s in members.values())
|
|
189
|
+
return communities, _modularity(graph, communities, deg, m2)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
Direction = Literal["under_split", "over_split", "matched"]
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _entropy(labels: Mapping[str, Hashable]) -> float:
|
|
196
|
+
"""Shannon entropy (nats) of a partition's label distribution."""
|
|
197
|
+
n = len(labels)
|
|
198
|
+
counts = Counter(labels.values())
|
|
199
|
+
return -sum((c / n) * math.log(c / n) for c in counts.values())
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _mutual_information(a: Mapping[str, Hashable], b: Mapping[str, Hashable]) -> float:
|
|
203
|
+
"""Mutual information (nats) between two partitions over the same key set."""
|
|
204
|
+
n = len(a)
|
|
205
|
+
ca, cb = Counter(a.values()), Counter(b.values())
|
|
206
|
+
joint = Counter((a[k], b[k]) for k in a)
|
|
207
|
+
mi = 0.0
|
|
208
|
+
for (x, y), nxy in joint.items():
|
|
209
|
+
p_xy = nxy / n
|
|
210
|
+
mi += p_xy * math.log(p_xy / ((ca[x] / n) * (cb[y] / n)))
|
|
211
|
+
return mi
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def partition_divergence(p_comm: Mapping[str, Hashable], p_dir: Mapping[str, Hashable]) -> float:
|
|
215
|
+
"""1 - NMI between two partitions of the same file set, in [0, 1].
|
|
216
|
+
|
|
217
|
+
NMI = I(X;Y) / mean(H(X), H(Y)); defined as 1.0 (so D = 0) when both
|
|
218
|
+
partitions are trivial (single cluster each). A non-trivial partition
|
|
219
|
+
against a trivial one has MI 0 -> NMI 0 -> D = 1 (the flat-package case).
|
|
220
|
+
"""
|
|
221
|
+
h_a, h_b = _entropy(p_comm), _entropy(p_dir)
|
|
222
|
+
if h_a <= 0.0 and h_b <= 0.0: # both trivial (entropy is non-negative; -0.0 counts)
|
|
223
|
+
return 0.0
|
|
224
|
+
nmi = _mutual_information(p_comm, p_dir) / ((h_a + h_b) / 2)
|
|
225
|
+
return 1.0 - nmi
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def layout_direction(p_comm: Mapping[str, Hashable], p_dir: Mapping[str, Hashable]) -> Direction:
|
|
229
|
+
"""Direction of the layout/community mismatch by distinct-group count.
|
|
230
|
+
|
|
231
|
+
``under_split`` when the directory layout is flatter than the communities
|
|
232
|
+
(fewer dir groups than communities); ``over_split`` when finer; ``matched``
|
|
233
|
+
when equal.
|
|
234
|
+
"""
|
|
235
|
+
n_dir, n_comm = len(set(p_dir.values())), len(set(p_comm.values()))
|
|
236
|
+
if n_dir < n_comm:
|
|
237
|
+
return "under_split"
|
|
238
|
+
if n_dir > n_comm:
|
|
239
|
+
return "over_split"
|
|
240
|
+
return "matched"
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
Verdict = Literal["split", "consolidate", "aligned", "leave", "borderline", "no_signal"]
|
|
244
|
+
|
|
245
|
+
#: Default thresholds. ``split``/``leave`` are cross-validated for Q; ``divergence``
|
|
246
|
+
#: is provisional (#242 spec). ``min_connected`` is the minimum fraction of files
|
|
247
|
+
#: with an intra-package edge before a split/borderline verdict is trusted —
|
|
248
|
+
#: modularity Q is unreliable on a near-disconnected graph, where a single tiny
|
|
249
|
+
#: cluster inflates Q on a package of otherwise-independent files (calibrated on
|
|
250
|
+
#: owner-api/utils 0.42 vs cgis.query 0.68 — see #242 follow-up).
|
|
251
|
+
THRESHOLDS: dict[str, float] = {
|
|
252
|
+
"split": 0.35,
|
|
253
|
+
"leave": 0.25,
|
|
254
|
+
"divergence": 0.2,
|
|
255
|
+
"min_connected": 0.5,
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def classify_verdict(
|
|
260
|
+
*, q: float, d: float, direction: Direction, thresholds: Mapping[str, float]
|
|
261
|
+
) -> Verdict:
|
|
262
|
+
"""Verdict from modularity Q, divergence D, and mismatch direction.
|
|
263
|
+
|
|
264
|
+
``no_signal`` is decided upstream (no files / no edges); this maps a scored
|
|
265
|
+
package. Gated on BOTH Q (structure is real) and D (layout disagrees), with
|
|
266
|
+
direction disambiguating split (flatter layout) from consolidate (finer).
|
|
267
|
+
"""
|
|
268
|
+
if q < thresholds["leave"]:
|
|
269
|
+
return "leave"
|
|
270
|
+
if d < thresholds["divergence"]:
|
|
271
|
+
return "aligned"
|
|
272
|
+
if q >= thresholds["split"]:
|
|
273
|
+
if direction == "under_split":
|
|
274
|
+
return "split"
|
|
275
|
+
if direction == "over_split":
|
|
276
|
+
return "consolidate"
|
|
277
|
+
return "borderline"
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Per-node health metrics: fan_in, fan_out, depth, in_cycle."""
|
|
2
|
+
|
|
3
|
+
from collections import deque
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
from cgis.core.models import Edge, EdgeType, Node, NodeType
|
|
7
|
+
from cgis.query.drift._scc import build_adjacency, tarjan_scc
|
|
8
|
+
|
|
9
|
+
_BEHAVIORAL = frozenset({EdgeType.CALLS, EdgeType.IMPORTS, EdgeType.REFERENCES, EdgeType.USES})
|
|
10
|
+
_STRUCTURAL = frozenset({EdgeType.CONTAINS, EdgeType.DECLARES})
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class NodeHealth:
|
|
15
|
+
"""Health metrics for a single node."""
|
|
16
|
+
|
|
17
|
+
fan_in: int
|
|
18
|
+
fan_out: int
|
|
19
|
+
depth: int
|
|
20
|
+
in_cycle: bool
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class HealthScorer:
|
|
24
|
+
"""Compute health metrics for every node in a graph snapshot."""
|
|
25
|
+
|
|
26
|
+
def __init__(self, nodes: list[Node], edges: list[Edge]) -> None:
|
|
27
|
+
"""Accept the full node/edge lists from an ingestion run."""
|
|
28
|
+
self._nodes = nodes
|
|
29
|
+
self._edges = edges
|
|
30
|
+
|
|
31
|
+
def _compute_fan_counts(self) -> tuple[dict[str, int], dict[str, int]]:
|
|
32
|
+
"""Count incoming and outgoing behavioral edges (CALLS, IMPORTS, REFERENCES, USES) per node.
|
|
33
|
+
|
|
34
|
+
Returns fan_in and fan_out dicts keyed by node ID.
|
|
35
|
+
"""
|
|
36
|
+
fan_in: dict[str, int] = {n.id: 0 for n in self._nodes}
|
|
37
|
+
fan_out: dict[str, int] = {n.id: 0 for n in self._nodes}
|
|
38
|
+
for edge in self._edges:
|
|
39
|
+
if edge.type not in _BEHAVIORAL:
|
|
40
|
+
continue
|
|
41
|
+
if edge.source in fan_out:
|
|
42
|
+
fan_out[edge.source] += 1
|
|
43
|
+
if edge.target in fan_in:
|
|
44
|
+
fan_in[edge.target] += 1
|
|
45
|
+
return fan_in, fan_out
|
|
46
|
+
|
|
47
|
+
def _compute_depths(self) -> dict[str, int]:
|
|
48
|
+
"""BFS from FILE nodes via CONTAINS/DECLARES to assign structural depth."""
|
|
49
|
+
children: dict[str, list[str]] = {}
|
|
50
|
+
for edge in self._edges:
|
|
51
|
+
if edge.type in _STRUCTURAL:
|
|
52
|
+
children.setdefault(edge.source, []).append(edge.target)
|
|
53
|
+
|
|
54
|
+
file_ids = {n.id for n in self._nodes if n.type == NodeType.FILE}
|
|
55
|
+
depths: dict[str, int] = {}
|
|
56
|
+
queue: deque[tuple[str, int]] = deque((fid, 0) for fid in file_ids)
|
|
57
|
+
while queue:
|
|
58
|
+
node_id, d = queue.popleft()
|
|
59
|
+
if node_id in depths:
|
|
60
|
+
continue
|
|
61
|
+
depths[node_id] = d
|
|
62
|
+
for child_id in children.get(node_id, []):
|
|
63
|
+
if child_id not in depths:
|
|
64
|
+
queue.append((child_id, d + 1))
|
|
65
|
+
return depths
|
|
66
|
+
|
|
67
|
+
def _compute_cycles(self) -> set[str]:
|
|
68
|
+
"""Return the set of FILE/MODULE node IDs that participate in IMPORTS cycles."""
|
|
69
|
+
target_types = {NodeType.FILE, NodeType.MODULE}
|
|
70
|
+
file_and_module_ids = {n.id for n in self._nodes if n.type in target_types}
|
|
71
|
+
adj = build_adjacency(self._edges, frozenset({EdgeType.IMPORTS}))
|
|
72
|
+
adj = {
|
|
73
|
+
k: [v for v in vs if v in file_and_module_ids]
|
|
74
|
+
for k, vs in adj.items()
|
|
75
|
+
if k in file_and_module_ids
|
|
76
|
+
}
|
|
77
|
+
cyclic: set[str] = set()
|
|
78
|
+
for scc in tarjan_scc(adj):
|
|
79
|
+
if len(scc) > 1:
|
|
80
|
+
cyclic.update(scc)
|
|
81
|
+
return cyclic
|
|
82
|
+
|
|
83
|
+
def _compute_file_ancestors(self) -> dict[str, str]:
|
|
84
|
+
"""Map every descendant node ID to the FILE or MODULE root it belongs to (transitive)."""
|
|
85
|
+
children: dict[str, list[str]] = {}
|
|
86
|
+
for edge in self._edges:
|
|
87
|
+
if edge.type in _STRUCTURAL:
|
|
88
|
+
children.setdefault(edge.source, []).append(edge.target)
|
|
89
|
+
|
|
90
|
+
root_types = {NodeType.FILE, NodeType.MODULE}
|
|
91
|
+
file_ids = {n.id for n in self._nodes if n.type in root_types}
|
|
92
|
+
ancestor: dict[str, str] = {}
|
|
93
|
+
queue: deque[tuple[str, str]] = deque((fid, fid) for fid in file_ids)
|
|
94
|
+
while queue:
|
|
95
|
+
node_id, root_file = queue.popleft()
|
|
96
|
+
if node_id in ancestor:
|
|
97
|
+
continue
|
|
98
|
+
ancestor[node_id] = root_file
|
|
99
|
+
for child_id in children.get(node_id, []):
|
|
100
|
+
if child_id not in ancestor:
|
|
101
|
+
queue.append((child_id, root_file))
|
|
102
|
+
return ancestor
|
|
103
|
+
|
|
104
|
+
def enrich(self) -> list[Node]:
|
|
105
|
+
"""Return a new list of Nodes with health metrics merged into `metadata`."""
|
|
106
|
+
fan_in, fan_out = self._compute_fan_counts()
|
|
107
|
+
depths = self._compute_depths()
|
|
108
|
+
cyclic_file_ids = self._compute_cycles()
|
|
109
|
+
file_ancestor = self._compute_file_ancestors()
|
|
110
|
+
|
|
111
|
+
result: list[Node] = []
|
|
112
|
+
for node in self._nodes:
|
|
113
|
+
ancestor_file = file_ancestor.get(node.id, node.id)
|
|
114
|
+
in_cycle = ancestor_file in cyclic_file_ids
|
|
115
|
+
result.append(
|
|
116
|
+
node.model_copy(
|
|
117
|
+
update={
|
|
118
|
+
"metadata": {
|
|
119
|
+
**node.metadata,
|
|
120
|
+
"fan_in": fan_in.get(node.id, 0),
|
|
121
|
+
"fan_out": fan_out.get(node.id, 0),
|
|
122
|
+
"depth": depths.get(node.id, 0),
|
|
123
|
+
"in_cycle": in_cycle,
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
)
|
|
127
|
+
)
|
|
128
|
+
return result
|