pycode-kg 0.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pycode_kg/.DS_Store +0 -0
- pycode_kg/__init__.py +91 -0
- pycode_kg/__main__.py +11 -0
- pycode_kg/analysis/__init__.py +15 -0
- pycode_kg/analysis/bridge.py +108 -0
- pycode_kg/analysis/centrality.py +412 -0
- pycode_kg/analysis/framework_detector.py +103 -0
- pycode_kg/analysis/hybrid_rank.py +53 -0
- pycode_kg/app.py +1335 -0
- pycode_kg/architecture.py +624 -0
- pycode_kg/build_pycodekg_lancedb.py +15 -0
- pycode_kg/build_pycodekg_sqlite.py +15 -0
- pycode_kg/cli/__init__.py +29 -0
- pycode_kg/cli/cmd_analyze.py +96 -0
- pycode_kg/cli/cmd_architecture.py +150 -0
- pycode_kg/cli/cmd_bridges.py +26 -0
- pycode_kg/cli/cmd_build.py +154 -0
- pycode_kg/cli/cmd_build_full.py +242 -0
- pycode_kg/cli/cmd_centrality.py +131 -0
- pycode_kg/cli/cmd_explain.py +180 -0
- pycode_kg/cli/cmd_framework_nodes.py +18 -0
- pycode_kg/cli/cmd_hooks.py +137 -0
- pycode_kg/cli/cmd_init.py +312 -0
- pycode_kg/cli/cmd_mcp.py +71 -0
- pycode_kg/cli/cmd_model.py +53 -0
- pycode_kg/cli/cmd_query.py +211 -0
- pycode_kg/cli/cmd_snapshot.py +421 -0
- pycode_kg/cli/cmd_viz.py +180 -0
- pycode_kg/cli/main.py +23 -0
- pycode_kg/cli/options.py +63 -0
- pycode_kg/config.py +78 -0
- pycode_kg/graph.py +125 -0
- pycode_kg/index.py +542 -0
- pycode_kg/kg.py +220 -0
- pycode_kg/layout3d.py +470 -0
- pycode_kg/mcp/bridge_tools.py +19 -0
- pycode_kg/mcp/framework_tools.py +18 -0
- pycode_kg/mcp_server.py +1965 -0
- pycode_kg/module/__init__.py +83 -0
- pycode_kg/module/base.py +720 -0
- pycode_kg/module/extractor.py +276 -0
- pycode_kg/module/types.py +532 -0
- pycode_kg/pycodekg.py +543 -0
- pycode_kg/pycodekg_query.py +1 -0
- pycode_kg/pycodekg_snippet_packer.py +1 -0
- pycode_kg/pycodekg_thorough_analysis.py +2751 -0
- pycode_kg/pycodekg_viz.py +1 -0
- pycode_kg/pycodekg_viz3d.py +1 -0
- pycode_kg/ranking/__init__.py +1 -0
- pycode_kg/ranking/cli_rank.py +92 -0
- pycode_kg/ranking/coderank.py +555 -0
- pycode_kg/snapshots.py +612 -0
- pycode_kg/sql/004_add_centrality_table.sql +12 -0
- pycode_kg/store.py +766 -0
- pycode_kg/utils.py +39 -0
- pycode_kg/visitor.py +413 -0
- pycode_kg/viz3d.py +1353 -0
- pycode_kg/viz3d_timeline.py +364 -0
- pycode_kg-0.16.0.dist-info/METADATA +305 -0
- pycode_kg-0.16.0.dist-info/RECORD +63 -0
- pycode_kg-0.16.0.dist-info/WHEEL +4 -0
- pycode_kg-0.16.0.dist-info/entry_points.txt +19 -0
- pycode_kg-0.16.0.dist-info/licenses/LICENSE +94 -0
|
@@ -0,0 +1,2751 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
PyCodeKG Thorough Repository Analysis Tool
|
|
4
|
+
|
|
5
|
+
Performs comprehensive architectural analysis of Python repositories using PyCodeKG's
|
|
6
|
+
graph traversal capabilities. Analyzes:
|
|
7
|
+
- Complexity hotspots (highest fan-in/fan-out functions)
|
|
8
|
+
- Architectural patterns (core modules, integration points)
|
|
9
|
+
- Dependency analysis (circular deps, tight coupling)
|
|
10
|
+
- Code quality signals (dead code, orphaned functions)
|
|
11
|
+
|
|
12
|
+
Operational behavior:
|
|
13
|
+
- Entry point and configuration defaults: resolves ``repo_root`` and defaults
|
|
14
|
+
``db_path``/``lancedb_path`` to ``.pycodekg/graph.sqlite`` and
|
|
15
|
+
``.pycodekg/lancedb``.
|
|
16
|
+
- Logging approach: uses Rich console output for user-facing status and
|
|
17
|
+
standard ``logging`` for non-fatal diagnostic warnings.
|
|
18
|
+
- Error handling strategy: degrades gracefully when optional data is missing
|
|
19
|
+
(for example missing database, git metadata, or snapshot history) and emits
|
|
20
|
+
actionable warnings instead of failing hard where possible.
|
|
21
|
+
|
|
22
|
+
Usage:
|
|
23
|
+
python pycodekg_thorough_analysis.py /path/to/repo /path/to/db .pycodekg/lancedb
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
import datetime
|
|
27
|
+
import json
|
|
28
|
+
import logging
|
|
29
|
+
import os
|
|
30
|
+
import platform
|
|
31
|
+
import subprocess
|
|
32
|
+
import time
|
|
33
|
+
from collections import defaultdict
|
|
34
|
+
from collections.abc import Callable
|
|
35
|
+
from dataclasses import asdict, dataclass
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
|
|
38
|
+
from rich.console import Console
|
|
39
|
+
from rich.panel import Panel
|
|
40
|
+
from rich.table import Table
|
|
41
|
+
|
|
42
|
+
from pycode_kg.snapshots import SnapshotManager
|
|
43
|
+
|
|
44
|
+
# Setup logging
|
|
45
|
+
logging.basicConfig(level=logging.INFO)
|
|
46
|
+
logger = logging.getLogger(__name__)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass
|
|
50
|
+
class FunctionMetrics:
|
|
51
|
+
"""Metrics for a single function or class.
|
|
52
|
+
|
|
53
|
+
:param node_id: Stable node identifier
|
|
54
|
+
:param name: Function/class name
|
|
55
|
+
:param module: Module path containing this definition
|
|
56
|
+
:param kind: Kind of node (function, method, class)
|
|
57
|
+
:param fan_in: Count of callers (how many call this)
|
|
58
|
+
:param fan_out: Count of callees (how many this calls)
|
|
59
|
+
:param lines: Approximate line count
|
|
60
|
+
:param docstring: Docstring text if available
|
|
61
|
+
:param risk_level: Risk assessment (low, medium, high, critical)
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
node_id: str
|
|
65
|
+
name: str
|
|
66
|
+
module: str
|
|
67
|
+
kind: str
|
|
68
|
+
fan_in: int
|
|
69
|
+
fan_out: int
|
|
70
|
+
lines: int
|
|
71
|
+
docstring: str | None = None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass
|
|
75
|
+
class ModuleMetrics:
|
|
76
|
+
"""Metrics for a module.
|
|
77
|
+
|
|
78
|
+
:param path: Module file path
|
|
79
|
+
:param functions: Count of functions defined
|
|
80
|
+
:param classes: Count of classes defined
|
|
81
|
+
:param methods: Count of methods defined
|
|
82
|
+
:param incoming_deps: Modules that import this one
|
|
83
|
+
:param outgoing_deps: Modules this one imports
|
|
84
|
+
:param total_fan_in: Sum of all callers to functions in module
|
|
85
|
+
:param cohesion_score: Internal coupling strength (0-1)
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
path: str
|
|
89
|
+
functions: int
|
|
90
|
+
classes: int
|
|
91
|
+
methods: int
|
|
92
|
+
incoming_deps: list[str]
|
|
93
|
+
outgoing_deps: list[str]
|
|
94
|
+
total_fan_in: int
|
|
95
|
+
cohesion_score: float
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass
|
|
99
|
+
class CallChain:
|
|
100
|
+
"""Represents a notable call chain.
|
|
101
|
+
|
|
102
|
+
:param chain: List of function names in call order
|
|
103
|
+
:param depth: Length of the chain
|
|
104
|
+
:param total_callers: Sum of all callers in chain
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
chain: list[str]
|
|
108
|
+
depth: int
|
|
109
|
+
total_callers: int
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class PyCodeKGAnalyzer:
|
|
113
|
+
"""Thorough repository analyzer using PyCodeKG graph.
|
|
114
|
+
|
|
115
|
+
:param kg: PyCodeKG instance for graph queries
|
|
116
|
+
:param console: Rich console for output (creates new if None)
|
|
117
|
+
:param snapshot_mgr: Optional SnapshotManager for temporal history
|
|
118
|
+
:param include_dirs: Directories included in the analysis (empty = all)
|
|
119
|
+
:param exclude_dirs: Directories excluded from the analysis (empty = none)
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
def __init__(
|
|
123
|
+
self,
|
|
124
|
+
kg,
|
|
125
|
+
console: Console | None = None,
|
|
126
|
+
snapshot_mgr: SnapshotManager | None = None,
|
|
127
|
+
include_dirs: set[str] | None = None,
|
|
128
|
+
exclude_dirs: set[str] | None = None,
|
|
129
|
+
):
|
|
130
|
+
"""Initialize analyzer with PyCodeKG instance.
|
|
131
|
+
|
|
132
|
+
:param kg: PyCodeKG instance
|
|
133
|
+
:param console: Rich console for terminal output
|
|
134
|
+
:param snapshot_mgr: Optional SnapshotManager; when provided, snapshot
|
|
135
|
+
history is loaded and included in the report.
|
|
136
|
+
:param include_dirs: Set of top-level directory names that were indexed.
|
|
137
|
+
When empty/None, all directories were analyzed.
|
|
138
|
+
:param exclude_dirs: Set of directory names that were excluded from indexing.
|
|
139
|
+
"""
|
|
140
|
+
self.kg = kg
|
|
141
|
+
self.console = console or Console()
|
|
142
|
+
self.snapshot_mgr = snapshot_mgr
|
|
143
|
+
self.include_dirs: set[str] = include_dirs or set()
|
|
144
|
+
self.exclude_dirs: set[str] = exclude_dirs or set()
|
|
145
|
+
self.stats: dict = {}
|
|
146
|
+
self.function_metrics: dict[str, FunctionMetrics] = {}
|
|
147
|
+
self.module_metrics: dict[str, ModuleMetrics] = {}
|
|
148
|
+
self.orphaned_functions: list[FunctionMetrics] = []
|
|
149
|
+
self.high_fanout_functions: list[FunctionMetrics] = []
|
|
150
|
+
self.critical_paths: list[CallChain] = []
|
|
151
|
+
self.circular_deps: list[tuple[str, str]] = []
|
|
152
|
+
self.public_apis: list[FunctionMetrics] = []
|
|
153
|
+
self.issues: list[str] = []
|
|
154
|
+
self.strengths: list[str] = []
|
|
155
|
+
self.docstring_coverage: dict = {} # populated by _analyze_docstring_coverage
|
|
156
|
+
self.inheritance_analysis: dict = {} # populated by _analyze_inheritance
|
|
157
|
+
self.snapshot_history: list[dict] = [] # populated by _analyze_snapshots
|
|
158
|
+
self.centrality_records: list = [] # populated by _analyze_centrality (node-level)
|
|
159
|
+
self.centrality_modules: list[dict] = [] # populated by _analyze_centrality (module-level)
|
|
160
|
+
# Option A: CodeRank scores computed early and reused across phases
|
|
161
|
+
self.coderank_scores: dict[str, float] = {} # node_id → global PageRank score
|
|
162
|
+
self.coderank_top_nodes: list[dict] = [] # top-25 real nodes by CodeRank
|
|
163
|
+
# Option D: concern-based hybrid ranking results
|
|
164
|
+
self.concern_analysis: list[dict] = [] # populated by _analyze_concerns
|
|
165
|
+
# Single-line phase output: each phase fn writes its summary here
|
|
166
|
+
self._phase_result: str = ""
|
|
167
|
+
|
|
168
|
+
# ── total phase count (update if phases are added/removed) ────────────────
|
|
169
|
+
_TOTAL_PHASES = 15
|
|
170
|
+
|
|
171
|
+
def _run_phase(self, num: int, name: str, fn: Callable[[], None]) -> None:
|
|
172
|
+
"""Run ``fn()`` and emit a single summary line with elapsed time.
|
|
173
|
+
|
|
174
|
+
Each phase function writes its result summary to ``self._phase_result``
|
|
175
|
+
instead of printing directly, keeping the terminal output to one line
|
|
176
|
+
per phase.
|
|
177
|
+
|
|
178
|
+
:param num: Phase number (1-based).
|
|
179
|
+
:param name: Human-readable phase label.
|
|
180
|
+
:param fn: Zero-argument callable to execute.
|
|
181
|
+
"""
|
|
182
|
+
self._phase_result = ""
|
|
183
|
+
t0 = time.monotonic()
|
|
184
|
+
fn()
|
|
185
|
+
elapsed = time.monotonic() - t0
|
|
186
|
+
result = f" {self._phase_result}" if self._phase_result else ""
|
|
187
|
+
self.console.print(
|
|
188
|
+
f" [cyan]▶ Phase {num:2d}/{self._TOTAL_PHASES}:[/cyan]"
|
|
189
|
+
f" {name}{result} [green]({elapsed:.1f}s)[/green]"
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
def run_analysis(
|
|
193
|
+
self,
|
|
194
|
+
report_path: str | None = None,
|
|
195
|
+
*,
|
|
196
|
+
persist_centrality: bool = False,
|
|
197
|
+
) -> dict:
|
|
198
|
+
"""Run complete multi-phase analysis.
|
|
199
|
+
|
|
200
|
+
Phase ordering:
|
|
201
|
+
1. Baseline metrics
|
|
202
|
+
1b. CodeRank (Option A) — computed early so all later phases can use scores
|
|
203
|
+
2. Fan-in analysis — seeded by CodeRank top nodes (not semantic search)
|
|
204
|
+
3. Fan-out analysis
|
|
205
|
+
4. Dependency analysis
|
|
206
|
+
5. Pattern detection
|
|
207
|
+
6. Module coupling
|
|
208
|
+
7. Critical paths — seeded by CodeRank-ranked top functions
|
|
209
|
+
8. Public API identification
|
|
210
|
+
9. Docstring coverage
|
|
211
|
+
10. Inheritance hierarchy
|
|
212
|
+
11. Generate insights
|
|
213
|
+
12. Snapshot history
|
|
214
|
+
13. Structural centrality (SIR PageRank)
|
|
215
|
+
14. CodeRank top-nodes report section (Option D)
|
|
216
|
+
15. Concern-based hybrid ranking (Option D)
|
|
217
|
+
|
|
218
|
+
:param report_path: Optional path to write markdown report
|
|
219
|
+
:param persist_centrality: When ``True``, persist centrality scores to
|
|
220
|
+
the ``centrality_scores`` table in the SQLite graph DB.
|
|
221
|
+
:return: dictionary of analysis results
|
|
222
|
+
"""
|
|
223
|
+
_start_time = datetime.datetime.now(datetime.UTC)
|
|
224
|
+
try:
|
|
225
|
+
self._run_phase(1, "Baseline metrics", self._analyze_baseline)
|
|
226
|
+
self._run_phase(2, "CodeRank (global PageRank)", self._compute_coderank)
|
|
227
|
+
self._run_phase(3, "Fan-in analysis", self._analyze_fan_in)
|
|
228
|
+
self._run_phase(4, "Fan-out analysis", self._analyze_fan_out)
|
|
229
|
+
self._run_phase(5, "Dependency analysis", self._analyze_dependencies)
|
|
230
|
+
self._run_phase(6, "Pattern detection", self._detect_patterns)
|
|
231
|
+
self._run_phase(7, "Module coupling", self._analyze_module_coupling)
|
|
232
|
+
self._run_phase(8, "Critical paths", self._analyze_critical_paths)
|
|
233
|
+
self._run_phase(9, "Public API identification", self._identify_public_apis)
|
|
234
|
+
self._run_phase(10, "Docstring coverage", self._analyze_docstring_coverage)
|
|
235
|
+
self._run_phase(11, "Inheritance hierarchy", self._analyze_inheritance)
|
|
236
|
+
self._run_phase(12, "Generate insights", self._generate_insights)
|
|
237
|
+
self._run_phase(13, "Snapshot history", self._analyze_snapshots)
|
|
238
|
+
self._run_phase(14, "Structural centrality (SIR)", self._analyze_centrality)
|
|
239
|
+
if persist_centrality and self.centrality_records:
|
|
240
|
+
from pycode_kg.analysis.centrality import ( # pylint: disable=import-outside-toplevel
|
|
241
|
+
StructuralImportanceRanker,
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
StructuralImportanceRanker(self.kg.db_path).write_scores(self.centrality_records)
|
|
245
|
+
|
|
246
|
+
self._run_phase(15, "Concern-based ranking", self._analyze_concerns)
|
|
247
|
+
|
|
248
|
+
# Optional: write report
|
|
249
|
+
if report_path:
|
|
250
|
+
elapsed = (datetime.datetime.now(datetime.UTC) - _start_time).total_seconds()
|
|
251
|
+
self._write_report(report_path, elapsed_seconds=elapsed)
|
|
252
|
+
|
|
253
|
+
return self._compile_results()
|
|
254
|
+
|
|
255
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
256
|
+
self.console.print(f"[red]Analysis failed: {e}[/red]")
|
|
257
|
+
logger.exception("Analysis failed")
|
|
258
|
+
raise
|
|
259
|
+
|
|
260
|
+
def _analyze_baseline(self) -> None:
|
|
261
|
+
"""Phase 1: Get overall codebase metrics.
|
|
262
|
+
|
|
263
|
+
Queries graph_stats() to establish baseline counts by node kind
|
|
264
|
+
and edge relationship type.
|
|
265
|
+
"""
|
|
266
|
+
try:
|
|
267
|
+
self.stats = self.kg.stats()
|
|
268
|
+
n = self.stats.get("total_nodes", "?")
|
|
269
|
+
e = self.stats.get("total_edges", "?")
|
|
270
|
+
self._phase_result = f"nodes={n} edges={e}"
|
|
271
|
+
|
|
272
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
273
|
+
logger.warning(f"Could not get baseline stats: {e}")
|
|
274
|
+
self.console.print(f"[yellow]WARN[/yellow] Could not get baseline stats: {e}")
|
|
275
|
+
|
|
276
|
+
def _analyze_fan_in(self) -> None:
|
|
277
|
+
"""Phase 2: Find most-called functions (fan-in).
|
|
278
|
+
|
|
279
|
+
Option A: Uses CodeRank scores (computed in Phase 1b) to seed the
|
|
280
|
+
candidate set instead of fragile semantic search. Falls back to a
|
|
281
|
+
direct SQL query over all function/method/class nodes when CodeRank
|
|
282
|
+
is unavailable.
|
|
283
|
+
|
|
284
|
+
For each candidate node, the exact caller count is obtained via
|
|
285
|
+
``kg.callers(node_id, rel="CALLS")``. The top-15 by fan-in are
|
|
286
|
+
stored in ``self.function_metrics``.
|
|
287
|
+
"""
|
|
288
|
+
|
|
289
|
+
try:
|
|
290
|
+
# --- Option A: seed from CodeRank top nodes (deterministic, no LanceDB) ---
|
|
291
|
+
if self.coderank_scores:
|
|
292
|
+
# Pull ALL real function/method/class nodes from SQLite, sorted by
|
|
293
|
+
# CodeRank score descending. This is O(nodes) but avoids the k-limit
|
|
294
|
+
# and semantic-search noise of the old approach.
|
|
295
|
+
rows = self.kg.store.con.execute(
|
|
296
|
+
"""
|
|
297
|
+
SELECT id, name, kind, module_path, docstring, lineno, end_lineno
|
|
298
|
+
FROM nodes
|
|
299
|
+
WHERE kind IN ('function', 'method', 'class')
|
|
300
|
+
AND id NOT LIKE 'sym:%'
|
|
301
|
+
ORDER BY module_path, name
|
|
302
|
+
"""
|
|
303
|
+
).fetchall()
|
|
304
|
+
|
|
305
|
+
# Attach CodeRank scores and sort by score descending
|
|
306
|
+
scored: list[tuple[float, tuple]] = []
|
|
307
|
+
for row in rows:
|
|
308
|
+
node_id = row[0]
|
|
309
|
+
score = self.coderank_scores.get(node_id, 0.0)
|
|
310
|
+
scored.append((score, row))
|
|
311
|
+
scored.sort(key=lambda x: x[0], reverse=True)
|
|
312
|
+
|
|
313
|
+
# Compute actual fan-in for top-100 by CodeRank (covers all meaningful nodes)
|
|
314
|
+
fan_in_data: list[tuple] = []
|
|
315
|
+
for _score, row in scored[:100]:
|
|
316
|
+
node_id, name, kind, module_path, docstring, lineno, end_lineno = row
|
|
317
|
+
try:
|
|
318
|
+
caller_list = self.kg.callers(node_id, rel="CALLS")
|
|
319
|
+
caller_count = len(caller_list)
|
|
320
|
+
metrics = FunctionMetrics(
|
|
321
|
+
node_id=node_id,
|
|
322
|
+
name=name or "unknown",
|
|
323
|
+
module=module_path or "unknown",
|
|
324
|
+
kind=kind or "unknown",
|
|
325
|
+
fan_in=caller_count,
|
|
326
|
+
fan_out=0,
|
|
327
|
+
lines=max(0, (end_lineno or 0) - (lineno or 0)),
|
|
328
|
+
docstring=docstring,
|
|
329
|
+
)
|
|
330
|
+
fan_in_data.append((node_id, metrics))
|
|
331
|
+
except (AttributeError, ValueError, RuntimeError, TypeError) as e:
|
|
332
|
+
logger.debug(f"Could not analyze node {node_id}: {e}")
|
|
333
|
+
|
|
334
|
+
else:
|
|
335
|
+
# --- Fallback: direct SQL over all nodes (no CodeRank available) ---
|
|
336
|
+
logger.info("CodeRank unavailable — falling back to SQL fan-in scan")
|
|
337
|
+
rows = self.kg.store.con.execute(
|
|
338
|
+
"""
|
|
339
|
+
SELECT id, name, kind, module_path, docstring, lineno, end_lineno
|
|
340
|
+
FROM nodes
|
|
341
|
+
WHERE kind IN ('function', 'method', 'class')
|
|
342
|
+
AND id NOT LIKE 'sym:%'
|
|
343
|
+
ORDER BY module_path, name
|
|
344
|
+
"""
|
|
345
|
+
).fetchall()
|
|
346
|
+
|
|
347
|
+
fan_in_data = []
|
|
348
|
+
for row in rows:
|
|
349
|
+
node_id, name, kind, module_path, docstring, lineno, end_lineno = row
|
|
350
|
+
try:
|
|
351
|
+
caller_list = self.kg.callers(node_id, rel="CALLS")
|
|
352
|
+
caller_count = len(caller_list)
|
|
353
|
+
metrics = FunctionMetrics(
|
|
354
|
+
node_id=node_id,
|
|
355
|
+
name=name or "unknown",
|
|
356
|
+
module=module_path or "unknown",
|
|
357
|
+
kind=kind or "unknown",
|
|
358
|
+
fan_in=caller_count,
|
|
359
|
+
fan_out=0,
|
|
360
|
+
lines=max(0, (end_lineno or 0) - (lineno or 0)),
|
|
361
|
+
docstring=docstring,
|
|
362
|
+
)
|
|
363
|
+
fan_in_data.append((node_id, metrics))
|
|
364
|
+
except (AttributeError, ValueError, RuntimeError, TypeError) as e:
|
|
365
|
+
logger.debug(f"Could not analyze node {node_id}: {e}")
|
|
366
|
+
|
|
367
|
+
# Sort by fan-in descending, keep top 15
|
|
368
|
+
fan_in_data.sort(key=lambda x: x[1].fan_in, reverse=True)
|
|
369
|
+
for node_id, metrics in fan_in_data[:15]:
|
|
370
|
+
self.function_metrics[node_id] = metrics
|
|
371
|
+
|
|
372
|
+
seed = "CodeRank-seeded" if self.coderank_scores else "SQL fallback"
|
|
373
|
+
self._phase_result = f"top {len(self.function_metrics)} by fan-in ({seed})"
|
|
374
|
+
|
|
375
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
376
|
+
logger.warning(f"Fan-in analysis incomplete: {e}")
|
|
377
|
+
self.console.print(f"[yellow]WARN[/yellow] Fan-in analysis incomplete: {e}")
|
|
378
|
+
|
|
379
|
+
def _analyze_fan_out(self) -> None:
|
|
380
|
+
"""Phase 3: Find functions that call many others (fan-out).
|
|
381
|
+
|
|
382
|
+
Analyzes functions in the function_metrics already identified,
|
|
383
|
+
computing their actual fan-out by reverse-querying callee lists.
|
|
384
|
+
Also identifies additional high-fanout orchestrator functions.
|
|
385
|
+
"""
|
|
386
|
+
|
|
387
|
+
try:
|
|
388
|
+
# For functions already identified, compute their fan-out
|
|
389
|
+
for node_id, metrics in self.function_metrics.items():
|
|
390
|
+
try:
|
|
391
|
+
# Get the node to see what it calls
|
|
392
|
+
node = self.kg.node(node_id)
|
|
393
|
+
if node is None:
|
|
394
|
+
continue
|
|
395
|
+
|
|
396
|
+
# Count CALLS edges outgoing from this node
|
|
397
|
+
# This is a rough estimate via the store; exact count requires
|
|
398
|
+
# querying the store directly
|
|
399
|
+
fanout_count = 0
|
|
400
|
+
# Try to get edges from the node
|
|
401
|
+
if hasattr(self.kg, "_store"):
|
|
402
|
+
edges = self.kg._store.edges_from(node_id, rel="CALLS", limit=100)
|
|
403
|
+
fanout_count = len(edges) if edges else 0
|
|
404
|
+
|
|
405
|
+
metrics.fan_out = fanout_count
|
|
406
|
+
|
|
407
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
408
|
+
logger.debug(f"Could not compute fan-out for {node_id}: {e}")
|
|
409
|
+
|
|
410
|
+
# Query for additional orchestrator functions
|
|
411
|
+
try:
|
|
412
|
+
result = self.kg.query(
|
|
413
|
+
"coordinator orchestrator manager init constructor setup",
|
|
414
|
+
k=20,
|
|
415
|
+
hop=0,
|
|
416
|
+
rels=("CONTAINS",),
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
for node in result.nodes:
|
|
420
|
+
node_id = node.get("id")
|
|
421
|
+
if not node_id or node_id in self.function_metrics:
|
|
422
|
+
continue
|
|
423
|
+
|
|
424
|
+
if node.get("kind") not in ["function", "method"]:
|
|
425
|
+
continue
|
|
426
|
+
|
|
427
|
+
# Estimate fan-out for new functions
|
|
428
|
+
fanout_count = 0
|
|
429
|
+
if hasattr(self.kg, "_store"):
|
|
430
|
+
try:
|
|
431
|
+
edges = self.kg._store.edges_from(node_id, rel="CALLS", limit=100)
|
|
432
|
+
fanout_count = len(edges) if edges else 0
|
|
433
|
+
except (AttributeError, ValueError, RuntimeError):
|
|
434
|
+
pass
|
|
435
|
+
|
|
436
|
+
if fanout_count > 25:
|
|
437
|
+
metrics = FunctionMetrics(
|
|
438
|
+
node_id=node_id,
|
|
439
|
+
name=node.get("name", "unknown"),
|
|
440
|
+
module=node.get("module_path", "unknown"),
|
|
441
|
+
kind=node.get("kind", "unknown"),
|
|
442
|
+
fan_in=0,
|
|
443
|
+
fan_out=fanout_count,
|
|
444
|
+
lines=max(
|
|
445
|
+
0,
|
|
446
|
+
(node.get("end_lineno") or 0) - (node.get("lineno") or 0),
|
|
447
|
+
),
|
|
448
|
+
)
|
|
449
|
+
|
|
450
|
+
self.high_fanout_functions.append(metrics)
|
|
451
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
452
|
+
logger.debug(f"Could not query orchestrators: {e}")
|
|
453
|
+
|
|
454
|
+
self._phase_result = f"{len(self.high_fanout_functions)} high-fanout functions"
|
|
455
|
+
|
|
456
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
457
|
+
logger.warning(f"Fan-out analysis incomplete: {e}")
|
|
458
|
+
self.console.print(f"[yellow]WARN[/yellow] Fan-out analysis incomplete: {e}")
|
|
459
|
+
|
|
460
|
+
def _is_special_entry_point(self, node: dict) -> bool:
|
|
461
|
+
"""Check if a function is a special entry point that should be excluded from orphan detection.
|
|
462
|
+
|
|
463
|
+
Special entry points are intentional zero-callers that serve as:
|
|
464
|
+
- Protocol methods (__init__, __str__, __exit__, __enter__, etc.)
|
|
465
|
+
- Property methods (@property decorator, accessed as attributes)
|
|
466
|
+
- MCP tool functions decorated with @mcp.tool() in mcp_server.py
|
|
467
|
+
- CLI command handlers decorated with @click.command or @cli.command
|
|
468
|
+
|
|
469
|
+
These functions have zero internal callers because they're invoked by
|
|
470
|
+
external frameworks (MCP protocol, Click CLI, Python runtime), not by
|
|
471
|
+
regular code. Flagging them as orphans is a false positive.
|
|
472
|
+
|
|
473
|
+
:param node: Node dictionary with keys: name, module, kind
|
|
474
|
+
:return: True if this node should be excluded from orphan detection
|
|
475
|
+
"""
|
|
476
|
+
name = node.get("name", "")
|
|
477
|
+
module = node.get("module_path", "")
|
|
478
|
+
|
|
479
|
+
# ===== PROTOCOL METHODS =====
|
|
480
|
+
# Special methods like __init__, __str__, __exit__, __enter__, etc.
|
|
481
|
+
# are invoked by Python's runtime machinery, not explicit code calls.
|
|
482
|
+
if name.startswith("__") and name.endswith("__"):
|
|
483
|
+
return True
|
|
484
|
+
|
|
485
|
+
# ===== PROPERTY METHODS =====
|
|
486
|
+
# Methods decorated with @property are accessed as attributes (snapshot.key),
|
|
487
|
+
# not called as functions. The call graph only captures func() invocations,
|
|
488
|
+
# so properties appear to have zero callers even when used extensively.
|
|
489
|
+
# Known properties in the codebase:
|
|
490
|
+
property_names = {"key"} # Snapshot.key (accessed in snapshots.py: 3+ places)
|
|
491
|
+
if name in property_names:
|
|
492
|
+
return True
|
|
493
|
+
|
|
494
|
+
# ===== MCP TOOL FUNCTIONS =====
|
|
495
|
+
# Functions in mcp_server.py decorated with @mcp.tool() are dispatched
|
|
496
|
+
# by the MCP protocol router, not by explicit Python call sites.
|
|
497
|
+
if "mcp_server" in module:
|
|
498
|
+
return True
|
|
499
|
+
|
|
500
|
+
# ===== CLI COMMAND HANDLERS =====
|
|
501
|
+
# Functions in cli modules that serve as command entry points.
|
|
502
|
+
# These are decorated with @click.command or @cli.command and are
|
|
503
|
+
# dispatched by Click's CLI router when users invoke the tool.
|
|
504
|
+
if "/cli/" in module or module.endswith("cli.py"):
|
|
505
|
+
return True
|
|
506
|
+
|
|
507
|
+
return False
|
|
508
|
+
|
|
509
|
+
def _analyze_dependencies(self) -> None:
|
|
510
|
+
"""Phase 4: Analyze module-level dependencies.
|
|
511
|
+
|
|
512
|
+
Detects orphaned functions (zero callers), import cycles,
|
|
513
|
+
and tight coupling patterns. Excludes special entry points (protocol
|
|
514
|
+
methods and CLI commands) which have zero callers by design.
|
|
515
|
+
"""
|
|
516
|
+
|
|
517
|
+
try:
|
|
518
|
+
# Query for potential orphaned code
|
|
519
|
+
result = self.kg.query(
|
|
520
|
+
"unused dead code deprecated helper utility internal",
|
|
521
|
+
k=15,
|
|
522
|
+
hop=0,
|
|
523
|
+
rels=("CONTAINS",),
|
|
524
|
+
)
|
|
525
|
+
|
|
526
|
+
for node in result.nodes:
|
|
527
|
+
if node.get("kind") not in ["function", "method", "class"]:
|
|
528
|
+
continue
|
|
529
|
+
|
|
530
|
+
node_id = node.get("id")
|
|
531
|
+
if not node_id:
|
|
532
|
+
continue
|
|
533
|
+
|
|
534
|
+
try:
|
|
535
|
+
caller_list = self.kg.callers(node_id, rel="CALLS")
|
|
536
|
+
|
|
537
|
+
# Also check ATTR_ACCESS edges: a bound method reference
|
|
538
|
+
# (e.g. ``self._foo`` passed as a callback to ``_run_phase``)
|
|
539
|
+
# produces ATTR_ACCESS edges, not CALLS edges. A node with
|
|
540
|
+
# ATTR_ACCESS callers is in active use and must not be flagged
|
|
541
|
+
# as orphaned.
|
|
542
|
+
try:
|
|
543
|
+
attr_list = self.kg.callers(node_id, rel="ATTR_ACCESS")
|
|
544
|
+
except (AttributeError, ValueError, RuntimeError):
|
|
545
|
+
attr_list = []
|
|
546
|
+
|
|
547
|
+
# Functions with zero callers are flagged as orphaned,
|
|
548
|
+
# UNLESS they're special entry points (skip these).
|
|
549
|
+
if len(caller_list) == 0 and len(attr_list) == 0:
|
|
550
|
+
# Exclude protocol methods and CLI entry points
|
|
551
|
+
if self._is_special_entry_point(node):
|
|
552
|
+
logger.debug(
|
|
553
|
+
f"Skipping {node.get('name')} — "
|
|
554
|
+
f"special entry point (framework-driven)"
|
|
555
|
+
)
|
|
556
|
+
continue
|
|
557
|
+
|
|
558
|
+
metrics = FunctionMetrics(
|
|
559
|
+
node_id=node_id,
|
|
560
|
+
name=node.get("name", "unknown"),
|
|
561
|
+
module=node.get("module_path", "unknown"),
|
|
562
|
+
kind=node.get("kind", "unknown"),
|
|
563
|
+
fan_in=0,
|
|
564
|
+
fan_out=0,
|
|
565
|
+
lines=max(
|
|
566
|
+
0,
|
|
567
|
+
(node.get("end_lineno") or 0) - (node.get("lineno") or 0),
|
|
568
|
+
),
|
|
569
|
+
)
|
|
570
|
+
self.orphaned_functions.append(metrics)
|
|
571
|
+
|
|
572
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
573
|
+
logger.debug(f"Could not check callers for {node_id}: {e}")
|
|
574
|
+
|
|
575
|
+
self._phase_result = f"{len(self.orphaned_functions)} orphaned functions"
|
|
576
|
+
|
|
577
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
578
|
+
logger.warning(f"Dependency analysis incomplete: {e}")
|
|
579
|
+
self.console.print(f"[yellow]WARN[/yellow] Dependency analysis incomplete: {e}")
|
|
580
|
+
|
|
581
|
+
def _detect_patterns(self) -> None:
|
|
582
|
+
"""Phase 5: Detect architectural patterns.
|
|
583
|
+
|
|
584
|
+
Identifies core modules, integration points, layering violations,
|
|
585
|
+
and design patterns (singletons, managers, etc.).
|
|
586
|
+
"""
|
|
587
|
+
|
|
588
|
+
try:
|
|
589
|
+
# Identify core modules by aggregating fan-in
|
|
590
|
+
module_call_counts: dict[str, int] = defaultdict(int)
|
|
591
|
+
for metrics in self.function_metrics.values():
|
|
592
|
+
# Group by top-level module
|
|
593
|
+
module = metrics.module.split("/")[0] if "/" in metrics.module else metrics.module
|
|
594
|
+
module_call_counts[module] += metrics.fan_in
|
|
595
|
+
|
|
596
|
+
core_modules = sorted(
|
|
597
|
+
module_call_counts.items(),
|
|
598
|
+
key=lambda x: x[1],
|
|
599
|
+
reverse=True,
|
|
600
|
+
)[:5]
|
|
601
|
+
|
|
602
|
+
if core_modules:
|
|
603
|
+
self._phase_result = f"{len(core_modules)} core modules"
|
|
604
|
+
|
|
605
|
+
# Identify tight coupling patterns
|
|
606
|
+
high_fanout = sorted(
|
|
607
|
+
list(self.function_metrics.values()) + self.high_fanout_functions,
|
|
608
|
+
key=lambda m: m.fan_out,
|
|
609
|
+
reverse=True,
|
|
610
|
+
)[:10]
|
|
611
|
+
|
|
612
|
+
for func in high_fanout:
|
|
613
|
+
if func.fan_out > 50 and func.name != "__init__":
|
|
614
|
+
self.issues.append(
|
|
615
|
+
f"[HIGH] {func.name} has high fan-out ({func.fan_out} calls) "
|
|
616
|
+
"-- consider breaking into smaller functions"
|
|
617
|
+
)
|
|
618
|
+
|
|
619
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
620
|
+
logger.warning(f"Pattern detection incomplete: {e}")
|
|
621
|
+
|
|
622
|
+
def _analyze_module_coupling(self) -> None:
|
|
623
|
+
"""Phase 6: Analyze module-level coupling and dependencies.
|
|
624
|
+
|
|
625
|
+
Uses IMPORTS edges to identify module interdependencies and
|
|
626
|
+
calculate cohesion metrics.
|
|
627
|
+
|
|
628
|
+
Two bulk SQL queries replace the former per-module loop so the phase
|
|
629
|
+
runs in O(1) round-trips regardless of module count.
|
|
630
|
+
"""
|
|
631
|
+
try:
|
|
632
|
+
con = self.kg.store.con
|
|
633
|
+
|
|
634
|
+
# Query all modules directly from the store — reliable, no semantic k-limit
|
|
635
|
+
module_rows = con.execute(
|
|
636
|
+
"SELECT id, module_path FROM nodes WHERE kind = 'module' ORDER BY module_path"
|
|
637
|
+
).fetchall()
|
|
638
|
+
|
|
639
|
+
# ── Bulk import-edge query ──────────────────────────────────────────
|
|
640
|
+
# One pass through IMPORTS→RESOLVES_TO to get (importer, importee)
|
|
641
|
+
# module-path pairs. Both incoming and outgoing can be derived from
|
|
642
|
+
# this single result set.
|
|
643
|
+
import_pairs = con.execute(
|
|
644
|
+
"""
|
|
645
|
+
SELECT DISTINCT ni.module_path AS importer, nd.module_path AS importee
|
|
646
|
+
FROM edges ei
|
|
647
|
+
JOIN nodes ns ON ei.dst = ns.id
|
|
648
|
+
JOIN edges er ON er.src = ns.id AND er.rel = 'RESOLVES_TO'
|
|
649
|
+
JOIN nodes nd ON er.dst = nd.id AND nd.module_path IS NOT NULL
|
|
650
|
+
JOIN nodes ni ON ei.src = ni.id AND ni.kind = 'module'
|
|
651
|
+
WHERE ei.rel = 'IMPORTS'
|
|
652
|
+
AND ni.module_path != nd.module_path
|
|
653
|
+
"""
|
|
654
|
+
).fetchall()
|
|
655
|
+
|
|
656
|
+
# Build lookup dicts from the bulk result
|
|
657
|
+
incoming_map: dict[str, set[str]] = defaultdict(set) # importee → importers
|
|
658
|
+
outgoing_map: dict[str, set[str]] = defaultdict(set) # importer → importees
|
|
659
|
+
for importer, importee in import_pairs:
|
|
660
|
+
if importer and importee:
|
|
661
|
+
incoming_map[importee].add(importer)
|
|
662
|
+
outgoing_map[importer].add(importee)
|
|
663
|
+
|
|
664
|
+
# ── Bulk node-count query ───────────────────────────────────────────
|
|
665
|
+
count_rows = con.execute(
|
|
666
|
+
"SELECT module_path, kind, COUNT(*) FROM nodes"
|
|
667
|
+
" WHERE kind IN ('function', 'class', 'method')"
|
|
668
|
+
" GROUP BY module_path, kind"
|
|
669
|
+
).fetchall()
|
|
670
|
+
|
|
671
|
+
kind_counts: dict[str, dict[str, int]] = defaultdict(dict)
|
|
672
|
+
for mod_path, kind, cnt in count_rows:
|
|
673
|
+
if mod_path:
|
|
674
|
+
kind_counts[mod_path][kind] = cnt
|
|
675
|
+
|
|
676
|
+
# ── Assemble metrics ────────────────────────────────────────────────
|
|
677
|
+
for _, module_path in module_rows:
|
|
678
|
+
module_path = module_path or "unknown"
|
|
679
|
+
incoming = list(incoming_map.get(module_path, set()))
|
|
680
|
+
outgoing = list(outgoing_map.get(module_path, set()))
|
|
681
|
+
cohesion = min(1.0, len(outgoing) / (len(incoming) + len(outgoing) + 1))
|
|
682
|
+
counts = kind_counts.get(module_path, {})
|
|
683
|
+
self.module_metrics[module_path] = ModuleMetrics(
|
|
684
|
+
path=module_path,
|
|
685
|
+
functions=counts.get("function", 0),
|
|
686
|
+
classes=counts.get("class", 0),
|
|
687
|
+
methods=counts.get("method", 0),
|
|
688
|
+
incoming_deps=incoming,
|
|
689
|
+
outgoing_deps=outgoing,
|
|
690
|
+
total_fan_in=len(incoming),
|
|
691
|
+
cohesion_score=cohesion,
|
|
692
|
+
)
|
|
693
|
+
|
|
694
|
+
self._phase_result = f"{len(self.module_metrics)} modules"
|
|
695
|
+
|
|
696
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
697
|
+
logger.warning(f"Module coupling analysis incomplete: {e}")
|
|
698
|
+
|
|
699
|
+
def _analyze_critical_paths(self) -> None:
|
|
700
|
+
"""Phase 7: Identify key call chains.
|
|
701
|
+
|
|
702
|
+
Traces real call paths: finds high-fan-in functions, follows CALLS
|
|
703
|
+
edges forward to build actual execution chains, and prepends one
|
|
704
|
+
representative caller for context.
|
|
705
|
+
"""
|
|
706
|
+
|
|
707
|
+
try:
|
|
708
|
+
# Start from high fan-in functions (exclude dunder methods).
|
|
709
|
+
top_functions = [
|
|
710
|
+
m
|
|
711
|
+
for m in sorted(
|
|
712
|
+
self.function_metrics.values(),
|
|
713
|
+
key=lambda m: m.fan_in,
|
|
714
|
+
reverse=True,
|
|
715
|
+
)
|
|
716
|
+
if not (m.name.startswith("__") and m.name.endswith("__"))
|
|
717
|
+
][:5]
|
|
718
|
+
|
|
719
|
+
for func in top_functions:
|
|
720
|
+
try:
|
|
721
|
+
callers = self.kg.callers(func.node_id, rel="CALLS")
|
|
722
|
+
|
|
723
|
+
# Trace forward from func through real CALLS edges.
|
|
724
|
+
chain_names = [func.name]
|
|
725
|
+
chain_modules = [func.module]
|
|
726
|
+
seen_ids: set[str] = {func.node_id}
|
|
727
|
+
current_id = func.node_id
|
|
728
|
+
|
|
729
|
+
for _ in range(6):
|
|
730
|
+
edges = self.kg.store.edges_from(current_id, rel="CALLS", limit=5)
|
|
731
|
+
callee = None
|
|
732
|
+
for edge in edges:
|
|
733
|
+
dst_id = edge["dst"]
|
|
734
|
+
# Resolve sym: stubs via RESOLVES_TO.
|
|
735
|
+
if dst_id.startswith("sym:"):
|
|
736
|
+
resolved = self.kg.store.con.execute(
|
|
737
|
+
"SELECT dst FROM edges"
|
|
738
|
+
" WHERE src = ? AND rel = 'RESOLVES_TO' LIMIT 1",
|
|
739
|
+
(dst_id,),
|
|
740
|
+
).fetchone()
|
|
741
|
+
if resolved:
|
|
742
|
+
dst_id = resolved[0]
|
|
743
|
+
if dst_id in seen_ids:
|
|
744
|
+
continue
|
|
745
|
+
node = self.kg.store.node(dst_id)
|
|
746
|
+
if node and node.get("module_path"):
|
|
747
|
+
callee = node
|
|
748
|
+
seen_ids.add(dst_id)
|
|
749
|
+
current_id = dst_id
|
|
750
|
+
break
|
|
751
|
+
if callee:
|
|
752
|
+
chain_names.append(callee.get("name", "?"))
|
|
753
|
+
chain_modules.append(callee.get("module_path", ""))
|
|
754
|
+
else:
|
|
755
|
+
break
|
|
756
|
+
|
|
757
|
+
# Prepend top caller to give the chain context.
|
|
758
|
+
if callers:
|
|
759
|
+
caller_module = callers[0].get("module_path", "")
|
|
760
|
+
chain_names = [callers[0].get("name", "?"), *chain_names]
|
|
761
|
+
chain_modules = [caller_module, *chain_modules]
|
|
762
|
+
|
|
763
|
+
crosses_module = len(set(chain_modules)) > 1
|
|
764
|
+
if len(chain_names) >= 4 or crosses_module:
|
|
765
|
+
call_chain = CallChain(
|
|
766
|
+
chain=chain_names,
|
|
767
|
+
depth=len(chain_names),
|
|
768
|
+
total_callers=len(callers),
|
|
769
|
+
)
|
|
770
|
+
self.critical_paths.append(call_chain)
|
|
771
|
+
|
|
772
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
773
|
+
logger.debug(f"Could not trace path for {func.name}: {e}")
|
|
774
|
+
|
|
775
|
+
self._phase_result = f"{len(self.critical_paths)} key call chains"
|
|
776
|
+
|
|
777
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
778
|
+
logger.warning(f"Call chain analysis incomplete: {e}")
|
|
779
|
+
|
|
780
|
+
def _identify_public_apis(self) -> None:
|
|
781
|
+
"""Phase 8: Identify public APIs (module-level exports).
|
|
782
|
+
|
|
783
|
+
Strategy (in priority order):
|
|
784
|
+
1. Parse every ``__init__.py`` under the repo root with AST to find
|
|
785
|
+
re-exported names (``from X import Y`` / ``__all__`` entries).
|
|
786
|
+
Look each name up in the graph and record it as a public API.
|
|
787
|
+
2. Supplement with non-private functions/classes in ``function_metrics``
|
|
788
|
+
that have at least one caller and are not already included.
|
|
789
|
+
"""
|
|
790
|
+
|
|
791
|
+
try:
|
|
792
|
+
import ast as _ast # pylint: disable=import-outside-toplevel
|
|
793
|
+
|
|
794
|
+
already_ids: set[str] = set()
|
|
795
|
+
|
|
796
|
+
# --- Step 1: walk __init__.py files and collect re-exported names ---
|
|
797
|
+
try:
|
|
798
|
+
repo_root = Path(self.kg.repo_root)
|
|
799
|
+
for init_path in sorted(repo_root.rglob("__init__.py")):
|
|
800
|
+
try:
|
|
801
|
+
src = init_path.read_text(encoding="utf-8")
|
|
802
|
+
tree = _ast.parse(src)
|
|
803
|
+
except (OSError, SyntaxError):
|
|
804
|
+
continue
|
|
805
|
+
|
|
806
|
+
exported_names: set[str] = set()
|
|
807
|
+
|
|
808
|
+
for node in _ast.walk(tree):
|
|
809
|
+
# from X import Y, Z → Y, Z are re-exported
|
|
810
|
+
if isinstance(node, _ast.ImportFrom):
|
|
811
|
+
for alias in node.names:
|
|
812
|
+
name = alias.asname or alias.name
|
|
813
|
+
if name and not name.startswith("_"):
|
|
814
|
+
exported_names.add(name)
|
|
815
|
+
# __all__ = ["Foo", "bar"]
|
|
816
|
+
elif (
|
|
817
|
+
isinstance(node, _ast.Assign)
|
|
818
|
+
and any(
|
|
819
|
+
isinstance(t, _ast.Name) and t.id == "__all__" for t in node.targets
|
|
820
|
+
)
|
|
821
|
+
and isinstance(node.value, _ast.List | _ast.Tuple)
|
|
822
|
+
):
|
|
823
|
+
for elt in node.value.elts:
|
|
824
|
+
if isinstance(elt, _ast.Constant) and isinstance(elt.value, str):
|
|
825
|
+
exported_names.add(elt.value)
|
|
826
|
+
|
|
827
|
+
for name in exported_names:
|
|
828
|
+
rows = self.kg.store.con.execute(
|
|
829
|
+
"SELECT id, name, kind, module_path, docstring FROM nodes"
|
|
830
|
+
" WHERE name = ? AND kind IN ('class', 'function')"
|
|
831
|
+
" AND SUBSTR(name, 1, 1) != '_'",
|
|
832
|
+
(name,),
|
|
833
|
+
).fetchall()
|
|
834
|
+
for row in rows:
|
|
835
|
+
node_id, nm, kind, module_path, docstring = row
|
|
836
|
+
if node_id not in already_ids:
|
|
837
|
+
try:
|
|
838
|
+
fan_in = len(self.kg.callers(node_id, rel="CALLS"))
|
|
839
|
+
except (AttributeError, ValueError, RuntimeError):
|
|
840
|
+
fan_in = 0
|
|
841
|
+
self.public_apis.append(
|
|
842
|
+
FunctionMetrics(
|
|
843
|
+
node_id=node_id,
|
|
844
|
+
name=nm,
|
|
845
|
+
module=module_path or "",
|
|
846
|
+
kind=kind,
|
|
847
|
+
fan_in=fan_in,
|
|
848
|
+
fan_out=0,
|
|
849
|
+
lines=0,
|
|
850
|
+
docstring=docstring,
|
|
851
|
+
)
|
|
852
|
+
)
|
|
853
|
+
already_ids.add(node_id)
|
|
854
|
+
except (OSError, AttributeError, ValueError, RuntimeError):
|
|
855
|
+
pass
|
|
856
|
+
|
|
857
|
+
# --- Step 2: supplement from function_metrics ---
|
|
858
|
+
for func in sorted(
|
|
859
|
+
self.function_metrics.values(), key=lambda m: m.fan_in, reverse=True
|
|
860
|
+
):
|
|
861
|
+
if (
|
|
862
|
+
func.kind in ("function", "class")
|
|
863
|
+
and func.fan_in >= 1
|
|
864
|
+
and not func.name.startswith("_")
|
|
865
|
+
and func.node_id not in already_ids
|
|
866
|
+
):
|
|
867
|
+
self.public_apis.append(func)
|
|
868
|
+
already_ids.add(func.node_id)
|
|
869
|
+
|
|
870
|
+
# Sort by fan-in descending
|
|
871
|
+
self.public_apis.sort(key=lambda m: m.fan_in, reverse=True)
|
|
872
|
+
|
|
873
|
+
self._phase_result = f"{len(self.public_apis)} public API functions"
|
|
874
|
+
|
|
875
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
876
|
+
logger.warning(f"Public API identification incomplete: {e}")
|
|
877
|
+
|
|
878
|
+
def _analyze_docstring_coverage(self) -> None:
|
|
879
|
+
"""Phase 9: Measure docstring coverage across the codebase.
|
|
880
|
+
|
|
881
|
+
Queries the SQLite nodes table directly to count how many
|
|
882
|
+
functions, methods, classes, and modules have a non-empty
|
|
883
|
+
docstring versus their total. Results are stored in
|
|
884
|
+
``self.docstring_coverage`` for use by ``_generate_insights``
|
|
885
|
+
and ``_write_report``.
|
|
886
|
+
"""
|
|
887
|
+
|
|
888
|
+
try:
|
|
889
|
+
if not hasattr(self.kg, "_store"):
|
|
890
|
+
self.console.print("[yellow]WARN[/yellow] Docstring coverage skipped (no store)")
|
|
891
|
+
return
|
|
892
|
+
|
|
893
|
+
rows = self.kg._store.con.execute(
|
|
894
|
+
"""
|
|
895
|
+
SELECT
|
|
896
|
+
kind,
|
|
897
|
+
COUNT(*) AS total,
|
|
898
|
+
SUM(
|
|
899
|
+
CASE WHEN docstring IS NOT NULL AND TRIM(docstring) != ''
|
|
900
|
+
THEN 1 ELSE 0 END
|
|
901
|
+
) AS with_doc
|
|
902
|
+
FROM nodes
|
|
903
|
+
WHERE kind IN ('function', 'method', 'class', 'module')
|
|
904
|
+
GROUP BY kind
|
|
905
|
+
ORDER BY kind
|
|
906
|
+
"""
|
|
907
|
+
).fetchall()
|
|
908
|
+
|
|
909
|
+
by_kind: dict[str, dict[str, int]] = {}
|
|
910
|
+
overall_total = 0
|
|
911
|
+
overall_with_doc = 0
|
|
912
|
+
|
|
913
|
+
for kind, total, with_doc in rows:
|
|
914
|
+
by_kind[kind] = {"total": total, "with_doc": with_doc}
|
|
915
|
+
overall_total += total
|
|
916
|
+
overall_with_doc += with_doc
|
|
917
|
+
|
|
918
|
+
overall_pct = (overall_with_doc / overall_total * 100) if overall_total else 0.0
|
|
919
|
+
|
|
920
|
+
self.docstring_coverage = {
|
|
921
|
+
"by_kind": by_kind,
|
|
922
|
+
"total": overall_total,
|
|
923
|
+
"with_doc": overall_with_doc,
|
|
924
|
+
"coverage_pct": round(overall_pct, 1),
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
self._phase_result = f"{overall_with_doc}/{overall_total} nodes ({overall_pct:.1f}%)"
|
|
928
|
+
|
|
929
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
930
|
+
logger.warning(f"Docstring coverage analysis incomplete: {e}")
|
|
931
|
+
self.console.print(f"[yellow]WARN[/yellow] Docstring coverage incomplete: {e}")
|
|
932
|
+
|
|
933
|
+
def _analyze_inheritance(self) -> None:
|
|
934
|
+
"""Phase 10: Analyze class inheritance hierarchy.
|
|
935
|
+
|
|
936
|
+
Queries all INHERITS edges from the SQLite store and builds a complete
|
|
937
|
+
class hierarchy. Reports depth per class, multiple-inheritance usage,
|
|
938
|
+
and diamond patterns (two distinct inheritance paths to a common
|
|
939
|
+
ancestor, which can cause MRO surprises in Python).
|
|
940
|
+
"""
|
|
941
|
+
|
|
942
|
+
try:
|
|
943
|
+
rows = self.kg.store.con.execute(
|
|
944
|
+
"SELECT src, dst FROM edges WHERE rel = 'INHERITS'"
|
|
945
|
+
).fetchall()
|
|
946
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
947
|
+
logger.warning(f"Inheritance analysis incomplete: {e}")
|
|
948
|
+
self.console.print(f"[yellow]WARN[/yellow] Inheritance analysis incomplete: {e}")
|
|
949
|
+
return
|
|
950
|
+
|
|
951
|
+
if not rows:
|
|
952
|
+
self.inheritance_analysis = {
|
|
953
|
+
"total_inherits_edges": 0,
|
|
954
|
+
"classes": [],
|
|
955
|
+
"max_depth": 0,
|
|
956
|
+
"multiple_inheritance": [],
|
|
957
|
+
"diamonds": [],
|
|
958
|
+
}
|
|
959
|
+
self._phase_result = "no inheritance edges"
|
|
960
|
+
return
|
|
961
|
+
|
|
962
|
+
# INHERITS edge direction: src = child (the subclass), dst = parent base class.
|
|
963
|
+
# Resolve sym: stubs to real class IDs via RESOLVES_TO edges (same pattern as
|
|
964
|
+
# callers_of) so that in-repo inheritance (e.g. AlliumLayout → Layout3D) is
|
|
965
|
+
# tracked correctly even when the base class is referenced through an import stub.
|
|
966
|
+
con = self.kg.store.con
|
|
967
|
+
|
|
968
|
+
def _resolve_dst(dst: str) -> list[str]:
|
|
969
|
+
"""Return real node IDs for *dst*, resolving sym: stubs if needed."""
|
|
970
|
+
if not dst.startswith("sym:"):
|
|
971
|
+
return [dst]
|
|
972
|
+
resolved = con.execute(
|
|
973
|
+
"SELECT dst FROM edges WHERE src = ? AND rel = 'RESOLVES_TO'",
|
|
974
|
+
(dst,),
|
|
975
|
+
).fetchall()
|
|
976
|
+
# Keep only class/module nodes; fall back to the stub if nothing resolves
|
|
977
|
+
real = [r[0] for r in resolved if not r[0].startswith("sym:")]
|
|
978
|
+
return real if real else [dst]
|
|
979
|
+
|
|
980
|
+
parents: dict[str, set[str]] = {} # child_id → set of parent_ids (resolved)
|
|
981
|
+
children: dict[str, set[str]] = {} # parent_id → set of child_ids
|
|
982
|
+
all_classes: set[str] = set()
|
|
983
|
+
raw_parent_count: dict[str, int] = {} # child_id → number of declared bases
|
|
984
|
+
|
|
985
|
+
for src, dst in rows:
|
|
986
|
+
raw_parent_count[src] = raw_parent_count.get(src, 0) + 1
|
|
987
|
+
for real_dst in _resolve_dst(dst):
|
|
988
|
+
parents.setdefault(src, set()).add(real_dst)
|
|
989
|
+
if not real_dst.startswith("sym:"):
|
|
990
|
+
children.setdefault(real_dst, set()).add(src)
|
|
991
|
+
all_classes.add(real_dst)
|
|
992
|
+
all_classes.add(src)
|
|
993
|
+
|
|
994
|
+
def _all_internal_ancestors(cls_id: str) -> set[str]:
|
|
995
|
+
"""BFS to collect all non-sym: ancestors of *cls_id*."""
|
|
996
|
+
visited: set[str] = set()
|
|
997
|
+
queue = list(parents.get(cls_id, set()))
|
|
998
|
+
while queue:
|
|
999
|
+
cur = queue.pop()
|
|
1000
|
+
if cur.startswith("sym:") or cur in visited:
|
|
1001
|
+
continue
|
|
1002
|
+
visited.add(cur)
|
|
1003
|
+
queue.extend(parents.get(cur, set()))
|
|
1004
|
+
return visited
|
|
1005
|
+
|
|
1006
|
+
def _compute_depth(cls_id: str, memo: dict[str, int]) -> int:
|
|
1007
|
+
if cls_id in memo:
|
|
1008
|
+
# -1 sentinel means "currently being computed" — cycle detected, return 0
|
|
1009
|
+
return max(memo[cls_id], 0)
|
|
1010
|
+
internal_ps = {p for p in parents.get(cls_id, set()) if not p.startswith("sym:")}
|
|
1011
|
+
if not internal_ps:
|
|
1012
|
+
memo[cls_id] = 0
|
|
1013
|
+
return 0
|
|
1014
|
+
memo[cls_id] = -1 # mark in-progress before recursing
|
|
1015
|
+
depth = 1 + max(_compute_depth(p, memo) for p in internal_ps)
|
|
1016
|
+
memo[cls_id] = depth
|
|
1017
|
+
return depth
|
|
1018
|
+
|
|
1019
|
+
depth_memo: dict[str, int] = {}
|
|
1020
|
+
class_data: list[dict] = []
|
|
1021
|
+
multiple_inheritance: list[dict] = []
|
|
1022
|
+
diamonds: list[dict] = []
|
|
1023
|
+
|
|
1024
|
+
for cls_id in sorted(all_classes):
|
|
1025
|
+
node = self.kg.node(cls_id)
|
|
1026
|
+
name = node.get("name", cls_id) if node else cls_id.split(":")[-1]
|
|
1027
|
+
module = node.get("module_path", "") if node else ""
|
|
1028
|
+
|
|
1029
|
+
cls_parents = parents.get(cls_id, set())
|
|
1030
|
+
internal_parents = {p for p in cls_parents if not p.startswith("sym:")}
|
|
1031
|
+
declared_count = raw_parent_count.get(cls_id, len(cls_parents))
|
|
1032
|
+
depth = _compute_depth(cls_id, depth_memo)
|
|
1033
|
+
|
|
1034
|
+
class_data.append(
|
|
1035
|
+
{
|
|
1036
|
+
"node_id": cls_id,
|
|
1037
|
+
"name": name,
|
|
1038
|
+
"module": module,
|
|
1039
|
+
"depth": depth,
|
|
1040
|
+
"parent_count": declared_count,
|
|
1041
|
+
"child_count": len(children.get(cls_id, set())),
|
|
1042
|
+
"external_bases": max(0, declared_count - len(internal_parents)),
|
|
1043
|
+
}
|
|
1044
|
+
)
|
|
1045
|
+
|
|
1046
|
+
# Multiple inheritance: more than one declared base
|
|
1047
|
+
if declared_count > 1:
|
|
1048
|
+
parent_names = []
|
|
1049
|
+
for p in sorted(cls_parents):
|
|
1050
|
+
pn = self.kg.node(p) if not p.startswith("sym:") else None
|
|
1051
|
+
parent_names.append(pn.get("name", p) if pn else p.split(":")[-1])
|
|
1052
|
+
multiple_inheritance.append(
|
|
1053
|
+
{
|
|
1054
|
+
"class": name,
|
|
1055
|
+
"module": module,
|
|
1056
|
+
"bases": sorted(parent_names),
|
|
1057
|
+
}
|
|
1058
|
+
)
|
|
1059
|
+
|
|
1060
|
+
# Diamond detection: two distinct internal parents sharing a common ancestor
|
|
1061
|
+
if len(internal_parents) >= 2:
|
|
1062
|
+
ip_list = sorted(internal_parents)
|
|
1063
|
+
ancestor_sets = [_all_internal_ancestors(p) | {p} for p in ip_list]
|
|
1064
|
+
reported = False
|
|
1065
|
+
for i in range(len(ancestor_sets)):
|
|
1066
|
+
if reported:
|
|
1067
|
+
break
|
|
1068
|
+
for j in range(i + 1, len(ancestor_sets)):
|
|
1069
|
+
shared = ancestor_sets[i] & ancestor_sets[j]
|
|
1070
|
+
if shared:
|
|
1071
|
+
shared_names = []
|
|
1072
|
+
for s in sorted(shared):
|
|
1073
|
+
sn = self.kg.node(s)
|
|
1074
|
+
shared_names.append(sn.get("name", s) if sn else s.split(":")[-1])
|
|
1075
|
+
diamonds.append(
|
|
1076
|
+
{
|
|
1077
|
+
"class": name,
|
|
1078
|
+
"module": module,
|
|
1079
|
+
"common_ancestors": shared_names,
|
|
1080
|
+
}
|
|
1081
|
+
)
|
|
1082
|
+
reported = True
|
|
1083
|
+
break
|
|
1084
|
+
|
|
1085
|
+
max_depth = max((e["depth"] for e in class_data), default=0)
|
|
1086
|
+
self.inheritance_analysis = {
|
|
1087
|
+
"total_inherits_edges": len(rows),
|
|
1088
|
+
"classes": sorted(class_data, key=lambda x: x["depth"], reverse=True),
|
|
1089
|
+
"max_depth": max_depth,
|
|
1090
|
+
"multiple_inheritance": multiple_inheritance,
|
|
1091
|
+
"diamonds": diamonds,
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
self._phase_result = f"{len(all_classes)} classes max-depth={max_depth}"
|
|
1095
|
+
|
|
1096
|
+
# -------------------------------------------------------------------------
|
|
1097
|
+
# Option A: CodeRank — computed early, reused by fan-in and critical paths
|
|
1098
|
+
# -------------------------------------------------------------------------
|
|
1099
|
+
|
|
1100
|
+
def _compute_coderank(self) -> None:
|
|
1101
|
+
"""Phase 1b: Compute global CodeRank (weighted PageRank) over the graph.
|
|
1102
|
+
|
|
1103
|
+
Builds a directed weighted graph from the SQLite store using
|
|
1104
|
+
``build_code_graph`` and runs ``compute_coderank``. Results are stored
|
|
1105
|
+
in ``self.coderank_scores`` (node_id → score) and
|
|
1106
|
+
``self.coderank_top_nodes`` (top-25 real nodes, sym: stubs excluded).
|
|
1107
|
+
|
|
1108
|
+
These scores are reused in Phase 2 (fan-in seed discovery) and Phase 7
|
|
1109
|
+
(critical path prioritisation) so that both phases are driven by
|
|
1110
|
+
structural importance rather than fragile semantic search.
|
|
1111
|
+
"""
|
|
1112
|
+
|
|
1113
|
+
try:
|
|
1114
|
+
from pycode_kg.ranking.coderank import ( # pylint: disable=import-outside-toplevel
|
|
1115
|
+
build_code_graph,
|
|
1116
|
+
compute_coderank,
|
|
1117
|
+
)
|
|
1118
|
+
|
|
1119
|
+
graph = build_code_graph(
|
|
1120
|
+
str(self.kg.db_path),
|
|
1121
|
+
include_relations=("CALLS", "IMPORTS", "INHERITS"),
|
|
1122
|
+
exclude_test_paths=True,
|
|
1123
|
+
)
|
|
1124
|
+
self.coderank_scores = compute_coderank(graph)
|
|
1125
|
+
|
|
1126
|
+
# Build top-25 real nodes (exclude sym: stubs) with node metadata
|
|
1127
|
+
sorted_nodes = sorted(self.coderank_scores.items(), key=lambda kv: kv[1], reverse=True)
|
|
1128
|
+
top_nodes: list[dict] = []
|
|
1129
|
+
for node_id, score in sorted_nodes:
|
|
1130
|
+
if node_id.startswith("sym:"):
|
|
1131
|
+
continue
|
|
1132
|
+
attrs = graph.nodes.get(node_id, {})
|
|
1133
|
+
kind = attrs.get("kind", "")
|
|
1134
|
+
if kind not in ("function", "method", "class", "module"):
|
|
1135
|
+
continue
|
|
1136
|
+
top_nodes.append(
|
|
1137
|
+
{
|
|
1138
|
+
"node_id": node_id,
|
|
1139
|
+
"score": score,
|
|
1140
|
+
"kind": kind,
|
|
1141
|
+
"name": attrs.get("name", node_id.split(":")[-1]),
|
|
1142
|
+
"qualname": attrs.get("qualname", ""),
|
|
1143
|
+
"module_path": attrs.get("module_path", ""),
|
|
1144
|
+
}
|
|
1145
|
+
)
|
|
1146
|
+
if len(top_nodes) >= 25:
|
|
1147
|
+
break
|
|
1148
|
+
|
|
1149
|
+
self.coderank_top_nodes = top_nodes
|
|
1150
|
+
|
|
1151
|
+
if top_nodes:
|
|
1152
|
+
self._phase_result = (
|
|
1153
|
+
f"{len(self.coderank_scores)} nodes top=`{top_nodes[0]['name']}`"
|
|
1154
|
+
)
|
|
1155
|
+
else:
|
|
1156
|
+
self._phase_result = f"{len(self.coderank_scores)} nodes"
|
|
1157
|
+
|
|
1158
|
+
except (AttributeError, ValueError, RuntimeError, ImportError) as e:
|
|
1159
|
+
logger.warning(f"CodeRank computation incomplete: {e}")
|
|
1160
|
+
self.console.print(f"[yellow]WARN[/yellow] CodeRank incomplete: {e}")
|
|
1161
|
+
|
|
1162
|
+
# -------------------------------------------------------------------------
|
|
1163
|
+
# Option D: Phase 14 — CodeRank top-nodes report section
|
|
1164
|
+
# -------------------------------------------------------------------------
|
|
1165
|
+
|
|
1166
|
+
def _analyze_coderank_section(self) -> None:
|
|
1167
|
+
"""Phase 14: Prepare CodeRank top-nodes for the report.
|
|
1168
|
+
|
|
1169
|
+
``self.coderank_top_nodes`` was already populated by ``_compute_coderank``
|
|
1170
|
+
in Phase 1b. This phase is a no-op if CodeRank failed; it exists so
|
|
1171
|
+
the report-writing logic has a clear hook and the phase numbering stays
|
|
1172
|
+
consistent.
|
|
1173
|
+
"""
|
|
1174
|
+
if self.coderank_top_nodes:
|
|
1175
|
+
self._phase_result = f"{len(self.coderank_top_nodes)} top nodes"
|
|
1176
|
+
else:
|
|
1177
|
+
self._phase_result = "skipped (no data)"
|
|
1178
|
+
|
|
1179
|
+
# -------------------------------------------------------------------------
|
|
1180
|
+
# Option D: Phase 15 — Concern-based hybrid ranking
|
|
1181
|
+
# -------------------------------------------------------------------------
|
|
1182
|
+
|
|
1183
|
+
def _analyze_concerns(self) -> None:
|
|
1184
|
+
"""Phase 15: Run query_ranked for architectural concern queries.
|
|
1185
|
+
|
|
1186
|
+
Executes ``rank_query_hybrid`` for a fixed set of architectural concern
|
|
1187
|
+
queries using the pre-computed global CodeRank scores as the centrality
|
|
1188
|
+
signal. For each concern, the top-5 structurally-dominant nodes are
|
|
1189
|
+
recorded.
|
|
1190
|
+
|
|
1191
|
+
Results are stored in ``self.concern_analysis`` as a list of dicts, where
|
|
1192
|
+
each entry contains a ``concern`` label (e.g. error handling, data
|
|
1193
|
+
persistence, graph traversal) and a ``top_nodes`` list of ranked result
|
|
1194
|
+
dicts with ``rank``, ``node_id``, ``name``, ``module``, ``kind``,
|
|
1195
|
+
``score``, and ``why`` fields.
|
|
1196
|
+
|
|
1197
|
+
Requires the LanceDB vector index to be available (``self.kg`` must
|
|
1198
|
+
support ``query()``). Degrades gracefully if the index is unavailable.
|
|
1199
|
+
"""
|
|
1200
|
+
|
|
1201
|
+
CONCERNS = [
|
|
1202
|
+
"configuration loading initialization setup",
|
|
1203
|
+
"data persistence storage database",
|
|
1204
|
+
"query search retrieval semantic",
|
|
1205
|
+
"graph traversal node edge",
|
|
1206
|
+
]
|
|
1207
|
+
|
|
1208
|
+
try:
|
|
1209
|
+
from pycode_kg.ranking.coderank import ( # pylint: disable=import-outside-toplevel
|
|
1210
|
+
build_code_graph,
|
|
1211
|
+
rank_query_hybrid,
|
|
1212
|
+
)
|
|
1213
|
+
|
|
1214
|
+
# Reuse the graph if already built; otherwise build a fresh one.
|
|
1215
|
+
# We need the full graph object for rank_query_hybrid.
|
|
1216
|
+
graph = build_code_graph(
|
|
1217
|
+
str(self.kg.db_path),
|
|
1218
|
+
include_relations=("CALLS", "IMPORTS", "INHERITS"),
|
|
1219
|
+
exclude_test_paths=True,
|
|
1220
|
+
)
|
|
1221
|
+
|
|
1222
|
+
results: list[dict] = []
|
|
1223
|
+
|
|
1224
|
+
for concern in CONCERNS:
|
|
1225
|
+
try:
|
|
1226
|
+
# Get semantic scores from the vector index.
|
|
1227
|
+
# Suppress tqdm progress bars from LanceDB embedding model loading.
|
|
1228
|
+
import os as _os # pylint: disable=import-outside-toplevel
|
|
1229
|
+
|
|
1230
|
+
_old_disable = _os.environ.get("TQDM_DISABLE")
|
|
1231
|
+
_os.environ["TQDM_DISABLE"] = "1"
|
|
1232
|
+
try:
|
|
1233
|
+
query_result = self.kg.query(concern, k=8, hop=0, rels=("CONTAINS",))
|
|
1234
|
+
finally:
|
|
1235
|
+
if _old_disable is None:
|
|
1236
|
+
_os.environ.pop("TQDM_DISABLE", None)
|
|
1237
|
+
else:
|
|
1238
|
+
_os.environ["TQDM_DISABLE"] = _old_disable
|
|
1239
|
+
semantic_scores: dict[str, float] = {}
|
|
1240
|
+
for node in query_result.nodes:
|
|
1241
|
+
node_id = node.get("id")
|
|
1242
|
+
score = (node.get("relevance") or {}).get("score", 0.0)
|
|
1243
|
+
if node_id and score > 0:
|
|
1244
|
+
semantic_scores[node_id] = float(score)
|
|
1245
|
+
|
|
1246
|
+
if not semantic_scores:
|
|
1247
|
+
logger.debug(f"No semantic seeds for concern: {concern!r}")
|
|
1248
|
+
continue
|
|
1249
|
+
|
|
1250
|
+
ranked = rank_query_hybrid(
|
|
1251
|
+
graph,
|
|
1252
|
+
semantic_scores,
|
|
1253
|
+
global_coderank=self.coderank_scores or None,
|
|
1254
|
+
radius=2,
|
|
1255
|
+
top_k=5,
|
|
1256
|
+
)
|
|
1257
|
+
|
|
1258
|
+
top_nodes = []
|
|
1259
|
+
for rank_idx, r in enumerate(ranked, 1):
|
|
1260
|
+
if r.kind not in ("function", "method", "class"):
|
|
1261
|
+
continue
|
|
1262
|
+
top_nodes.append(
|
|
1263
|
+
{
|
|
1264
|
+
"rank": rank_idx,
|
|
1265
|
+
"node_id": r.node_id,
|
|
1266
|
+
"name": r.qualname or r.node_id.split(":")[-1],
|
|
1267
|
+
"module": r.module_path or "",
|
|
1268
|
+
"kind": r.kind or "",
|
|
1269
|
+
"score": round(r.adjusted_score, 4),
|
|
1270
|
+
"why": list(r.why),
|
|
1271
|
+
}
|
|
1272
|
+
)
|
|
1273
|
+
if len(top_nodes) >= 5:
|
|
1274
|
+
break
|
|
1275
|
+
|
|
1276
|
+
if top_nodes:
|
|
1277
|
+
results.append({"concern": concern, "top_nodes": top_nodes})
|
|
1278
|
+
|
|
1279
|
+
except (AttributeError, ValueError, RuntimeError) as e:
|
|
1280
|
+
logger.debug(f"Concern query failed for {concern!r}: {e}")
|
|
1281
|
+
|
|
1282
|
+
self.concern_analysis = results
|
|
1283
|
+
self._phase_result = f"{len(results)}/{len(CONCERNS)} concerns resolved"
|
|
1284
|
+
|
|
1285
|
+
except (AttributeError, ValueError, RuntimeError, ImportError) as e:
|
|
1286
|
+
logger.warning(f"Concern analysis incomplete: {e}")
|
|
1287
|
+
self.console.print(f"[yellow]WARN[/yellow] Concern analysis incomplete: {e}")
|
|
1288
|
+
|
|
1289
|
+
# -------------------------------------------------------------------------
|
|
1290
|
+
# Phase 2 override: SQL-seeded fan-in using CodeRank top nodes (Option A)
|
|
1291
|
+
# -------------------------------------------------------------------------
|
|
1292
|
+
|
|
1293
|
+
def _analyze_centrality(self) -> None:
|
|
1294
|
+
"""Phase 13: Compute Structural Importance Ranking (SIR).
|
|
1295
|
+
|
|
1296
|
+
Runs weighted PageRank over the resolved graph using CALLS, INHERITS,
|
|
1297
|
+
IMPORTS, and CONTAINS edges, with cross-module and private-symbol
|
|
1298
|
+
adjustments. Results are stored in ``self.centrality_records`` for
|
|
1299
|
+
inclusion in reports and optional DB persistence.
|
|
1300
|
+
"""
|
|
1301
|
+
|
|
1302
|
+
try:
|
|
1303
|
+
from pycode_kg.analysis.centrality import ( # pylint: disable=import-outside-toplevel
|
|
1304
|
+
StructuralImportanceRanker,
|
|
1305
|
+
aggregate_module_scores,
|
|
1306
|
+
)
|
|
1307
|
+
|
|
1308
|
+
ranker = StructuralImportanceRanker(self.kg.db_path)
|
|
1309
|
+
# Compute all nodes (no top cap) so module aggregates are accurate,
|
|
1310
|
+
# then store top-25 for node-level display.
|
|
1311
|
+
all_records = ranker.compute()
|
|
1312
|
+
self.centrality_records = all_records[:25]
|
|
1313
|
+
self.centrality_modules = aggregate_module_scores(all_records)
|
|
1314
|
+
|
|
1315
|
+
self._phase_result = f"{len(all_records)} nodes {len(self.centrality_modules)} modules"
|
|
1316
|
+
except (AttributeError, ValueError, RuntimeError, ImportError) as e:
|
|
1317
|
+
logger.warning(f"Centrality analysis incomplete: {e}")
|
|
1318
|
+
self.console.print(f"[yellow]WARN[/yellow] Centrality analysis incomplete: {e}")
|
|
1319
|
+
|
|
1320
|
+
def _analyze_snapshots(self) -> None:
|
|
1321
|
+
"""Phase 12: Load snapshot history for temporal comparison.
|
|
1322
|
+
|
|
1323
|
+
Calls ``list_snapshots()`` on the SnapshotManager (if provided) and
|
|
1324
|
+
stores the most recent entries in ``self.snapshot_history`` for
|
|
1325
|
+
inclusion in the report. Silently skips when no SnapshotManager was
|
|
1326
|
+
supplied or the snapshots directory doesn't exist yet.
|
|
1327
|
+
"""
|
|
1328
|
+
if self.snapshot_mgr is None:
|
|
1329
|
+
self._phase_result = "skipped (no snapshot manager)"
|
|
1330
|
+
return
|
|
1331
|
+
|
|
1332
|
+
try:
|
|
1333
|
+
self.snapshot_history = self.snapshot_mgr.list_snapshots(limit=10)
|
|
1334
|
+
self._phase_result = f"{len(self.snapshot_history)} snapshot(s)"
|
|
1335
|
+
except (AttributeError, ValueError, RuntimeError, OSError) as e:
|
|
1336
|
+
logger.warning(f"Snapshot history unavailable: {e}")
|
|
1337
|
+
self.console.print(f"[yellow]WARN[/yellow] Snapshot history unavailable: {e}")
|
|
1338
|
+
|
|
1339
|
+
def _generate_insights(self) -> None:
|
|
1340
|
+
"""Phase 10: Generate actionable insights.
|
|
1341
|
+
|
|
1342
|
+
Compiles issues and strengths based on metrics collected
|
|
1343
|
+
in earlier phases.
|
|
1344
|
+
"""
|
|
1345
|
+
# Strengths
|
|
1346
|
+
if len(self.function_metrics) > 0:
|
|
1347
|
+
self.strengths.append(
|
|
1348
|
+
f"Well-structured with {len(self.function_metrics)} core functions identified"
|
|
1349
|
+
)
|
|
1350
|
+
|
|
1351
|
+
if len(self.orphaned_functions) == 0:
|
|
1352
|
+
self.strengths.append("No obvious dead code detected")
|
|
1353
|
+
|
|
1354
|
+
if len(self.high_fanout_functions) == 0:
|
|
1355
|
+
self.strengths.append("No god objects or god functions detected")
|
|
1356
|
+
|
|
1357
|
+
# Docstring coverage signals
|
|
1358
|
+
cov = self.docstring_coverage
|
|
1359
|
+
if cov:
|
|
1360
|
+
pct = cov["coverage_pct"]
|
|
1361
|
+
if pct >= 80:
|
|
1362
|
+
self.strengths.append(
|
|
1363
|
+
f"Good docstring coverage: {pct}% of functions/methods/classes/modules documented"
|
|
1364
|
+
)
|
|
1365
|
+
elif pct >= 50:
|
|
1366
|
+
self.issues.append(
|
|
1367
|
+
f"[WARN] Moderate docstring coverage ({pct}%) — semantic retrieval quality is degraded "
|
|
1368
|
+
"for undocumented nodes; BM25 is as effective as embeddings without docstrings"
|
|
1369
|
+
)
|
|
1370
|
+
else:
|
|
1371
|
+
self.issues.append(
|
|
1372
|
+
f"[LOW] Low docstring coverage ({pct}%) — semantic query quality will be poor; "
|
|
1373
|
+
"embedding undocumented nodes yields only structured identifiers, not NL-searchable text. "
|
|
1374
|
+
"Prioritize docstrings on high-fan-in functions first."
|
|
1375
|
+
)
|
|
1376
|
+
|
|
1377
|
+
# Centrality cross-reference insights
|
|
1378
|
+
if self.centrality_modules and self.module_metrics:
|
|
1379
|
+
# High-SIR modules that are also tightly coupled = highest architectural risk
|
|
1380
|
+
sir_by_path = {m["module_path"]: m for m in self.centrality_modules[:10]}
|
|
1381
|
+
risky = [
|
|
1382
|
+
m
|
|
1383
|
+
for path, m in sir_by_path.items()
|
|
1384
|
+
if path in self.module_metrics
|
|
1385
|
+
and (
|
|
1386
|
+
len(self.module_metrics[path].incoming_deps)
|
|
1387
|
+
+ len(self.module_metrics[path].outgoing_deps)
|
|
1388
|
+
)
|
|
1389
|
+
> 4
|
|
1390
|
+
]
|
|
1391
|
+
if risky:
|
|
1392
|
+
names = ", ".join(f"`{m['module_path'].split('/')[-1]}`" for m in risky[:3])
|
|
1393
|
+
self.issues.append(
|
|
1394
|
+
f"[WARN] High-SIR modules with tight coupling: {names} -- "
|
|
1395
|
+
"structurally central AND heavily connected; changes here ripple broadly"
|
|
1396
|
+
)
|
|
1397
|
+
else:
|
|
1398
|
+
top_name = self.centrality_modules[0]["module_path"].split("/")[-1]
|
|
1399
|
+
self.strengths.append(
|
|
1400
|
+
f"Top-ranked module (`{top_name}`) has manageable coupling -- "
|
|
1401
|
+
"structural importance is not compounded by excessive dependencies"
|
|
1402
|
+
)
|
|
1403
|
+
|
|
1404
|
+
# High-SIR nodes missing docstrings = highest-priority documentation targets
|
|
1405
|
+
if self.centrality_records:
|
|
1406
|
+
top_ids = [r.node_id for r in self.centrality_records[:15]]
|
|
1407
|
+
try:
|
|
1408
|
+
placeholders = ",".join("?" * len(top_ids))
|
|
1409
|
+
rows = self.kg.store.con.execute(
|
|
1410
|
+
f"SELECT id FROM nodes WHERE id IN ({placeholders})" # noqa: S608
|
|
1411
|
+
" AND (docstring IS NULL OR TRIM(docstring) = '')",
|
|
1412
|
+
top_ids,
|
|
1413
|
+
).fetchall()
|
|
1414
|
+
undocumented_ids = {row[0] for row in rows}
|
|
1415
|
+
undocumented = [
|
|
1416
|
+
r for r in self.centrality_records[:15] if r.node_id in undocumented_ids
|
|
1417
|
+
]
|
|
1418
|
+
if undocumented:
|
|
1419
|
+
names = ", ".join(f"`{r.name}`" for r in undocumented[:4])
|
|
1420
|
+
self.issues.append(
|
|
1421
|
+
f"[WARN] High-SIR nodes lack docstrings: {names} -- "
|
|
1422
|
+
"structurally critical but invisible to the semantic index; "
|
|
1423
|
+
"document these first for maximum retrieval improvement"
|
|
1424
|
+
)
|
|
1425
|
+
except (AttributeError, ValueError, RuntimeError):
|
|
1426
|
+
pass
|
|
1427
|
+
|
|
1428
|
+
# Issues
|
|
1429
|
+
if len(self.orphaned_functions) > 0:
|
|
1430
|
+
names = ", ".join(f"`{f.name}`" for f in self.orphaned_functions)
|
|
1431
|
+
self.issues.append(
|
|
1432
|
+
f"[WARN] {len(self.orphaned_functions)} orphaned functions found "
|
|
1433
|
+
f"({names}) -- consider archiving or documenting"
|
|
1434
|
+
)
|
|
1435
|
+
|
|
1436
|
+
if len(self.high_fanout_functions) > 0:
|
|
1437
|
+
self.issues.append(
|
|
1438
|
+
f"[WARN] {len(self.high_fanout_functions)} functions with high fan-out "
|
|
1439
|
+
"-- potential orchestrators or god objects"
|
|
1440
|
+
)
|
|
1441
|
+
|
|
1442
|
+
# Module size checks — flag modules with >30 functions/methods/classes
|
|
1443
|
+
try:
|
|
1444
|
+
large_modules = self.kg.store.con.execute(
|
|
1445
|
+
"""
|
|
1446
|
+
SELECT module_path, COUNT(*) AS cnt
|
|
1447
|
+
FROM nodes
|
|
1448
|
+
WHERE kind IN ('function', 'method', 'class')
|
|
1449
|
+
AND module_path IS NOT NULL
|
|
1450
|
+
GROUP BY module_path
|
|
1451
|
+
HAVING cnt > 30
|
|
1452
|
+
ORDER BY cnt DESC
|
|
1453
|
+
LIMIT 5
|
|
1454
|
+
"""
|
|
1455
|
+
).fetchall()
|
|
1456
|
+
for mod_path, cnt in large_modules:
|
|
1457
|
+
mod_name = mod_path.split("/")[-1] if mod_path else "?"
|
|
1458
|
+
self.issues.append(
|
|
1459
|
+
f"[WARN] `{mod_name}` has {cnt} functions/methods/classes "
|
|
1460
|
+
"-- consider splitting into focused submodules"
|
|
1461
|
+
)
|
|
1462
|
+
except (AttributeError, ValueError, RuntimeError):
|
|
1463
|
+
pass
|
|
1464
|
+
|
|
1465
|
+
# Inheritance insights
|
|
1466
|
+
inh = self.inheritance_analysis
|
|
1467
|
+
if inh:
|
|
1468
|
+
if inh.get("diamonds"):
|
|
1469
|
+
d_names = ", ".join(f"`{d['class']}`" for d in inh["diamonds"])
|
|
1470
|
+
self.issues.append(
|
|
1471
|
+
f"[WARN] Diamond inheritance detected: {d_names} -- "
|
|
1472
|
+
"verify MRO is intentional (C3 linearisation)"
|
|
1473
|
+
)
|
|
1474
|
+
if inh.get("max_depth", 0) > 3:
|
|
1475
|
+
self.issues.append(
|
|
1476
|
+
f"[WARN] Deep inheritance hierarchy (max depth {inh['max_depth']}) -- "
|
|
1477
|
+
"consider flattening via composition"
|
|
1478
|
+
)
|
|
1479
|
+
if inh.get("multiple_inheritance") and not inh.get("diamonds"):
|
|
1480
|
+
self.strengths.append(
|
|
1481
|
+
f"Multiple inheritance used in {len(inh['multiple_inheritance'])} class(es) "
|
|
1482
|
+
"without diamond patterns"
|
|
1483
|
+
)
|
|
1484
|
+
|
|
1485
|
+
self._phase_result = f"{len(self.issues)} issues {len(self.strengths)} strengths"
|
|
1486
|
+
|
|
1487
|
+
def _compute_quality_grade(self) -> tuple[float, str, str]:
|
|
1488
|
+
"""Compute an overall quality score, letter grade, and label.
|
|
1489
|
+
|
|
1490
|
+
Scoring (100 points total):
|
|
1491
|
+
- Docstring coverage (0–40 pts): ≥80% → 40, ≥50% → 20, else 0
|
|
1492
|
+
- Orphaned functions (0–25 pts): 0 → 25, 1–2 → 15, 3–5 → 5, else 0
|
|
1493
|
+
- High fan-out functions (0–20 pts): 0 → 20, 1–2 → 12, else 4
|
|
1494
|
+
- Circular dependencies (0–15 pts): 0 → 15, 1 → 8, else 0
|
|
1495
|
+
|
|
1496
|
+
:return: Tuple of (score 0–100, letter grade A–F, label string)
|
|
1497
|
+
"""
|
|
1498
|
+
score = 0.0
|
|
1499
|
+
|
|
1500
|
+
# Docstring coverage
|
|
1501
|
+
cov = self.docstring_coverage
|
|
1502
|
+
if cov:
|
|
1503
|
+
pct = cov.get("coverage_pct", 0)
|
|
1504
|
+
if pct >= 80:
|
|
1505
|
+
score += 40
|
|
1506
|
+
elif pct >= 50:
|
|
1507
|
+
score += 20
|
|
1508
|
+
|
|
1509
|
+
# Orphaned functions
|
|
1510
|
+
n_orphaned = len(self.orphaned_functions)
|
|
1511
|
+
if n_orphaned == 0:
|
|
1512
|
+
score += 25
|
|
1513
|
+
elif n_orphaned <= 2:
|
|
1514
|
+
score += 15
|
|
1515
|
+
elif n_orphaned <= 5:
|
|
1516
|
+
score += 5
|
|
1517
|
+
|
|
1518
|
+
# High fan-out functions
|
|
1519
|
+
n_fanout = len(self.high_fanout_functions)
|
|
1520
|
+
if n_fanout == 0:
|
|
1521
|
+
score += 20
|
|
1522
|
+
elif n_fanout <= 2:
|
|
1523
|
+
score += 12
|
|
1524
|
+
else:
|
|
1525
|
+
score += 4
|
|
1526
|
+
|
|
1527
|
+
# Circular dependencies
|
|
1528
|
+
n_circular = len(self.circular_deps)
|
|
1529
|
+
if n_circular == 0:
|
|
1530
|
+
score += 15
|
|
1531
|
+
elif n_circular == 1:
|
|
1532
|
+
score += 8
|
|
1533
|
+
|
|
1534
|
+
if score >= 90:
|
|
1535
|
+
grade, label = "A", "Excellent"
|
|
1536
|
+
elif score >= 75:
|
|
1537
|
+
grade, label = "B", "Good"
|
|
1538
|
+
elif score >= 60:
|
|
1539
|
+
grade, label = "C", "Fair"
|
|
1540
|
+
elif score >= 45:
|
|
1541
|
+
grade, label = "D", "Needs Work"
|
|
1542
|
+
else:
|
|
1543
|
+
grade, label = "F", "Critical"
|
|
1544
|
+
|
|
1545
|
+
return score, grade, label
|
|
1546
|
+
|
|
1547
|
+
def _build_recommendations(self) -> str:
|
|
1548
|
+
"""Build tailored recommendations from actual analysis results.
|
|
1549
|
+
|
|
1550
|
+
:return: Markdown string with immediate, medium-term, and long-term sections.
|
|
1551
|
+
"""
|
|
1552
|
+
immediate: list[str] = []
|
|
1553
|
+
medium: list[str] = []
|
|
1554
|
+
long_term: list[str] = []
|
|
1555
|
+
|
|
1556
|
+
cov = self.docstring_coverage
|
|
1557
|
+
if cov:
|
|
1558
|
+
pct = cov.get("coverage_pct", 0)
|
|
1559
|
+
if pct < 80:
|
|
1560
|
+
undocumented = cov.get("total", 0) - cov.get("with_doc", 0)
|
|
1561
|
+
immediate.append(
|
|
1562
|
+
f"**Improve docstring coverage** — {undocumented} nodes lack docstrings; "
|
|
1563
|
+
"prioritize high-fan-in functions and public APIs first for maximum semantic retrieval gain"
|
|
1564
|
+
)
|
|
1565
|
+
|
|
1566
|
+
if self.orphaned_functions:
|
|
1567
|
+
names = ", ".join(f"`{f.name}`" for f in self.orphaned_functions[:5])
|
|
1568
|
+
suffix = (
|
|
1569
|
+
f" (and {len(self.orphaned_functions) - 5} more)"
|
|
1570
|
+
if len(self.orphaned_functions) > 5
|
|
1571
|
+
else ""
|
|
1572
|
+
)
|
|
1573
|
+
immediate.append(
|
|
1574
|
+
f"**Remove or archive orphaned functions** — {names}{suffix} have zero callers "
|
|
1575
|
+
"and add maintenance burden"
|
|
1576
|
+
)
|
|
1577
|
+
|
|
1578
|
+
if self.high_fanout_functions:
|
|
1579
|
+
top = self.high_fanout_functions[0]
|
|
1580
|
+
immediate.append(
|
|
1581
|
+
f"**Refactor high fan-out orchestrators** — `{top.name}` calls {top.fan_out} functions; "
|
|
1582
|
+
"consider splitting into smaller, focused coordinators"
|
|
1583
|
+
)
|
|
1584
|
+
|
|
1585
|
+
if self.circular_deps:
|
|
1586
|
+
pairs = "; ".join(f"`{a}` ↔ `{b}`" for a, b in self.circular_deps[:3])
|
|
1587
|
+
immediate.append(
|
|
1588
|
+
f"**Resolve circular dependencies** — {pairs}; introduce an abstraction layer or invert the dependency"
|
|
1589
|
+
)
|
|
1590
|
+
|
|
1591
|
+
# High fan-in functions need documentation/thread-safety review
|
|
1592
|
+
top_fanin = sorted(self.function_metrics.values(), key=lambda m: m.fan_in, reverse=True)[:3]
|
|
1593
|
+
if top_fanin and top_fanin[0].fan_in > 1:
|
|
1594
|
+
names = ", ".join(f"`{m.name}`" for m in top_fanin)
|
|
1595
|
+
medium.append(
|
|
1596
|
+
f"**Harden high fan-in functions** — {names} are widely depended upon; "
|
|
1597
|
+
"review for thread safety, clear contracts, and stable interfaces"
|
|
1598
|
+
)
|
|
1599
|
+
|
|
1600
|
+
if self.module_metrics:
|
|
1601
|
+
tightly_coupled = [
|
|
1602
|
+
m
|
|
1603
|
+
for m in self.module_metrics.values()
|
|
1604
|
+
if len(m.incoming_deps) + len(m.outgoing_deps) > 5
|
|
1605
|
+
]
|
|
1606
|
+
if tightly_coupled:
|
|
1607
|
+
medium.append(
|
|
1608
|
+
"**Reduce module coupling** — consider splitting tightly coupled modules "
|
|
1609
|
+
"or introducing interface boundaries"
|
|
1610
|
+
)
|
|
1611
|
+
|
|
1612
|
+
if self.critical_paths:
|
|
1613
|
+
medium.append(
|
|
1614
|
+
"**Add tests for key call chains** — the identified call chains represent "
|
|
1615
|
+
"well-traveled execution paths that benefit most from regression coverage"
|
|
1616
|
+
)
|
|
1617
|
+
|
|
1618
|
+
if self.public_apis:
|
|
1619
|
+
long_term.append(
|
|
1620
|
+
"**Version and stabilize the public API** — document breaking-change policies "
|
|
1621
|
+
f"for {', '.join(f'`{a.name}`' for a in self.public_apis[:3])}"
|
|
1622
|
+
)
|
|
1623
|
+
|
|
1624
|
+
long_term.append(
|
|
1625
|
+
"**Enforce layer boundaries** — add linting or CI checks to prevent unexpected "
|
|
1626
|
+
"cross-module dependencies as the codebase grows"
|
|
1627
|
+
)
|
|
1628
|
+
long_term.append(
|
|
1629
|
+
"**Monitor hot paths** — instrument the high fan-in functions identified here "
|
|
1630
|
+
"to catch performance regressions early"
|
|
1631
|
+
)
|
|
1632
|
+
|
|
1633
|
+
if not immediate and not medium:
|
|
1634
|
+
immediate.append(
|
|
1635
|
+
"**Maintain current quality** — no critical issues detected; keep coverage and structure healthy"
|
|
1636
|
+
)
|
|
1637
|
+
|
|
1638
|
+
lines = []
|
|
1639
|
+
if immediate:
|
|
1640
|
+
lines.append("### Immediate Actions")
|
|
1641
|
+
for i, rec in enumerate(immediate, 1):
|
|
1642
|
+
lines.append(f"{i}. {rec}")
|
|
1643
|
+
lines.append("")
|
|
1644
|
+
if medium:
|
|
1645
|
+
lines.append("### Medium-term Refactoring")
|
|
1646
|
+
for i, rec in enumerate(medium, 1):
|
|
1647
|
+
lines.append(f"{i}. {rec}")
|
|
1648
|
+
lines.append("")
|
|
1649
|
+
if long_term:
|
|
1650
|
+
lines.append("### Long-term Architecture")
|
|
1651
|
+
for i, rec in enumerate(long_term, 1):
|
|
1652
|
+
lines.append(f"{i}. {rec}")
|
|
1653
|
+
return "\n".join(lines)
|
|
1654
|
+
|
|
1655
|
+
def _get_report_metadata(self, elapsed_seconds: float = 0.0) -> str:
|
|
1656
|
+
"""Build a Markdown metadata block for the top of the report.
|
|
1657
|
+
|
|
1658
|
+
Collects the generation timestamp, PyCodeKG package version, current Git
|
|
1659
|
+
commit SHA and branch, host platform details, and a summary of the
|
|
1660
|
+
graph snapshot metrics (total nodes/edges). All Git, import, and
|
|
1661
|
+
platform operations fail gracefully so the method is safe to call
|
|
1662
|
+
outside of a Git working tree or before the package is installed.
|
|
1663
|
+
|
|
1664
|
+
:param elapsed_seconds: Total analysis elapsed time in seconds
|
|
1665
|
+
:return: Formatted Markdown string to prepend to the report.
|
|
1666
|
+
"""
|
|
1667
|
+
# --- timestamp ---
|
|
1668
|
+
now = datetime.datetime.now(datetime.UTC)
|
|
1669
|
+
generated = now.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
1670
|
+
|
|
1671
|
+
# --- version ---
|
|
1672
|
+
version = "unknown"
|
|
1673
|
+
try:
|
|
1674
|
+
from pycode_kg import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel
|
|
1675
|
+
__version__ as _v,
|
|
1676
|
+
)
|
|
1677
|
+
|
|
1678
|
+
version = f"pycode-kg {_v}"
|
|
1679
|
+
except (ImportError, AttributeError):
|
|
1680
|
+
try:
|
|
1681
|
+
from importlib.metadata import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel
|
|
1682
|
+
version as _pkg_version,
|
|
1683
|
+
)
|
|
1684
|
+
|
|
1685
|
+
version = f"pycode-kg {_pkg_version('pycode-kg')}"
|
|
1686
|
+
except (ImportError, AttributeError):
|
|
1687
|
+
pass
|
|
1688
|
+
|
|
1689
|
+
# --- git commit (prefer CI env vars, fall back to subprocess) ---
|
|
1690
|
+
commit = os.environ.get("GITHUB_SHA", "")
|
|
1691
|
+
if commit:
|
|
1692
|
+
commit = commit[:7]
|
|
1693
|
+
else:
|
|
1694
|
+
try:
|
|
1695
|
+
result = subprocess.run(
|
|
1696
|
+
["git", "rev-parse", "--short", "HEAD"],
|
|
1697
|
+
capture_output=True,
|
|
1698
|
+
text=True,
|
|
1699
|
+
timeout=5,
|
|
1700
|
+
)
|
|
1701
|
+
if result.returncode == 0:
|
|
1702
|
+
commit = result.stdout.strip()
|
|
1703
|
+
except (OSError, FileNotFoundError):
|
|
1704
|
+
pass
|
|
1705
|
+
if not commit:
|
|
1706
|
+
commit = "unknown"
|
|
1707
|
+
|
|
1708
|
+
# --- git branch (prefer CI env vars, fall back to subprocess) ---
|
|
1709
|
+
branch = ""
|
|
1710
|
+
github_ref = os.environ.get("GITHUB_REF", "")
|
|
1711
|
+
if github_ref.startswith("refs/heads/"):
|
|
1712
|
+
branch = github_ref[len("refs/heads/") :]
|
|
1713
|
+
elif github_ref.startswith("refs/pull/"):
|
|
1714
|
+
# Pull-request ref: refs/pull/<number>/merge
|
|
1715
|
+
branch = github_ref
|
|
1716
|
+
if not branch:
|
|
1717
|
+
try:
|
|
1718
|
+
result = subprocess.run(
|
|
1719
|
+
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
|
1720
|
+
capture_output=True,
|
|
1721
|
+
text=True,
|
|
1722
|
+
timeout=5,
|
|
1723
|
+
)
|
|
1724
|
+
if result.returncode == 0:
|
|
1725
|
+
branch = result.stdout.strip()
|
|
1726
|
+
except (OSError, FileNotFoundError):
|
|
1727
|
+
pass
|
|
1728
|
+
if not branch:
|
|
1729
|
+
branch = "unknown"
|
|
1730
|
+
|
|
1731
|
+
commit_ref = f"{commit} ({branch})"
|
|
1732
|
+
|
|
1733
|
+
# --- platform ---
|
|
1734
|
+
try:
|
|
1735
|
+
_sys = platform.system()
|
|
1736
|
+
_mac = platform.mac_ver()[0]
|
|
1737
|
+
_os = f"macOS {_mac}" if _mac else f"{_sys} {platform.release()}"
|
|
1738
|
+
_arch = platform.machine() # arm64 / x86_64
|
|
1739
|
+
_cpu = platform.processor() or _arch # e.g. "arm" / "i386"
|
|
1740
|
+
_host = platform.node()
|
|
1741
|
+
_py = platform.python_version()
|
|
1742
|
+
plat = f"{_os} | {_arch} ({_cpu}) | {_host} | Python {_py}"
|
|
1743
|
+
except Exception: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
|
1744
|
+
plat = "unknown"
|
|
1745
|
+
|
|
1746
|
+
# --- graph snapshot metrics ---
|
|
1747
|
+
stats = self.stats or {}
|
|
1748
|
+
total_nodes = stats.get("total_nodes", "?")
|
|
1749
|
+
total_edges = stats.get("total_edges", "?")
|
|
1750
|
+
meaningful = stats.get("meaningful_nodes")
|
|
1751
|
+
graph_line = f"{total_nodes} nodes · {total_edges} edges"
|
|
1752
|
+
if meaningful is not None:
|
|
1753
|
+
graph_line += f" ({meaningful} meaningful)"
|
|
1754
|
+
|
|
1755
|
+
# --- included/excluded directories ---
|
|
1756
|
+
if self.include_dirs:
|
|
1757
|
+
dirs_line = ", ".join(sorted(self.include_dirs))
|
|
1758
|
+
else:
|
|
1759
|
+
dirs_line = "all"
|
|
1760
|
+
|
|
1761
|
+
if self.exclude_dirs:
|
|
1762
|
+
exclude_line = ", ".join(sorted(self.exclude_dirs))
|
|
1763
|
+
else:
|
|
1764
|
+
exclude_line = "none"
|
|
1765
|
+
|
|
1766
|
+
# --- elapsed time ---
|
|
1767
|
+
elapsed_str = ""
|
|
1768
|
+
if elapsed_seconds > 0:
|
|
1769
|
+
mins, secs = divmod(int(elapsed_seconds), 60)
|
|
1770
|
+
if mins > 0:
|
|
1771
|
+
elapsed_str = f"{mins}m {secs}s"
|
|
1772
|
+
else:
|
|
1773
|
+
elapsed_str = f"{secs}s"
|
|
1774
|
+
|
|
1775
|
+
return (
|
|
1776
|
+
"> **Analysis Report Metadata** \n"
|
|
1777
|
+
f"> - **Generated:** {generated} \n"
|
|
1778
|
+
f"> - **Version:** {version} \n"
|
|
1779
|
+
f"> - **Commit:** {commit_ref} \n"
|
|
1780
|
+
f"> - **Platform:** {plat} \n"
|
|
1781
|
+
f"> - **Graph:** {graph_line} \n"
|
|
1782
|
+
f"> - **Included directories:** {dirs_line} \n"
|
|
1783
|
+
f"> - **Excluded directories:** {exclude_line} \n"
|
|
1784
|
+
+ (f"> - **Elapsed time:** {elapsed_str} \n" if elapsed_str else "")
|
|
1785
|
+
+ "\n"
|
|
1786
|
+
)
|
|
1787
|
+
|
|
1788
|
+
def _write_report(self, report_path: str, elapsed_seconds: float = 0.0) -> None:
|
|
1789
|
+
"""Generate comprehensive markdown report with tables and analysis.
|
|
1790
|
+
|
|
1791
|
+
:param report_path: Path to write the markdown report to
|
|
1792
|
+
:param elapsed_seconds: Total analysis duration in seconds
|
|
1793
|
+
"""
|
|
1794
|
+
report_date = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S UTC")
|
|
1795
|
+
stats = self.stats
|
|
1796
|
+
repo_name = self.kg.repo_root.name
|
|
1797
|
+
quality_score, quality_grade, quality_label = self._compute_quality_grade()
|
|
1798
|
+
grade_emoji = {"A": "[A]", "B": "[B]", "C": "[C]", "D": "[D]", "F": "[F]"}.get(
|
|
1799
|
+
quality_grade, "[ ]"
|
|
1800
|
+
)
|
|
1801
|
+
|
|
1802
|
+
metadata_block = self._get_report_metadata(elapsed_seconds=elapsed_seconds)
|
|
1803
|
+
|
|
1804
|
+
# Build comprehensive report
|
|
1805
|
+
report = (
|
|
1806
|
+
metadata_block
|
|
1807
|
+
+ f"""# {repo_name} Analysis
|
|
1808
|
+
|
|
1809
|
+
**Generated:** {report_date}
|
|
1810
|
+
|
|
1811
|
+
---
|
|
1812
|
+
|
|
1813
|
+
## Executive Summary
|
|
1814
|
+
|
|
1815
|
+
This report provides a comprehensive architectural analysis of the **{repo_name}** repository using PyCodeKG's knowledge graph. The analysis covers complexity hotspots, module coupling, key call chains, and code quality signals to guide refactoring and architecture decisions.
|
|
1816
|
+
|
|
1817
|
+
| Overall Quality | Grade | Score |
|
|
1818
|
+
|----------------|-------|-------|
|
|
1819
|
+
| {grade_emoji} **{quality_label}** | **{quality_grade}** | {quality_score:.0f} / 100 |
|
|
1820
|
+
|
|
1821
|
+
---
|
|
1822
|
+
|
|
1823
|
+
## Baseline Metrics
|
|
1824
|
+
|
|
1825
|
+
| Metric | Value |
|
|
1826
|
+
|--------|-------|
|
|
1827
|
+
| **Total Nodes** | {stats.get("total_nodes", "N/A")} |
|
|
1828
|
+
| **Total Edges** | {stats.get("total_edges", "N/A")} |
|
|
1829
|
+
| **Modules** | {len(self.module_metrics)} (of {stats.get("node_counts", {}).get("module", "?")} total) |
|
|
1830
|
+
| **Functions** | {stats.get("node_counts", {}).get("function", "N/A")} |
|
|
1831
|
+
| **Classes** | {stats.get("node_counts", {}).get("class", "N/A")} |
|
|
1832
|
+
| **Methods** | {stats.get("node_counts", {}).get("method", "N/A")} |
|
|
1833
|
+
|
|
1834
|
+
### Edge Distribution
|
|
1835
|
+
|
|
1836
|
+
| Relationship Type | Count |
|
|
1837
|
+
|-------------------|-------|
|
|
1838
|
+
| CALLS | {stats.get("edge_counts", {}).get("CALLS", 0)} |
|
|
1839
|
+
| CONTAINS | {stats.get("edge_counts", {}).get("CONTAINS", 0)} |
|
|
1840
|
+
| IMPORTS | {stats.get("edge_counts", {}).get("IMPORTS", 0)} |
|
|
1841
|
+
| ATTR_ACCESS | {stats.get("edge_counts", {}).get("ATTR_ACCESS", 0)} |
|
|
1842
|
+
| INHERITS | {stats.get("edge_counts", {}).get("INHERITS", 0)} |
|
|
1843
|
+
|
|
1844
|
+
---
|
|
1845
|
+
|
|
1846
|
+
## Fan-In Ranking
|
|
1847
|
+
|
|
1848
|
+
Most-called functions are potential bottlenecks or core functionality. These functions are heavily depended upon across the codebase.
|
|
1849
|
+
|
|
1850
|
+
| # | Function | Module | Callers |
|
|
1851
|
+
|---|----------|--------|---------|
|
|
1852
|
+
"""
|
|
1853
|
+
)
|
|
1854
|
+
|
|
1855
|
+
for i, metrics in enumerate(
|
|
1856
|
+
sorted(self.function_metrics.values(), key=lambda m: m.fan_in, reverse=True)[:15],
|
|
1857
|
+
1,
|
|
1858
|
+
):
|
|
1859
|
+
report += f"| {i} | `{metrics.name}()` | {metrics.module} | **{metrics.fan_in}** |\n"
|
|
1860
|
+
|
|
1861
|
+
report += """
|
|
1862
|
+
|
|
1863
|
+
**Insight:** Functions with high fan-in are either core APIs or bottlenecks. Review these for:
|
|
1864
|
+
- Thread safety and performance
|
|
1865
|
+
- Clear documentation and contracts
|
|
1866
|
+
- Potential for breaking changes
|
|
1867
|
+
|
|
1868
|
+
---
|
|
1869
|
+
|
|
1870
|
+
## High Fan-Out Functions (Orchestrators)
|
|
1871
|
+
|
|
1872
|
+
Functions that call many others may indicate complex orchestration logic or poor separation of concerns.
|
|
1873
|
+
|
|
1874
|
+
"""
|
|
1875
|
+
|
|
1876
|
+
if self.high_fanout_functions:
|
|
1877
|
+
report += """| # | Function | Module | Calls | Type |
|
|
1878
|
+
|---|----------|--------|-------|------|
|
|
1879
|
+
"""
|
|
1880
|
+
for i, func in enumerate(
|
|
1881
|
+
sorted(self.high_fanout_functions, key=lambda f: f.fan_out, reverse=True)[:10],
|
|
1882
|
+
1,
|
|
1883
|
+
):
|
|
1884
|
+
func_type = "Orchestrator" if func.fan_out > 50 else "Coordinator"
|
|
1885
|
+
report += (
|
|
1886
|
+
f"| {i} | `{func.name}()` | {func.module} | "
|
|
1887
|
+
f"**{func.fan_out}** | {func_type} |\n"
|
|
1888
|
+
)
|
|
1889
|
+
report += "\n"
|
|
1890
|
+
else:
|
|
1891
|
+
report += "No extreme high fan-out functions detected. Well-balanced architecture.\n\n"
|
|
1892
|
+
|
|
1893
|
+
report += """---
|
|
1894
|
+
|
|
1895
|
+
## Module Architecture
|
|
1896
|
+
|
|
1897
|
+
Top modules by dependency coupling and cohesion (showing up to 10 with activity).
|
|
1898
|
+
Cohesion = incoming / (incoming + outgoing + 1); higher = more internally focused.
|
|
1899
|
+
|
|
1900
|
+
"""
|
|
1901
|
+
|
|
1902
|
+
if self.module_metrics:
|
|
1903
|
+
report += """| Module | Functions | Classes | Incoming | Outgoing | Cohesion |
|
|
1904
|
+
|--------|-----------|---------|----------|----------|----------|
|
|
1905
|
+
"""
|
|
1906
|
+
for module, module_metric in sorted(
|
|
1907
|
+
self.module_metrics.items(),
|
|
1908
|
+
key=lambda x: x[1].functions + x[1].classes + x[1].methods,
|
|
1909
|
+
reverse=True,
|
|
1910
|
+
)[:10]:
|
|
1911
|
+
report += (
|
|
1912
|
+
f"| `{module}` | {module_metric.functions} | {module_metric.classes} | "
|
|
1913
|
+
f"{len(module_metric.incoming_deps)} | {len(module_metric.outgoing_deps)} | "
|
|
1914
|
+
f"{module_metric.cohesion_score:.2f} |\n"
|
|
1915
|
+
)
|
|
1916
|
+
report += "\n"
|
|
1917
|
+
|
|
1918
|
+
report += """---
|
|
1919
|
+
|
|
1920
|
+
## Key Call Chains
|
|
1921
|
+
|
|
1922
|
+
Deepest call chains in the codebase.
|
|
1923
|
+
|
|
1924
|
+
"""
|
|
1925
|
+
|
|
1926
|
+
if self.critical_paths:
|
|
1927
|
+
for i, chain in enumerate(self.critical_paths[:5], 1):
|
|
1928
|
+
chain_str = " → ".join(chain.chain)
|
|
1929
|
+
report += f"**Chain {i}** (depth: {chain.depth})\n\n```\n{chain_str}\n```\n\n"
|
|
1930
|
+
else:
|
|
1931
|
+
report += "No deep call chains detected.\n\n"
|
|
1932
|
+
|
|
1933
|
+
report += """---
|
|
1934
|
+
|
|
1935
|
+
## Public API Surface
|
|
1936
|
+
|
|
1937
|
+
Identified public APIs (module-level functions with high usage).
|
|
1938
|
+
|
|
1939
|
+
"""
|
|
1940
|
+
|
|
1941
|
+
if self.public_apis:
|
|
1942
|
+
report += """| Function | Module | Fan-In | Type |
|
|
1943
|
+
|----------|--------|--------|------|
|
|
1944
|
+
"""
|
|
1945
|
+
for api in sorted(self.public_apis, key=lambda a: a.fan_in, reverse=True)[:10]:
|
|
1946
|
+
report += f"| `{api.name}()` | {api.module} | {api.fan_in} | {api.kind} |\n"
|
|
1947
|
+
else:
|
|
1948
|
+
report += "No public APIs identified.\n\n"
|
|
1949
|
+
|
|
1950
|
+
# --- Docstring Coverage section ---
|
|
1951
|
+
cov = self.docstring_coverage
|
|
1952
|
+
if cov:
|
|
1953
|
+
overall_pct = cov["coverage_pct"]
|
|
1954
|
+
pct_bar = "[OK]" if overall_pct >= 80 else "[WARN]" if overall_pct >= 50 else "[LOW]"
|
|
1955
|
+
report += """---
|
|
1956
|
+
|
|
1957
|
+
## Docstring Coverage
|
|
1958
|
+
|
|
1959
|
+
Docstring coverage directly determines semantic retrieval quality. Nodes without
|
|
1960
|
+
docstrings embed only structured identifiers (`KIND/NAME/QUALNAME/MODULE`), where
|
|
1961
|
+
keyword search is as effective as vector embeddings. The semantic model earns its
|
|
1962
|
+
value only when a docstring is present.
|
|
1963
|
+
|
|
1964
|
+
| Kind | Documented | Total | Coverage |
|
|
1965
|
+
|------|-----------|-------|----------|
|
|
1966
|
+
"""
|
|
1967
|
+
for kind in ("function", "method", "class", "module"):
|
|
1968
|
+
if kind in cov["by_kind"]:
|
|
1969
|
+
k = cov["by_kind"][kind]
|
|
1970
|
+
kind_pct = (k["with_doc"] / k["total"] * 100) if k["total"] else 0.0
|
|
1971
|
+
kind_bar = "[OK]" if kind_pct >= 80 else "[WARN]" if kind_pct >= 50 else "[LOW]"
|
|
1972
|
+
report += (
|
|
1973
|
+
f"| `{kind}` | {k['with_doc']} | {k['total']} | "
|
|
1974
|
+
f"{kind_bar} {kind_pct:.1f}% |\n"
|
|
1975
|
+
)
|
|
1976
|
+
report += (
|
|
1977
|
+
f"| **total** | **{cov['with_doc']}** | **{cov['total']}** | "
|
|
1978
|
+
f"**{pct_bar} {overall_pct:.1f}%** |\n\n"
|
|
1979
|
+
)
|
|
1980
|
+
|
|
1981
|
+
if overall_pct < 80:
|
|
1982
|
+
undocumented = cov["total"] - cov["with_doc"]
|
|
1983
|
+
report += (
|
|
1984
|
+
f"> **Recommendation:** {undocumented} nodes lack docstrings. "
|
|
1985
|
+
"Prioritize documenting high-fan-in functions and public API surface "
|
|
1986
|
+
"first — these have the highest impact on query accuracy.\n\n"
|
|
1987
|
+
)
|
|
1988
|
+
else:
|
|
1989
|
+
report += "---\n\n## Docstring Coverage\n\nCoverage data not available.\n\n"
|
|
1990
|
+
|
|
1991
|
+
# --- Structural Importance Ranking section (module-level) ---
|
|
1992
|
+
report += "---\n\n## Structural Importance Ranking (SIR)\n\n"
|
|
1993
|
+
if self.centrality_modules:
|
|
1994
|
+
report += (
|
|
1995
|
+
"Weighted PageRank aggregated by module — reveals architectural spine. "
|
|
1996
|
+
"Cross-module edges boosted 1.5×; private symbols penalized 0.85×. "
|
|
1997
|
+
"Node-level detail: `pycodekg centrality --top 25`\n\n"
|
|
1998
|
+
)
|
|
1999
|
+
report += "| Rank | Score | Members | Module |\n"
|
|
2000
|
+
report += "|------|-------|---------|--------|\n"
|
|
2001
|
+
for m in self.centrality_modules[:15]:
|
|
2002
|
+
report += (
|
|
2003
|
+
f"| {m['rank']} | {m['score']:.6f} | {m['member_count']} "
|
|
2004
|
+
f"| `{m['module_path']}` |\n"
|
|
2005
|
+
)
|
|
2006
|
+
report += "\n"
|
|
2007
|
+
else:
|
|
2008
|
+
report += "Centrality data not available.\n\n"
|
|
2009
|
+
|
|
2010
|
+
issues_text = (
|
|
2011
|
+
"\n".join(f"- {issue}" for issue in self.issues)
|
|
2012
|
+
if self.issues
|
|
2013
|
+
else "- No major issues detected"
|
|
2014
|
+
)
|
|
2015
|
+
strengths_text = (
|
|
2016
|
+
"\n".join(f"- {strength}" for strength in self.strengths)
|
|
2017
|
+
if self.strengths
|
|
2018
|
+
else "- Continue monitoring code quality"
|
|
2019
|
+
)
|
|
2020
|
+
|
|
2021
|
+
report += f"""
|
|
2022
|
+
|
|
2023
|
+
---
|
|
2024
|
+
|
|
2025
|
+
## Code Quality Issues
|
|
2026
|
+
|
|
2027
|
+
{issues_text}
|
|
2028
|
+
|
|
2029
|
+
---
|
|
2030
|
+
|
|
2031
|
+
## Architectural Strengths
|
|
2032
|
+
|
|
2033
|
+
{strengths_text}
|
|
2034
|
+
|
|
2035
|
+
---
|
|
2036
|
+
|
|
2037
|
+
## Recommendations
|
|
2038
|
+
|
|
2039
|
+
{self._build_recommendations()}
|
|
2040
|
+
|
|
2041
|
+
---
|
|
2042
|
+
|
|
2043
|
+
## Inheritance Hierarchy
|
|
2044
|
+
|
|
2045
|
+
"""
|
|
2046
|
+
inh = self.inheritance_analysis
|
|
2047
|
+
if inh and inh.get("total_inherits_edges", 0) > 0:
|
|
2048
|
+
report += (
|
|
2049
|
+
f"**{inh['total_inherits_edges']}** INHERITS edges across "
|
|
2050
|
+
f"**{len(inh['classes'])}** classes. "
|
|
2051
|
+
f"Max depth: **{inh['max_depth']}**.\n\n"
|
|
2052
|
+
)
|
|
2053
|
+
report += "| Class | Module | Depth | Parents | Children |\n"
|
|
2054
|
+
report += "|-------|--------|-------|---------|----------|\n"
|
|
2055
|
+
for cls in inh["classes"][:20]:
|
|
2056
|
+
report += (
|
|
2057
|
+
f"| `{cls['name']}` | {cls['module']} "
|
|
2058
|
+
f"| {cls['depth']} | {cls['parent_count']} | {cls['child_count']} |\n"
|
|
2059
|
+
)
|
|
2060
|
+
if inh.get("multiple_inheritance"):
|
|
2061
|
+
report += (
|
|
2062
|
+
f"\n### Multiple Inheritance ({len(inh['multiple_inheritance'])} classes)\n\n"
|
|
2063
|
+
)
|
|
2064
|
+
for mi in inh["multiple_inheritance"]:
|
|
2065
|
+
bases = ", ".join(f"`{b}`" for b in mi["bases"])
|
|
2066
|
+
report += f"- `{mi['class']}` ({mi['module']}) inherits from {bases}\n"
|
|
2067
|
+
if inh.get("diamonds"):
|
|
2068
|
+
report += f"\n### Diamond Patterns ({len(inh['diamonds'])} detected)\n\n"
|
|
2069
|
+
for d in inh["diamonds"]:
|
|
2070
|
+
common = ", ".join(f"`{a}`" for a in d["common_ancestors"])
|
|
2071
|
+
report += f"- `{d['class']}` ({d['module']}) — common ancestor(s): {common}\n"
|
|
2072
|
+
else:
|
|
2073
|
+
report += "No inheritance edges (no class hierarchies).\n"
|
|
2074
|
+
|
|
2075
|
+
# --- Snapshot History section ---
|
|
2076
|
+
report += "\n\n---\n\n## Snapshot History\n\n"
|
|
2077
|
+
if self.snapshot_history:
|
|
2078
|
+
report += (
|
|
2079
|
+
"Recent snapshots in reverse chronological order. "
|
|
2080
|
+
"Δ columns show change vs. the immediately preceding snapshot.\n\n"
|
|
2081
|
+
)
|
|
2082
|
+
report += (
|
|
2083
|
+
"| # | Timestamp | Branch | Version | Nodes | Edges | Coverage"
|
|
2084
|
+
" | Δ Nodes | Δ Edges | Δ Coverage |\n"
|
|
2085
|
+
)
|
|
2086
|
+
report += (
|
|
2087
|
+
"|---|-----------|--------|---------|-------|-------|----------"
|
|
2088
|
+
"|---------|---------|------------|\n"
|
|
2089
|
+
)
|
|
2090
|
+
for i, snap in enumerate(self.snapshot_history, 1):
|
|
2091
|
+
ts = snap.get("timestamp", "")[:19].replace("T", " ")
|
|
2092
|
+
branch = snap.get("branch", "?")
|
|
2093
|
+
version = snap.get("version", "?")
|
|
2094
|
+
m = snap.get("metrics", {})
|
|
2095
|
+
nodes = m.get("total_nodes", "?")
|
|
2096
|
+
edges = m.get("total_edges", "?")
|
|
2097
|
+
cov_raw = m.get("docstring_coverage", None)
|
|
2098
|
+
cov_str = f"{cov_raw * 100:.1f}%" if cov_raw is not None else "?"
|
|
2099
|
+
|
|
2100
|
+
delta = (snap.get("deltas") or {}).get("vs_previous") or {}
|
|
2101
|
+
dn = delta.get("nodes")
|
|
2102
|
+
de = delta.get("edges")
|
|
2103
|
+
dc = delta.get("coverage_delta")
|
|
2104
|
+
dn_str = f"{dn:+d}" if dn is not None else "—"
|
|
2105
|
+
de_str = f"{de:+d}" if de is not None else "—"
|
|
2106
|
+
dc_str = f"{dc * 100:+.1f}%" if dc is not None else "—"
|
|
2107
|
+
|
|
2108
|
+
report += (
|
|
2109
|
+
f"| {i} | {ts} | {branch} | {version} | {nodes} | {edges}"
|
|
2110
|
+
f" | {cov_str} | {dn_str} | {de_str} | {dc_str} |\n"
|
|
2111
|
+
)
|
|
2112
|
+
else:
|
|
2113
|
+
report += "No snapshots found. Run `pycodekg snapshot save <version>` to capture one.\n"
|
|
2114
|
+
|
|
2115
|
+
report += """
|
|
2116
|
+
|
|
2117
|
+
---
|
|
2118
|
+
|
|
2119
|
+
## Appendix: Orphaned Code
|
|
2120
|
+
|
|
2121
|
+
Functions with zero callers (potential dead code):
|
|
2122
|
+
|
|
2123
|
+
"""
|
|
2124
|
+
|
|
2125
|
+
if self.orphaned_functions:
|
|
2126
|
+
report += """| Function | Module | Lines |
|
|
2127
|
+
|----------|--------|-------|
|
|
2128
|
+
"""
|
|
2129
|
+
for func in sorted(self.orphaned_functions, key=lambda f: f.lines, reverse=True)[:15]:
|
|
2130
|
+
report += f"| `{func.name}()` | {func.module} | {func.lines} |\n"
|
|
2131
|
+
else:
|
|
2132
|
+
report += "No orphaned functions detected.\n"
|
|
2133
|
+
|
|
2134
|
+
# --- CodeRank Top Nodes section (Option D) ---
|
|
2135
|
+
report += "---\n\n## CodeRank -- Global Structural Importance\n\n"
|
|
2136
|
+
if self.coderank_top_nodes:
|
|
2137
|
+
report += (
|
|
2138
|
+
"Weighted PageRank over CALLS + IMPORTS + INHERITS edges "
|
|
2139
|
+
"(test paths excluded). Scores are normalized to sum to 1.0. "
|
|
2140
|
+
"This ranking seeds Phase 2 fan-in discovery and Phase 15 concern queries.\n\n"
|
|
2141
|
+
)
|
|
2142
|
+
report += "| Rank | Score | Kind | Name | Module |\n"
|
|
2143
|
+
report += "|------|-------|------|------|--------|\n"
|
|
2144
|
+
for i, n in enumerate(self.coderank_top_nodes[:20], 1):
|
|
2145
|
+
report += (
|
|
2146
|
+
f"| {i} | {n['score']:.6f} | {n['kind']} "
|
|
2147
|
+
f"| `{n['qualname'] or n['name']}` | {n['module_path']} |\n"
|
|
2148
|
+
)
|
|
2149
|
+
report += "\n"
|
|
2150
|
+
else:
|
|
2151
|
+
report += "CodeRank data not available.\n\n"
|
|
2152
|
+
|
|
2153
|
+
# --- Concern-based Hybrid Ranking section (Option D) ---
|
|
2154
|
+
report += "---\n\n## Concern-Based Hybrid Ranking\n\n"
|
|
2155
|
+
if self.concern_analysis:
|
|
2156
|
+
report += (
|
|
2157
|
+
"Top structurally-dominant nodes per architectural concern "
|
|
2158
|
+
"(0.60 × semantic + 0.25 × CodeRank + 0.15 × graph proximity).\n\n"
|
|
2159
|
+
)
|
|
2160
|
+
for entry in self.concern_analysis:
|
|
2161
|
+
concern = entry["concern"]
|
|
2162
|
+
report += f"### {concern.title()}\n\n"
|
|
2163
|
+
report += "| Rank | Score | Kind | Name | Module |\n"
|
|
2164
|
+
report += "|------|-------|------|------|--------|\n"
|
|
2165
|
+
for n in entry["top_nodes"]:
|
|
2166
|
+
report += (
|
|
2167
|
+
f"| {n['rank']} | {n['score']} | {n['kind']} "
|
|
2168
|
+
f"| `{n['name']}` | {n['module']} |\n"
|
|
2169
|
+
)
|
|
2170
|
+
report += "\n"
|
|
2171
|
+
else:
|
|
2172
|
+
report += "Concern analysis not available.\n\n"
|
|
2173
|
+
|
|
2174
|
+
elapsed_str = (
|
|
2175
|
+
f"{elapsed_seconds:.1f}s" if elapsed_seconds < 60 else f"{elapsed_seconds / 60:.1f}m"
|
|
2176
|
+
)
|
|
2177
|
+
report += f"""
|
|
2178
|
+
|
|
2179
|
+
---
|
|
2180
|
+
|
|
2181
|
+
*Report generated by PyCodeKG Thorough Analysis Tool — analysis completed in {elapsed_str}*
|
|
2182
|
+
"""
|
|
2183
|
+
|
|
2184
|
+
Path(report_path).write_text(report)
|
|
2185
|
+
self.console.print(f"[OK] Report written to {report_path}")
|
|
2186
|
+
|
|
2187
|
+
def _compile_results(self) -> dict:
|
|
2188
|
+
"""Compile analysis results into a dictionary.
|
|
2189
|
+
|
|
2190
|
+
Function metrics are sorted by risk descending (critical → high →
|
|
2191
|
+
medium → low) so the most dangerous functions appear first.
|
|
2192
|
+
|
|
2193
|
+
Module metrics exclude empty modules (no incoming or outgoing
|
|
2194
|
+
dependencies) to avoid cluttering output with __init__.py stubs.
|
|
2195
|
+
|
|
2196
|
+
:return: dictionary with all analysis data
|
|
2197
|
+
"""
|
|
2198
|
+
sorted_fn = sorted(
|
|
2199
|
+
self.function_metrics.items(),
|
|
2200
|
+
key=lambda kv: kv[1].fan_in,
|
|
2201
|
+
reverse=True,
|
|
2202
|
+
)
|
|
2203
|
+
active_modules = {
|
|
2204
|
+
k: v
|
|
2205
|
+
for k, v in self.module_metrics.items()
|
|
2206
|
+
if v.total_fan_in > 0 or len(v.outgoing_deps) > 0
|
|
2207
|
+
}
|
|
2208
|
+
return {
|
|
2209
|
+
"timestamp": datetime.datetime.now(datetime.UTC).isoformat(),
|
|
2210
|
+
"statistics": self.stats,
|
|
2211
|
+
"docstring_coverage": self.docstring_coverage,
|
|
2212
|
+
"function_metrics": {k: asdict(v) for k, v in sorted_fn},
|
|
2213
|
+
"module_metrics": {k: asdict(v) for k, v in active_modules.items()},
|
|
2214
|
+
"orphaned_functions": [asdict(f) for f in self.orphaned_functions],
|
|
2215
|
+
"high_fanout_functions": [asdict(f) for f in self.high_fanout_functions],
|
|
2216
|
+
"critical_paths": [asdict(c) for c in self.critical_paths],
|
|
2217
|
+
"public_apis": [asdict(a) for a in self.public_apis],
|
|
2218
|
+
"issues": self.issues,
|
|
2219
|
+
"strengths": self.strengths,
|
|
2220
|
+
"inheritance": self.inheritance_analysis,
|
|
2221
|
+
"snapshot_history": self.snapshot_history,
|
|
2222
|
+
"centrality": [
|
|
2223
|
+
{
|
|
2224
|
+
"rank": r.rank,
|
|
2225
|
+
"node_id": r.node_id,
|
|
2226
|
+
"kind": r.kind,
|
|
2227
|
+
"name": r.name,
|
|
2228
|
+
"module_path": r.module_path,
|
|
2229
|
+
"score": r.score,
|
|
2230
|
+
"inbound_count": r.inbound_count,
|
|
2231
|
+
"cross_module_inbound": r.cross_module_inbound,
|
|
2232
|
+
"rel_breakdown": r.rel_breakdown,
|
|
2233
|
+
}
|
|
2234
|
+
for r in self.centrality_records
|
|
2235
|
+
],
|
|
2236
|
+
"centrality_modules": self.centrality_modules,
|
|
2237
|
+
# Option A: CodeRank global scores (top-25 real nodes)
|
|
2238
|
+
"coderank_top_nodes": self.coderank_top_nodes,
|
|
2239
|
+
# Option D: concern-based hybrid ranking
|
|
2240
|
+
"concern_analysis": self.concern_analysis,
|
|
2241
|
+
}
|
|
2242
|
+
|
|
2243
|
+
def to_markdown(self) -> str:
|
|
2244
|
+
"""
|
|
2245
|
+
Render the analysis results as a Markdown context document.
|
|
2246
|
+
|
|
2247
|
+
Similar to `SnippetPack.to_markdown()`, this produces a structured
|
|
2248
|
+
Markdown document with sections for each analysis phase:
|
|
2249
|
+
baseline metrics, complexity hotspots, high fan-out functions,
|
|
2250
|
+
module architecture, critical paths, public APIs, docstring coverage,
|
|
2251
|
+
issues, and strengths.
|
|
2252
|
+
|
|
2253
|
+
The output is optimized for LLM ingestion with clear headers, tables,
|
|
2254
|
+
and organized sections.
|
|
2255
|
+
|
|
2256
|
+
:return: Markdown-formatted string representing the full analysis.
|
|
2257
|
+
"""
|
|
2258
|
+
out: list[str] = []
|
|
2259
|
+
stats = self.stats
|
|
2260
|
+
|
|
2261
|
+
out.append("# PyCodeKG Repository Analysis\n")
|
|
2262
|
+
out.append(f"**Generated:** {datetime.datetime.now(datetime.UTC).isoformat()} \n")
|
|
2263
|
+
out.append("\n---\n")
|
|
2264
|
+
|
|
2265
|
+
# Baseline Metrics
|
|
2266
|
+
out.append("## Baseline Metrics\n")
|
|
2267
|
+
out.append("| Metric | Value |")
|
|
2268
|
+
out.append("|--------|-------|")
|
|
2269
|
+
out.append(f"| Total Nodes | {stats.get('total_nodes', 'N/A')} |")
|
|
2270
|
+
out.append(f"| Total Edges | {stats.get('total_edges', 'N/A')} |")
|
|
2271
|
+
total_modules = stats.get("node_counts", {}).get("module", "?")
|
|
2272
|
+
out.append(f"| Modules | {len(self.module_metrics)} (of {total_modules} total) |")
|
|
2273
|
+
out.append(f"| Functions | {stats.get('node_counts', {}).get('function', 'N/A')} |")
|
|
2274
|
+
out.append(f"| Classes | {stats.get('node_counts', {}).get('class', 'N/A')} |")
|
|
2275
|
+
out.append(f"| Methods | {stats.get('node_counts', {}).get('method', 'N/A')} |")
|
|
2276
|
+
out.append("")
|
|
2277
|
+
|
|
2278
|
+
out.append("### Edge Distribution")
|
|
2279
|
+
out.append("| Relationship | Count |")
|
|
2280
|
+
out.append("|---|---|")
|
|
2281
|
+
for rel in ("CALLS", "CONTAINS", "IMPORTS", "ATTR_ACCESS", "INHERITS"):
|
|
2282
|
+
count = stats.get("edge_counts", {}).get(rel, 0)
|
|
2283
|
+
out.append(f"| {rel} | {count} |")
|
|
2284
|
+
out.append("")
|
|
2285
|
+
|
|
2286
|
+
# Fan-In Ranking
|
|
2287
|
+
out.append("## Fan-In Ranking\n")
|
|
2288
|
+
if self.function_metrics:
|
|
2289
|
+
out.append(
|
|
2290
|
+
"Most-called functions and methods "
|
|
2291
|
+
"(classes omitted — instantiation counts are not architectural signals).\n"
|
|
2292
|
+
)
|
|
2293
|
+
out.append("| # | Kind | Function | Module | Callers |")
|
|
2294
|
+
out.append("|---|---|---|---|---|")
|
|
2295
|
+
ranked_fan_in = [
|
|
2296
|
+
m
|
|
2297
|
+
for m in sorted(
|
|
2298
|
+
self.function_metrics.values(), key=lambda m: m.fan_in, reverse=True
|
|
2299
|
+
)
|
|
2300
|
+
if m.kind != "class"
|
|
2301
|
+
][:15]
|
|
2302
|
+
for i, metrics in enumerate(ranked_fan_in, 1):
|
|
2303
|
+
out.append(
|
|
2304
|
+
f"| {i} | {metrics.kind} | `{metrics.name}()` | {metrics.module} | {metrics.fan_in} |"
|
|
2305
|
+
)
|
|
2306
|
+
else:
|
|
2307
|
+
out.append("No high fan-in functions identified.\n")
|
|
2308
|
+
out.append("")
|
|
2309
|
+
|
|
2310
|
+
# High Fan-Out Functions
|
|
2311
|
+
out.append("## High Fan-Out Functions (Orchestrators)\n")
|
|
2312
|
+
if self.high_fanout_functions:
|
|
2313
|
+
out.append(
|
|
2314
|
+
"Functions that call many others may indicate complex orchestration logic.\n"
|
|
2315
|
+
)
|
|
2316
|
+
out.append("| # | Function | Module | Calls | Type |")
|
|
2317
|
+
out.append("|---|---|---|---|---|")
|
|
2318
|
+
for i, func in enumerate(
|
|
2319
|
+
sorted(self.high_fanout_functions, key=lambda f: f.fan_out, reverse=True)[:10],
|
|
2320
|
+
1,
|
|
2321
|
+
):
|
|
2322
|
+
func_type = "Orchestrator" if func.fan_out > 50 else "Coordinator"
|
|
2323
|
+
out.append(
|
|
2324
|
+
f"| {i} | `{func.name}()` | {func.module} | {func.fan_out} | {func_type} |"
|
|
2325
|
+
)
|
|
2326
|
+
else:
|
|
2327
|
+
out.append("No high fan-out functions detected. Well-balanced architecture.\n")
|
|
2328
|
+
out.append("")
|
|
2329
|
+
|
|
2330
|
+
# Module Architecture
|
|
2331
|
+
out.append("## Module Architecture\n")
|
|
2332
|
+
out.append(
|
|
2333
|
+
"Cohesion = incoming / (incoming + outgoing + 1); higher = more internally focused.\n"
|
|
2334
|
+
)
|
|
2335
|
+
if self.module_metrics:
|
|
2336
|
+
total_modules = len(self.module_metrics)
|
|
2337
|
+
cap = min(10, total_modules)
|
|
2338
|
+
out.append(
|
|
2339
|
+
f"Top modules by dependency coupling and cohesion (showing {cap} of {total_modules} total).\n"
|
|
2340
|
+
)
|
|
2341
|
+
out.append("| Module | Functions | Classes | Incoming | Outgoing | Cohesion |")
|
|
2342
|
+
out.append("|---|---|---|---|---|---|")
|
|
2343
|
+
for module, module_metric in sorted(
|
|
2344
|
+
self.module_metrics.items(),
|
|
2345
|
+
key=lambda x: x[1].functions + x[1].classes + x[1].methods,
|
|
2346
|
+
reverse=True,
|
|
2347
|
+
)[:cap]:
|
|
2348
|
+
out.append(
|
|
2349
|
+
f"| `{module}` | {module_metric.functions} | {module_metric.classes} | "
|
|
2350
|
+
f"{len(module_metric.incoming_deps)} | {len(module_metric.outgoing_deps)} | "
|
|
2351
|
+
f"{module_metric.cohesion_score:.2f} |"
|
|
2352
|
+
)
|
|
2353
|
+
else:
|
|
2354
|
+
out.append("No module metrics available.\n")
|
|
2355
|
+
out.append("")
|
|
2356
|
+
|
|
2357
|
+
# Key Call Chains
|
|
2358
|
+
out.append("## Key Call Chains\n")
|
|
2359
|
+
if self.critical_paths:
|
|
2360
|
+
out.append("Deepest call chains in the codebase.\n")
|
|
2361
|
+
for i, chain in enumerate(self.critical_paths[:5], 1):
|
|
2362
|
+
chain_str = " → ".join(chain.chain)
|
|
2363
|
+
out.append(f"**Chain {i}** (depth: {chain.depth})\n")
|
|
2364
|
+
out.append(f"```\n{chain_str}\n```\n")
|
|
2365
|
+
else:
|
|
2366
|
+
out.append("No deep call chains detected.\n")
|
|
2367
|
+
out.append("")
|
|
2368
|
+
|
|
2369
|
+
# Public API Surface
|
|
2370
|
+
out.append("## Public API Surface\n")
|
|
2371
|
+
if self.public_apis:
|
|
2372
|
+
out.append("| Function | Module | Fan-In |")
|
|
2373
|
+
out.append("|---|---|---|")
|
|
2374
|
+
for api in sorted(self.public_apis, key=lambda a: a.fan_in, reverse=True)[:10]:
|
|
2375
|
+
out.append(f"| `{api.name}()` | {api.module} | {api.fan_in} |")
|
|
2376
|
+
else:
|
|
2377
|
+
out.append("No public APIs identified.\n")
|
|
2378
|
+
out.append("")
|
|
2379
|
+
|
|
2380
|
+
# Docstring Coverage
|
|
2381
|
+
out.append("## Docstring Coverage\n")
|
|
2382
|
+
cov = self.docstring_coverage
|
|
2383
|
+
if cov:
|
|
2384
|
+
overall_pct = cov["coverage_pct"]
|
|
2385
|
+
pct_bar = "[OK]" if overall_pct >= 80 else "[WARN]" if overall_pct >= 50 else "[LOW]"
|
|
2386
|
+
out.append("| Kind | Documented | Total | Coverage |")
|
|
2387
|
+
out.append("|---|---|---|---|")
|
|
2388
|
+
for kind in ("function", "method", "class", "module"):
|
|
2389
|
+
if kind in cov["by_kind"]:
|
|
2390
|
+
k = cov["by_kind"][kind]
|
|
2391
|
+
kind_pct = (k["with_doc"] / k["total"] * 100) if k["total"] else 0.0
|
|
2392
|
+
kind_bar = "[OK]" if kind_pct >= 80 else "[WARN]" if kind_pct >= 50 else "[LOW]"
|
|
2393
|
+
out.append(
|
|
2394
|
+
f"| `{kind}` | {k['with_doc']} | {k['total']} | {kind_bar} {kind_pct:.1f}% |"
|
|
2395
|
+
)
|
|
2396
|
+
out.append(
|
|
2397
|
+
f"| **total** | **{cov['with_doc']}** | **{cov['total']}** | "
|
|
2398
|
+
f"**{pct_bar} {overall_pct:.1f}%** |"
|
|
2399
|
+
)
|
|
2400
|
+
else:
|
|
2401
|
+
out.append("Coverage data not available.\n")
|
|
2402
|
+
out.append("")
|
|
2403
|
+
|
|
2404
|
+
# Structural Importance Ranking (module-level)
|
|
2405
|
+
out.append("## Structural Importance Ranking (SIR)\n")
|
|
2406
|
+
if self.centrality_modules:
|
|
2407
|
+
out.append(
|
|
2408
|
+
"Weighted PageRank aggregated by module — reveals architectural spine. "
|
|
2409
|
+
"Cross-module edges boosted 1.5×; private symbols penalized 0.85×. "
|
|
2410
|
+
"Node-level detail: `pycodekg centrality --top 25`\n"
|
|
2411
|
+
)
|
|
2412
|
+
out.append("| Rank | Score | Members | Module |")
|
|
2413
|
+
out.append("|------|-------|---------|--------|")
|
|
2414
|
+
for m in self.centrality_modules[:15]:
|
|
2415
|
+
out.append(
|
|
2416
|
+
f"| {m['rank']} | {m['score']:.6f} | {m['member_count']} "
|
|
2417
|
+
f"| `{m['module_path']}` |"
|
|
2418
|
+
)
|
|
2419
|
+
else:
|
|
2420
|
+
out.append("Centrality data not available.\n")
|
|
2421
|
+
out.append("")
|
|
2422
|
+
|
|
2423
|
+
# Issues
|
|
2424
|
+
out.append("## Code Quality Issues\n")
|
|
2425
|
+
if self.issues:
|
|
2426
|
+
for issue in self.issues:
|
|
2427
|
+
out.append(f"- {issue}")
|
|
2428
|
+
else:
|
|
2429
|
+
out.append("- No major issues detected")
|
|
2430
|
+
out.append("")
|
|
2431
|
+
|
|
2432
|
+
# Strengths
|
|
2433
|
+
out.append("## Architectural Strengths\n")
|
|
2434
|
+
if self.strengths:
|
|
2435
|
+
for strength in self.strengths:
|
|
2436
|
+
out.append(f"- {strength}")
|
|
2437
|
+
else:
|
|
2438
|
+
out.append("- Continue monitoring code quality")
|
|
2439
|
+
out.append("")
|
|
2440
|
+
|
|
2441
|
+
# Inheritance Hierarchy
|
|
2442
|
+
inh = self.inheritance_analysis
|
|
2443
|
+
out.append("## Inheritance Hierarchy\n")
|
|
2444
|
+
if inh and inh.get("total_inherits_edges", 0) > 0:
|
|
2445
|
+
out.append(
|
|
2446
|
+
f"**{inh['total_inherits_edges']}** INHERITS edges across "
|
|
2447
|
+
f"**{len(inh['classes'])}** classes. "
|
|
2448
|
+
f"Max depth: **{inh['max_depth']}**.\n"
|
|
2449
|
+
)
|
|
2450
|
+
out.append("| Class | Module | Depth | Parents | Children |")
|
|
2451
|
+
out.append("|-------|--------|-------|---------|----------|")
|
|
2452
|
+
for cls in inh["classes"][:20]:
|
|
2453
|
+
out.append(
|
|
2454
|
+
f"| `{cls['name']}` | {cls['module']} "
|
|
2455
|
+
f"| {cls['depth']} | {cls['parent_count']} | {cls['child_count']} |"
|
|
2456
|
+
)
|
|
2457
|
+
out.append("")
|
|
2458
|
+
if inh.get("multiple_inheritance"):
|
|
2459
|
+
out.append(
|
|
2460
|
+
f"### Multiple Inheritance ({len(inh['multiple_inheritance'])} classes)\n"
|
|
2461
|
+
)
|
|
2462
|
+
for mi in inh["multiple_inheritance"]:
|
|
2463
|
+
bases = ", ".join(f"`{b}`" for b in mi["bases"])
|
|
2464
|
+
out.append(f"- `{mi['class']}` ({mi['module']}) inherits from {bases}")
|
|
2465
|
+
out.append("")
|
|
2466
|
+
if inh.get("diamonds"):
|
|
2467
|
+
out.append(f"### Diamond Patterns ({len(inh['diamonds'])} detected)\n")
|
|
2468
|
+
for d in inh["diamonds"]:
|
|
2469
|
+
common = ", ".join(f"`{a}`" for a in d["common_ancestors"])
|
|
2470
|
+
out.append(f"- `{d['class']}` ({d['module']}) — common ancestor(s): {common}")
|
|
2471
|
+
out.append("")
|
|
2472
|
+
else:
|
|
2473
|
+
out.append("No inheritance edges (no class hierarchies in this codebase).\n")
|
|
2474
|
+
out.append("")
|
|
2475
|
+
|
|
2476
|
+
# Snapshot History
|
|
2477
|
+
out.append("## Snapshot History\n")
|
|
2478
|
+
if self.snapshot_history:
|
|
2479
|
+
out.append(
|
|
2480
|
+
"Recent snapshots (reverse chronological). "
|
|
2481
|
+
"Δ columns show change vs. the immediately preceding snapshot.\n"
|
|
2482
|
+
)
|
|
2483
|
+
out.append(
|
|
2484
|
+
"| # | Timestamp | Branch | Version | Nodes | Edges | Coverage"
|
|
2485
|
+
" | Δ Nodes | Δ Edges | Δ Coverage |"
|
|
2486
|
+
)
|
|
2487
|
+
out.append(
|
|
2488
|
+
"|---|-----------|--------|---------|-------|-------|----------"
|
|
2489
|
+
"|---------|---------|------------|"
|
|
2490
|
+
)
|
|
2491
|
+
for i, snap in enumerate(self.snapshot_history, 1):
|
|
2492
|
+
ts = snap.get("timestamp", "")[:19].replace("T", " ")
|
|
2493
|
+
branch = snap.get("branch", "?")
|
|
2494
|
+
version = snap.get("version", "?")
|
|
2495
|
+
m = snap.get("metrics", {})
|
|
2496
|
+
nodes = m.get("total_nodes", "?")
|
|
2497
|
+
edges = m.get("total_edges", "?")
|
|
2498
|
+
cov_raw = m.get("docstring_coverage", None)
|
|
2499
|
+
cov_str = f"{cov_raw * 100:.1f}%" if cov_raw is not None else "?"
|
|
2500
|
+
|
|
2501
|
+
delta = (snap.get("deltas") or {}).get("vs_previous") or {}
|
|
2502
|
+
dn = delta.get("nodes")
|
|
2503
|
+
de = delta.get("edges")
|
|
2504
|
+
dc = delta.get("coverage_delta")
|
|
2505
|
+
dn_str = f"{dn:+d}" if dn is not None else "—"
|
|
2506
|
+
de_str = f"{de:+d}" if de is not None else "—"
|
|
2507
|
+
dc_str = f"{dc * 100:+.1f}%" if dc is not None else "—"
|
|
2508
|
+
|
|
2509
|
+
out.append(
|
|
2510
|
+
f"| {i} | {ts} | {branch} | {version} | {nodes} | {edges}"
|
|
2511
|
+
f" | {cov_str} | {dn_str} | {de_str} | {dc_str} |"
|
|
2512
|
+
)
|
|
2513
|
+
else:
|
|
2514
|
+
out.append(
|
|
2515
|
+
"No snapshots found. Run `pycodekg snapshot save <version>` to capture one.\n"
|
|
2516
|
+
)
|
|
2517
|
+
out.append("")
|
|
2518
|
+
|
|
2519
|
+
# Orphaned Functions
|
|
2520
|
+
out.append("## Orphaned Code\n")
|
|
2521
|
+
if self.orphaned_functions:
|
|
2522
|
+
out.append("Functions with zero callers (potential dead code).\n")
|
|
2523
|
+
out.append("| Function | Module | Lines |")
|
|
2524
|
+
out.append("|---|---|---|")
|
|
2525
|
+
for func in sorted(self.orphaned_functions, key=lambda f: f.lines, reverse=True)[:15]:
|
|
2526
|
+
out.append(f"| `{func.name}()` | {func.module} | {func.lines} |")
|
|
2527
|
+
else:
|
|
2528
|
+
out.append("No orphaned functions detected.\n")
|
|
2529
|
+
out.append("")
|
|
2530
|
+
|
|
2531
|
+
# CodeRank Top Nodes (Option D)
|
|
2532
|
+
out.append("## CodeRank -- Global Structural Importance\n")
|
|
2533
|
+
if self.coderank_top_nodes:
|
|
2534
|
+
out.append(
|
|
2535
|
+
"Weighted PageRank over CALLS + IMPORTS + INHERITS edges "
|
|
2536
|
+
"(test paths excluded). Scores normalized to sum to 1.0.\n"
|
|
2537
|
+
)
|
|
2538
|
+
out.append("| Rank | Score | Kind | Name | Module |")
|
|
2539
|
+
out.append("|------|-------|------|------|--------|")
|
|
2540
|
+
for i, n in enumerate(self.coderank_top_nodes[:20], 1):
|
|
2541
|
+
out.append(
|
|
2542
|
+
f"| {i} | {n['score']:.6f} | {n['kind']} "
|
|
2543
|
+
f"| `{n['qualname'] or n['name']}` | {n['module_path']} |"
|
|
2544
|
+
)
|
|
2545
|
+
else:
|
|
2546
|
+
out.append("CodeRank data not available.\n")
|
|
2547
|
+
out.append("")
|
|
2548
|
+
|
|
2549
|
+
# Concern-based Hybrid Ranking (Option D)
|
|
2550
|
+
out.append("## Concern-Based Hybrid Ranking\n")
|
|
2551
|
+
if self.concern_analysis:
|
|
2552
|
+
out.append(
|
|
2553
|
+
"Top structurally-dominant nodes per architectural concern "
|
|
2554
|
+
"(0.60 × semantic + 0.25 × CodeRank + 0.15 × graph proximity).\n"
|
|
2555
|
+
)
|
|
2556
|
+
for entry in self.concern_analysis:
|
|
2557
|
+
concern = entry["concern"]
|
|
2558
|
+
out.append(f"### {concern.title()}\n")
|
|
2559
|
+
out.append("| Rank | Score | Kind | Name | Module |")
|
|
2560
|
+
out.append("|------|-------|------|------|--------|")
|
|
2561
|
+
for n in entry["top_nodes"]:
|
|
2562
|
+
out.append(
|
|
2563
|
+
f"| {n['rank']} | {n['score']} | {n['kind']} "
|
|
2564
|
+
f"| `{n['name']}` | {n['module']} |"
|
|
2565
|
+
)
|
|
2566
|
+
out.append("")
|
|
2567
|
+
else:
|
|
2568
|
+
out.append("Concern analysis not available.\n")
|
|
2569
|
+
out.append("")
|
|
2570
|
+
|
|
2571
|
+
return "\n".join(out)
|
|
2572
|
+
|
|
2573
|
+
def print_summary(self) -> None:
|
|
2574
|
+
"""Print analysis summary to console."""
|
|
2575
|
+
# Header
|
|
2576
|
+
self.console.print()
|
|
2577
|
+
self.console.print(
|
|
2578
|
+
Panel.fit(
|
|
2579
|
+
"[bold cyan]PyCodeKG Repository Analysis[/bold cyan]",
|
|
2580
|
+
border_style="cyan",
|
|
2581
|
+
)
|
|
2582
|
+
)
|
|
2583
|
+
self.console.print()
|
|
2584
|
+
|
|
2585
|
+
# Stats table
|
|
2586
|
+
stats_table = Table(title="Baseline Metrics", show_header=True)
|
|
2587
|
+
stats_table.add_column("Metric", style="dim")
|
|
2588
|
+
stats_table.add_column("Value")
|
|
2589
|
+
|
|
2590
|
+
for key, value in self.stats.items():
|
|
2591
|
+
stats_table.add_row(key, str(value))
|
|
2592
|
+
|
|
2593
|
+
self.console.print(stats_table)
|
|
2594
|
+
self.console.print()
|
|
2595
|
+
|
|
2596
|
+
# Most called functions
|
|
2597
|
+
if self.function_metrics:
|
|
2598
|
+
calls_table = Table(title="Most Called Functions (Fan-In)", show_header=True)
|
|
2599
|
+
calls_table.add_column("Function", style="cyan")
|
|
2600
|
+
calls_table.add_column("Callers", justify="right")
|
|
2601
|
+
|
|
2602
|
+
for metrics in sorted(
|
|
2603
|
+
self.function_metrics.values(),
|
|
2604
|
+
key=lambda m: m.fan_in,
|
|
2605
|
+
reverse=True,
|
|
2606
|
+
)[:10]:
|
|
2607
|
+
calls_table.add_row(
|
|
2608
|
+
metrics.name,
|
|
2609
|
+
str(metrics.fan_in),
|
|
2610
|
+
)
|
|
2611
|
+
|
|
2612
|
+
self.console.print(calls_table)
|
|
2613
|
+
self.console.print()
|
|
2614
|
+
|
|
2615
|
+
# Issues
|
|
2616
|
+
if self.issues:
|
|
2617
|
+
self.console.print("[bold yellow]Issues Found:[/bold yellow]")
|
|
2618
|
+
for issue in self.issues:
|
|
2619
|
+
self.console.print(f" {issue}")
|
|
2620
|
+
self.console.print()
|
|
2621
|
+
|
|
2622
|
+
# Strengths
|
|
2623
|
+
if self.strengths:
|
|
2624
|
+
self.console.print("[bold green]Strengths:[/bold green]")
|
|
2625
|
+
for strength in self.strengths:
|
|
2626
|
+
self.console.print(f" {strength}")
|
|
2627
|
+
self.console.print()
|
|
2628
|
+
|
|
2629
|
+
|
|
2630
|
+
def _default_report_name(repo_root: Path) -> str:
|
|
2631
|
+
"""Derive a timestamped default markdown report path under ``analysis/``.
|
|
2632
|
+
|
|
2633
|
+
:param repo_root: Repository root directory
|
|
2634
|
+
:return: Path string like ``analysis/myrepo_analysis_20260224.md``
|
|
2635
|
+
"""
|
|
2636
|
+
repo_name = repo_root.resolve().name
|
|
2637
|
+
date_str = datetime.datetime.now(datetime.UTC).strftime("%Y%m%d")
|
|
2638
|
+
return str(Path("analysis") / f"{repo_name}_analysis_{date_str}.md")
|
|
2639
|
+
|
|
2640
|
+
|
|
2641
|
+
def main(
|
|
2642
|
+
repo_root: str = ".",
|
|
2643
|
+
db_path: str | None = None,
|
|
2644
|
+
lancedb_path: str | None = None,
|
|
2645
|
+
report_path: str | None = None,
|
|
2646
|
+
json_path: str | None = None,
|
|
2647
|
+
quiet: bool = False,
|
|
2648
|
+
include: set[str] | None = None,
|
|
2649
|
+
exclude: set[str] | None = None,
|
|
2650
|
+
persist_centrality: bool = False,
|
|
2651
|
+
) -> None:
|
|
2652
|
+
"""Main entry point.
|
|
2653
|
+
|
|
2654
|
+
Paths for ``db_path`` and ``lancedb_path`` default to the standard
|
|
2655
|
+
``.pycodekg/`` layout inside ``repo_root`` when not provided.
|
|
2656
|
+
The markdown report defaults to ``<repo>_analysis_<YYYYMMDD>.md``
|
|
2657
|
+
in the current working directory. The JSON snapshot always writes
|
|
2658
|
+
to ``~/.claude/pycodekg_analysis_latest.json`` unless overridden.
|
|
2659
|
+
|
|
2660
|
+
Error handling strategy:
|
|
2661
|
+
- Missing graph database prints a warning and allows execution to continue,
|
|
2662
|
+
so callers still get useful diagnostics.
|
|
2663
|
+
- Import/runtime failures are reported with clear remediation guidance.
|
|
2664
|
+
|
|
2665
|
+
Logging approach:
|
|
2666
|
+
- Rich console output is used for visible run status and outputs.
|
|
2667
|
+
- ``logging`` warnings are used for partial-analysis fallbacks.
|
|
2668
|
+
|
|
2669
|
+
:param repo_root: Root directory of the repository (default: ``"."``)
|
|
2670
|
+
:param db_path: Path to SQLite knowledge graph; default ``.pycodekg/graph.sqlite``
|
|
2671
|
+
:param lancedb_path: Path to LanceDB vector index; default ``.pycodekg/lancedb``
|
|
2672
|
+
:param report_path: Markdown report output path; auto-named when ``None``
|
|
2673
|
+
:param json_path: JSON snapshot output path; when ``None`` (default) no JSON is written
|
|
2674
|
+
:param quiet: Suppress console summary table when ``True``
|
|
2675
|
+
:param include: Set of top-level directory names to include in analysis.
|
|
2676
|
+
When empty/None, all directories are analyzed.
|
|
2677
|
+
:param exclude: Set of directory names that were excluded during indexing.
|
|
2678
|
+
Recorded in the report metadata for traceability.
|
|
2679
|
+
:param persist_centrality: When ``True``, persist SIR scores to the
|
|
2680
|
+
``centrality_scores`` table in the SQLite graph DB.
|
|
2681
|
+
"""
|
|
2682
|
+
console = Console()
|
|
2683
|
+
root = Path(repo_root).resolve()
|
|
2684
|
+
db = Path(db_path) if db_path else root / ".pycodekg" / "graph.sqlite"
|
|
2685
|
+
lancedb = Path(lancedb_path) if lancedb_path else root / ".pycodekg" / "lancedb"
|
|
2686
|
+
md_out = report_path or _default_report_name(root)
|
|
2687
|
+
Path(md_out).parent.mkdir(parents=True, exist_ok=True)
|
|
2688
|
+
json_out = Path(json_path) if json_path else None
|
|
2689
|
+
|
|
2690
|
+
if not db.exists():
|
|
2691
|
+
console.print(
|
|
2692
|
+
f"[red]ERROR[/red] Database not found at [dim]{db}[/dim]\n"
|
|
2693
|
+
"Run [bold]pycodekg build --repo .[/bold] first, then re-run analyze."
|
|
2694
|
+
)
|
|
2695
|
+
return
|
|
2696
|
+
|
|
2697
|
+
try:
|
|
2698
|
+
from pycode_kg import PyCodeKG # pylint: disable=import-outside-toplevel
|
|
2699
|
+
|
|
2700
|
+
console.print(f"[dim]Repo : {root}[/dim]")
|
|
2701
|
+
console.print(f"[dim]DB : {db}[/dim]")
|
|
2702
|
+
console.print(f"[dim]Index : {lancedb}[/dim]")
|
|
2703
|
+
console.print(f"[dim]Report : {md_out}[/dim]")
|
|
2704
|
+
console.print()
|
|
2705
|
+
|
|
2706
|
+
kg = PyCodeKG(repo_root=root, db_path=db, lancedb_dir=lancedb)
|
|
2707
|
+
|
|
2708
|
+
snapshots_dir = root / ".pycodekg" / "snapshots"
|
|
2709
|
+
snap_mgr = SnapshotManager(snapshots_dir) if snapshots_dir.exists() else None
|
|
2710
|
+
|
|
2711
|
+
# Resolve effective include/exclude dirs: prefer explicit args, fall back to pyproject.toml
|
|
2712
|
+
from pycode_kg.config import ( # pylint: disable=import-outside-toplevel
|
|
2713
|
+
load_exclude_dirs,
|
|
2714
|
+
load_include_dirs,
|
|
2715
|
+
)
|
|
2716
|
+
|
|
2717
|
+
effective_include = include or load_include_dirs(root)
|
|
2718
|
+
effective_exclude = exclude or load_exclude_dirs(root)
|
|
2719
|
+
|
|
2720
|
+
analyzer = PyCodeKGAnalyzer(
|
|
2721
|
+
kg,
|
|
2722
|
+
console,
|
|
2723
|
+
snapshot_mgr=snap_mgr,
|
|
2724
|
+
include_dirs=effective_include,
|
|
2725
|
+
exclude_dirs=effective_exclude,
|
|
2726
|
+
)
|
|
2727
|
+
results = analyzer.run_analysis(
|
|
2728
|
+
report_path=md_out,
|
|
2729
|
+
persist_centrality=persist_centrality,
|
|
2730
|
+
)
|
|
2731
|
+
|
|
2732
|
+
if not quiet:
|
|
2733
|
+
analyzer.print_summary()
|
|
2734
|
+
|
|
2735
|
+
if json_out is not None:
|
|
2736
|
+
json_out.parent.mkdir(parents=True, exist_ok=True)
|
|
2737
|
+
json_out.write_text(json.dumps(results, indent=2))
|
|
2738
|
+
console.print(f"[dim]JSON : {json_out}[/dim]")
|
|
2739
|
+
|
|
2740
|
+
except ImportError as e:
|
|
2741
|
+
console.print(
|
|
2742
|
+
f"[red]Error: Could not import PyCodeKG[/red]\n"
|
|
2743
|
+
f"Details: {e}\n\n"
|
|
2744
|
+
"Make sure you are running inside the pycode_kg package environment."
|
|
2745
|
+
)
|
|
2746
|
+
logger.exception("Import error")
|
|
2747
|
+
raise
|
|
2748
|
+
|
|
2749
|
+
|
|
2750
|
+
if __name__ == "__main__":
|
|
2751
|
+
main()
|