codegraphx 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codegraph/__init__.py +37 -0
- codegraph/cli.py +106 -0
- codegraph/commands/__init__.py +1 -0
- codegraph/commands/conflicts.py +154 -0
- codegraph/commands/diff.py +189 -0
- codegraph/commands/export.py +141 -0
- codegraph/commands/graph.py +200 -0
- codegraph/commands/index.py +924 -0
- codegraph/commands/serve.py +62 -0
- codegraph/commands/why.py +645 -0
- codegraph/decisions/__init__.py +18 -0
- codegraph/decisions/conflicts.py +238 -0
- codegraph/decisions/extractor.py +698 -0
- codegraph/decisions/llm_extractor.py +161 -0
- codegraph/decisions/sources.py +298 -0
- codegraph/git/__init__.py +5 -0
- codegraph/git/diff.py +115 -0
- codegraph/git/history.py +156 -0
- codegraph/git/incremental.py +75 -0
- codegraph/git/refs.py +70 -0
- codegraph/models.py +211 -0
- codegraph/parser/__init__.py +1 -0
- codegraph/parser/base.py +164 -0
- codegraph/parser/common.py +147 -0
- codegraph/parser/parallel.py +96 -0
- codegraph/parser/registry.py +83 -0
- codegraph/parser/treesitter_cpp.py +102 -0
- codegraph/parser/treesitter_go.py +113 -0
- codegraph/parser/treesitter_java.py +79 -0
- codegraph/parser/treesitter_js.py +15 -0
- codegraph/parser/treesitter_py.py +163 -0
- codegraph/parser/treesitter_rust.py +120 -0
- codegraph/parser/treesitter_ts.py +178 -0
- codegraph/storage.py +767 -0
- codegraph/web/__init__.py +1 -0
- codegraph/web/app.py +271 -0
- codegraph/web/static/app.js +356 -0
- codegraph/web/static/index.html +59 -0
- codegraph/web/static/styles.css +92 -0
- codegraphx-0.1.0.dist-info/METADATA +262 -0
- codegraphx-0.1.0.dist-info/RECORD +44 -0
- codegraphx-0.1.0.dist-info/WHEEL +4 -0
- codegraphx-0.1.0.dist-info/entry_points.txt +2 -0
- codegraphx-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""Optional local LLM decision extraction via Ollama HTTP API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
from datetime import datetime, timezone
|
|
8
|
+
|
|
9
|
+
import httpx
|
|
10
|
+
|
|
11
|
+
from codegraph.models import Decision, DecisionSource, DecisionStatus
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
PROMPT_TEMPLATE = """You are analyzing a Git commit to extract a design decision.
|
|
16
|
+
Commit message:
|
|
17
|
+
{commit_message}
|
|
18
|
+
|
|
19
|
+
Diff summary:
|
|
20
|
+
{diff_summary}
|
|
21
|
+
|
|
22
|
+
Extract one decision if present. Output JSON with fields:
|
|
23
|
+
- content: one sentence
|
|
24
|
+
- reason: why
|
|
25
|
+
- alternatives: list (may be empty)
|
|
26
|
+
- confidence: 0.0-1.0
|
|
27
|
+
If no decision is present, output {{"content": null}}.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
MAX_LLM_CONFIDENCE = 0.7
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class LLMExtractor:
|
|
34
|
+
"""Rule-engine fallback that talks to a local Ollama server."""
|
|
35
|
+
|
|
36
|
+
def __init__(
|
|
37
|
+
self,
|
|
38
|
+
model: str = "llama3.2",
|
|
39
|
+
base_url: str = "http://localhost:11434",
|
|
40
|
+
timeout: float = 8.0,
|
|
41
|
+
) -> None:
|
|
42
|
+
"""Store model settings.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
model: Ollama model name.
|
|
46
|
+
base_url: Ollama HTTP root.
|
|
47
|
+
timeout: Request timeout seconds.
|
|
48
|
+
"""
|
|
49
|
+
self.model = model
|
|
50
|
+
self.base_url = base_url.rstrip("/")
|
|
51
|
+
self.timeout = timeout
|
|
52
|
+
|
|
53
|
+
def is_available(self) -> bool:
|
|
54
|
+
"""Return True when Ollama answers /api/tags."""
|
|
55
|
+
try:
|
|
56
|
+
res = httpx.get(f"{self.base_url}/api/tags", timeout=self.timeout)
|
|
57
|
+
return res.status_code == 200
|
|
58
|
+
except httpx.HTTPError as exc:
|
|
59
|
+
logger.debug("Ollama unavailable: %s", exc)
|
|
60
|
+
return False
|
|
61
|
+
|
|
62
|
+
def extract_decision(
|
|
63
|
+
self,
|
|
64
|
+
commit_message: str,
|
|
65
|
+
diff_summary: str,
|
|
66
|
+
*,
|
|
67
|
+
source_ref: str = "",
|
|
68
|
+
author: str = "",
|
|
69
|
+
file_path: str = "",
|
|
70
|
+
) -> Decision | None:
|
|
71
|
+
"""Ask the LLM for one structured decision.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
commit_message: Full commit message.
|
|
75
|
+
diff_summary: Short diff summary text.
|
|
76
|
+
source_ref: Provenance string (usually a commit sha).
|
|
77
|
+
author: Commit author.
|
|
78
|
+
file_path: Related file path.
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
A Decision with confidence capped at 0.7, or None.
|
|
82
|
+
"""
|
|
83
|
+
prompt = PROMPT_TEMPLATE.format(
|
|
84
|
+
commit_message=commit_message[:2000],
|
|
85
|
+
diff_summary=diff_summary[:1000] or "(none)",
|
|
86
|
+
)
|
|
87
|
+
raw = self._generate(prompt)
|
|
88
|
+
if not raw:
|
|
89
|
+
return None
|
|
90
|
+
return self._parse_response(
|
|
91
|
+
raw,
|
|
92
|
+
source_ref=source_ref,
|
|
93
|
+
author=author,
|
|
94
|
+
file_path=file_path,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
def _generate(self, prompt: str) -> str | None:
|
|
98
|
+
"""Call Ollama /api/generate and return response text."""
|
|
99
|
+
try:
|
|
100
|
+
res = httpx.post(
|
|
101
|
+
f"{self.base_url}/api/generate",
|
|
102
|
+
json={
|
|
103
|
+
"model": self.model,
|
|
104
|
+
"prompt": prompt,
|
|
105
|
+
"stream": False,
|
|
106
|
+
"format": "json",
|
|
107
|
+
},
|
|
108
|
+
timeout=self.timeout,
|
|
109
|
+
)
|
|
110
|
+
if res.status_code != 200:
|
|
111
|
+
logger.warning("Ollama HTTP %s", res.status_code)
|
|
112
|
+
return None
|
|
113
|
+
data = res.json()
|
|
114
|
+
return str(data.get("response") or "")
|
|
115
|
+
except httpx.HTTPError as exc:
|
|
116
|
+
logger.warning("Ollama request failed: %s", exc)
|
|
117
|
+
return None
|
|
118
|
+
except ValueError as exc:
|
|
119
|
+
logger.warning("Ollama JSON failed: %s", exc)
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
def _parse_response(
|
|
123
|
+
self,
|
|
124
|
+
raw: str,
|
|
125
|
+
*,
|
|
126
|
+
source_ref: str,
|
|
127
|
+
author: str,
|
|
128
|
+
file_path: str,
|
|
129
|
+
) -> Decision | None:
|
|
130
|
+
"""Parse LLM JSON into a Decision."""
|
|
131
|
+
try:
|
|
132
|
+
payload = json.loads(raw)
|
|
133
|
+
except json.JSONDecodeError:
|
|
134
|
+
logger.warning("LLM returned non-JSON output")
|
|
135
|
+
return None
|
|
136
|
+
content = payload.get("content")
|
|
137
|
+
if not content:
|
|
138
|
+
return None
|
|
139
|
+
try:
|
|
140
|
+
confidence = float(payload.get("confidence") or 0.5)
|
|
141
|
+
except (TypeError, ValueError):
|
|
142
|
+
confidence = 0.5
|
|
143
|
+
confidence = min(max(confidence, 0.0), MAX_LLM_CONFIDENCE)
|
|
144
|
+
alternatives = payload.get("alternatives") or []
|
|
145
|
+
if isinstance(alternatives, str):
|
|
146
|
+
alternatives = [alternatives]
|
|
147
|
+
return Decision(
|
|
148
|
+
uid=f"llm::{hash(str(content)) & 0xFFFFFFFF:08x}",
|
|
149
|
+
content=str(content)[:220],
|
|
150
|
+
reason=str(payload.get("reason") or content)[:500],
|
|
151
|
+
alternatives=[str(a) for a in alternatives][:5],
|
|
152
|
+
status=DecisionStatus.ACCEPTED,
|
|
153
|
+
source=DecisionSource.COMMIT,
|
|
154
|
+
source_ref=source_ref or "llm",
|
|
155
|
+
timestamp=datetime.now(timezone.utc),
|
|
156
|
+
author=author,
|
|
157
|
+
file_path=file_path,
|
|
158
|
+
line=None,
|
|
159
|
+
constraints=[],
|
|
160
|
+
confidence=confidence,
|
|
161
|
+
)
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
"""Extract Decisions from commits, ADR files, and CHANGELOG."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import re
|
|
7
|
+
from datetime import datetime, timezone
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from codegraph.decisions.extractor import (
|
|
11
|
+
decision_from_adr_sections,
|
|
12
|
+
decision_from_commit_subject,
|
|
13
|
+
decisions_from_commit_body,
|
|
14
|
+
extract_decisions_from_text,
|
|
15
|
+
parse_adr_markdown,
|
|
16
|
+
)
|
|
17
|
+
from codegraph.git.history import GitHistory
|
|
18
|
+
from codegraph.models import Decision, DecisionSource, DecisionTier
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
CHANGELOG_SECTION = re.compile(
|
|
23
|
+
r"^##\s+\[?(Changed|Removed|Deprecated|Breaking|Security)\]?",
|
|
24
|
+
re.IGNORECASE | re.MULTILINE,
|
|
25
|
+
)
|
|
26
|
+
CHANGELOG_NEXT = re.compile(r"^##\s+", re.MULTILINE)
|
|
27
|
+
|
|
28
|
+
# Python projects usually ship CHANGES.rst with reStructuredText headings, and
|
|
29
|
+
# reading only CHANGELOG.md made the tool blind to them: Flask's 537 release-note
|
|
30
|
+
# bullets produced one decision instead of the 26 that state a rationale.
|
|
31
|
+
CHANGELOG_NAMES = (
|
|
32
|
+
"CHANGELOG.md",
|
|
33
|
+
"CHANGELOG.rst",
|
|
34
|
+
"CHANGELOG.txt",
|
|
35
|
+
"CHANGES.md",
|
|
36
|
+
"CHANGES.rst",
|
|
37
|
+
"CHANGES.txt",
|
|
38
|
+
"HISTORY.md",
|
|
39
|
+
"HISTORY.rst",
|
|
40
|
+
"NEWS.md",
|
|
41
|
+
"NEWS.rst",
|
|
42
|
+
)
|
|
43
|
+
_BULLET = re.compile(r"^(?P<marker>[-*+])\s+(?P<text>\S.*)$")
|
|
44
|
+
_RST_UNDERLINE = re.compile(r"^(?P<title>\S.*)\n[-=~^\"'`#*+]{3,}\s*$", re.MULTILINE)
|
|
45
|
+
_MD_HEADING = re.compile(r"^(?P<hashes>#{1,4})\s+(?P<title>.+?)\s*#*$", re.MULTILINE)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _changelog_path(repo_path: Path) -> Path | None:
|
|
49
|
+
"""Return the first conventional changelog file that exists.
|
|
50
|
+
|
|
51
|
+
The comparison is case-insensitive on purpose: Express ships `History.md`
|
|
52
|
+
and a case-sensitive lookup would find it on Windows but not on macOS or
|
|
53
|
+
Linux, so the same repository produced different graphs per platform.
|
|
54
|
+
"""
|
|
55
|
+
if not repo_path.is_dir():
|
|
56
|
+
return None
|
|
57
|
+
by_lower = {path.name.lower(): path for path in repo_path.iterdir() if path.is_file()}
|
|
58
|
+
for name in CHANGELOG_NAMES:
|
|
59
|
+
candidate = by_lower.get(name.lower())
|
|
60
|
+
if candidate is not None:
|
|
61
|
+
return candidate
|
|
62
|
+
return None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _sections(text: str) -> list[tuple[str, str]]:
|
|
66
|
+
"""Split a changelog into (heading, body) pairs for markdown or rst."""
|
|
67
|
+
headings = [(m.start(), m.group("title")) for m in _RST_UNDERLINE.finditer(text)]
|
|
68
|
+
headings += [(m.start(), m.group("title")) for m in _MD_HEADING.finditer(text)]
|
|
69
|
+
if not headings:
|
|
70
|
+
return [("Changelog", text)]
|
|
71
|
+
headings.sort(key=lambda item: item[0])
|
|
72
|
+
out: list[tuple[str, str]] = []
|
|
73
|
+
for index, (start, title) in enumerate(headings):
|
|
74
|
+
end = headings[index + 1][0] if index + 1 < len(headings) else len(text)
|
|
75
|
+
out.append((title.strip()[:60], text[start:end]))
|
|
76
|
+
return out
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _bullets(body: str) -> list[str]:
|
|
80
|
+
"""Return bullet texts, joining indented continuation lines."""
|
|
81
|
+
bullets: list[str] = []
|
|
82
|
+
current: list[str] = []
|
|
83
|
+
for raw in body.splitlines():
|
|
84
|
+
match = _BULLET.match(raw.strip())
|
|
85
|
+
if match:
|
|
86
|
+
if current:
|
|
87
|
+
bullets.append(" ".join(current))
|
|
88
|
+
current = [match.group("text").strip()]
|
|
89
|
+
elif current and raw.startswith((" ", "\t")) and raw.strip():
|
|
90
|
+
current.append(raw.strip())
|
|
91
|
+
elif current and not raw.strip():
|
|
92
|
+
bullets.append(" ".join(current))
|
|
93
|
+
current = []
|
|
94
|
+
if current:
|
|
95
|
+
bullets.append(" ".join(current))
|
|
96
|
+
return [b for b in bullets if len(b) > 8]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def extract_from_changelog(repo_path: Path) -> list[Decision]:
|
|
100
|
+
"""Extract decision-like bullets from a changelog, markdown or rst.
|
|
101
|
+
|
|
102
|
+
Every bullet is a candidate and the tier classifier keeps only the ones that
|
|
103
|
+
state a reason or a rejected alternative, so listing all of them raises recall
|
|
104
|
+
without lowering precision.
|
|
105
|
+
"""
|
|
106
|
+
path = _changelog_path(repo_path)
|
|
107
|
+
if path is None:
|
|
108
|
+
return []
|
|
109
|
+
|
|
110
|
+
try:
|
|
111
|
+
text = path.read_text(encoding="utf-8")
|
|
112
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
113
|
+
logger.warning("Cannot read %s: %s", path.name, exc)
|
|
114
|
+
return []
|
|
115
|
+
|
|
116
|
+
out: list[Decision] = []
|
|
117
|
+
for heading, body in _sections(text):
|
|
118
|
+
for bullet in _bullets(body):
|
|
119
|
+
out.extend(
|
|
120
|
+
extract_decisions_from_text(
|
|
121
|
+
f"{heading}: {bullet}",
|
|
122
|
+
source=DecisionSource.COMMENT,
|
|
123
|
+
source_ref=f"{path.name} · {heading}",
|
|
124
|
+
timestamp=_file_time(path),
|
|
125
|
+
file_path=path.name,
|
|
126
|
+
confidence=0.6,
|
|
127
|
+
)
|
|
128
|
+
)
|
|
129
|
+
return out
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def extract_from_commits(
|
|
133
|
+
repo_path: Path,
|
|
134
|
+
depth: int | None = 100,
|
|
135
|
+
use_llm: bool = False,
|
|
136
|
+
llm_model: str = "llama3.2",
|
|
137
|
+
) -> list[Decision]:
|
|
138
|
+
"""Extract rationale-like decisions from commit messages."""
|
|
139
|
+
try:
|
|
140
|
+
history = GitHistory(repo_path)
|
|
141
|
+
except ValueError as exc:
|
|
142
|
+
logger.warning("%s", exc)
|
|
143
|
+
return []
|
|
144
|
+
|
|
145
|
+
out: list[Decision] = []
|
|
146
|
+
llm = None
|
|
147
|
+
if use_llm:
|
|
148
|
+
from codegraph.decisions.llm_extractor import LLMExtractor
|
|
149
|
+
|
|
150
|
+
llm = LLMExtractor(model=llm_model)
|
|
151
|
+
if not llm.is_available():
|
|
152
|
+
logger.warning("Ollama unavailable; continuing with rule-based extraction only")
|
|
153
|
+
llm = None
|
|
154
|
+
|
|
155
|
+
for commit in history.iter_commits(depth=depth):
|
|
156
|
+
try:
|
|
157
|
+
ref = f"commit {commit.sha}"
|
|
158
|
+
path = commit.changed_files[0] if commit.changed_files else ""
|
|
159
|
+
found: list[Decision] = []
|
|
160
|
+
subject = decision_from_commit_subject(
|
|
161
|
+
commit.message,
|
|
162
|
+
source_ref=ref,
|
|
163
|
+
timestamp=commit.timestamp,
|
|
164
|
+
author=commit.author,
|
|
165
|
+
file_path=path,
|
|
166
|
+
)
|
|
167
|
+
if subject:
|
|
168
|
+
found.append(subject)
|
|
169
|
+
found.extend(
|
|
170
|
+
decisions_from_commit_body(
|
|
171
|
+
commit.message,
|
|
172
|
+
source_ref=ref,
|
|
173
|
+
timestamp=commit.timestamp,
|
|
174
|
+
author=commit.author,
|
|
175
|
+
file_path=path,
|
|
176
|
+
)
|
|
177
|
+
)
|
|
178
|
+
if commit.message.lstrip().lower().startswith("merge pull request"):
|
|
179
|
+
found.extend(
|
|
180
|
+
extract_decisions_from_text(
|
|
181
|
+
commit.message,
|
|
182
|
+
source=DecisionSource.PR,
|
|
183
|
+
source_ref=ref,
|
|
184
|
+
timestamp=commit.timestamp,
|
|
185
|
+
author=commit.author,
|
|
186
|
+
file_path=path,
|
|
187
|
+
confidence=0.65,
|
|
188
|
+
)
|
|
189
|
+
)
|
|
190
|
+
if llm is not None and not _has_rationale(commit.message):
|
|
191
|
+
extra = llm.extract_decision(
|
|
192
|
+
commit.message,
|
|
193
|
+
", ".join(commit.changed_files[:20]),
|
|
194
|
+
source_ref=ref,
|
|
195
|
+
author=commit.author,
|
|
196
|
+
file_path=path,
|
|
197
|
+
)
|
|
198
|
+
if extra:
|
|
199
|
+
extra.tier = _llm_tier(extra)
|
|
200
|
+
found.append(extra)
|
|
201
|
+
out.extend(found)
|
|
202
|
+
except Exception as exc: # noqa: BLE001
|
|
203
|
+
logger.warning("Decision extract failed for %s: %s", commit.sha, exc)
|
|
204
|
+
return out
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _llm_tier(decision: Decision) -> DecisionTier:
|
|
208
|
+
"""LLM output is only a decision when it states a reason."""
|
|
209
|
+
from codegraph.decisions.extractor import classify_tier
|
|
210
|
+
|
|
211
|
+
return classify_tier(decision.content, decision.reason, decision.alternatives)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _has_rationale(message: str) -> bool:
|
|
215
|
+
"""True when the message already contains rationale keywords."""
|
|
216
|
+
from codegraph.decisions.extractor import looks_like_decision
|
|
217
|
+
|
|
218
|
+
return any(looks_like_decision(line) for line in (message or "").splitlines())
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def extract_from_adrs(repo_path: Path) -> list[Decision]:
|
|
222
|
+
"""Extract decisions from docs/adr/*.md files."""
|
|
223
|
+
adr_dir = repo_path / "docs" / "adr"
|
|
224
|
+
if not adr_dir.is_dir():
|
|
225
|
+
return []
|
|
226
|
+
|
|
227
|
+
out: list[Decision] = []
|
|
228
|
+
for path in sorted(adr_dir.glob("*.md")):
|
|
229
|
+
try:
|
|
230
|
+
text = path.read_text(encoding="utf-8")
|
|
231
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
232
|
+
logger.warning("Cannot read ADR %s: %s", path, exc)
|
|
233
|
+
continue
|
|
234
|
+
sections = parse_adr_markdown(text)
|
|
235
|
+
try:
|
|
236
|
+
rel = path.relative_to(repo_path).as_posix()
|
|
237
|
+
except ValueError:
|
|
238
|
+
rel = path.as_posix()
|
|
239
|
+
decision = decision_from_adr_sections(
|
|
240
|
+
sections,
|
|
241
|
+
source_ref=f"ADR {path.stem}",
|
|
242
|
+
timestamp=_file_time(path),
|
|
243
|
+
file_path=rel,
|
|
244
|
+
)
|
|
245
|
+
if decision:
|
|
246
|
+
out.append(decision)
|
|
247
|
+
return out
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _dedupe_key(decision: Decision) -> tuple[str, str]:
|
|
253
|
+
"""Key used to collapse the same content extracted by several rules."""
|
|
254
|
+
normalized = " ".join((decision.content or "").split()).lower()
|
|
255
|
+
return (decision.source_ref, normalized)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _prefer(a: Decision, b: Decision) -> Decision:
|
|
259
|
+
"""Keep the richer of two records for the same source and content."""
|
|
260
|
+
if (a.tier is DecisionTier.DECISION) != (b.tier is DecisionTier.DECISION):
|
|
261
|
+
return a if a.tier is DecisionTier.DECISION else b
|
|
262
|
+
return a if a.confidence >= b.confidence else b
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def extract_all(
|
|
266
|
+
repo_path: Path,
|
|
267
|
+
depth: int | None = 100,
|
|
268
|
+
use_llm: bool = False,
|
|
269
|
+
llm_model: str = "llama3.2",
|
|
270
|
+
) -> list[Decision]:
|
|
271
|
+
"""Extract records from all supported sources.
|
|
272
|
+
|
|
273
|
+
Records are de-duplicated twice: by uid, and by (source_ref, content) so the
|
|
274
|
+
same sentence cannot enter twice through different rules.
|
|
275
|
+
"""
|
|
276
|
+
merged: dict[str, Decision] = {}
|
|
277
|
+
by_key: dict[tuple[str, str], Decision] = {}
|
|
278
|
+
for source_items in (
|
|
279
|
+
extract_from_commits(repo_path, depth=depth, use_llm=use_llm, llm_model=llm_model),
|
|
280
|
+
extract_from_adrs(repo_path),
|
|
281
|
+
extract_from_changelog(repo_path),
|
|
282
|
+
):
|
|
283
|
+
for item in source_items:
|
|
284
|
+
merged.setdefault(item.uid, item)
|
|
285
|
+
key = _dedupe_key(item)
|
|
286
|
+
existing = by_key.get(key)
|
|
287
|
+
by_key[key] = item if existing is None else _prefer(existing, item)
|
|
288
|
+
|
|
289
|
+
keep = {id(d) for d in by_key.values()}
|
|
290
|
+
return [d for d in merged.values() if id(d) in keep]
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _file_time(path: Path) -> datetime:
|
|
294
|
+
"""Best-effort mtime as UTC datetime."""
|
|
295
|
+
try:
|
|
296
|
+
return datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc)
|
|
297
|
+
except OSError:
|
|
298
|
+
return datetime.now(timezone.utc)
|
codegraph/git/diff.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Parse unified diffs to find changed line numbers per commit."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import re
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
HUNK_RE = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")
|
|
12
|
+
|
|
13
|
+
# A repository is opened once per root, and one commit's diff is parsed once.
|
|
14
|
+
# Indexing a repository asks for the same commit repeatedly (a commit produces
|
|
15
|
+
# several decision records), and re-opening GitPython's Repo for every call was
|
|
16
|
+
# measurable: 85 records shared 50 commits on the `got` benchmark.
|
|
17
|
+
_REPOS: dict[str, object] = {}
|
|
18
|
+
_PATCH_CACHE: dict[tuple[str, str], dict[str, list[int]]] = {}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _parse_patch_lines(patch: str) -> list[int]:
|
|
22
|
+
"""Return new-file line numbers touched in a unified diff patch."""
|
|
23
|
+
lines: list[int] = []
|
|
24
|
+
new_ln = 0
|
|
25
|
+
in_hunk = False
|
|
26
|
+
for raw in patch.splitlines():
|
|
27
|
+
if raw.startswith("@@"):
|
|
28
|
+
m = HUNK_RE.match(raw)
|
|
29
|
+
if not m:
|
|
30
|
+
in_hunk = False
|
|
31
|
+
continue
|
|
32
|
+
new_ln = int(m.group(1))
|
|
33
|
+
in_hunk = True
|
|
34
|
+
continue
|
|
35
|
+
if not in_hunk:
|
|
36
|
+
continue
|
|
37
|
+
if raw.startswith("+") and not raw.startswith("+++"):
|
|
38
|
+
lines.append(new_ln)
|
|
39
|
+
new_ln += 1
|
|
40
|
+
elif raw.startswith("-") and not raw.startswith("---"):
|
|
41
|
+
continue
|
|
42
|
+
elif raw.startswith("\\"):
|
|
43
|
+
continue
|
|
44
|
+
else:
|
|
45
|
+
new_ln += 1
|
|
46
|
+
return lines
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def get_changed_lines(repo_path: Path, commit_sha: str) -> dict[str, list[int]]:
|
|
50
|
+
"""Return {file_path: [changed line numbers]} for one commit.
|
|
51
|
+
|
|
52
|
+
Uses GitPython to read the unified diff of commit vs its first parent
|
|
53
|
+
(or the full tree for a root commit). Results are cached per repository and
|
|
54
|
+
commit for the lifetime of the process: the indexer asks about the same
|
|
55
|
+
commit once per decision record it produced.
|
|
56
|
+
"""
|
|
57
|
+
root = repo_path.expanduser().resolve()
|
|
58
|
+
key = (str(root), commit_sha)
|
|
59
|
+
cached = _PATCH_CACHE.get(key)
|
|
60
|
+
if cached is not None:
|
|
61
|
+
return cached
|
|
62
|
+
|
|
63
|
+
from git import InvalidGitRepositoryError, Repo
|
|
64
|
+
|
|
65
|
+
repo = _REPOS.get(str(root))
|
|
66
|
+
if repo is None:
|
|
67
|
+
try:
|
|
68
|
+
repo = Repo(str(root), search_parent_directories=True)
|
|
69
|
+
except InvalidGitRepositoryError:
|
|
70
|
+
logger.warning("Not a git repo: %s", root)
|
|
71
|
+
return {}
|
|
72
|
+
_REPOS[str(root)] = repo
|
|
73
|
+
|
|
74
|
+
try:
|
|
75
|
+
commit = repo.commit(commit_sha) # type: ignore[attr-defined]
|
|
76
|
+
except Exception as exc: # noqa: BLE001
|
|
77
|
+
logger.warning("Cannot read commit %s: %s", commit_sha, exc)
|
|
78
|
+
return {}
|
|
79
|
+
|
|
80
|
+
out: dict[str, list[int]] = {}
|
|
81
|
+
try:
|
|
82
|
+
if commit.parents:
|
|
83
|
+
diffs = commit.parents[0].diff(commit, create_patch=True, unified=3)
|
|
84
|
+
else:
|
|
85
|
+
diffs = commit.diff(NULL_TREE(repo), create_patch=True, unified=3)
|
|
86
|
+
except Exception as exc: # noqa: BLE001
|
|
87
|
+
logger.warning("Diff failed for %s: %s", commit_sha, exc)
|
|
88
|
+
return {}
|
|
89
|
+
|
|
90
|
+
for d in diffs:
|
|
91
|
+
path = d.b_path or d.a_path
|
|
92
|
+
if not path:
|
|
93
|
+
continue
|
|
94
|
+
rel = str(path).replace("\\", "/")
|
|
95
|
+
patch = ""
|
|
96
|
+
try:
|
|
97
|
+
raw_patch = d.diff
|
|
98
|
+
if isinstance(raw_patch, bytes):
|
|
99
|
+
patch = raw_patch.decode("utf-8", errors="replace")
|
|
100
|
+
else:
|
|
101
|
+
patch = str(raw_patch or "")
|
|
102
|
+
except Exception: # noqa: BLE001
|
|
103
|
+
patch = ""
|
|
104
|
+
if not patch:
|
|
105
|
+
continue
|
|
106
|
+
touched = sorted(set(_parse_patch_lines(patch)))
|
|
107
|
+
if touched:
|
|
108
|
+
out[rel] = touched
|
|
109
|
+
_PATCH_CACHE[key] = out
|
|
110
|
+
return out
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def NULL_TREE(repo):
|
|
114
|
+
"""Empty tree object used as parent for root commits."""
|
|
115
|
+
return repo.tree("4b825dc642cb6eb9a060e54bf8d69288fbee4904")
|