devkb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devkb/__init__.py +11 -0
- devkb/audit.py +173 -0
- devkb/bench/add_packs.py +52 -0
- devkb/bench/cli_tasks.py +105 -0
- devkb/bench/confirm_tasks.py +149 -0
- devkb/bench/module_truth.py +249 -0
- devkb/bench/oco_tasks.py +216 -0
- devkb/bench/pipeline.py +313 -0
- devkb/bench/process_tasks.py +135 -0
- devkb/bench/report.py +74 -0
- devkb/bench/run.py +174 -0
- devkb/bench/truth_check.py +148 -0
- devkb/bench/workflow_report.py +64 -0
- devkb/bench/workflow_run.py +206 -0
- devkb/bench/workflow_tasks.py +227 -0
- devkb/build.py +258 -0
- devkb/build_symbols.py +1394 -0
- devkb/call_recall.py +159 -0
- devkb/cli.py +53 -0
- devkb/coverage.py +155 -0
- devkb/edge_truth.py +272 -0
- devkb/extractors/backlog.py +113 -0
- devkb/extractors/component_subsystem.py +116 -0
- devkb/extractors/configuration.py +267 -0
- devkb/extractors/files_tests.py +220 -0
- devkb/extractors/flows.py +117 -0
- devkb/extractors/lifecycle.py +258 -0
- devkb/extractors/members.py +160 -0
- devkb/extractors/pull_requests.py +271 -0
- devkb/extractors/releases.py +49 -0
- devkb/extractors/source_baseline.py +66 -0
- devkb/extractors/spec_docs.py +220 -0
- devkb/extractors/symbol_tests.py +131 -0
- devkb/extractors/test_inventory.py +38 -0
- devkb/extractors/ticket_similarity.py +193 -0
- devkb/extractors/tickets_git.py +151 -0
- devkb/extractors/warnings.py +230 -0
- devkb/extractors/work_units.py +90 -0
- devkb/github.py +515 -0
- devkb/gitutil.py +16 -0
- devkb/index.py +51 -0
- devkb/init.py +178 -0
- devkb/mcp_server.py +216 -0
- devkb/model.py +221 -0
- devkb/module_file_bridge.py +57 -0
- devkb/ontology.md +178 -0
- devkb/packs.py +462 -0
- devkb/process_api.py +1601 -0
- devkb/project_context.py +191 -0
- devkb/query.py +251 -0
- devkb/service.py +324 -0
- devkb/startup.py +103 -0
- devkb/test_discovery.py +76 -0
- devkb/ts_symbols.py +291 -0
- devkb-0.2.0.dist-info/METADATA +162 -0
- devkb-0.2.0.dist-info/RECORD +61 -0
- devkb-0.2.0.dist-info/WHEEL +5 -0
- devkb-0.2.0.dist-info/entry_points.txt +2 -0
- devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
- devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
- devkb-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""Ticket + Bug Lineage + git commit log extractor (devkb Phase 1).
|
|
2
|
+
|
|
3
|
+
Ports an earlier traceability script's parsing logic
|
|
4
|
+
(_parse_metadata, _parse_lineage, _git_commits, _git_files_changed) rather
|
|
5
|
+
than rewriting it -- that logic is already correct, proven against OCO's own
|
|
6
|
+
large ticket/commit history via `make trace`. The port's job is only to (a)
|
|
7
|
+
persist the result instead of re-deriving it every call, and (b) parameterize
|
|
8
|
+
the ticket prefix / issues dir / repo root per project instead of hardcoding
|
|
9
|
+
OCO's own.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import re
|
|
14
|
+
import subprocess
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from devkb import github
|
|
18
|
+
from devkb.model import DevKBGraph
|
|
19
|
+
from devkb.project_context import ProjectContext
|
|
20
|
+
|
|
21
|
+
_LINEAGE_VERBS = {
|
|
22
|
+
"caused_by": "CAUSED_BY",
|
|
23
|
+
"regression_of": "REGRESSION_OF",
|
|
24
|
+
"duplicate_of": "DUPLICATE_OF",
|
|
25
|
+
"blocks": "BLOCKS",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _run(cmd: list[str], cwd: Path) -> str:
|
|
30
|
+
try:
|
|
31
|
+
r = subprocess.run(cmd, capture_output=True, text=True, cwd=str(cwd), timeout=30)
|
|
32
|
+
return r.stdout.strip()
|
|
33
|
+
except (OSError, subprocess.SubprocessError):
|
|
34
|
+
return ""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _iter_ticket_files(ctx: ProjectContext) -> list[Path]:
|
|
38
|
+
"""Every ticket file for this project: issues_dir plus any
|
|
39
|
+
historical/sprints/*/closed-tickets/ archives -- same two-location search
|
|
40
|
+
traceability.py's _find_issue does per-ticket, generalized to "all"."""
|
|
41
|
+
files: list[Path] = []
|
|
42
|
+
prefix_glob = f"{ctx.ticket_prefix}-*.md"
|
|
43
|
+
if ctx.issues_dir.is_dir():
|
|
44
|
+
files.extend(sorted(ctx.issues_dir.glob(prefix_glob)))
|
|
45
|
+
if ctx.sprint_dir.is_dir():
|
|
46
|
+
for sprint in sorted(ctx.sprint_dir.iterdir()):
|
|
47
|
+
closed = sprint / "closed-tickets"
|
|
48
|
+
if closed.is_dir():
|
|
49
|
+
files.extend(sorted(closed.glob(prefix_glob)))
|
|
50
|
+
return files
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _parse_metadata(text: str, path: Path) -> dict[str, str]:
|
|
54
|
+
stem = path.stem # PREFIX-NNN_TYPE_PRI_SEV_CREATED_RESOLVED_STATUS_COMPONENT_ASSIGNEE
|
|
55
|
+
parts = stem.split("_")
|
|
56
|
+
fields = ("ticket", "type", "priority", "severity", "created", "resolved", "status",
|
|
57
|
+
"component", "assignee")
|
|
58
|
+
meta = {name: (parts[i] if len(parts) > i else "?") for i, name in enumerate(fields)}
|
|
59
|
+
m = re.search(r"^# .+?: (.+)$", text, re.MULTILINE)
|
|
60
|
+
meta["title"] = m.group(1).strip() if m else "?"
|
|
61
|
+
return meta
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _parse_section(text: str, header: str) -> list[str]:
|
|
65
|
+
pat = rf"^{re.escape(header)}\s*\n((?:[-*].+\n?)*)"
|
|
66
|
+
m = re.search(pat, text, re.MULTILINE)
|
|
67
|
+
if not m:
|
|
68
|
+
return []
|
|
69
|
+
return [ln.strip() for ln in m.group(1).splitlines() if ln.strip()]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _parse_lineage(text: str) -> dict[str, str]:
|
|
73
|
+
fields: dict[str, str] = {}
|
|
74
|
+
for line in _parse_section(text, "## Bug Lineage"):
|
|
75
|
+
fm = re.match(r"[-*] \*\*(\w+)\*\*:\s*(.+?)(?:\s+—.*)?$", line)
|
|
76
|
+
if fm:
|
|
77
|
+
fields[fm.group(1)] = fm.group(2).strip()
|
|
78
|
+
return fields
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _git_commits(ticket: str, repo_root: Path) -> list[dict[str, str]]:
|
|
82
|
+
out = _run(["git", "log", "--all", f"--grep={ticket}", "--format=%H|%ad|%an|%s",
|
|
83
|
+
"--date=short"], cwd=repo_root)
|
|
84
|
+
# git's own --grep is an unanchored substring/regex match, so
|
|
85
|
+
# --grep=OCO-461 also matches "OCO-4615", "OCO-4610"-"OCO-4619", etc --
|
|
86
|
+
# confirmed live against this repo's own history: OCO-461 alone pulled in
|
|
87
|
+
# a commit for OCO-4615 that relocated the whole factory/ submodule,
|
|
88
|
+
# spuriously attributing ~20k touched files to OCO-461 once devkb Phase 2
|
|
89
|
+
# (ODSF-032) built TOUCHES edges on top of this ticket's commit list. A
|
|
90
|
+
# \b-anchored post-filter rejects that false match ("461" has a word
|
|
91
|
+
# character immediately after it in "4615", so no boundary there) while
|
|
92
|
+
# still accepting real mentions followed by punctuation/whitespace/EOL.
|
|
93
|
+
pattern = re.compile(rf"\b{re.escape(ticket)}\b")
|
|
94
|
+
commits = []
|
|
95
|
+
for line in out.splitlines():
|
|
96
|
+
parts = line.split("|", 3)
|
|
97
|
+
if len(parts) == 4 and pattern.search(parts[3]):
|
|
98
|
+
commits.append({"hash": parts[0], "date": parts[1], "author": parts[2], "subject": parts[3]})
|
|
99
|
+
return commits
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def extract(ctx: ProjectContext) -> DevKBGraph:
|
|
103
|
+
g = DevKBGraph()
|
|
104
|
+
authors = github.settings(ctx.repo_root).authors
|
|
105
|
+
for path in _iter_ticket_files(ctx):
|
|
106
|
+
try:
|
|
107
|
+
text = path.read_text(errors="replace")
|
|
108
|
+
except OSError:
|
|
109
|
+
continue
|
|
110
|
+
meta = _parse_metadata(text, path)
|
|
111
|
+
ticket = meta["ticket"]
|
|
112
|
+
if ticket == "?":
|
|
113
|
+
continue
|
|
114
|
+
|
|
115
|
+
g.add_node(
|
|
116
|
+
"Ticket", ticket, source="tickets",
|
|
117
|
+
title=meta["title"], type=meta["type"],
|
|
118
|
+
# ADR-SDP-DEVKB-002 §2: the alias source for Epic/Story queries
|
|
119
|
+
# (`Ticket{ticket_type: EPIC|STORY}`), normalised so a lower-case
|
|
120
|
+
# filename type cannot fall out of the alias.
|
|
121
|
+
ticket_type=meta["type"].upper(), priority=meta["priority"],
|
|
122
|
+
severity=meta["severity"], status=meta["status"], component=meta["component"],
|
|
123
|
+
assignee=meta["assignee"], created=meta["created"], resolved=meta["resolved"],
|
|
124
|
+
issue_path=str(path.relative_to(ctx.repo_root)),
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
lineage = _parse_lineage(text)
|
|
128
|
+
evidence = f"{path.relative_to(ctx.repo_root)}#Bug-Lineage"
|
|
129
|
+
for field, verb in _LINEAGE_VERBS.items():
|
|
130
|
+
target = lineage.get(field)
|
|
131
|
+
if not target:
|
|
132
|
+
continue
|
|
133
|
+
for target_ticket in re.findall(rf"{re.escape(ctx.ticket_prefix)}-\d+", target):
|
|
134
|
+
g.add_edge(
|
|
135
|
+
verb, "Ticket", ticket, "Ticket", target_ticket,
|
|
136
|
+
source="bug-lineage", confidence=1.0, evidence_ref=evidence,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
commits = _git_commits(ticket, ctx.repo_root)
|
|
140
|
+
for c in commits:
|
|
141
|
+
g.add_node(
|
|
142
|
+
"Commit", c["hash"], source="git",
|
|
143
|
+
short_hash=c["hash"][:10], date=c["date"], author=c["author"],
|
|
144
|
+
subject=github.public_subject(c["subject"], authors),
|
|
145
|
+
)
|
|
146
|
+
g.add_edge(
|
|
147
|
+
"REFERENCES", "Commit", c["hash"], "Ticket", ticket,
|
|
148
|
+
source="git-log-grep", confidence=1.0,
|
|
149
|
+
evidence_ref=f"git log --grep={ticket}",
|
|
150
|
+
)
|
|
151
|
+
return g
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""Compliance-violation extractor (devkb Phase 3, ADR-SDP-DEVKB-001).
|
|
2
|
+
|
|
3
|
+
Ingests dev_compliance_monitor.py --json-report's already-structured
|
|
4
|
+
{rule, severity, detail, evidence} feed as-is -- zero new parsing invented,
|
|
5
|
+
per the roadmap's own stated goal. Each registered project carries its own
|
|
6
|
+
copy of dev_compliance_monitor.py under sdp/devsecops/compliance/ (its own
|
|
7
|
+
ROOT resolves via that copy's own __file__, so subprocess-invoking THAT
|
|
8
|
+
project's copy correctly checks THAT project's repo -- confirmed ODSF and
|
|
9
|
+
OCO each have a real copy; others have none). A project without the script
|
|
10
|
+
gets an empty Warning set, not an error -- same "can't check yet" framing
|
|
11
|
+
already used elsewhere in devkb for a missing discovery/graph.json.
|
|
12
|
+
|
|
13
|
+
Always passes --dry-run: dev_compliance_monitor.py's own main() unconditionally
|
|
14
|
+
calls create_issue() for every violation found when NOT in dry-run mode --
|
|
15
|
+
devkb's build must never have the side effect of filing real tickets.
|
|
16
|
+
|
|
17
|
+
Most violations have no parseable source-file reference (DEV-06/09's
|
|
18
|
+
evidence is "File: <ticket>.md" pointing at issues/, not source; DEV-01/02/
|
|
19
|
+
03/04/07/08/10's evidence is process-level free text -- a SHA, a "Run: make
|
|
20
|
+
..." hint, a coverage percentage). FLAGGED_BY is only emitted when a real,
|
|
21
|
+
on-disk repo-relative path can be confidently parsed out of `evidence`. The
|
|
22
|
+
Warning node itself is always created regardless -- a warning with no file
|
|
23
|
+
target is still a real, queryable finding ("list every open warning"), not
|
|
24
|
+
silently dropped.
|
|
25
|
+
"""
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import hashlib
|
|
29
|
+
import json
|
|
30
|
+
import re
|
|
31
|
+
import subprocess
|
|
32
|
+
import tempfile
|
|
33
|
+
from datetime import datetime, timezone
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
|
|
36
|
+
from devkb.model import DevKBGraph
|
|
37
|
+
from devkb.project_context import ProjectContext, run_repo_scripts
|
|
38
|
+
|
|
39
|
+
_COMPLIANCE_SCRIPT_REL = "sdp/devsecops/compliance/dev_compliance_monitor.py"
|
|
40
|
+
_TIMEOUT_S = 120
|
|
41
|
+
# Effectively "all time" -- devkb wants the true current state, not a
|
|
42
|
+
# rolling recent window (the script's own --since default is 24h, meant for
|
|
43
|
+
# its usual CI/cron usage, not a durable "what's currently flagged" answer).
|
|
44
|
+
_SINCE_WINDOW = "3650d"
|
|
45
|
+
|
|
46
|
+
# DEV-05's own evidence shape: "path:line: text[; path:line: text ...]".
|
|
47
|
+
_PATH_LINE_RE = re.compile(r"([\w./-]+):(\d+):")
|
|
48
|
+
# DEV-06/DEV-09's own evidence shape: "File: <path>".
|
|
49
|
+
_FILE_PREFIX_RE = re.compile(r"^File:\s*(\S+)")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _run_compliance_scan(ctx: ProjectContext) -> list[dict[str, str]]:
|
|
53
|
+
script = ctx.repo_root / _COMPLIANCE_SCRIPT_REL
|
|
54
|
+
if not script.is_file() or not run_repo_scripts(ctx.repo_root):
|
|
55
|
+
return []
|
|
56
|
+
with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp:
|
|
57
|
+
tmp_path = Path(tmp.name)
|
|
58
|
+
try:
|
|
59
|
+
try:
|
|
60
|
+
subprocess.run(
|
|
61
|
+
["python3", str(script), "--check-all", "--since", _SINCE_WINDOW,
|
|
62
|
+
"--dry-run", "--json-report", str(tmp_path)],
|
|
63
|
+
cwd=str(ctx.repo_root), capture_output=True, text=True, timeout=_TIMEOUT_S,
|
|
64
|
+
)
|
|
65
|
+
except (OSError, subprocess.SubprocessError):
|
|
66
|
+
return []
|
|
67
|
+
try:
|
|
68
|
+
data = json.loads(tmp_path.read_text())
|
|
69
|
+
except (OSError, ValueError):
|
|
70
|
+
return []
|
|
71
|
+
finally:
|
|
72
|
+
tmp_path.unlink(missing_ok=True)
|
|
73
|
+
violations = data.get("violations", [])
|
|
74
|
+
return violations if isinstance(violations, list) else []
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _extract_files(evidence: str, repo_root: Path) -> list[str]:
|
|
78
|
+
"""Best-effort, conservative: only returns paths that actually exist on
|
|
79
|
+
disk under repo_root -- avoids turning a misparsed "Run: make ..." hint
|
|
80
|
+
into a phantom File node."""
|
|
81
|
+
found: list[str] = []
|
|
82
|
+
for m in _PATH_LINE_RE.finditer(evidence):
|
|
83
|
+
candidate = m.group(1)
|
|
84
|
+
if candidate not in found and (repo_root / candidate).is_file():
|
|
85
|
+
found.append(candidate)
|
|
86
|
+
m = _FILE_PREFIX_RE.match(evidence)
|
|
87
|
+
if m and m.group(1) not in found and (repo_root / m.group(1)).is_file():
|
|
88
|
+
found.append(m.group(1))
|
|
89
|
+
return found
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def extract(ctx: ProjectContext) -> DevKBGraph:
|
|
93
|
+
g = DevKBGraph()
|
|
94
|
+
for v in _run_compliance_scan(ctx):
|
|
95
|
+
rule = v.get("rule", "")
|
|
96
|
+
detail = v.get("detail", "")
|
|
97
|
+
evidence = v.get("evidence", "")
|
|
98
|
+
if not rule:
|
|
99
|
+
continue
|
|
100
|
+
# Natural key: sha256(rule+detail+evidence) -- the ontology's original
|
|
101
|
+
# spec was sha256(rule+evidence) alone, widened here after confirming
|
|
102
|
+
# live that several rules' own `evidence` string is fixed boilerplate
|
|
103
|
+
# (e.g. DEV-07's is always the literal "No ADR issue found in issues/
|
|
104
|
+
# for recent schema change", identical across every DEV-07 hit) while
|
|
105
|
+
# `detail` is what actually varies per violation (the specific file,
|
|
106
|
+
# sha, ticket id). rule+evidence alone would silently collapse two
|
|
107
|
+
# distinct real violations into one Warning node the moment a second
|
|
108
|
+
# one occurred -- confirmed this had not yet happened in any
|
|
109
|
+
# registered project's real data, but was a live landmine, not a
|
|
110
|
+
# hypothetical.
|
|
111
|
+
natural_key = hashlib.sha256(f"{rule}:{detail}:{evidence}".encode()).hexdigest()
|
|
112
|
+
g.add_node(
|
|
113
|
+
"Warning", natural_key, source="dev-compliance-monitor",
|
|
114
|
+
rule=rule, severity=v.get("severity", ""), detail=detail,
|
|
115
|
+
evidence=evidence,
|
|
116
|
+
)
|
|
117
|
+
for path in _extract_files(evidence, ctx.repo_root):
|
|
118
|
+
# File may already exist from Phase 2's TOUCHES extraction --
|
|
119
|
+
# add_node() merges props onto the existing node, doesn't
|
|
120
|
+
# duplicate (DevKBGraph.add_node's own dedup-by-key behavior).
|
|
121
|
+
g.add_node("File", path, source="dev-compliance-monitor")
|
|
122
|
+
g.add_edge(
|
|
123
|
+
"FLAGGED_BY", "File", path, "Warning", natural_key,
|
|
124
|
+
source="dev-compliance-monitor", confidence=1.0,
|
|
125
|
+
evidence_ref=f"{rule}: {evidence[:120]}",
|
|
126
|
+
)
|
|
127
|
+
return g
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _history_path(ctx: ProjectContext) -> Path:
|
|
131
|
+
return ctx.devkb_dir / "warnings-history.jsonl"
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _load_history(history_path: Path) -> list[dict]: # type: ignore[type-arg]
|
|
135
|
+
if not history_path.exists():
|
|
136
|
+
return []
|
|
137
|
+
out: list[dict] = [] # type: ignore[type-arg]
|
|
138
|
+
for line in history_path.read_text().splitlines():
|
|
139
|
+
line = line.strip()
|
|
140
|
+
if not line:
|
|
141
|
+
continue
|
|
142
|
+
try:
|
|
143
|
+
out.append(json.loads(line))
|
|
144
|
+
except ValueError:
|
|
145
|
+
continue # a corrupt line doesn't invalidate the rest of the log
|
|
146
|
+
return out
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _append_history(history_path: Path, entry: dict) -> None: # type: ignore[type-arg]
|
|
150
|
+
history_path.parent.mkdir(parents=True, exist_ok=True)
|
|
151
|
+
with history_path.open("a") as f:
|
|
152
|
+
f.write(json.dumps(entry) + "\n")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _scan_time(ctx: ProjectContext) -> str:
|
|
156
|
+
"""The time this build's scan stands for: the HEAD commit's time, not the wall clock
|
|
157
|
+
(ODSF-230 / ADR-SDP-DEVKB-002 determinism) — rebuilding an unchanged repo yields the same
|
|
158
|
+
graph. Falls back to the clock only outside a git checkout."""
|
|
159
|
+
import subprocess # noqa: PLC0415
|
|
160
|
+
|
|
161
|
+
try:
|
|
162
|
+
out = subprocess.run(["git", "-C", str(ctx.repo_root), "log", "-1", "--format=%cI"],
|
|
163
|
+
capture_output=True, text=True, timeout=30, check=False).stdout.strip()
|
|
164
|
+
except (OSError, subprocess.TimeoutExpired, AttributeError):
|
|
165
|
+
out = ""
|
|
166
|
+
return out or datetime.now(timezone.utc).isoformat()
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def apply_lifecycle(ctx: ProjectContext, graph: DevKBGraph) -> None:
|
|
170
|
+
"""Adds first_seen/last_seen/resolved_at/status to every Warning node
|
|
171
|
+
the current build knows about, PLUS re-adds a node for any warning that
|
|
172
|
+
is in warnings-history.jsonl but absent from this run (i.e. resolved --
|
|
173
|
+
extract() only ever returns currently-open violations, so a fixed
|
|
174
|
+
warning would otherwise silently vanish from the graph the moment it's
|
|
175
|
+
fixed instead of leaving a "this was open from X to Y" record, the exact
|
|
176
|
+
lifecycle tracking this step exists for).
|
|
177
|
+
|
|
178
|
+
Must be called on the final MERGED graph, after every extractor's own
|
|
179
|
+
Warning nodes are already in it -- lifecycle needs the whole picture,
|
|
180
|
+
not one extractor's partial view. Appends this build's own scan to
|
|
181
|
+
warnings-history.jsonl as its last step (side effect, not idempotent to
|
|
182
|
+
call twice per build -- build.py calls it exactly once)."""
|
|
183
|
+
history_path = _history_path(ctx)
|
|
184
|
+
history = _load_history(history_path)
|
|
185
|
+
now_iso = _scan_time(ctx)
|
|
186
|
+
|
|
187
|
+
current: list[dict] = [
|
|
188
|
+
{
|
|
189
|
+
"natural_key": n.natural_key, "rule": n.props.get("rule", ""),
|
|
190
|
+
"severity": n.props.get("severity", ""), "detail": n.props.get("detail", ""),
|
|
191
|
+
"evidence": n.props.get("evidence", ""),
|
|
192
|
+
}
|
|
193
|
+
for n in graph.nodes("Warning")
|
|
194
|
+
]
|
|
195
|
+
|
|
196
|
+
scans = [*history, {"scanned_at": now_iso, "violations": current}]
|
|
197
|
+
|
|
198
|
+
# "" (not None) marks "still open" -- DevKBGraph.add_node() silently
|
|
199
|
+
# drops any prop whose value is None (so a partial update never clobbers
|
|
200
|
+
# an already-set value on merge), which would make resolved_at vanish
|
|
201
|
+
# from props entirely for every open warning instead of storing "not
|
|
202
|
+
# resolved". An empty string is a real, present, falsy value that
|
|
203
|
+
# survives that filter and still means "no resolution timestamp".
|
|
204
|
+
lifecycle: dict[str, dict] = {}
|
|
205
|
+
last_content: dict[str, dict] = {}
|
|
206
|
+
for scan in scans:
|
|
207
|
+
ts = scan.get("scanned_at", now_iso)
|
|
208
|
+
open_now = {v["natural_key"]: v for v in scan.get("violations", []) if v.get("natural_key")}
|
|
209
|
+
for key, rec in lifecycle.items():
|
|
210
|
+
if rec["resolved_at"] == "" and key not in open_now:
|
|
211
|
+
rec["resolved_at"] = ts
|
|
212
|
+
for key, content in open_now.items():
|
|
213
|
+
last_content[key] = content
|
|
214
|
+
if key not in lifecycle:
|
|
215
|
+
lifecycle[key] = {"first_seen": ts, "last_seen": ts, "resolved_at": ""}
|
|
216
|
+
else:
|
|
217
|
+
lifecycle[key]["last_seen"] = ts
|
|
218
|
+
lifecycle[key]["resolved_at"] = "" # reopened, if it had been resolved
|
|
219
|
+
|
|
220
|
+
for key, rec in lifecycle.items():
|
|
221
|
+
content = last_content[key]
|
|
222
|
+
graph.add_node(
|
|
223
|
+
"Warning", key, source="dev-compliance-monitor",
|
|
224
|
+
rule=content["rule"], severity=content["severity"], detail=content["detail"],
|
|
225
|
+
evidence=content["evidence"],
|
|
226
|
+
first_seen=rec["first_seen"], last_seen=rec["last_seen"], resolved_at=rec["resolved_at"],
|
|
227
|
+
status="open" if rec["resolved_at"] == "" else "resolved",
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
_append_history(history_path, {"scanned_at": now_iso, "violations": current})
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Commit-cluster inference (devkb BL-6, body/legs model) -- deterministic
|
|
2
|
+
clustering only. The LLM narrative step (a plain-language description per
|
|
3
|
+
cluster) is flagged as follow-up work requiring an explicit cost go-ahead,
|
|
4
|
+
not built here -- a WorkUnit is still real, queryable signal without it
|
|
5
|
+
(member commits, span, touched-file count).
|
|
6
|
+
|
|
7
|
+
For a repo with real commit history and zero real tickets: groups Commit
|
|
8
|
+
nodes with no REFERENCES edge to a real Ticket into WorkUnit nodes -- a
|
|
9
|
+
first approximation of "this was probably one unit of work." Deliberately
|
|
10
|
+
a DIFFERENT node type from Ticket, never merged or presented as
|
|
11
|
+
equivalent -- see devkb/ROADMAP.md's own body/legs model / Cross-phase
|
|
12
|
+
notes for why that distinction is permanent, not a naming detail.
|
|
13
|
+
|
|
14
|
+
Clustering signal is day-granularity (git log's own --date=short resolution,
|
|
15
|
+
already fetched elsewhere -- no extra subprocess call added just for finer
|
|
16
|
+
timestamps) + same author + any touched-file overlap. Confidence is 0.5 on
|
|
17
|
+
every GROUPS edge -- lower than any other edge in the graph -- because this
|
|
18
|
+
is a labeled guess, not derived fact.
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from devkb.model import DevKBGraph, Node
|
|
23
|
+
from devkb.project_context import ProjectContext
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _unreferenced_commits(graph: DevKBGraph) -> list[Node]:
|
|
27
|
+
referenced = {e.src for e in graph.edges("REFERENCES")}
|
|
28
|
+
commits = [c for c in graph.nodes("Commit") if c.natural_key not in referenced]
|
|
29
|
+
return sorted(commits, key=lambda c: c.props.get("date", ""))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _commit_to_files_index(graph: DevKBGraph) -> dict[str, set[str]]:
|
|
33
|
+
index: dict[str, set[str]] = {}
|
|
34
|
+
for e in graph.edges("TOUCHES"):
|
|
35
|
+
index.setdefault(e.src, set()).add(e.dst)
|
|
36
|
+
return index
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _cluster(commits: list[Node], files_by_commit: dict[str, set[str]]) -> list[list[Node]]:
|
|
40
|
+
clusters: list[list[Node]] = []
|
|
41
|
+
current: list[Node] = []
|
|
42
|
+
current_files: set[str] = set()
|
|
43
|
+
for c in commits:
|
|
44
|
+
this_files = files_by_commit.get(c.natural_key, set())
|
|
45
|
+
if not current:
|
|
46
|
+
current, current_files = [c], set(this_files)
|
|
47
|
+
continue
|
|
48
|
+
same_day = current[-1].props.get("date") == c.props.get("date")
|
|
49
|
+
same_author = current[-1].props.get("author") == c.props.get("author")
|
|
50
|
+
overlaps = bool(current_files & this_files) or not current_files or not this_files
|
|
51
|
+
if same_day and same_author and overlaps:
|
|
52
|
+
current.append(c)
|
|
53
|
+
current_files |= this_files
|
|
54
|
+
else:
|
|
55
|
+
clusters.append(current)
|
|
56
|
+
current, current_files = [c], set(this_files)
|
|
57
|
+
if current:
|
|
58
|
+
clusters.append(current)
|
|
59
|
+
return clusters
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def extract(ctx: ProjectContext, graph: DevKBGraph) -> DevKBGraph:
|
|
63
|
+
"""graph must already contain Commit nodes + REFERENCES edges (Phase 1)
|
|
64
|
+
+ TOUCHES edges (BL-1/Phase 2) -- build.py merges these before calling
|
|
65
|
+
this."""
|
|
66
|
+
g = DevKBGraph()
|
|
67
|
+
commits = _unreferenced_commits(graph)
|
|
68
|
+
if not commits:
|
|
69
|
+
return g
|
|
70
|
+
files_by_commit = _commit_to_files_index(graph)
|
|
71
|
+
|
|
72
|
+
for idx, cluster in enumerate(_cluster(commits, files_by_commit), start=1):
|
|
73
|
+
first, last = cluster[0], cluster[-1]
|
|
74
|
+
unit_key = f"{first.props.get('date', 'unknown')}-{idx}-{first.natural_key[:10]}"
|
|
75
|
+
touched: set[str] = set()
|
|
76
|
+
for c in cluster:
|
|
77
|
+
touched |= files_by_commit.get(c.natural_key, set())
|
|
78
|
+
g.add_node(
|
|
79
|
+
"WorkUnit", unit_key, source="commit-cluster-inference",
|
|
80
|
+
commit_count=len(cluster), first_date=first.props.get("date"),
|
|
81
|
+
last_date=last.props.get("date"), author=first.props.get("author"),
|
|
82
|
+
touched_file_count=len(touched),
|
|
83
|
+
)
|
|
84
|
+
for c in cluster:
|
|
85
|
+
g.add_edge(
|
|
86
|
+
"GROUPS", "WorkUnit", unit_key, "Commit", c.natural_key,
|
|
87
|
+
source="commit-cluster-inference", confidence=0.5,
|
|
88
|
+
evidence_ref=f"same day + same author + touched-file overlap, cluster {unit_key}",
|
|
89
|
+
)
|
|
90
|
+
return g
|