devkb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devkb/__init__.py +11 -0
- devkb/audit.py +173 -0
- devkb/bench/add_packs.py +52 -0
- devkb/bench/cli_tasks.py +105 -0
- devkb/bench/confirm_tasks.py +149 -0
- devkb/bench/module_truth.py +249 -0
- devkb/bench/oco_tasks.py +216 -0
- devkb/bench/pipeline.py +313 -0
- devkb/bench/process_tasks.py +135 -0
- devkb/bench/report.py +74 -0
- devkb/bench/run.py +174 -0
- devkb/bench/truth_check.py +148 -0
- devkb/bench/workflow_report.py +64 -0
- devkb/bench/workflow_run.py +206 -0
- devkb/bench/workflow_tasks.py +227 -0
- devkb/build.py +258 -0
- devkb/build_symbols.py +1394 -0
- devkb/call_recall.py +159 -0
- devkb/cli.py +53 -0
- devkb/coverage.py +155 -0
- devkb/edge_truth.py +272 -0
- devkb/extractors/backlog.py +113 -0
- devkb/extractors/component_subsystem.py +116 -0
- devkb/extractors/configuration.py +267 -0
- devkb/extractors/files_tests.py +220 -0
- devkb/extractors/flows.py +117 -0
- devkb/extractors/lifecycle.py +258 -0
- devkb/extractors/members.py +160 -0
- devkb/extractors/pull_requests.py +271 -0
- devkb/extractors/releases.py +49 -0
- devkb/extractors/source_baseline.py +66 -0
- devkb/extractors/spec_docs.py +220 -0
- devkb/extractors/symbol_tests.py +131 -0
- devkb/extractors/test_inventory.py +38 -0
- devkb/extractors/ticket_similarity.py +193 -0
- devkb/extractors/tickets_git.py +151 -0
- devkb/extractors/warnings.py +230 -0
- devkb/extractors/work_units.py +90 -0
- devkb/github.py +515 -0
- devkb/gitutil.py +16 -0
- devkb/index.py +51 -0
- devkb/init.py +178 -0
- devkb/mcp_server.py +216 -0
- devkb/model.py +221 -0
- devkb/module_file_bridge.py +57 -0
- devkb/ontology.md +178 -0
- devkb/packs.py +462 -0
- devkb/process_api.py +1601 -0
- devkb/project_context.py +191 -0
- devkb/query.py +251 -0
- devkb/service.py +324 -0
- devkb/startup.py +103 -0
- devkb/test_discovery.py +76 -0
- devkb/ts_symbols.py +291 -0
- devkb-0.2.0.dist-info/METADATA +162 -0
- devkb-0.2.0.dist-info/RECORD +61 -0
- devkb-0.2.0.dist-info/WHEEL +5 -0
- devkb-0.2.0.dist-info/entry_points.txt +2 -0
- devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
- devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
- devkb-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Work package + milestone extractor (devkb Phase 1).
|
|
2
|
+
|
|
3
|
+
Ports pms-backend/pmsapp/routers/pms.py's _parse_workpackages/_parse_milestones/
|
|
4
|
+
_milestone_members/wp-map.json reverse-lookup logic rather than rewriting it --
|
|
5
|
+
that parsing is already correct and exercised daily by the live PMS board. The
|
|
6
|
+
port's job is only to persist it as PART_OF/BELONGS_TO edges instead of
|
|
7
|
+
re-deriving it on every /workpackages or /milestones request.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from devkb.model import DevKBGraph
|
|
17
|
+
from devkb.project_context import ProjectContext
|
|
18
|
+
|
|
19
|
+
_WP_ROW_RE = re.compile(
|
|
20
|
+
r"\|\s*([A-Z]-\d{2,3}[a-z]?)\s*\|\s*([^|]+?)\s*\|\s*(P[012])\s*\|\s*([A-Z_]+)\s*\|"
|
|
21
|
+
r"\s*([SMXL]+)\s*\|\s*([A-Z+]+)\s*\|\s*(\d+)\s*\|"
|
|
22
|
+
)
|
|
23
|
+
_MILESTONE_ROW_RE = re.compile(
|
|
24
|
+
r"^\|\s*(M\d+)\s*—\s*([^|]+?)\s*\|\s*(.+?)\s*\|\s*([^|]+?)\s*\|\s*([^|]+?)\s*\|$"
|
|
25
|
+
)
|
|
26
|
+
_WP_ID_TOKEN_RE = re.compile(r"\b([A-Z]-\d{2,3}[a-z]?)\b")
|
|
27
|
+
_WP_ID_RANGE_RE = re.compile(r"[A-Z]-\d{2,3}[a-z]?\s*\.\.+\s*[A-Z]-\d{2,3}[a-z]?")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _read_json(path: Path, default: Any = None) -> Any:
|
|
31
|
+
try:
|
|
32
|
+
return json.loads(path.read_text())
|
|
33
|
+
except (OSError, ValueError):
|
|
34
|
+
return default
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _parse_workpackages(text: str) -> list[dict[str, Any]]:
|
|
38
|
+
by_id: dict[str, dict[str, Any]] = {}
|
|
39
|
+
for line in text.splitlines():
|
|
40
|
+
m = _WP_ROW_RE.match(line)
|
|
41
|
+
if not m:
|
|
42
|
+
continue
|
|
43
|
+
wp_id, title, priority, status, effort, owner, sprint = m.groups()
|
|
44
|
+
by_id[wp_id] = {
|
|
45
|
+
"id": wp_id, "stream": wp_id[0], "title": title.strip(),
|
|
46
|
+
"priority": priority.strip(), "status": status.strip(),
|
|
47
|
+
"effort": effort.strip(), "owner": owner.strip(), "sprint": sprint.strip(),
|
|
48
|
+
}
|
|
49
|
+
return list(by_id.values())
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _parse_milestones(text: str) -> list[dict[str, Any]]:
|
|
53
|
+
in_section = False
|
|
54
|
+
results: list[dict[str, Any]] = []
|
|
55
|
+
for line in text.splitlines():
|
|
56
|
+
if line.startswith("## MILESTONE SUMMARY"):
|
|
57
|
+
in_section = True
|
|
58
|
+
continue
|
|
59
|
+
if in_section and line.startswith("## "):
|
|
60
|
+
break
|
|
61
|
+
if not in_section:
|
|
62
|
+
continue
|
|
63
|
+
m = _MILESTONE_ROW_RE.match(line)
|
|
64
|
+
if m:
|
|
65
|
+
mid, title, exit_condition, owner, sprint = m.groups()
|
|
66
|
+
results.append({
|
|
67
|
+
"id": mid, "title": title.strip(), "exit_condition": exit_condition.strip(),
|
|
68
|
+
"owner": owner.strip(), "sprint": sprint.strip(),
|
|
69
|
+
})
|
|
70
|
+
return results
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _milestone_members(milestones: list[dict[str, Any]]) -> dict[str, set[str]]:
|
|
74
|
+
out: dict[str, set[str]] = {}
|
|
75
|
+
for m in milestones:
|
|
76
|
+
exit_cond = _WP_ID_RANGE_RE.sub(" ", m.get("exit_condition", ""))
|
|
77
|
+
out[m["id"]] = set(_WP_ID_TOKEN_RE.findall(exit_cond))
|
|
78
|
+
return out
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def extract(ctx: ProjectContext) -> DevKBGraph:
|
|
82
|
+
g = DevKBGraph()
|
|
83
|
+
if not ctx.board_file.exists():
|
|
84
|
+
return g
|
|
85
|
+
text = ctx.board_file.read_text()
|
|
86
|
+
board_evidence = str(ctx.board_file.relative_to(ctx.repo_root))
|
|
87
|
+
|
|
88
|
+
wps = _parse_workpackages(text)
|
|
89
|
+
milestones = _parse_milestones(text)
|
|
90
|
+
wp_map = _read_json(ctx.wp_map_file, {})
|
|
91
|
+
members = _milestone_members(milestones)
|
|
92
|
+
|
|
93
|
+
for m in milestones:
|
|
94
|
+
g.add_node("Milestone", m["id"], source="backlog", title=m["title"],
|
|
95
|
+
exit_condition=m["exit_condition"], owner=m["owner"], sprint=m["sprint"])
|
|
96
|
+
|
|
97
|
+
for wp in wps:
|
|
98
|
+
g.add_node("WorkPackage", wp["id"], source="backlog", title=wp["title"],
|
|
99
|
+
stream=wp["stream"], priority=wp["priority"], status=wp["status"],
|
|
100
|
+
effort=wp["effort"], owner=wp["owner"], sprint=wp["sprint"])
|
|
101
|
+
|
|
102
|
+
ticket = wp_map.get(f"wp:{wp['id']}")
|
|
103
|
+
if ticket:
|
|
104
|
+
g.add_edge("PART_OF", "Ticket", ticket, "WorkPackage", wp["id"],
|
|
105
|
+
source="wp-map", confidence=1.0,
|
|
106
|
+
evidence_ref=str(ctx.wp_map_file.relative_to(ctx.repo_root)))
|
|
107
|
+
|
|
108
|
+
for milestone_id, wp_ids in members.items():
|
|
109
|
+
for wp_id in wp_ids:
|
|
110
|
+
g.add_edge("BELONGS_TO", "WorkPackage", wp_id, "Milestone", milestone_id,
|
|
111
|
+
source="board-milestone-summary", confidence=1.0, evidence_ref=board_evidence)
|
|
112
|
+
|
|
113
|
+
return g
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Component + Subsystem containment (devkb BL-8, KNOWLEDGE_SPEC.md Section
|
|
2
|
+
4/5).
|
|
3
|
+
|
|
4
|
+
Per `12-sys-arch.pdf` (Cornell CS5150): a Component is the *implementation
|
|
5
|
+
of* a Subsystem, not a level above it -- Section 2's audit corrected an
|
|
6
|
+
earlier inverted framing. Both sides are real, already-existing data, zero
|
|
7
|
+
new hand-curated grouping:
|
|
8
|
+
- Component mirrors `discovery/`'s own already-built `Service` nodes
|
|
9
|
+
(`.onboarding/graph.json`) -- same natural_key, so the two stores stay
|
|
10
|
+
joinable without a lookup table.
|
|
11
|
+
- Subsystem is `pms.config.json`'s own real `subsystems` list.
|
|
12
|
+
- PART_OF (File -> Component): from each Module's own `service` property,
|
|
13
|
+
already computed by discovery, promoted to a real edge.
|
|
14
|
+
- REALIZES (Component -> Subsystem): a deterministic path-prefix match,
|
|
15
|
+
longest match wins when a Component's path falls under more than one
|
|
16
|
+
subsystem path candidate (skip, don't guess, when none match).
|
|
17
|
+
|
|
18
|
+
Needs no ticket, no commit history, no build_symbols.py output -- purely a
|
|
19
|
+
function of `.onboarding/graph.json` (if a discovery scan has been run for
|
|
20
|
+
this project) and `pms.config.json` (always present, `project_context.py`
|
|
21
|
+
already refuses to resolve a project without one). Missing
|
|
22
|
+
`.onboarding/graph.json` yields an empty graph, not an error -- same
|
|
23
|
+
"can't check yet" framing used everywhere else in devkb for an optional
|
|
24
|
+
upstream input.
|
|
25
|
+
"""
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import json
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
|
|
31
|
+
from devkb.model import DevKBGraph
|
|
32
|
+
from devkb.project_context import ProjectContext, load_config
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _read_json(path: Path) -> dict | None:
|
|
36
|
+
try:
|
|
37
|
+
return json.loads(path.read_text())
|
|
38
|
+
except (OSError, ValueError):
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _subsystem_for(component_path: str, subsystems: list[dict]) -> dict | None:
|
|
43
|
+
"""Longest-prefix match: component_path == subsystem path, or nested
|
|
44
|
+
one directory level or more under it. None when no subsystem's path is
|
|
45
|
+
a real prefix -- skipped, not guessed."""
|
|
46
|
+
best: dict | None = None
|
|
47
|
+
for s in subsystems:
|
|
48
|
+
sp = s.get("path", "").strip("/")
|
|
49
|
+
if not sp:
|
|
50
|
+
continue
|
|
51
|
+
if component_path == sp or component_path.startswith(f"{sp}/"):
|
|
52
|
+
if best is None or len(sp) > len(best.get("path", "").strip("/")):
|
|
53
|
+
best = s
|
|
54
|
+
return best
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def extract(ctx: ProjectContext) -> DevKBGraph:
|
|
58
|
+
g = DevKBGraph()
|
|
59
|
+
|
|
60
|
+
graph_path = ctx.repo_root / ".onboarding" / "graph.json"
|
|
61
|
+
discovery = _read_json(graph_path)
|
|
62
|
+
if discovery is None:
|
|
63
|
+
return g
|
|
64
|
+
|
|
65
|
+
nodes = discovery.get("nodes", [])
|
|
66
|
+
services = {n["natural_key"]: n for n in nodes if n.get("type") == "Service"}
|
|
67
|
+
modules = [n for n in nodes if n.get("type") == "Module"]
|
|
68
|
+
if not services:
|
|
69
|
+
return g
|
|
70
|
+
|
|
71
|
+
cfg = load_config(ctx.repo_root)
|
|
72
|
+
subsystems = cfg.get("subsystems", [])
|
|
73
|
+
|
|
74
|
+
# Component nodes + REALIZES -- one per real discovery Service, path
|
|
75
|
+
# relative to the repo root (needed both for REALIZES' own prefix
|
|
76
|
+
# match and as the Component's own queryable `path` property).
|
|
77
|
+
component_paths: dict[str, str] = {}
|
|
78
|
+
for natural_key, svc in services.items():
|
|
79
|
+
root = svc.get("root")
|
|
80
|
+
if not root:
|
|
81
|
+
continue
|
|
82
|
+
try:
|
|
83
|
+
rel_path = str(Path(root).resolve().relative_to(ctx.repo_root)).replace("\\", "/")
|
|
84
|
+
except ValueError:
|
|
85
|
+
continue # a Service root outside this repo -- not this project's own component
|
|
86
|
+
component_paths[natural_key] = rel_path
|
|
87
|
+
g.add_node("Component", natural_key, source="discovery-service", label=natural_key, path=rel_path)
|
|
88
|
+
|
|
89
|
+
match = _subsystem_for(rel_path, subsystems)
|
|
90
|
+
if match:
|
|
91
|
+
g.add_node("Subsystem", match["key"], source="pms.config.json",
|
|
92
|
+
label=match.get("label", match["key"]), path=match.get("path", ""))
|
|
93
|
+
g.add_edge(
|
|
94
|
+
"REALIZES", "Component", natural_key, "Subsystem", match["key"],
|
|
95
|
+
source="path-prefix-match", confidence=1.0,
|
|
96
|
+
evidence_ref=f"{rel_path} under subsystem path {match.get('path')}",
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
# PART_OF (File -> Component) -- from each Module's own `service` +
|
|
100
|
+
# `file` (service-root-relative), promoted to a real edge against a
|
|
101
|
+
# repo-relative File natural_key devkb's own File nodes already use.
|
|
102
|
+
for mod in modules:
|
|
103
|
+
service_name = mod.get("service")
|
|
104
|
+
mod_file = mod.get("file")
|
|
105
|
+
component_path = component_paths.get(service_name)
|
|
106
|
+
if not service_name or not mod_file or component_path is None:
|
|
107
|
+
continue
|
|
108
|
+
file_path = f"{component_path}/{mod_file}" if component_path else mod_file
|
|
109
|
+
g.add_node("File", file_path, source="discovery-module")
|
|
110
|
+
g.add_edge(
|
|
111
|
+
"PART_OF", "File", file_path, "Component", service_name,
|
|
112
|
+
source="discovery-module", confidence=1.0,
|
|
113
|
+
evidence_ref=f"Module {mod.get('natural_key')} (service={service_name})",
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
return g
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
"""Configuration as first-class graph knowledge (ADR-SDP-DEVKB-005 §4-5, ODSF-312).
|
|
2
|
+
|
|
3
|
+
ConfigKey one named setting: an environment variable, or a top-level key of the root pms.config.json
|
|
4
|
+
READS Symbol -> ConfigKey (File -> ConfigKey for a read at module level): os.environ[...], os.environ.get,
|
|
5
|
+
os.environ.setdefault, os.getenv, `"X" in os.environ`, and the same through `from os import environ`
|
|
6
|
+
DECLARES File -> ConfigKey: *.env.example / .env.template / .env.sample, compose `environment:` and `${VAR}`
|
|
7
|
+
references, systemd `Environment=`, Dockerfile `ENV` and `ARG`, pms.config.json top-level keys
|
|
8
|
+
Component each compose service, systemd unit and Dockerfile (a second source beside discovery's Services)
|
|
9
|
+
DEPLOYS File -> Component, from the file that defines it
|
|
10
|
+
|
|
11
|
+
Only names are recorded. A value can be a secret, so no value is read into the graph, and an untracked `.env` file
|
|
12
|
+
is never opened: the file list is `git ls-files`, and among env files only example/template/sample ones are read.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import ast
|
|
17
|
+
import json
|
|
18
|
+
import re
|
|
19
|
+
import subprocess
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from devkb.model import DevKBGraph
|
|
23
|
+
|
|
24
|
+
try: # compose files are YAML; without PyYAML the rest of the extractor still runs
|
|
25
|
+
import yaml
|
|
26
|
+
except ImportError: # pragma: no cover - PyYAML is present wherever the factory runs
|
|
27
|
+
yaml = None
|
|
28
|
+
|
|
29
|
+
SOURCE = "configuration"
|
|
30
|
+
_NAME = re.compile(r"^[A-Z][A-Z0-9_]{1,63}$")
|
|
31
|
+
_ENV_LINE = re.compile(r"^\s*(?:export\s+)?([A-Z][A-Z0-9_]{1,63})\s*=")
|
|
32
|
+
_INTERP = re.compile(r"\$\{([A-Z][A-Z0-9_]{1,63})(?::?[-?][^}]*)?\}")
|
|
33
|
+
_ENV_EXAMPLE = re.compile(r"(^|/)(\.env\.(example|template|sample|tmpl)|[^/]+\.env\.(example|template|sample|tmpl)|"
|
|
34
|
+
r"\.env\.dist|env\.example)$")
|
|
35
|
+
_COMPOSE = re.compile(r"(^|/)(docker-)?compose[^/]*\.ya?ml$")
|
|
36
|
+
_UNIT = re.compile(r"\.(service|timer|socket)$")
|
|
37
|
+
_DOCKERFILE = re.compile(r"(^|/)(Dockerfile([.-][^/]*)?|[^/]+\.Dockerfile)$")
|
|
38
|
+
_TEST = re.compile(r"(^|/)(tests?/|test_[^/]*\.py$|[^/]*_test\.py$|conftest\.py$)")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def tracked_files(repo_root: Path) -> list[str]:
|
|
42
|
+
out = subprocess.run(["git", "ls-files", "--recurse-submodules"], cwd=repo_root, capture_output=True, text=True,
|
|
43
|
+
check=False).stdout
|
|
44
|
+
return [f for f in out.splitlines() if f.strip()]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# ── code that reads a setting ────────────────────────────────────────────────
|
|
48
|
+
class _Reads(ast.NodeVisitor):
|
|
49
|
+
"""(qualified function name or "", setting name) for each literal environment read."""
|
|
50
|
+
|
|
51
|
+
def __init__(self) -> None:
|
|
52
|
+
self.stack: list[str] = []
|
|
53
|
+
self.fn_depth = 0
|
|
54
|
+
self.environ_names = {"environ"} # `from os import environ` binds it bare
|
|
55
|
+
self.found: list[tuple[str, str]] = []
|
|
56
|
+
|
|
57
|
+
def _scope(self) -> str:
|
|
58
|
+
return ".".join(self.stack) if self.fn_depth else ""
|
|
59
|
+
|
|
60
|
+
def visit_ClassDef(self, node):
|
|
61
|
+
self.stack.append(node.name)
|
|
62
|
+
self.generic_visit(node)
|
|
63
|
+
self.stack.pop()
|
|
64
|
+
|
|
65
|
+
def _fn(self, node):
|
|
66
|
+
self.stack.append(node.name)
|
|
67
|
+
self.fn_depth += 1
|
|
68
|
+
self.generic_visit(node)
|
|
69
|
+
self.fn_depth -= 1
|
|
70
|
+
self.stack.pop()
|
|
71
|
+
|
|
72
|
+
visit_FunctionDef = visit_AsyncFunctionDef = _fn
|
|
73
|
+
|
|
74
|
+
def _is_environ(self, node) -> bool:
|
|
75
|
+
return (isinstance(node, ast.Attribute) and node.attr == "environ" and isinstance(node.value, ast.Name)
|
|
76
|
+
and node.value.id == "os") or (isinstance(node, ast.Name) and node.id in self.environ_names)
|
|
77
|
+
|
|
78
|
+
def _add(self, arg) -> None:
|
|
79
|
+
if isinstance(arg, ast.Constant) and isinstance(arg.value, str) and _NAME.match(arg.value):
|
|
80
|
+
self.found.append((self._scope(), arg.value))
|
|
81
|
+
|
|
82
|
+
def visit_Subscript(self, node):
|
|
83
|
+
if self._is_environ(node.value):
|
|
84
|
+
self._add(node.slice)
|
|
85
|
+
self.generic_visit(node)
|
|
86
|
+
|
|
87
|
+
def visit_Call(self, node):
|
|
88
|
+
f = node.func
|
|
89
|
+
if node.args and isinstance(f, ast.Attribute):
|
|
90
|
+
if f.attr in ("get", "setdefault", "pop") and self._is_environ(f.value):
|
|
91
|
+
self._add(node.args[0])
|
|
92
|
+
elif f.attr == "getenv" and isinstance(f.value, ast.Name) and f.value.id == "os":
|
|
93
|
+
self._add(node.args[0])
|
|
94
|
+
elif node.args and isinstance(f, ast.Name) and f.id == "getenv":
|
|
95
|
+
self._add(node.args[0])
|
|
96
|
+
self.generic_visit(node)
|
|
97
|
+
|
|
98
|
+
def visit_Compare(self, node):
|
|
99
|
+
if (len(node.ops) == 1 and isinstance(node.ops[0], (ast.In, ast.NotIn)) and self._is_environ(node.comparators[0])):
|
|
100
|
+
self._add(node.left)
|
|
101
|
+
self.generic_visit(node)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def python_reads(text: str) -> list[tuple[str, str]]:
|
|
105
|
+
try:
|
|
106
|
+
tree = ast.parse(text)
|
|
107
|
+
except (SyntaxError, ValueError, RecursionError):
|
|
108
|
+
return []
|
|
109
|
+
v = _Reads()
|
|
110
|
+
v.visit(tree)
|
|
111
|
+
return v.found
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# ── files that declare a setting or define a service ─────────────────────────
|
|
115
|
+
def env_example_names(text: str) -> list[str]:
|
|
116
|
+
return [m.group(1) for line in text.splitlines() if (m := _ENV_LINE.match(line)) and not line.lstrip().startswith("#")]
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def compose_services(text: str) -> list[dict]:
|
|
120
|
+
"""[{"name", "image", "ports", "settings"}] for each service; names in `environment:` and `${VAR}` references."""
|
|
121
|
+
if yaml is None:
|
|
122
|
+
return []
|
|
123
|
+
try:
|
|
124
|
+
doc = yaml.safe_load(text)
|
|
125
|
+
except yaml.YAMLError:
|
|
126
|
+
return []
|
|
127
|
+
out = []
|
|
128
|
+
services = (doc or {}).get("services") if isinstance(doc, dict) else None
|
|
129
|
+
for name, svc in (services or {}).items() if isinstance(services, dict) else []:
|
|
130
|
+
svc = svc if isinstance(svc, dict) else {}
|
|
131
|
+
env = svc.get("environment") or {}
|
|
132
|
+
keys = list(env) if isinstance(env, dict) else [str(e).split("=", 1)[0] for e in env if isinstance(e, str)]
|
|
133
|
+
ports = [str(p) if not isinstance(p, dict) else f"{p.get('published', '')}:{p.get('target', '')}" for p in svc.get("ports") or []]
|
|
134
|
+
out.append({"name": str(name), "image": str(svc.get("image") or ""), "ports": ports,
|
|
135
|
+
"settings": sorted({k for k in keys if _NAME.match(k)} | set(_INTERP.findall(json.dumps(svc))))})
|
|
136
|
+
return out
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def unit_settings(text: str) -> tuple[list[str], str]:
|
|
140
|
+
"""Environment= names and the ExecStart program of a systemd unit."""
|
|
141
|
+
names, exec_start = [], ""
|
|
142
|
+
for line in text.splitlines():
|
|
143
|
+
line = line.strip()
|
|
144
|
+
if line.startswith("Environment="):
|
|
145
|
+
for part in re.findall(r'"([^"]*)"|(\S+)', line[len("Environment="):]):
|
|
146
|
+
name = (part[0] or part[1]).split("=", 1)[0]
|
|
147
|
+
if _NAME.match(name):
|
|
148
|
+
names.append(name)
|
|
149
|
+
elif line.startswith("ExecStart=") and not exec_start:
|
|
150
|
+
exec_start = line[len("ExecStart="):].lstrip("-@+!").split()[0] if line[len("ExecStart="):].split() else ""
|
|
151
|
+
return names, exec_start
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def dockerfile_facts(text: str) -> tuple[list[str], str, list[str]]:
|
|
155
|
+
"""ENV and ARG names, the first FROM image, EXPOSE ports."""
|
|
156
|
+
names, base, ports = [], "", []
|
|
157
|
+
for line in text.splitlines():
|
|
158
|
+
words = line.strip().split()
|
|
159
|
+
if not words:
|
|
160
|
+
continue
|
|
161
|
+
op = words[0].upper()
|
|
162
|
+
if op == "FROM" and not base and len(words) > 1:
|
|
163
|
+
base = words[1]
|
|
164
|
+
elif op in ("ENV", "ARG"):
|
|
165
|
+
for w in words[1:]:
|
|
166
|
+
name = w.split("=", 1)[0]
|
|
167
|
+
if _NAME.match(name):
|
|
168
|
+
names.append(name)
|
|
169
|
+
if op == "ENV" and "=" not in w:
|
|
170
|
+
break # `ENV KEY value` form: one name, the rest is its value
|
|
171
|
+
elif op == "EXPOSE":
|
|
172
|
+
ports += words[1:]
|
|
173
|
+
return names, base, ports
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def extract(ctx) -> DevKBGraph:
|
|
177
|
+
g = DevKBGraph()
|
|
178
|
+
root = ctx.repo_root
|
|
179
|
+
|
|
180
|
+
def setting(name: str, kind: str = "env") -> str:
|
|
181
|
+
g.add_node("ConfigKey", name, source=SOURCE, kind=kind)
|
|
182
|
+
return name
|
|
183
|
+
|
|
184
|
+
for rel in tracked_files(root):
|
|
185
|
+
path = root / rel
|
|
186
|
+
if rel.endswith(".py"):
|
|
187
|
+
if _TEST.search(rel):
|
|
188
|
+
continue # a test setting an env var for itself is not the product reading a setting
|
|
189
|
+
try:
|
|
190
|
+
reads = python_reads(path.read_text(errors="replace"))
|
|
191
|
+
except OSError:
|
|
192
|
+
continue
|
|
193
|
+
for scope, name in reads:
|
|
194
|
+
setting(name)
|
|
195
|
+
if scope:
|
|
196
|
+
g.add_edge("READS", "Symbol", f"{rel}::{scope}", "ConfigKey", name, source=SOURCE, confidence=0.9,
|
|
197
|
+
evidence_ref=f"{rel}: environment read")
|
|
198
|
+
else:
|
|
199
|
+
g.add_edge("READS", "File", rel, "ConfigKey", name, source=SOURCE, confidence=0.9,
|
|
200
|
+
evidence_ref=f"{rel}: module-level environment read")
|
|
201
|
+
continue
|
|
202
|
+
try:
|
|
203
|
+
if _ENV_EXAMPLE.search(rel):
|
|
204
|
+
names = env_example_names(path.read_text(errors="replace"))
|
|
205
|
+
if names:
|
|
206
|
+
g.add_node("File", rel, source=SOURCE)
|
|
207
|
+
for name in names:
|
|
208
|
+
g.add_edge("DECLARES", "File", rel, "ConfigKey", setting(name), source=SOURCE, evidence_ref=f"{rel}: KEY=")
|
|
209
|
+
elif _COMPOSE.search(rel):
|
|
210
|
+
for svc in compose_services(path.read_text(errors="replace")):
|
|
211
|
+
comp = f"{rel}#{svc['name']}"
|
|
212
|
+
g.add_node("File", rel, source=SOURCE)
|
|
213
|
+
g.add_node("Component", comp, source=SOURCE, kind="compose", label=svc["name"], file=rel,
|
|
214
|
+
image=svc["image"] or None, ports=svc["ports"] or None)
|
|
215
|
+
g.add_edge("DEPLOYS", "File", rel, "Component", comp, source=SOURCE, evidence_ref=f"{rel}: services.{svc['name']}")
|
|
216
|
+
for name in svc["settings"]:
|
|
217
|
+
g.add_edge("DECLARES", "File", rel, "ConfigKey", setting(name), source=SOURCE,
|
|
218
|
+
evidence_ref=f"{rel}: services.{svc['name']}.environment")
|
|
219
|
+
elif _UNIT.search(rel):
|
|
220
|
+
names, exec_start = unit_settings(path.read_text(errors="replace"))
|
|
221
|
+
comp = f"systemd:{Path(rel).name}"
|
|
222
|
+
g.add_node("File", rel, source=SOURCE)
|
|
223
|
+
g.add_node("Component", comp, source=SOURCE, kind="systemd", label=Path(rel).name, file=rel,
|
|
224
|
+
exec=Path(exec_start).name if exec_start else None)
|
|
225
|
+
g.add_edge("DEPLOYS", "File", rel, "Component", comp, source=SOURCE, evidence_ref=f"{rel}: unit")
|
|
226
|
+
for name in names:
|
|
227
|
+
g.add_edge("DECLARES", "File", rel, "ConfigKey", setting(name), source=SOURCE, evidence_ref=f"{rel}: Environment=")
|
|
228
|
+
elif _DOCKERFILE.search(rel):
|
|
229
|
+
names, base, ports = dockerfile_facts(path.read_text(errors="replace"))
|
|
230
|
+
g.add_node("File", rel, source=SOURCE)
|
|
231
|
+
g.add_node("Component", rel, source=SOURCE, kind="docker", label=rel, file=rel, image=base or None,
|
|
232
|
+
ports=ports or None)
|
|
233
|
+
g.add_edge("DEPLOYS", "File", rel, "Component", rel, source=SOURCE, evidence_ref=f"{rel}: FROM {base}")
|
|
234
|
+
for name in names:
|
|
235
|
+
g.add_edge("DECLARES", "File", rel, "ConfigKey", setting(name), source=SOURCE, evidence_ref=f"{rel}: ENV/ARG")
|
|
236
|
+
except OSError:
|
|
237
|
+
continue
|
|
238
|
+
|
|
239
|
+
cfg_path = root / "pms.config.json"
|
|
240
|
+
try:
|
|
241
|
+
cfg = json.loads(cfg_path.read_text())
|
|
242
|
+
except (OSError, ValueError):
|
|
243
|
+
cfg = {}
|
|
244
|
+
for key in cfg if isinstance(cfg, dict) else []:
|
|
245
|
+
if not key.startswith("_"): # `_comment` keys are prose, not settings
|
|
246
|
+
g.add_node("File", "pms.config.json", source=SOURCE)
|
|
247
|
+
name = setting(f"pms.config.json:{key}", kind="config_file")
|
|
248
|
+
g.add_edge("DECLARES", "File", "pms.config.json", "ConfigKey", name, source=SOURCE, evidence_ref="pms.config.json top-level key")
|
|
249
|
+
return g
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def settle(graph: DevKBGraph) -> int:
|
|
253
|
+
"""Run after the merge: a READS from a function the symbol extractor did not index is re-attached to its file, so a
|
|
254
|
+
read is never dropped because two extractors name one function differently. Returns how many were re-attached."""
|
|
255
|
+
symbols = {n.natural_key for n in graph.nodes("Symbol")}
|
|
256
|
+
files = {n.natural_key for n in graph.nodes("File")}
|
|
257
|
+
moved = 0
|
|
258
|
+
for e in [e for e in graph.edges("READS") if e.src_type == "Symbol" and e.src not in symbols]:
|
|
259
|
+
graph._edges.discard(e)
|
|
260
|
+
rel = e.src.split("::", 1)[0]
|
|
261
|
+
if rel not in files:
|
|
262
|
+
graph.add_node("File", rel, source=SOURCE)
|
|
263
|
+
files.add(rel)
|
|
264
|
+
graph.add_edge("READS", "File", rel, "ConfigKey", e.dst, source=SOURCE, confidence=e.confidence,
|
|
265
|
+
evidence_ref=e.evidence_ref + " (function not indexed; attributed to its file)")
|
|
266
|
+
moved += 1
|
|
267
|
+
return moved
|