datagraph-core 0.8.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datagraph/__init__.py +59 -0
- datagraph/ai/__init__.py +6 -0
- datagraph/ai/explain.py +52 -0
- datagraph/ai/lineage.py +162 -0
- datagraph/ai/providers.py +197 -0
- datagraph/analysis/__init__.py +5 -0
- datagraph/analysis/impact.py +85 -0
- datagraph/analysis/modeling.py +514 -0
- datagraph/analysis/relationships.py +64 -0
- datagraph/analysis/risk.py +70 -0
- datagraph/analysis/tests_recommender.py +55 -0
- datagraph/cli.py +831 -0
- datagraph/extractors/__init__.py +31 -0
- datagraph/extractors/airflow_extractor.py +223 -0
- datagraph/extractors/base.py +21 -0
- datagraph/extractors/datahub_extractor.py +162 -0
- datagraph/extractors/dbt_extractor.py +321 -0
- datagraph/extractors/git_extractor.py +118 -0
- datagraph/extractors/js_extractor.py +146 -0
- datagraph/extractors/lambda_extractor.py +215 -0
- datagraph/extractors/lineage_file_extractor.py +138 -0
- datagraph/extractors/openlineage_extractor.py +130 -0
- datagraph/extractors/python_extractor.py +231 -0
- datagraph/extractors/registry.py +94 -0
- datagraph/extractors/sql_extractor.py +256 -0
- datagraph/extractors/sql_in_code.py +88 -0
- datagraph/extractors/warehouse_extractor.py +240 -0
- datagraph/graph/__init__.py +16 -0
- datagraph/graph/graph.py +546 -0
- datagraph/graph/model.py +106 -0
- datagraph/html_report.py +240 -0
- datagraph/knowledge.py +210 -0
- datagraph/maintenance.py +109 -0
- datagraph/mcp_server.py +120 -0
- datagraph/profiling.py +202 -0
- datagraph/report.py +162 -0
- datagraph/security.py +96 -0
- datagraph_core-0.8.2.dist-info/METADATA +427 -0
- datagraph_core-0.8.2.dist-info/RECORD +43 -0
- datagraph_core-0.8.2.dist-info/WHEEL +5 -0
- datagraph_core-0.8.2.dist-info/entry_points.txt +2 -0
- datagraph_core-0.8.2.dist-info/licenses/LICENSE +21 -0
- datagraph_core-0.8.2.dist-info/top_level.txt +1 -0
datagraph/__init__.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""datagraph — AI-powered Change Impact Graph for data and code systems.
|
|
2
|
+
|
|
3
|
+
Build a deterministic dependency graph from real engineering artifacts
|
|
4
|
+
(Python AST, dbt manifest, SQL lineage, git diff, OpenLineage / DataHub
|
|
5
|
+
lineage files, warehouse information_schema), then ask:
|
|
6
|
+
"if I change this, what can break?"
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from .graph import EXTRACTED, INFERRED, LLM, DataGraph, Edge, EdgeType, ImpactGraph, Node, NodeType, diff_graphs
|
|
10
|
+
from .analysis import ImpactAnalysis, analyze_impact
|
|
11
|
+
from .knowledge import build_wiki, context
|
|
12
|
+
from .profiling import profile_warehouse
|
|
13
|
+
from .analysis.modeling import classify_tables, propose_from_table, star_schema
|
|
14
|
+
from .extractors.registry import ExtractorPlugin, register
|
|
15
|
+
from .extractors import (
|
|
16
|
+
DbtExtractor,
|
|
17
|
+
LineageFileExtractor,
|
|
18
|
+
OpenLineageExtractor,
|
|
19
|
+
PythonExtractor,
|
|
20
|
+
SqlExtractor,
|
|
21
|
+
WarehouseExtractor,
|
|
22
|
+
AirflowExtractor,
|
|
23
|
+
LambdaExtractor,
|
|
24
|
+
JsExtractor,
|
|
25
|
+
DataHubExtractor,
|
|
26
|
+
changed_node_ids,
|
|
27
|
+
collect_changes,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
__version__ = "0.8.2"
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
"build_wiki", "context", "profile_warehouse", "classify_tables", "star_schema", "propose_from_table", "ExtractorPlugin", "register",
|
|
34
|
+
"Edge",
|
|
35
|
+
"EdgeType",
|
|
36
|
+
"ImpactGraph",
|
|
37
|
+
"DataGraph",
|
|
38
|
+
"Node",
|
|
39
|
+
"NodeType",
|
|
40
|
+
"EXTRACTED",
|
|
41
|
+
"INFERRED",
|
|
42
|
+
"LLM",
|
|
43
|
+
"diff_graphs",
|
|
44
|
+
"ImpactAnalysis",
|
|
45
|
+
"analyze_impact",
|
|
46
|
+
"DbtExtractor",
|
|
47
|
+
"PythonExtractor",
|
|
48
|
+
"SqlExtractor",
|
|
49
|
+
"OpenLineageExtractor",
|
|
50
|
+
"LineageFileExtractor",
|
|
51
|
+
"WarehouseExtractor",
|
|
52
|
+
"AirflowExtractor",
|
|
53
|
+
"LambdaExtractor",
|
|
54
|
+
"JsExtractor",
|
|
55
|
+
"DataHubExtractor",
|
|
56
|
+
"changed_node_ids",
|
|
57
|
+
"collect_changes",
|
|
58
|
+
"__version__",
|
|
59
|
+
]
|
datagraph/ai/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
from .explain import explain_impact
|
|
2
|
+
from .lineage import apply_suggestions, schema_summary, suggest_lineage
|
|
3
|
+
from .providers import AnthropicProvider, BedrockProvider, LLMProvider, OpenAICompatibleProvider, get_provider
|
|
4
|
+
|
|
5
|
+
__all__ = ["explain_impact", "suggest_lineage", "apply_suggestions", "schema_summary",
|
|
6
|
+
"get_provider", "LLMProvider", "AnthropicProvider", "BedrockProvider", "OpenAICompatibleProvider"]
|
datagraph/ai/explain.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Optional AI layer: explain a deterministic impact analysis in plain language.
|
|
2
|
+
|
|
3
|
+
The graph itself is never built by an LLM. The model only receives the
|
|
4
|
+
already-computed blast radius and turns it into an explanation, a risk
|
|
5
|
+
narrative, and a review checklist.
|
|
6
|
+
|
|
7
|
+
Providers: Anthropic (default), Amazon Bedrock (Nova, Claude on Bedrock, ...), or any
|
|
8
|
+
OpenAI-compatible endpoint — see ``datagraph.ai.providers``. Install with
|
|
9
|
+
``pip install datagraph[ai]`` (Anthropic) or ``datagraph[bedrock]`` and set the
|
|
10
|
+
provider's credentials in the environment.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
from typing import Any, Optional
|
|
17
|
+
|
|
18
|
+
from ..analysis import ImpactAnalysis
|
|
19
|
+
from ..security import UNTRUSTED_NOTICE, wrap_untrusted
|
|
20
|
+
from .providers import get_provider
|
|
21
|
+
|
|
22
|
+
SYSTEM_PROMPT = (
|
|
23
|
+
"You are a senior data platform engineer reviewing a proposed change. "
|
|
24
|
+
"You are given a deterministic change-impact analysis computed from real "
|
|
25
|
+
"engineering artifacts (Python AST, dbt manifest, SQL lineage, git diff). "
|
|
26
|
+
"Do not invent nodes or dependencies that are not in the analysis. "
|
|
27
|
+
"Write for the engineer about to merge this change: explain what was "
|
|
28
|
+
"changed, what can break and why (walk the propagation paths), whether the "
|
|
29
|
+
"stated risk level seems right, and what to verify before and after "
|
|
30
|
+
"deploying. Keep it focused and concrete. " + UNTRUSTED_NOTICE
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def explain_impact(
|
|
35
|
+
analysis: ImpactAnalysis,
|
|
36
|
+
model: Optional[str] = None,
|
|
37
|
+
api_key: Optional[str] = None,
|
|
38
|
+
max_tokens: int = 16000,
|
|
39
|
+
provider: Optional[Any] = None,
|
|
40
|
+
) -> str:
|
|
41
|
+
"""Return a plain-language explanation of an impact analysis.
|
|
42
|
+
|
|
43
|
+
``provider`` is an ``LLMProvider`` instance or a name ('anthropic' | 'bedrock' | 'openai');
|
|
44
|
+
omitted -> ``$DATAGRAPH_LLM_PROVIDER`` or Anthropic. ``model`` overrides the provider default.
|
|
45
|
+
"""
|
|
46
|
+
llm = get_provider(provider, model=model, api_key=api_key)
|
|
47
|
+
payload = json.dumps(analysis.to_dict(), indent=2, sort_keys=True)
|
|
48
|
+
user = ("Here is the change impact analysis as JSON:\n\n"
|
|
49
|
+
+ wrap_untrusted(f"```json\n{payload}\n```")
|
|
50
|
+
+ "\n\nExplain the impact of this change.")
|
|
51
|
+
text = llm.complete(SYSTEM_PROMPT, user, max_tokens=max_tokens)
|
|
52
|
+
return text or "(The model declined to analyze this request.)"
|
datagraph/ai/lineage.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""LLM-assisted lineage — a clearly-labelled FALLBACK, never the primary source.
|
|
2
|
+
|
|
3
|
+
Deterministic extractors come first. When they cannot derive a relationship
|
|
4
|
+
(SQL that sqlglot cannot parse, stored procedures, dynamic SQL in code, tables
|
|
5
|
+
with no declared foreign keys but obvious naming conventions), ``suggest_lineage``
|
|
6
|
+
asks Claude for candidate relationships using **structured outputs**, and
|
|
7
|
+
``apply_suggestions`` adds them as edges tagged ``provenance: "llm"`` with a
|
|
8
|
+
confidence and a reason. Those edges are shown with an ``(llm)`` marker,
|
|
9
|
+
excluded by ``--no-inferred``, and can be reviewed in ``relationships --json``.
|
|
10
|
+
|
|
11
|
+
Providers: Anthropic (default), Amazon Bedrock (Nova / Claude on Bedrock / ...), or any
|
|
12
|
+
OpenAI-compatible endpoint - see ``datagraph.ai.providers``.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
from typing import Dict, List, Optional
|
|
19
|
+
|
|
20
|
+
from ..security import UNTRUSTED_NOTICE, sanitize_text, wrap_untrusted
|
|
21
|
+
from .providers import extract_json, get_provider
|
|
22
|
+
from ..graph import LLM, Edge, EdgeType, ImpactGraph, Node, NodeType
|
|
23
|
+
|
|
24
|
+
SYSTEM_PROMPT = (
|
|
25
|
+
"You are a careful data engineer. You are given a database schema (tables with their columns), "
|
|
26
|
+
"the relationships that are ALREADY known from deterministic sources, and optionally SQL snippets "
|
|
27
|
+
"that an automatic parser could not handle. Suggest ADDITIONAL likely relationships only: "
|
|
28
|
+
"(a) column-to-column dependencies you can read from the SQL, and (b) foreign-key-like links "
|
|
29
|
+
"implied by naming conventions (e.g. orders.customer_id -> customers.id). "
|
|
30
|
+
"Use the exact node ids given. Never repeat a known relationship. Give a confidence between 0 and 1 "
|
|
31
|
+
"and a one-sentence reason. If you are not reasonably confident, omit the relationship."
|
|
32
|
+
+ " " + UNTRUSTED_NOTICE
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
SUGGESTION_SCHEMA = {
|
|
36
|
+
"type": "object",
|
|
37
|
+
"properties": {
|
|
38
|
+
"relationships": {
|
|
39
|
+
"type": "array",
|
|
40
|
+
"items": {
|
|
41
|
+
"type": "object",
|
|
42
|
+
"properties": {
|
|
43
|
+
"kind": {"type": "string", "enum": ["table", "column"]},
|
|
44
|
+
"source": {"type": "string", "description": "node id that depends on / is derived from the target"},
|
|
45
|
+
"target": {"type": "string", "description": "node id that the source depends on"},
|
|
46
|
+
"confidence": {"type": "number"},
|
|
47
|
+
"reason": {"type": "string"},
|
|
48
|
+
},
|
|
49
|
+
"required": ["kind", "source", "target", "confidence", "reason"],
|
|
50
|
+
"additionalProperties": False,
|
|
51
|
+
},
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
"required": ["relationships"],
|
|
55
|
+
"additionalProperties": False,
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
_TABLE_TYPES = {NodeType.TABLE, NodeType.VIEW, NodeType.DBT_MODEL, NodeType.DBT_SOURCE, NodeType.DBT_SEED, NodeType.DBT_SNAPSHOT}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def schema_summary(graph: ImpactGraph, max_tables: Optional[int] = None) -> Dict:
|
|
62
|
+
"""Compact, deterministic description of the graph the model will reason about."""
|
|
63
|
+
tables = []
|
|
64
|
+
known = []
|
|
65
|
+
by_id = {n.id: n for n in graph.nodes()}
|
|
66
|
+
for n in sorted(graph.nodes(), key=lambda x: x.id):
|
|
67
|
+
if n.type not in _TABLE_TYPES:
|
|
68
|
+
continue
|
|
69
|
+
cols = []
|
|
70
|
+
for c in graph.nodes(NodeType.COLUMN):
|
|
71
|
+
if c.meta.get("parent") != n.id:
|
|
72
|
+
continue
|
|
73
|
+
entry = {"id": c.id, "name": c.name, "type": c.meta.get("data_type")}
|
|
74
|
+
prof = c.meta.get("profile") or {}
|
|
75
|
+
if prof: # distinct counts / ranges help the model spot join keys
|
|
76
|
+
entry["profile"] = {k: prof.get(k) for k in ("distinct", "null_pct", "min", "max") if prof.get(k) is not None}
|
|
77
|
+
cols.append(entry)
|
|
78
|
+
cols.sort(key=lambda c: c["id"])
|
|
79
|
+
t_entry = {"id": n.id, "type": n.type.value, "columns": cols}
|
|
80
|
+
if (n.meta.get("profile") or {}).get("row_count") is not None:
|
|
81
|
+
t_entry["row_count"] = n.meta["profile"]["row_count"]
|
|
82
|
+
tables.append(t_entry)
|
|
83
|
+
if max_tables is not None:
|
|
84
|
+
tables = tables[:max_tables]
|
|
85
|
+
for e in graph.edges():
|
|
86
|
+
if e.type in (EdgeType.DEPENDS_ON, EdgeType.WRITES_TO):
|
|
87
|
+
s, d = by_id.get(e.src), by_id.get(e.dst)
|
|
88
|
+
if s and d and (s.type in _TABLE_TYPES or s.type == NodeType.COLUMN):
|
|
89
|
+
known.append({"source": e.src, "target": e.dst, "via": e.meta.get("via", e.type.value)})
|
|
90
|
+
return {"tables": tables, "known_relationships": sorted(known, key=lambda r: (r["source"], r["target"]))}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def suggest_lineage(
|
|
94
|
+
graph: ImpactGraph,
|
|
95
|
+
unparsed_sql: Optional[List[Dict]] = None,
|
|
96
|
+
model: Optional[str] = None,
|
|
97
|
+
api_key: Optional[str] = None,
|
|
98
|
+
max_tables: Optional[int] = None,
|
|
99
|
+
max_tokens: int = 16000,
|
|
100
|
+
provider=None,
|
|
101
|
+
) -> List[Dict]:
|
|
102
|
+
"""Ask the configured LLM for candidate relationships. Returns a list of suggestion dicts
|
|
103
|
+
(kind, source, target, confidence, reason) - nothing is added to the graph yet.
|
|
104
|
+
``provider``: LLMProvider instance or name ('anthropic' | 'bedrock' | 'openai'); default Anthropic."""
|
|
105
|
+
llm = get_provider(provider, model=model, api_key=api_key)
|
|
106
|
+
payload = schema_summary(graph, max_tables=max_tables)
|
|
107
|
+
payload["unparsed_sql"] = [
|
|
108
|
+
{"where": sanitize_text(u.get("where"), 300), "sql": sanitize_text(u.get("sql", ""), 4000)} for u in (unparsed_sql or [])
|
|
109
|
+
][:50]
|
|
110
|
+
user = ("Schema, known relationships and unparsed SQL as JSON:\n\n"
|
|
111
|
+
+ wrap_untrusted(f"```json\n{json.dumps(payload, indent=2, sort_keys=True)}\n```")
|
|
112
|
+
+ "\n\nSuggest additional relationships.")
|
|
113
|
+
text = llm.complete(SYSTEM_PROMPT, user, max_tokens=max_tokens, json_schema=SUGGESTION_SCHEMA)
|
|
114
|
+
data = extract_json(text)
|
|
115
|
+
if not isinstance(data, dict):
|
|
116
|
+
return []
|
|
117
|
+
out = []
|
|
118
|
+
for r in data.get("relationships", []):
|
|
119
|
+
try:
|
|
120
|
+
out.append({"kind": r["kind"], "source": r["source"], "target": r["target"],
|
|
121
|
+
"confidence": float(r["confidence"]), "reason": str(r.get("reason", ""))})
|
|
122
|
+
except (KeyError, TypeError, ValueError):
|
|
123
|
+
continue
|
|
124
|
+
return out
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def apply_suggestions(graph: ImpactGraph, suggestions: List[Dict], min_confidence: float = 0.6) -> int:
|
|
128
|
+
"""Add accepted suggestions as ``llm``-provenance DEPENDS_ON edges. Returns how many were added.
|
|
129
|
+
|
|
130
|
+
Only nodes that already exist in the graph (or columns of existing tables) are
|
|
131
|
+
linked — the model cannot invent tables."""
|
|
132
|
+
added = 0
|
|
133
|
+
for s in suggestions:
|
|
134
|
+
if s.get("confidence", 0) < min_confidence:
|
|
135
|
+
continue
|
|
136
|
+
src, dst = s["source"], s["target"]
|
|
137
|
+
if src == dst:
|
|
138
|
+
continue
|
|
139
|
+
for nid in (src, dst):
|
|
140
|
+
if nid not in graph:
|
|
141
|
+
if nid.startswith("column:") and "." in nid:
|
|
142
|
+
parent_name = nid[len("column:"):].rpartition(".")[0]
|
|
143
|
+
parent = next((n.id for n in graph.nodes() if n.id.split(":", 1)[-1] == parent_name), None)
|
|
144
|
+
if parent is None:
|
|
145
|
+
break
|
|
146
|
+
graph.add_node(Node(id=nid, type=NodeType.COLUMN, name=nid.rsplit(".", 1)[-1], meta={"parent": parent}))
|
|
147
|
+
graph.add_edge(Edge(src=parent, dst=nid, type=EdgeType.CONTAINS))
|
|
148
|
+
else:
|
|
149
|
+
break
|
|
150
|
+
else:
|
|
151
|
+
before = len(graph.edges())
|
|
152
|
+
graph.add_edge(Edge(src=src, dst=dst, type=EdgeType.DEPENDS_ON,
|
|
153
|
+
meta={"provenance": LLM, "via": "llm", "confidence": s.get("confidence"), "reason": s.get("reason", "")}))
|
|
154
|
+
if len(graph.edges()) > before:
|
|
155
|
+
added += 1
|
|
156
|
+
if s.get("kind") == "column":
|
|
157
|
+
# also link the owning tables so table-level views show the relationship
|
|
158
|
+
sp, dp = graph.get_node(src), graph.get_node(dst)
|
|
159
|
+
if sp and dp and sp.meta.get("parent") and dp.meta.get("parent") and sp.meta["parent"] != dp.meta["parent"]:
|
|
160
|
+
graph.add_edge(Edge(src=sp.meta["parent"], dst=dp.meta["parent"], type=EdgeType.DEPENDS_ON,
|
|
161
|
+
meta={"provenance": LLM, "via": "llm", "confidence": s.get("confidence")}))
|
|
162
|
+
return added
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""LLM providers for the optional AI layer.
|
|
2
|
+
|
|
3
|
+
The graph is never built by an LLM; these providers only *explain* results or *suggest*
|
|
4
|
+
relationships (tagged ``llm``). Three backends, one tiny interface:
|
|
5
|
+
|
|
6
|
+
* ``anthropic`` - Claude via the Anthropic API (``pip install datagraph[ai]``, ``ANTHROPIC_API_KEY``)
|
|
7
|
+
* ``bedrock`` - Amazon Bedrock Converse API: Amazon Nova, Claude on Bedrock, Llama, Mistral ...
|
|
8
|
+
(``pip install datagraph[bedrock]``; standard AWS credentials / ``AWS_REGION``)
|
|
9
|
+
* ``openai`` - any OpenAI-compatible chat endpoint (OpenAI, Azure OpenAI, Ollama, vLLM, LM Studio, Groq ...)
|
|
10
|
+
via plain HTTPS - no extra dependency (``DATAGRAPH_LLM_BASE_URL`` / ``DATAGRAPH_LLM_API_KEY``)
|
|
11
|
+
|
|
12
|
+
Selection: explicit ``provider=`` argument, else ``DATAGRAPH_LLM_PROVIDER``, else ``anthropic``.
|
|
13
|
+
Model: explicit ``model=``, else ``DATAGRAPH_LLM_MODEL``, else the provider default.
|
|
14
|
+
Secrets are read from the environment / the cloud SDK's credential chain - never from the graph.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
from typing import Any, Dict, Optional
|
|
23
|
+
|
|
24
|
+
DEFAULT_MODELS = {
|
|
25
|
+
"anthropic": "claude-opus-5",
|
|
26
|
+
"bedrock": "amazon.nova-pro-v1:0",
|
|
27
|
+
"openai": "gpt-4o-mini",
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
_JSON_INSTRUCTION = (
|
|
31
|
+
"\n\nRespond with ONLY a single JSON object (no prose, no code fences) that conforms to this JSON schema:\n"
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class LLMProvider:
|
|
36
|
+
"""Minimal interface: complete(system, user) -> text. Subclasses set ``name`` and ``model``."""
|
|
37
|
+
|
|
38
|
+
name = "base"
|
|
39
|
+
model = ""
|
|
40
|
+
|
|
41
|
+
def complete(self, system: str, user: str, *, max_tokens: int = 4096, json_schema: Optional[Dict] = None) -> str:
|
|
42
|
+
raise NotImplementedError
|
|
43
|
+
|
|
44
|
+
# shared helper for providers without native structured output
|
|
45
|
+
@staticmethod
|
|
46
|
+
def _with_schema(user: str, json_schema: Optional[Dict]) -> str:
|
|
47
|
+
if not json_schema:
|
|
48
|
+
return user
|
|
49
|
+
return user + _JSON_INSTRUCTION + json.dumps(json_schema)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def extract_json(text: str) -> Optional[Any]:
|
|
53
|
+
"""Parse JSON from a model reply, tolerating code fences and surrounding prose."""
|
|
54
|
+
if not text:
|
|
55
|
+
return None
|
|
56
|
+
t = text.strip()
|
|
57
|
+
m = re.search(r"```(?:json)?\s*(.*?)```", t, re.S)
|
|
58
|
+
if m:
|
|
59
|
+
t = m.group(1).strip()
|
|
60
|
+
try:
|
|
61
|
+
return json.loads(t)
|
|
62
|
+
except json.JSONDecodeError:
|
|
63
|
+
pass
|
|
64
|
+
start, end = t.find("{"), t.rfind("}")
|
|
65
|
+
if start != -1 and end > start:
|
|
66
|
+
try:
|
|
67
|
+
return json.loads(t[start:end + 1])
|
|
68
|
+
except json.JSONDecodeError:
|
|
69
|
+
return None
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class AnthropicProvider(LLMProvider):
|
|
74
|
+
name = "anthropic"
|
|
75
|
+
|
|
76
|
+
def __init__(self, model: Optional[str] = None, api_key: Optional[str] = None, client=None):
|
|
77
|
+
self.model = model or DEFAULT_MODELS["anthropic"]
|
|
78
|
+
if client is not None:
|
|
79
|
+
self.client = client
|
|
80
|
+
else:
|
|
81
|
+
try:
|
|
82
|
+
import anthropic
|
|
83
|
+
except ImportError as e:
|
|
84
|
+
raise ImportError("The Anthropic provider requires the 'anthropic' package: pip install datagraph[ai]") from e
|
|
85
|
+
self.client = anthropic.Anthropic(api_key=api_key) if api_key else anthropic.Anthropic()
|
|
86
|
+
|
|
87
|
+
def complete(self, system: str, user: str, *, max_tokens: int = 4096, json_schema: Optional[Dict] = None) -> str:
|
|
88
|
+
kwargs: Dict[str, Any] = dict(model=self.model, max_tokens=max_tokens, system=system,
|
|
89
|
+
messages=[{"role": "user", "content": user}])
|
|
90
|
+
if json_schema:
|
|
91
|
+
kwargs["output_config"] = {"format": {"type": "json_schema", "schema": json_schema}}
|
|
92
|
+
if max_tokens > 8000 and not json_schema:
|
|
93
|
+
with self.client.messages.stream(**kwargs) as stream: # long outputs: stream to avoid timeouts
|
|
94
|
+
response = stream.get_final_message()
|
|
95
|
+
else:
|
|
96
|
+
response = self.client.messages.create(**kwargs)
|
|
97
|
+
if getattr(response, "stop_reason", None) == "refusal":
|
|
98
|
+
return ""
|
|
99
|
+
return "".join(getattr(b, "text", "") for b in response.content if getattr(b, "type", "") == "text")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class BedrockProvider(LLMProvider):
|
|
103
|
+
"""Amazon Bedrock Converse API - works for Amazon Nova, Anthropic Claude on Bedrock, Llama, Mistral, ..."""
|
|
104
|
+
|
|
105
|
+
name = "bedrock"
|
|
106
|
+
|
|
107
|
+
def __init__(self, model: Optional[str] = None, region: Optional[str] = None, client=None, profile: Optional[str] = None):
|
|
108
|
+
self.model = model or os.environ.get("DATAGRAPH_LLM_MODEL") or DEFAULT_MODELS["bedrock"]
|
|
109
|
+
if client is not None:
|
|
110
|
+
self.client = client
|
|
111
|
+
else:
|
|
112
|
+
try:
|
|
113
|
+
import boto3
|
|
114
|
+
except ImportError as e:
|
|
115
|
+
raise ImportError("The Bedrock provider requires 'boto3': pip install datagraph[bedrock]") from e
|
|
116
|
+
session = boto3.Session(profile_name=profile) if profile else boto3.Session()
|
|
117
|
+
self.client = session.client("bedrock-runtime", region_name=region or os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or "us-east-1")
|
|
118
|
+
|
|
119
|
+
# Bedrock models have per-model output caps (Nova: 10k, many others 4k-8k); clamp and retry once on the model's limit.
|
|
120
|
+
DEFAULT_MAX_TOKENS = int(os.environ.get("DATAGRAPH_LLM_MAX_TOKENS", "8000"))
|
|
121
|
+
|
|
122
|
+
def complete(self, system: str, user: str, *, max_tokens: int = 4096, json_schema: Optional[Dict] = None) -> str:
|
|
123
|
+
budget = min(int(max_tokens), self.DEFAULT_MAX_TOKENS)
|
|
124
|
+
for attempt in (1, 2):
|
|
125
|
+
try:
|
|
126
|
+
response = self.client.converse(
|
|
127
|
+
modelId=self.model,
|
|
128
|
+
system=[{"text": system}],
|
|
129
|
+
messages=[{"role": "user", "content": [{"text": self._with_schema(user, json_schema)}]}],
|
|
130
|
+
inferenceConfig={"maxTokens": budget, "temperature": 0.0},
|
|
131
|
+
)
|
|
132
|
+
break
|
|
133
|
+
except Exception as e: # botocore ValidationException carries the limit in its message
|
|
134
|
+
m = re.search(r"model limit of (\d+)", str(e))
|
|
135
|
+
if attempt == 1 and m and int(m.group(1)) < budget:
|
|
136
|
+
budget = max(256, int(m.group(1)) - 1)
|
|
137
|
+
continue
|
|
138
|
+
raise
|
|
139
|
+
content = ((response.get("output") or {}).get("message") or {}).get("content") or []
|
|
140
|
+
return "".join(part.get("text", "") for part in content if isinstance(part, dict))
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class OpenAICompatibleProvider(LLMProvider):
|
|
144
|
+
"""Any /v1/chat/completions endpoint (OpenAI, Azure OpenAI, Ollama, vLLM, Groq, LM Studio ...). No SDK needed."""
|
|
145
|
+
|
|
146
|
+
name = "openai"
|
|
147
|
+
|
|
148
|
+
def __init__(self, model: Optional[str] = None, api_key: Optional[str] = None, base_url: Optional[str] = None, transport=None):
|
|
149
|
+
self.model = model or os.environ.get("DATAGRAPH_LLM_MODEL") or DEFAULT_MODELS["openai"]
|
|
150
|
+
self.api_key = api_key or os.environ.get("DATAGRAPH_LLM_API_KEY") or os.environ.get("OPENAI_API_KEY") or ""
|
|
151
|
+
self.base_url = (base_url or os.environ.get("DATAGRAPH_LLM_BASE_URL") or os.environ.get("OPENAI_BASE_URL") or "https://api.openai.com/v1").rstrip("/")
|
|
152
|
+
self._transport = transport # callable(url, headers, payload) -> dict, injectable for tests
|
|
153
|
+
|
|
154
|
+
def _post(self, url: str, headers: Dict[str, str], payload: Dict) -> Dict:
|
|
155
|
+
if self._transport:
|
|
156
|
+
return self._transport(url, headers, payload)
|
|
157
|
+
import urllib.request
|
|
158
|
+
|
|
159
|
+
req = urllib.request.Request(url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST")
|
|
160
|
+
with urllib.request.urlopen(req, timeout=300) as resp: # nosec - URL comes from configuration, not data
|
|
161
|
+
return json.loads(resp.read().decode("utf-8"))
|
|
162
|
+
|
|
163
|
+
def complete(self, system: str, user: str, *, max_tokens: int = 4096, json_schema: Optional[Dict] = None) -> str:
|
|
164
|
+
headers = {"Content-Type": "application/json"}
|
|
165
|
+
if self.api_key:
|
|
166
|
+
headers["Authorization"] = f"Bearer {self.api_key}"
|
|
167
|
+
payload: Dict[str, Any] = {
|
|
168
|
+
"model": self.model, "max_tokens": int(max_tokens), "temperature": 0,
|
|
169
|
+
"messages": [{"role": "system", "content": system}, {"role": "user", "content": self._with_schema(user, json_schema)}],
|
|
170
|
+
}
|
|
171
|
+
if json_schema:
|
|
172
|
+
payload["response_format"] = {"type": "json_object"}
|
|
173
|
+
data = self._post(f"{self.base_url}/chat/completions", headers, payload)
|
|
174
|
+
choices = data.get("choices") or []
|
|
175
|
+
if not choices:
|
|
176
|
+
return ""
|
|
177
|
+
msg = choices[0].get("message") or {}
|
|
178
|
+
content = msg.get("content")
|
|
179
|
+
if isinstance(content, list): # some servers return content parts
|
|
180
|
+
content = "".join(p.get("text", "") for p in content if isinstance(p, dict))
|
|
181
|
+
return content or ""
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def get_provider(provider: Optional[Any] = None, model: Optional[str] = None, api_key: Optional[str] = None, **kwargs) -> LLMProvider:
|
|
185
|
+
"""Resolve a provider: an LLMProvider instance is returned as-is; a name ('anthropic' | 'bedrock' | 'openai')
|
|
186
|
+
or None (-> $DATAGRAPH_LLM_PROVIDER, default anthropic) builds one."""
|
|
187
|
+
if isinstance(provider, LLMProvider):
|
|
188
|
+
return provider
|
|
189
|
+
name = (provider or os.environ.get("DATAGRAPH_LLM_PROVIDER") or "anthropic").lower()
|
|
190
|
+
model = model or os.environ.get("DATAGRAPH_LLM_MODEL")
|
|
191
|
+
if name == "anthropic":
|
|
192
|
+
return AnthropicProvider(model=model, api_key=api_key, **kwargs)
|
|
193
|
+
if name in ("bedrock", "aws", "nova"):
|
|
194
|
+
return BedrockProvider(model=model, **kwargs)
|
|
195
|
+
if name in ("openai", "openai-compatible", "ollama", "azure", "vllm", "groq"):
|
|
196
|
+
return OpenAICompatibleProvider(model=model, api_key=api_key, **kwargs)
|
|
197
|
+
raise ValueError(f"unknown LLM provider '{name}' (use anthropic | bedrock | openai)")
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""High-level impact analysis: blast radius + risk + owners + test plan in one call."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Dict, List, Optional
|
|
7
|
+
|
|
8
|
+
from ..graph import ImpactGraph
|
|
9
|
+
from .risk import risk_score
|
|
10
|
+
from .tests_recommender import recommend_tests
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class ImpactAnalysis:
|
|
15
|
+
changed: List[str]
|
|
16
|
+
affected: Dict[str, int] # node id -> depth
|
|
17
|
+
risk: Dict # {"score": float, "level": str}
|
|
18
|
+
recommended_tests: List[str]
|
|
19
|
+
trees: List[Dict] = field(default_factory=list)
|
|
20
|
+
owners: Dict[str, List[str]] = field(default_factory=dict) # owner -> affected node names
|
|
21
|
+
include_inferred: bool = True
|
|
22
|
+
_graph: Optional[ImpactGraph] = None
|
|
23
|
+
|
|
24
|
+
def summary_by_type(self) -> Dict[str, int]:
|
|
25
|
+
counts: Dict[str, int] = {}
|
|
26
|
+
if self._graph is None:
|
|
27
|
+
return counts
|
|
28
|
+
for node_id in self.affected:
|
|
29
|
+
node = self._graph.get_node(node_id)
|
|
30
|
+
if node:
|
|
31
|
+
counts[node.type.value] = counts.get(node.type.value, 0) + 1
|
|
32
|
+
return counts
|
|
33
|
+
|
|
34
|
+
def to_dict(self) -> Dict:
|
|
35
|
+
return {
|
|
36
|
+
"changed": self.changed,
|
|
37
|
+
"affected": self.affected,
|
|
38
|
+
"affected_by_type": self.summary_by_type(),
|
|
39
|
+
"risk": self.risk,
|
|
40
|
+
"owners": self.owners,
|
|
41
|
+
"recommended_tests": self.recommended_tests,
|
|
42
|
+
"include_inferred": self.include_inferred,
|
|
43
|
+
"trees": self.trees,
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def analyze_impact(
|
|
48
|
+
graph: ImpactGraph,
|
|
49
|
+
changed: List[str],
|
|
50
|
+
max_depth: Optional[int] = None,
|
|
51
|
+
include_inferred: bool = True,
|
|
52
|
+
) -> ImpactAnalysis:
|
|
53
|
+
"""Compute the blast radius, risk level, owners to notify, and test plan."""
|
|
54
|
+
resolved: List[str] = []
|
|
55
|
+
for ref in changed:
|
|
56
|
+
node = graph.resolve(ref)
|
|
57
|
+
if node is not None:
|
|
58
|
+
if node.id not in resolved:
|
|
59
|
+
resolved.append(node.id)
|
|
60
|
+
elif ref in graph and ref not in resolved:
|
|
61
|
+
resolved.append(ref)
|
|
62
|
+
|
|
63
|
+
affected = graph.impact(resolved, max_depth=max_depth, include_inferred=include_inferred)
|
|
64
|
+
risk = risk_score(graph, affected)
|
|
65
|
+
tests = recommend_tests(graph, affected)
|
|
66
|
+
trees = [graph.impact_tree(nid, max_depth=max_depth, include_inferred=include_inferred) for nid in resolved]
|
|
67
|
+
|
|
68
|
+
owners: Dict[str, List[str]] = {}
|
|
69
|
+
for nid in affected:
|
|
70
|
+
node = graph.get_node(nid)
|
|
71
|
+
if node and node.owner:
|
|
72
|
+
owners.setdefault(str(node.owner), []).append(node.name)
|
|
73
|
+
for k in owners:
|
|
74
|
+
owners[k] = sorted(set(owners[k]))
|
|
75
|
+
|
|
76
|
+
return ImpactAnalysis(
|
|
77
|
+
changed=resolved,
|
|
78
|
+
affected=affected,
|
|
79
|
+
risk=risk,
|
|
80
|
+
recommended_tests=tests,
|
|
81
|
+
trees=trees,
|
|
82
|
+
owners=dict(sorted(owners.items())),
|
|
83
|
+
include_inferred=include_inferred,
|
|
84
|
+
_graph=graph,
|
|
85
|
+
)
|