prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,652 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Natural Language Interface for ProGraph.
|
|
3
|
+
|
|
4
|
+
Enables querying ProGraph using natural language, with support for
|
|
5
|
+
multi-turn conversations and automatic query retry on errors.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import logging
|
|
10
|
+
import re
|
|
11
|
+
import uuid
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
from prograph.nlp.client import (
|
|
18
|
+
CompletionResponse,
|
|
19
|
+
LLMClient,
|
|
20
|
+
LLMError,
|
|
21
|
+
Message,
|
|
22
|
+
)
|
|
23
|
+
from prograph.nlp.schema_context import SchemaContextBuilder
|
|
24
|
+
from prograph.nlp.prompts.system import (
|
|
25
|
+
get_explanation_prompt,
|
|
26
|
+
get_retry_prompt,
|
|
27
|
+
)
|
|
28
|
+
from prograph.rules import home
|
|
29
|
+
|
|
30
|
+
if TYPE_CHECKING:
|
|
31
|
+
from prograph import ProGraph
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class QueryResult:
|
|
36
|
+
"""
|
|
37
|
+
Result from a natural language query.
|
|
38
|
+
|
|
39
|
+
Attributes:
|
|
40
|
+
question: Original natural language question.
|
|
41
|
+
generated_query: The query that was generated.
|
|
42
|
+
language: Query language used (cypher or sparql).
|
|
43
|
+
results: Query execution results.
|
|
44
|
+
explanation: Natural language explanation of results.
|
|
45
|
+
execution_plan: The SQL the query runs as (`ProGraph.to_sql`), or None.
|
|
46
|
+
conversation_id: ID for multi-turn conversation tracking.
|
|
47
|
+
error: Error message if query failed.
|
|
48
|
+
retries: Number of retries needed.
|
|
49
|
+
attempts: Each attempt that failed and was retried, in order: its query and the error that
|
|
50
|
+
caused the retry (`{"query", "error"}`); the last attempt is `generated_query`.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
question: str
|
|
54
|
+
generated_query: str
|
|
55
|
+
language: str
|
|
56
|
+
results: List[Dict[str, Any]]
|
|
57
|
+
explanation: str = ""
|
|
58
|
+
execution_plan: Optional[str] = None
|
|
59
|
+
conversation_id: Optional[str] = None
|
|
60
|
+
error: Optional[str] = None
|
|
61
|
+
retries: int = 0
|
|
62
|
+
attempts: List[Dict[str, str]] = field(default_factory=list)
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def success(self) -> bool:
|
|
66
|
+
"""Check if the query executed successfully."""
|
|
67
|
+
return self.error is None
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def result_count(self) -> int:
|
|
71
|
+
"""Get number of results returned."""
|
|
72
|
+
return len(self.results)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class ConversationContext:
|
|
77
|
+
"""
|
|
78
|
+
Tracks multi-turn conversation state.
|
|
79
|
+
|
|
80
|
+
Maintains message history and query context to enable
|
|
81
|
+
follow-up questions like "Now filter to France only".
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
id: str
|
|
85
|
+
messages: List[Message] = field(default_factory=list)
|
|
86
|
+
query_history: List[str] = field(default_factory=list)
|
|
87
|
+
result_history: List[List[Dict[str, Any]]] = field(default_factory=list)
|
|
88
|
+
|
|
89
|
+
def add_exchange(
|
|
90
|
+
self,
|
|
91
|
+
question: str,
|
|
92
|
+
query: str,
|
|
93
|
+
results: List[Dict[str, Any]],
|
|
94
|
+
) -> None:
|
|
95
|
+
"""Add a question/answer exchange to the conversation."""
|
|
96
|
+
self.messages.append(Message.user(question))
|
|
97
|
+
self.messages.append(Message.assistant(query))
|
|
98
|
+
self.query_history.append(query)
|
|
99
|
+
self.result_history.append(results)
|
|
100
|
+
|
|
101
|
+
def get_context_messages(self) -> List[Message]:
|
|
102
|
+
"""Get all messages for context."""
|
|
103
|
+
return self.messages.copy()
|
|
104
|
+
|
|
105
|
+
def get_last_query(self) -> Optional[str]:
|
|
106
|
+
"""Get the most recent query."""
|
|
107
|
+
return self.query_history[-1] if self.query_history else None
|
|
108
|
+
|
|
109
|
+
# The languages with a system prompt and few-shot examples.
|
|
110
|
+
# What `NaturalLanguageInterface._is_error_response` reads a model's reply by.
|
|
111
|
+
_REFUSAL_MARKER = re.compile(r"\s*(?:#|//|--)\s*error\s*:", re.IGNORECASE)
|
|
112
|
+
_REFUSAL_PHRASES = ("cannot be generated", "cannot answer", "not enough information")
|
|
113
|
+
_COMMENT_LINE = ("#", "//", "--")
|
|
114
|
+
_PROLOGUE_LINE = re.compile(r"\s*(?:(?:PREFIX\s+\S*:|BASE)\s*<[^<>\s]*>\s*)+\Z", re.IGNORECASE)
|
|
115
|
+
_QUERY_START = re.compile(
|
|
116
|
+
r"\s*(?:SELECT|CONSTRUCT|ASK|DESCRIBE|PREFIX|BASE|INSERT|DELETE|LOAD|CLEAR|DROP|ADD|MOVE|COPY"
|
|
117
|
+
r"|WITH|MATCH|OPTIONAL|RETURN|UNWIND|CALL|CREATE|MERGE|SET|DETACH|REMOVE|FOREACH|USE)\b",
|
|
118
|
+
re.IGNORECASE)
|
|
119
|
+
# A query's strings, quoted names and IRIs, then its comments: left to right,
|
|
120
|
+
# so a `#` in an IRI or a string starts no comment.
|
|
121
|
+
_NOT_QUERY_WORDS = re.compile(
|
|
122
|
+
r'"""(?:.|\n)*?"""|\'\'\'(?:.|\n)*?\'\'\''
|
|
123
|
+
r'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'|`[^`\n]*`|<[^<>\s]*>'
|
|
124
|
+
r'|(?:#|//)[^\n]*|/\*(?:.|\n)*?\*/')
|
|
125
|
+
|
|
126
|
+
# A fenced block: its opening fence, label and the rest of that line, then its body up to a closing
|
|
127
|
+
# fence on a line of its own, or to the end of the reply when the model was cut off.
|
|
128
|
+
_FENCE = re.compile(r"^[ \t]*```[ \t]*([\w+-]*)[^\n]*\n(.*?)(?:^[ \t]*```[ \t]*$|\Z)",
|
|
129
|
+
re.MULTILINE | re.DOTALL)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@home("query-in-reply", "The query in a model's reply",
|
|
133
|
+
terms=("fence", "markdown", "prose", "LLM reply", "ask", "query_of_reply", "_extract_query", "cypher fence", "fence label"))
|
|
134
|
+
def query_of_reply(text: str, language: str = "sparql") -> str:
|
|
135
|
+
"""The query a model's reply holds (docs/architecture/rules.md, "The query in a model's reply").
|
|
136
|
+
|
|
137
|
+
Models write the query in a fenced block, often with prose before or after it. The query is
|
|
138
|
+
the first block labelled with the language asked for (`sparql`, `cypher`), else the first fenced
|
|
139
|
+
block of any label, else the whole reply. A reply that opens with the refusal marker (`# ERROR:` or `// ERROR:`)
|
|
140
|
+
is kept whole, so `_is_error_response` reads it as a refusal even when a query follows in a fence;
|
|
141
|
+
a refusal inside a fence is that block's text and is read the same way."""
|
|
142
|
+
text = text.strip()
|
|
143
|
+
if _REFUSAL_MARKER.match(text):
|
|
144
|
+
return text
|
|
145
|
+
blocks = [(m.group(1).lower(), m.group(2)) for m in _FENCE.finditer(text)]
|
|
146
|
+
for label, body in blocks:
|
|
147
|
+
if label == language:
|
|
148
|
+
return body.strip()
|
|
149
|
+
return blocks[0][1].strip() if blocks else text
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# The languages ask writes, each with its prompt (`sparql_prompt`, `cypher_prompt`).
|
|
153
|
+
SUPPORTED_LANGUAGES = ("sparql", "cypher")
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _supported_language(language: str, engine: Any = None) -> str:
|
|
157
|
+
"""`language` as ask writes it; refused by name where ask writes no such language
|
|
158
|
+
(`AskLanguageError`), or for Cypher over a schema with no property-graph view, an R2RML mapping
|
|
159
|
+
(`PropertyGraphViewError`, before any model call)."""
|
|
160
|
+
lang = language.lower()
|
|
161
|
+
if lang not in SUPPORTED_LANGUAGES:
|
|
162
|
+
from prograph.utils.exceptions import AskLanguageError
|
|
163
|
+
raise AskLanguageError(language)
|
|
164
|
+
if lang == "cypher" and engine is not None:
|
|
165
|
+
from prograph.r2rml.schema import require_property_graph
|
|
166
|
+
require_property_graph(getattr(engine, "_schema", None), "Cypher ask")
|
|
167
|
+
return lang
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class NaturalLanguageInterface:
|
|
171
|
+
"""
|
|
172
|
+
Natural language query interface for ProGraph.
|
|
173
|
+
|
|
174
|
+
Converts natural language questions into Cypher or SPARQL queries
|
|
175
|
+
using an LLM, executes them against ProGraph, and returns results
|
|
176
|
+
with explanations.
|
|
177
|
+
|
|
178
|
+
Example:
|
|
179
|
+
>>> from prograph import ProGraph
|
|
180
|
+
>>> from prograph.nlp import NaturalLanguageInterface
|
|
181
|
+
>>> from prograph.nlp.providers import DatabricksLLMClient
|
|
182
|
+
|
|
183
|
+
>>> engine = ProGraph(backend='spark', schema=schema, dataframes=dfs)
|
|
184
|
+
>>> llm = DatabricksLLMClient()
|
|
185
|
+
>>> nl = NaturalLanguageInterface(engine, llm)
|
|
186
|
+
|
|
187
|
+
>>> # Single question
|
|
188
|
+
>>> result = nl.ask("Find customers who placed more than 10 orders")
|
|
189
|
+
>>> print(result.generated_query)
|
|
190
|
+
>>> print(result.results)
|
|
191
|
+
|
|
192
|
+
>>> # Multi-turn conversation
|
|
193
|
+
>>> result1 = nl.ask("Find high-value customers")
|
|
194
|
+
>>> result2 = nl.ask("Now filter to France only",
|
|
195
|
+
... conversation_id=result1.conversation_id)
|
|
196
|
+
"""
|
|
197
|
+
|
|
198
|
+
def __init__(
|
|
199
|
+
self,
|
|
200
|
+
engine: "ProGraph",
|
|
201
|
+
llm_client: LLMClient,
|
|
202
|
+
language: str = "sparql",
|
|
203
|
+
domain: str = "generic",
|
|
204
|
+
include_explanation: bool = True,
|
|
205
|
+
validate_before_execute: bool = True,
|
|
206
|
+
auto_retry_on_error: bool = True,
|
|
207
|
+
max_retries: int = 3,
|
|
208
|
+
temperature: Optional[float] = 0.0,
|
|
209
|
+
executor: Optional[Callable[[str, str], List[Dict[str, Any]]]] = None,
|
|
210
|
+
):
|
|
211
|
+
"""
|
|
212
|
+
Initialize natural language interface.
|
|
213
|
+
|
|
214
|
+
Args:
|
|
215
|
+
engine: ProGraph engine instance.
|
|
216
|
+
llm_client: LLM client for query generation.
|
|
217
|
+
language: Default query language: "sparql" (the default) or "cypher".
|
|
218
|
+
domain: Domain for few-shot examples (generic, aml, social, etc.).
|
|
219
|
+
include_explanation: Whether to generate result explanations.
|
|
220
|
+
validate_before_execute: Validate query syntax before execution.
|
|
221
|
+
auto_retry_on_error: Automatically retry on query errors.
|
|
222
|
+
max_retries: Maximum retry attempts.
|
|
223
|
+
temperature: LLM sampling temperature; None sends none. A model
|
|
224
|
+
that refuses one is asked without it (the client's rule).
|
|
225
|
+
executor: Runs a query, `executor(query, language)`, returning
|
|
226
|
+
its rows and raising on an error (default: the engine's
|
|
227
|
+
`execute`). The MCP `ask` tool runs it as `run_sparql`
|
|
228
|
+
does, so its answer carries the SQL that ran.
|
|
229
|
+
"""
|
|
230
|
+
self.engine = engine
|
|
231
|
+
self.llm = llm_client
|
|
232
|
+
self.language = _supported_language(language, engine)
|
|
233
|
+
self.domain = domain
|
|
234
|
+
self.include_explanation = include_explanation
|
|
235
|
+
self.validate_before_execute = validate_before_execute
|
|
236
|
+
self.auto_retry = auto_retry_on_error
|
|
237
|
+
self.max_retries = max_retries
|
|
238
|
+
self.temperature = temperature
|
|
239
|
+
self._executor = executor
|
|
240
|
+
|
|
241
|
+
# Initialize schema context builder
|
|
242
|
+
self.schema_context = SchemaContextBuilder(engine)
|
|
243
|
+
|
|
244
|
+
# Conversation storage
|
|
245
|
+
self._conversations: Dict[str, ConversationContext] = {}
|
|
246
|
+
|
|
247
|
+
def ask(
|
|
248
|
+
self,
|
|
249
|
+
question: str,
|
|
250
|
+
conversation_id: Optional[str] = None,
|
|
251
|
+
language: Optional[str] = None,
|
|
252
|
+
) -> QueryResult:
|
|
253
|
+
"""
|
|
254
|
+
Convert natural language to graph query and execute.
|
|
255
|
+
|
|
256
|
+
Args:
|
|
257
|
+
question: Natural language question.
|
|
258
|
+
conversation_id: Optional ID for follow-up questions.
|
|
259
|
+
language: Override default query language.
|
|
260
|
+
|
|
261
|
+
Returns:
|
|
262
|
+
QueryResult with generated query, results, and explanation.
|
|
263
|
+
"""
|
|
264
|
+
lang = _supported_language(language or self.language, self.engine)
|
|
265
|
+
|
|
266
|
+
# Get or create conversation context
|
|
267
|
+
if conversation_id and conversation_id in self._conversations:
|
|
268
|
+
conversation = self._conversations[conversation_id]
|
|
269
|
+
else:
|
|
270
|
+
conversation = self._create_conversation()
|
|
271
|
+
|
|
272
|
+
# Build messages
|
|
273
|
+
messages = self._build_messages(question, conversation, lang)
|
|
274
|
+
|
|
275
|
+
# Generate query
|
|
276
|
+
try:
|
|
277
|
+
response = self.llm.complete(
|
|
278
|
+
messages=messages,
|
|
279
|
+
temperature=self.temperature,
|
|
280
|
+
)
|
|
281
|
+
query = self._extract_query(response.content, lang)
|
|
282
|
+
except LLMError as e:
|
|
283
|
+
return QueryResult(
|
|
284
|
+
question=question,
|
|
285
|
+
generated_query="",
|
|
286
|
+
language=lang,
|
|
287
|
+
results=[],
|
|
288
|
+
error=f"LLM error: {e}",
|
|
289
|
+
conversation_id=conversation.id,
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
# Check for error response from LLM
|
|
293
|
+
if self._is_error_response(query):
|
|
294
|
+
return QueryResult(
|
|
295
|
+
question=question,
|
|
296
|
+
generated_query=query,
|
|
297
|
+
language=lang,
|
|
298
|
+
results=[],
|
|
299
|
+
error=query,
|
|
300
|
+
conversation_id=conversation.id,
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
# Execute query with retry logic; `query` becomes the one that ran
|
|
304
|
+
# last, the model's correction after a failure.
|
|
305
|
+
attempts: List[Dict[str, str]] = []
|
|
306
|
+
query, results, error, retries = self._execute_with_retry(
|
|
307
|
+
question=question,
|
|
308
|
+
query=query,
|
|
309
|
+
language=lang,
|
|
310
|
+
conversation=conversation,
|
|
311
|
+
attempts=attempts,
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
# Generate explanation if requested
|
|
315
|
+
explanation = ""
|
|
316
|
+
if self.include_explanation and results and not error:
|
|
317
|
+
explanation = self._generate_explanation(query, results)
|
|
318
|
+
|
|
319
|
+
# Get execution plan if available
|
|
320
|
+
execution_plan = None
|
|
321
|
+
if not error:
|
|
322
|
+
try:
|
|
323
|
+
execution_plan = self.engine.to_sql(query, language=lang)
|
|
324
|
+
except Exception as e:
|
|
325
|
+
logger.debug("explain_plan_failed", extra={"query": query[:100], "error": str(e)})
|
|
326
|
+
pass
|
|
327
|
+
|
|
328
|
+
# Update conversation context
|
|
329
|
+
if not error:
|
|
330
|
+
conversation.add_exchange(question, query, results)
|
|
331
|
+
|
|
332
|
+
return QueryResult(
|
|
333
|
+
question=question,
|
|
334
|
+
generated_query=query,
|
|
335
|
+
language=lang,
|
|
336
|
+
results=results,
|
|
337
|
+
explanation=explanation,
|
|
338
|
+
execution_plan=execution_plan,
|
|
339
|
+
conversation_id=conversation.id,
|
|
340
|
+
error=error,
|
|
341
|
+
retries=retries,
|
|
342
|
+
attempts=attempts,
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
def _build_messages(
|
|
346
|
+
self,
|
|
347
|
+
question: str,
|
|
348
|
+
conversation: ConversationContext,
|
|
349
|
+
language: str,
|
|
350
|
+
) -> List[Message]:
|
|
351
|
+
"""Build message list for LLM."""
|
|
352
|
+
# The rules, the schema and examples generated from the mapping: for SPARQL the prompt the MCP
|
|
353
|
+
# server offers as `write-sparql` too; for Cypher the property-graph view's.
|
|
354
|
+
from prograph.nlp.prompts.system import cypher_prompt, sparql_prompt
|
|
355
|
+
system_prompt = (sparql_prompt(self.engine.schema) if language == "sparql"
|
|
356
|
+
else cypher_prompt(self.engine))
|
|
357
|
+
|
|
358
|
+
messages = [Message.system(system_prompt)]
|
|
359
|
+
|
|
360
|
+
# Add conversation history for context
|
|
361
|
+
if conversation.messages:
|
|
362
|
+
messages.extend(conversation.get_context_messages())
|
|
363
|
+
|
|
364
|
+
# Add current question
|
|
365
|
+
messages.append(Message.user(question))
|
|
366
|
+
|
|
367
|
+
return messages
|
|
368
|
+
|
|
369
|
+
def _extract_query(self, content: str, language: str = "sparql") -> str:
|
|
370
|
+
"""Extract query from LLM response."""
|
|
371
|
+
content = query_of_reply(content, language)
|
|
372
|
+
|
|
373
|
+
# An answer with no text, as when every part was reasoning and
|
|
374
|
+
# max_tokens ran out, is no query: run as SPARQL it read as "no
|
|
375
|
+
# rows" (1495).
|
|
376
|
+
if not content:
|
|
377
|
+
raise LLMError(
|
|
378
|
+
"the model returned no text (an empty answer or only "
|
|
379
|
+
"reasoning; max_tokens may be too low)"
|
|
380
|
+
)
|
|
381
|
+
return content
|
|
382
|
+
|
|
383
|
+
@home("non-query-reply", "A model's reply that is no query",
|
|
384
|
+
terms=("# ERROR:", "refusal", "LLM reply", "ask", "_is_error_response"))
|
|
385
|
+
def _is_error_response(self, query: str) -> bool:
|
|
386
|
+
"""Whether a model's reply says the question cannot be answered, and
|
|
387
|
+
so is the answer and is never run (docs/architecture/rules.md, "A
|
|
388
|
+
model's reply that is no query").
|
|
389
|
+
|
|
390
|
+
- A reply that opens with the prompts' marker (`# ERROR:`, in any
|
|
391
|
+
case or spacing). A query after the marker changes nothing: the
|
|
392
|
+
model's first word is that it cannot answer, and running what it
|
|
393
|
+
offers beside that would pass off a query it disowns as the answer.
|
|
394
|
+
The marker anywhere later is a comment in a query.
|
|
395
|
+
- A reply with no query in it: only comments, with or without the
|
|
396
|
+
PREFIX lines the rules ask it to start with. SPARQL parses that as
|
|
397
|
+
an empty Update, and the model would be asked until retries ran out.
|
|
398
|
+
- A phrase of refusal in text that is not a query, or outside a
|
|
399
|
+
query's strings, names and comments: `FILTER(?n = "cannot answer")`
|
|
400
|
+
is a query.
|
|
401
|
+
|
|
402
|
+
Any other text goes to the engine, whose error goes back to the model.
|
|
403
|
+
"""
|
|
404
|
+
if not query.strip():
|
|
405
|
+
return False
|
|
406
|
+
if _REFUSAL_MARKER.match(query):
|
|
407
|
+
return True
|
|
408
|
+
kept = [line for line in query.splitlines() if line.strip()]
|
|
409
|
+
body = [line for line in kept if not line.lstrip().startswith(_COMMENT_LINE)]
|
|
410
|
+
if not body or (len(body) < len(kept) and all(_PROLOGUE_LINE.match(line) for line in body)):
|
|
411
|
+
return True
|
|
412
|
+
text = "\n".join(body)
|
|
413
|
+
if _QUERY_START.match(text):
|
|
414
|
+
text = _NOT_QUERY_WORDS.sub(" ", text)
|
|
415
|
+
text = text.lower()
|
|
416
|
+
return any(phrase in text for phrase in _REFUSAL_PHRASES)
|
|
417
|
+
|
|
418
|
+
def _check_vocabulary(self, query: str) -> None:
|
|
419
|
+
"""Raise for a class or predicate the schema does not declare: SPARQL
|
|
420
|
+
answers it with no rows, which the model would take for the data's
|
|
421
|
+
answer (`undeclared_iris`)."""
|
|
422
|
+
from prograph.schema.context import undeclared_iris
|
|
423
|
+
found = undeclared_iris(self.engine.schema, query, ontology=self.engine.ontology)
|
|
424
|
+
if found:
|
|
425
|
+
raise ValueError("; ".join(
|
|
426
|
+
f"{w['message']}" + (f" (did you mean {', '.join('<' + n + '>' for n in w['nearest'])}?)"
|
|
427
|
+
if w["nearest"] else "")
|
|
428
|
+
for w in found))
|
|
429
|
+
|
|
430
|
+
def _execute_with_retry(
|
|
431
|
+
self,
|
|
432
|
+
question: str,
|
|
433
|
+
query: str,
|
|
434
|
+
language: str,
|
|
435
|
+
conversation: ConversationContext,
|
|
436
|
+
attempts: Optional[List[Dict[str, str]]] = None,
|
|
437
|
+
) -> tuple[str, List[Dict[str, Any]], Optional[str], int]:
|
|
438
|
+
"""
|
|
439
|
+
Execute query with automatic retry on errors. Each attempt that fails and is retried is
|
|
440
|
+
appended to `attempts` (its query and its error), so the queries that did not answer are
|
|
441
|
+
kept beside the one that did.
|
|
442
|
+
|
|
443
|
+
Returns:
|
|
444
|
+
Tuple of (last query tried, results, error, retry_count).
|
|
445
|
+
"""
|
|
446
|
+
retries = 0
|
|
447
|
+
current_query = query
|
|
448
|
+
last_error = None
|
|
449
|
+
|
|
450
|
+
while retries <= self.max_retries:
|
|
451
|
+
try:
|
|
452
|
+
# ask is read-only: a Cypher query that writes is refused, before it is planned.
|
|
453
|
+
if language == "cypher":
|
|
454
|
+
from prograph.read_only import refuse_ask_write
|
|
455
|
+
refuse_ask_write(current_query)
|
|
456
|
+
|
|
457
|
+
# Validate syntax if enabled
|
|
458
|
+
if self.validate_before_execute:
|
|
459
|
+
try:
|
|
460
|
+
from prograph.plan.ir import entry
|
|
461
|
+
entry.check(self.engine, current_query, language)
|
|
462
|
+
except Exception as e:
|
|
463
|
+
raise ValueError(f"Query syntax error: {e}")
|
|
464
|
+
if language == "sparql":
|
|
465
|
+
self._check_vocabulary(current_query)
|
|
466
|
+
|
|
467
|
+
# Execute query
|
|
468
|
+
if self._executor is not None:
|
|
469
|
+
results = self._executor(current_query, language)
|
|
470
|
+
else:
|
|
471
|
+
results = self.engine.execute(current_query, language=language, return_type="list")
|
|
472
|
+
return current_query, results, None, retries
|
|
473
|
+
|
|
474
|
+
except Exception as e:
|
|
475
|
+
last_error = str(e)
|
|
476
|
+
|
|
477
|
+
if not self.auto_retry or retries >= self.max_retries:
|
|
478
|
+
break
|
|
479
|
+
|
|
480
|
+
# Try to fix the query
|
|
481
|
+
retry_prompt = get_retry_prompt(question, last_error)
|
|
482
|
+
messages = self._build_messages(question, conversation, language)
|
|
483
|
+
messages.append(Message.assistant(current_query))
|
|
484
|
+
messages.append(Message.user(retry_prompt))
|
|
485
|
+
|
|
486
|
+
try:
|
|
487
|
+
response = self.llm.complete(
|
|
488
|
+
messages=messages,
|
|
489
|
+
temperature=self.temperature,
|
|
490
|
+
)
|
|
491
|
+
failed, current_query = current_query, self._extract_query(response.content, language)
|
|
492
|
+
retries += 1
|
|
493
|
+
except LLMError:
|
|
494
|
+
break
|
|
495
|
+
# Kept once the model has given its correction: a correction that never came is no retry.
|
|
496
|
+
if attempts is not None:
|
|
497
|
+
attempts.append({"query": failed, "error": last_error})
|
|
498
|
+
logger.debug("ask_retry", extra={"attempt": retries, "query": failed[:200],
|
|
499
|
+
"error": last_error[:200]})
|
|
500
|
+
# A correction that says it cannot be done is the model's
|
|
501
|
+
# answer, as a first reply is in `ask`: it is never run.
|
|
502
|
+
if self._is_error_response(current_query):
|
|
503
|
+
return current_query, [], current_query, retries
|
|
504
|
+
|
|
505
|
+
return current_query, [], last_error, retries
|
|
506
|
+
|
|
507
|
+
def _generate_explanation(
|
|
508
|
+
self,
|
|
509
|
+
query: str,
|
|
510
|
+
results: List[Dict[str, Any]],
|
|
511
|
+
) -> str:
|
|
512
|
+
"""Generate natural language explanation of results."""
|
|
513
|
+
if not results:
|
|
514
|
+
return "No results found."
|
|
515
|
+
|
|
516
|
+
# Format results for the prompt
|
|
517
|
+
if len(results) <= 5:
|
|
518
|
+
results_str = json.dumps(results, indent=2, default=str)
|
|
519
|
+
else:
|
|
520
|
+
results_str = json.dumps(results[:5], indent=2, default=str)
|
|
521
|
+
results_str += f"\n... and {len(results) - 5} more results"
|
|
522
|
+
|
|
523
|
+
prompt = get_explanation_prompt(query, results_str)
|
|
524
|
+
|
|
525
|
+
try:
|
|
526
|
+
response = self.llm.complete(
|
|
527
|
+
messages=[Message.user(prompt)],
|
|
528
|
+
temperature=0.0,
|
|
529
|
+
)
|
|
530
|
+
return response.content.strip()
|
|
531
|
+
except LLMError:
|
|
532
|
+
return f"Found {len(results)} results."
|
|
533
|
+
|
|
534
|
+
def _create_conversation(self) -> ConversationContext:
|
|
535
|
+
"""Create a new conversation context."""
|
|
536
|
+
conv_id = str(uuid.uuid4())[:8]
|
|
537
|
+
conversation = ConversationContext(id=conv_id)
|
|
538
|
+
self._conversations[conv_id] = conversation
|
|
539
|
+
return conversation
|
|
540
|
+
|
|
541
|
+
def start_conversation(self) -> str:
|
|
542
|
+
"""
|
|
543
|
+
Start a new conversation.
|
|
544
|
+
|
|
545
|
+
Returns:
|
|
546
|
+
Conversation ID for follow-up questions.
|
|
547
|
+
"""
|
|
548
|
+
conversation = self._create_conversation()
|
|
549
|
+
return conversation.id
|
|
550
|
+
|
|
551
|
+
def end_conversation(self, conversation_id: str) -> None:
|
|
552
|
+
"""
|
|
553
|
+
End and clean up a conversation.
|
|
554
|
+
|
|
555
|
+
Args:
|
|
556
|
+
conversation_id: ID of conversation to end.
|
|
557
|
+
"""
|
|
558
|
+
if conversation_id in self._conversations:
|
|
559
|
+
del self._conversations[conversation_id]
|
|
560
|
+
|
|
561
|
+
def get_conversation(self, conversation_id: str) -> Optional[ConversationContext]:
|
|
562
|
+
"""
|
|
563
|
+
Get conversation context by ID.
|
|
564
|
+
|
|
565
|
+
Args:
|
|
566
|
+
conversation_id: Conversation ID.
|
|
567
|
+
|
|
568
|
+
Returns:
|
|
569
|
+
ConversationContext or None if not found.
|
|
570
|
+
"""
|
|
571
|
+
return self._conversations.get(conversation_id)
|
|
572
|
+
|
|
573
|
+
def explain(self, question: str, language: Optional[str] = None) -> str:
|
|
574
|
+
"""
|
|
575
|
+
Generate query explanation without executing.
|
|
576
|
+
|
|
577
|
+
Args:
|
|
578
|
+
question: Natural language question.
|
|
579
|
+
language: Override default query language.
|
|
580
|
+
|
|
581
|
+
Returns:
|
|
582
|
+
Explanation of what query would do.
|
|
583
|
+
"""
|
|
584
|
+
lang = _supported_language(language or self.language, self.engine)
|
|
585
|
+
conversation = self._create_conversation()
|
|
586
|
+
messages = self._build_messages(question, conversation, lang)
|
|
587
|
+
|
|
588
|
+
try:
|
|
589
|
+
response = self.llm.complete(
|
|
590
|
+
messages=messages,
|
|
591
|
+
temperature=self.temperature,
|
|
592
|
+
)
|
|
593
|
+
query = self._extract_query(response.content, lang)
|
|
594
|
+
|
|
595
|
+
# Get execution plan
|
|
596
|
+
try:
|
|
597
|
+
sql = self.engine.to_sql(query, language=lang)
|
|
598
|
+
return f"Query: {query}\n\nSQL:\n{sql}"
|
|
599
|
+
except Exception as e:
|
|
600
|
+
logger.debug("explain_plan_failed", extra={"query": query[:100], "error": str(e)})
|
|
601
|
+
return f"Query: {query}"
|
|
602
|
+
|
|
603
|
+
except LLMError as e:
|
|
604
|
+
return f"Error generating query: {e}"
|
|
605
|
+
finally:
|
|
606
|
+
self.end_conversation(conversation.id)
|
|
607
|
+
|
|
608
|
+
def suggest_queries(self, n: int = 5) -> List[str]:
|
|
609
|
+
"""
|
|
610
|
+
Suggest example questions based on schema.
|
|
611
|
+
|
|
612
|
+
Args:
|
|
613
|
+
n: Number of suggestions to generate.
|
|
614
|
+
|
|
615
|
+
Returns:
|
|
616
|
+
List of suggested questions.
|
|
617
|
+
"""
|
|
618
|
+
if self.language == "cypher":
|
|
619
|
+
from prograph.nlp.graph_context import cypher_examples, describe_graph
|
|
620
|
+
return [ex.question for ex in cypher_examples(describe_graph(self.engine), n=n)]
|
|
621
|
+
examples = self.schema_context.build_few_shot_examples(language=self.language, n=n)
|
|
622
|
+
return [ex["question"] for ex in examples]
|
|
623
|
+
|
|
624
|
+
def generate_query_only(
|
|
625
|
+
self,
|
|
626
|
+
question: str,
|
|
627
|
+
language: Optional[str] = None,
|
|
628
|
+
) -> str:
|
|
629
|
+
"""
|
|
630
|
+
Generate query without executing.
|
|
631
|
+
|
|
632
|
+
Args:
|
|
633
|
+
question: Natural language question.
|
|
634
|
+
language: Override default query language.
|
|
635
|
+
|
|
636
|
+
Returns:
|
|
637
|
+
Generated query string.
|
|
638
|
+
"""
|
|
639
|
+
lang = _supported_language(language or self.language, self.engine)
|
|
640
|
+
conversation = self._create_conversation()
|
|
641
|
+
messages = self._build_messages(question, conversation, lang)
|
|
642
|
+
|
|
643
|
+
try:
|
|
644
|
+
response = self.llm.complete(
|
|
645
|
+
messages=messages,
|
|
646
|
+
temperature=self.temperature,
|
|
647
|
+
)
|
|
648
|
+
return self._extract_query(response.content, lang)
|
|
649
|
+
except LLMError as e:
|
|
650
|
+
return f"-- ERROR: {e}"
|
|
651
|
+
finally:
|
|
652
|
+
self.end_conversation(conversation.id)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Prompt templates and examples for ProGraph NL interface.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from prograph.nlp.prompts.system import (
|
|
6
|
+
CYPHER_SYSTEM_PROMPT,
|
|
7
|
+
get_system_prompt,
|
|
8
|
+
)
|
|
9
|
+
from prograph.nlp.prompts.examples import (
|
|
10
|
+
CYPHER_EXAMPLES,
|
|
11
|
+
get_examples_for_domain,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"CYPHER_SYSTEM_PROMPT",
|
|
16
|
+
"get_system_prompt",
|
|
17
|
+
"CYPHER_EXAMPLES",
|
|
18
|
+
"get_examples_for_domain",
|
|
19
|
+
]
|