prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/mcp/sparql.py
ADDED
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
"""The SPARQL tools of the MCP server: one core for both ways an
|
|
2
|
+
agent works.
|
|
3
|
+
|
|
4
|
+
- The agent writes SPARQL: `get_schema` (the vocabulary, its prefixes and
|
|
5
|
+
examples from the mapping), then `run_sparql`, fixing the query from the
|
|
6
|
+
error it gets back.
|
|
7
|
+
- ProGraph writes it: `ask` (`prograph.nlp` turns the question into SPARQL,
|
|
8
|
+
validated and run with the same errors fed back).
|
|
9
|
+
|
|
10
|
+
`explain_sparql` says what would run, without running it.
|
|
11
|
+
|
|
12
|
+
Every answer carries the SPARQL and the SQL that ran (`run_record`: the
|
|
13
|
+
statement the database was sent, or null with the reason the query
|
|
14
|
+
answered on the Python executor). Errors are data: `error`, `reason` and
|
|
15
|
+
`suggestions`, so the agent can fix the query and try again. Output is
|
|
16
|
+
bounded as `execute_query`'s is (`tools.DEFAULT_ROW_LIMIT`, at most
|
|
17
|
+
`tools.MAX_ROW_LIMIT`), with the real total. Updates are always refused:
|
|
18
|
+
ProGraph is read-only over the data it maps.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
|
|
24
|
+
|
|
25
|
+
from prograph.mcp.tools import _clamp_limit
|
|
26
|
+
|
|
27
|
+
if TYPE_CHECKING:
|
|
28
|
+
from prograph import ProGraph
|
|
29
|
+
|
|
30
|
+
# ──────────────────────────────────────────────────────────────────────
|
|
31
|
+
# get_schema
|
|
32
|
+
# ──────────────────────────────────────────────────────────────────────
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get_schema(engine: "ProGraph", examples: int = 5) -> Dict[str, Any]:
|
|
36
|
+
"""The schema as the vocabulary a SPARQL query is written in: the
|
|
37
|
+
PREFIX lines, classes and their hierarchy, predicates with their
|
|
38
|
+
datatypes or object classes (`render_sparql`), the same as data,
|
|
39
|
+
example queries generated from the mapping, and `query_rules`: the
|
|
40
|
+
rules `ask` gives its model for writing the query (`SPARQL_QUERY_RULES`,
|
|
41
|
+
one copy for both)."""
|
|
42
|
+
from prograph.nlp.prompts.system import render_sparql_rules
|
|
43
|
+
from prograph.schema.context import describe_schema, render_sparql, sparql_prefixes
|
|
44
|
+
from prograph.schema.examples import sparql_examples
|
|
45
|
+
|
|
46
|
+
if engine.schema is None:
|
|
47
|
+
return {"error": "this engine has no schema", "reason": "no_schema"}
|
|
48
|
+
desc = describe_schema(engine.schema)
|
|
49
|
+
vocab = desc.vocabulary
|
|
50
|
+
response: Dict[str, Any] = {
|
|
51
|
+
"vocabulary": render_sparql(desc),
|
|
52
|
+
"query_rules": render_sparql_rules(),
|
|
53
|
+
"prefixes": sparql_prefixes(vocab),
|
|
54
|
+
"classes": [{"iri": c.iri, "within": c.within, "individuals": c.individuals}
|
|
55
|
+
for c in vocab.classes],
|
|
56
|
+
"predicates": [{"iri": t.iri, "kind": t.kind, "subjects": t.subject_classes,
|
|
57
|
+
"datatype": t.datatype, "objects": t.object_classes}
|
|
58
|
+
for t in vocab.predicates],
|
|
59
|
+
"examples": [{"question": e.question, "sparql": e.query}
|
|
60
|
+
for e in sparql_examples(desc, n=examples)],
|
|
61
|
+
}
|
|
62
|
+
if engine.ontology is not None:
|
|
63
|
+
from prograph.mcp.tools import get_ontology
|
|
64
|
+
response["ontology"] = get_ontology(engine)
|
|
65
|
+
return response
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# ──────────────────────────────────────────────────────────────────────
|
|
69
|
+
# run_sparql
|
|
70
|
+
# ──────────────────────────────────────────────────────────────────────
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def run_sparql(engine: "ProGraph", query: str, limit: Optional[int] = None, *,
|
|
74
|
+
row_cap: Optional[int] = None,
|
|
75
|
+
timeout: Optional[float] = None, context: Any = None) -> Dict[str, Any]:
|
|
76
|
+
"""Run a SPARQL query: `{sparql, sql, ran_in, rows, row_count,
|
|
77
|
+
total_row_count, truncated, limit_applied}`, with `reason` where it ran
|
|
78
|
+
in Python and `warnings` for IRIs the schema does not declare (no rows
|
|
79
|
+
for them is SPARQL's answer, not an error). An error is `{sparql,
|
|
80
|
+
error, reason, suggestions}`, and `sql` where the database rejected it.
|
|
81
|
+
|
|
82
|
+
`row_cap` and `timeout` are a server's: a result
|
|
83
|
+
past `row_cap` rows, or a run past `timeout` seconds, is an error
|
|
84
|
+
(`reason` 'row_cap' or 'timeout'), whatever `limit` is; `limit` only
|
|
85
|
+
says how many of the rows the answer shows.
|
|
86
|
+
|
|
87
|
+
`context`: the run's `QueryContext`, whose cancel() stops it; a cancelled
|
|
88
|
+
run is the error `reason` 'cancelled'."""
|
|
89
|
+
from prograph.execution import QueryCancelledException
|
|
90
|
+
from prograph.plan.run_record import observe
|
|
91
|
+
|
|
92
|
+
form, refused = _form(query)
|
|
93
|
+
if refused is not None:
|
|
94
|
+
return refused
|
|
95
|
+
effective = _clamp_limit(limit)
|
|
96
|
+
warnings = unknown_iri_warnings(engine, query)
|
|
97
|
+
with observe() as observed:
|
|
98
|
+
try:
|
|
99
|
+
result = _execute(engine, query, form, row_cap, timeout, context)
|
|
100
|
+
except _OverLimit as over:
|
|
101
|
+
return {"sparql": query, "sql": None, "error": str(over), "reason": over.reason,
|
|
102
|
+
"suggestions": ["narrow the query, or add a LIMIT"]}
|
|
103
|
+
except QueryCancelledException as cancelled:
|
|
104
|
+
return {"sparql": query, "sql": None, "error": str(cancelled), "reason": "cancelled",
|
|
105
|
+
"suggestions": []}
|
|
106
|
+
except Exception as error: # noqa: BLE001 - the error is the answer
|
|
107
|
+
failed = None
|
|
108
|
+
response = error_response(engine, query, error)
|
|
109
|
+
if observed.run is not None:
|
|
110
|
+
failed = observed.run.failed
|
|
111
|
+
response["sql"] = getattr(error, "sql", None) or failed
|
|
112
|
+
if warnings:
|
|
113
|
+
response["warnings"] = warnings
|
|
114
|
+
response["hint"] = UNDECLARED_HINT
|
|
115
|
+
return response
|
|
116
|
+
run = observed.run
|
|
117
|
+
rows, total = result
|
|
118
|
+
response = {
|
|
119
|
+
"sparql": query,
|
|
120
|
+
"sql": run.sql,
|
|
121
|
+
"ran_in": run.ran_in,
|
|
122
|
+
"rows": rows[:effective],
|
|
123
|
+
"row_count": min(total, effective),
|
|
124
|
+
"total_row_count": total,
|
|
125
|
+
"truncated": total > effective,
|
|
126
|
+
"limit_applied": effective,
|
|
127
|
+
}
|
|
128
|
+
if run.ran_in != "sql":
|
|
129
|
+
response["reason"] = run.reason
|
|
130
|
+
if warnings:
|
|
131
|
+
response["warnings"] = warnings
|
|
132
|
+
response["hint"] = UNDECLARED_HINT
|
|
133
|
+
return response
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _form(query: str) -> Tuple[Optional[str], Optional[Dict[str, Any]]]:
|
|
137
|
+
"""The query's form ('select', 'ask', 'construct', 'describe' or
|
|
138
|
+
'update'), or the refusal of an update or a parse error. An update is
|
|
139
|
+
refused, to run or to explain: ProGraph is read-only over the data it
|
|
140
|
+
maps."""
|
|
141
|
+
from prograph.plan.ir.sparql_front import query_form
|
|
142
|
+
from prograph.read_only import message
|
|
143
|
+
try:
|
|
144
|
+
form = query_form(query)
|
|
145
|
+
except Exception as error: # noqa: BLE001
|
|
146
|
+
return None, {"sparql": query, "sql": None, **_error(error)}
|
|
147
|
+
if form == "update":
|
|
148
|
+
return form, {"sparql": query, "sql": None,
|
|
149
|
+
"error": message("a SPARQL Update"),
|
|
150
|
+
"reason": "read_only",
|
|
151
|
+
"suggestions": ["write a SELECT, ASK or CONSTRUCT query"]}
|
|
152
|
+
return form, None
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class _OverLimit(Exception):
|
|
156
|
+
"""A run past the server's row cap or timeout."""
|
|
157
|
+
|
|
158
|
+
def __init__(self, reason: str, text: str):
|
|
159
|
+
self.reason = reason
|
|
160
|
+
super().__init__(text)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _execute(engine: "ProGraph", query: str, form: Optional[str], row_cap: Optional[int] = None,
|
|
164
|
+
timeout: Optional[float] = None, context: Any = None,
|
|
165
|
+
language: str = "sparql") -> Tuple[List[Dict[str, Any]], int]:
|
|
166
|
+
"""The answer as rows, and their total: a SELECT's solutions, an ASK's
|
|
167
|
+
one row `{"boolean": ...}`, a CONSTRUCT's triples as `{s, p, o}`. A
|
|
168
|
+
SELECT is read batch by batch and stops past `row_cap`; the time is
|
|
169
|
+
checked after the run and between batches, as the endpoint's /sparql
|
|
170
|
+
checks it (`prograph.endpoint.app.Query`)."""
|
|
171
|
+
from prograph.execution import statement_deadline
|
|
172
|
+
with statement_deadline(timeout):
|
|
173
|
+
try:
|
|
174
|
+
return _execute_timed(engine, query, form, row_cap, timeout,
|
|
175
|
+
{"context": context} if context is not None else {}, language)
|
|
176
|
+
except TimeoutError as error:
|
|
177
|
+
raise _OverLimit("timeout", f"the query passed the timeout of {timeout} s: {error}") from None
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _execute_timed(engine: "ProGraph", query: str, form: Optional[str], row_cap: Optional[int],
|
|
181
|
+
timeout: Optional[float], running: Dict[str, Any],
|
|
182
|
+
language: str = "sparql") -> Tuple[List[Dict[str, Any]], int]:
|
|
183
|
+
import time
|
|
184
|
+
started = time.monotonic()
|
|
185
|
+
|
|
186
|
+
def check_time() -> None:
|
|
187
|
+
if timeout is not None and time.monotonic() - started > timeout:
|
|
188
|
+
raise _OverLimit("timeout", f"the query passed the timeout of {timeout} s")
|
|
189
|
+
|
|
190
|
+
def check_rows(count: int) -> None:
|
|
191
|
+
if row_cap is not None and count > row_cap:
|
|
192
|
+
raise _OverLimit("row_cap", f"the query passed the row cap of {row_cap}")
|
|
193
|
+
if form in ("construct", "describe"):
|
|
194
|
+
graph = engine.execute(query, language="sparql", **running)
|
|
195
|
+
check_time()
|
|
196
|
+
rows = [{"s": str(s), "p": str(p), "o": str(o)} for s, p, o in graph]
|
|
197
|
+
check_rows(len(rows))
|
|
198
|
+
return rows, len(rows)
|
|
199
|
+
if form == "ask":
|
|
200
|
+
rows = engine.execute(query, language="sparql", return_type="list", **running)
|
|
201
|
+
check_time()
|
|
202
|
+
return [{"boolean": bool(rows and rows[0].get("boolean"))}], 1
|
|
203
|
+
reader = engine.execute(query, language=language, return_type="batches", **running)
|
|
204
|
+
rows: List[Dict[str, Any]] = []
|
|
205
|
+
for batch in reader:
|
|
206
|
+
check_time()
|
|
207
|
+
rows.extend(batch.to_pylist(maps_as_pydicts="strict"))
|
|
208
|
+
check_rows(len(rows))
|
|
209
|
+
check_time()
|
|
210
|
+
return rows, len(rows)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def run_cypher(engine: "ProGraph", query: str, limit: Optional[int] = None, *,
|
|
214
|
+
row_cap: Optional[int] = None, timeout: Optional[float] = None,
|
|
215
|
+
context: Any = None) -> Dict[str, Any]:
|
|
216
|
+
"""Run a Cypher query that only reads, as `run_sparql` runs SPARQL: `{cypher, sql, ran_in, rows,
|
|
217
|
+
row_count, total_row_count, truncated, limit_applied}`; an error is `{cypher, sql, error, reason,
|
|
218
|
+
suggestions}`. A query that writes is refused (`reason` 'read_only') before it is planned: `ask`
|
|
219
|
+
only reads, and the owned store would take the write otherwise."""
|
|
220
|
+
from prograph.execution import QueryCancelledException
|
|
221
|
+
from prograph.plan.run_record import observe
|
|
222
|
+
from prograph.read_only import cypher_write_clauses
|
|
223
|
+
|
|
224
|
+
if cypher_write_clauses(query):
|
|
225
|
+
from prograph.read_only import ASK_WRITE
|
|
226
|
+
return {"cypher": query, "sql": None,
|
|
227
|
+
"error": ASK_WRITE.format(clauses=", ".join(cypher_write_clauses(query))),
|
|
228
|
+
"reason": "read_only", "suggestions": ["write a query that only reads: MATCH ... RETURN"]}
|
|
229
|
+
effective = _clamp_limit(limit)
|
|
230
|
+
with observe() as observed:
|
|
231
|
+
try:
|
|
232
|
+
rows, total = _execute(engine, query, "select", row_cap, timeout, context, language="cypher")
|
|
233
|
+
except _OverLimit as over:
|
|
234
|
+
return {"cypher": query, "sql": None, "error": str(over), "reason": over.reason,
|
|
235
|
+
"suggestions": ["narrow the query, or add a LIMIT"]}
|
|
236
|
+
except QueryCancelledException as cancelled:
|
|
237
|
+
return {"cypher": query, "sql": None, "error": str(cancelled), "reason": "cancelled",
|
|
238
|
+
"suggestions": []}
|
|
239
|
+
except Exception as error: # noqa: BLE001 - the error is the answer
|
|
240
|
+
failed = observed.run.failed if observed.run is not None else None
|
|
241
|
+
return {"cypher": query, "sql": getattr(error, "sql", None) or failed, **_error(error),
|
|
242
|
+
"suggestions": []}
|
|
243
|
+
run = observed.run
|
|
244
|
+
response = {"cypher": query, "sql": run.sql if run is not None else None,
|
|
245
|
+
"ran_in": run.ran_in if run is not None else None,
|
|
246
|
+
"rows": rows[:effective], "row_count": min(total, effective), "total_row_count": total,
|
|
247
|
+
"truncated": total > effective, "limit_applied": effective}
|
|
248
|
+
if run is not None and run.ran_in != "sql":
|
|
249
|
+
response["reason"] = run.reason
|
|
250
|
+
return response
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
# ──────────────────────────────────────────────────────────────────────
|
|
254
|
+
# ask
|
|
255
|
+
# ──────────────────────────────────────────────────────────────────────
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
class _Failed(Exception):
|
|
259
|
+
"""A run_sparql error, raised so the NLP retry loop feeds it back."""
|
|
260
|
+
|
|
261
|
+
def __init__(self, response: Dict[str, Any]):
|
|
262
|
+
self.response = response
|
|
263
|
+
text = response["error"]
|
|
264
|
+
if response.get("suggestions"):
|
|
265
|
+
text += " (suggestions: " + "; ".join(response["suggestions"]) + ")"
|
|
266
|
+
super().__init__(text)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def ask(engine: "ProGraph", question: str, llm: Any, limit: Optional[int] = None, *,
|
|
270
|
+
explain: bool = True, max_retries: int = 3, row_cap: Optional[int] = None,
|
|
271
|
+
timeout: Optional[float] = None, language: str = "sparql") -> Dict[str, Any]:
|
|
272
|
+
"""ProGraph writes the query, in SPARQL (the default) or Cypher (`language`): `prograph.nlp` turns the
|
|
273
|
+
question into a query over the schema, checks it (it parses; for SPARQL every class and predicate is
|
|
274
|
+
declared; for Cypher it only reads) and runs it as `run_sparql` or `run_cypher` does, feeding an error
|
|
275
|
+
back to the model up to `max_retries` times. The answer is that run's, with the question, the
|
|
276
|
+
`language`, the model's `explanation` of the rows (one more model call, when `explain`), the `retries`
|
|
277
|
+
and, after a retry, `attempts`: each earlier query (under the language's key, `sparql` or `cypher`) with
|
|
278
|
+
the error that caused the retry; an error is `{question, language, <language>, error, reason, retries}`
|
|
279
|
+
(and `attempts`)."""
|
|
280
|
+
from prograph.nlp import NaturalLanguageInterface
|
|
281
|
+
from prograph.nlp.interface import _supported_language
|
|
282
|
+
from prograph.read_only import cypher_write_clauses
|
|
283
|
+
|
|
284
|
+
from prograph.r2rml.schema import PropertyGraphViewError
|
|
285
|
+
try:
|
|
286
|
+
lang = _supported_language(language, engine)
|
|
287
|
+
except PropertyGraphViewError as refused:
|
|
288
|
+
return {"question": question, "language": language.lower(), language.lower(): None, "sql": None,
|
|
289
|
+
"error": str(refused), "reason": "no_property_graph",
|
|
290
|
+
"suggestions": ["ask in SPARQL (language 'sparql')"], "retries": 0}
|
|
291
|
+
last: Dict[str, Any] = {}
|
|
292
|
+
runner = run_sparql if lang == "sparql" else run_cypher
|
|
293
|
+
|
|
294
|
+
def run(query: str, language: str) -> List[Dict[str, Any]]:
|
|
295
|
+
response = runner(engine, query, limit, row_cap=row_cap, timeout=timeout)
|
|
296
|
+
last.clear()
|
|
297
|
+
last.update(response)
|
|
298
|
+
if "error" in response:
|
|
299
|
+
raise _Failed(response)
|
|
300
|
+
return response["rows"]
|
|
301
|
+
|
|
302
|
+
nli = NaturalLanguageInterface(engine, llm, language=lang, include_explanation=explain,
|
|
303
|
+
max_retries=max_retries, executor=run)
|
|
304
|
+
result = nli.ask(question)
|
|
305
|
+
if result.error:
|
|
306
|
+
if nli._is_error_response(result.generated_query):
|
|
307
|
+
# The model's last word is that it cannot be answered: what an
|
|
308
|
+
# earlier attempt ran and why it failed is not this answer's.
|
|
309
|
+
last.clear()
|
|
310
|
+
reason = last.get("reason") if "error" in last else "no_query"
|
|
311
|
+
if lang == "cypher" and "error" not in last and cypher_write_clauses(result.generated_query or ""):
|
|
312
|
+
# Refused by ask itself before it ran (`refuse_ask_write`), not by a run.
|
|
313
|
+
reason = "read_only"
|
|
314
|
+
return {"question": question, "language": lang, lang: result.generated_query or None,
|
|
315
|
+
"sql": last.get("sql"), "error": result.error, "reason": reason,
|
|
316
|
+
"suggestions": last.get("suggestions", []), "retries": result.retries,
|
|
317
|
+
**_attempts(result, lang)}
|
|
318
|
+
return {"question": question, "language": lang, **last, "explanation": result.explanation or None,
|
|
319
|
+
"retries": result.retries, **_attempts(result, lang)}
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _attempts(result: Any, language: str = "sparql") -> Dict[str, Any]:
|
|
323
|
+
"""`attempts` after a retry: each earlier query, under the language's key, and the error that caused the retry."""
|
|
324
|
+
return {"attempts": [{language: a["query"], "error": a["error"]} for a in result.attempts]} if result.attempts else {}
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
# ──────────────────────────────────────────────────────────────────────
|
|
328
|
+
# explain_sparql
|
|
329
|
+
# ──────────────────────────────────────────────────────────────────────
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def explain_sparql(engine: "ProGraph", query: str) -> Dict[str, Any]:
|
|
333
|
+
"""What a query would run, without running it: `{sparql, sql, ran_in}`,
|
|
334
|
+
and `warnings` for undeclared IRIs; an error response where the plan
|
|
335
|
+
does not build it."""
|
|
336
|
+
form, refused = _form(query)
|
|
337
|
+
if refused is not None:
|
|
338
|
+
return refused
|
|
339
|
+
try:
|
|
340
|
+
sql = engine.to_sql(query, language="sparql")
|
|
341
|
+
except Exception as error: # noqa: BLE001
|
|
342
|
+
return error_response(engine, query, error)
|
|
343
|
+
response: Dict[str, Any] = {"sparql": query, "sql": sql, "ran_in": "sql"}
|
|
344
|
+
warnings = unknown_iri_warnings(engine, query)
|
|
345
|
+
if warnings:
|
|
346
|
+
response["warnings"] = warnings
|
|
347
|
+
response["hint"] = UNDECLARED_HINT
|
|
348
|
+
return response
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
# ──────────────────────────────────────────────────────────────────────
|
|
352
|
+
# Errors and undeclared IRIs
|
|
353
|
+
# ──────────────────────────────────────────────────────────────────────
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _error(error: Exception) -> Dict[str, Any]:
|
|
357
|
+
"""`error` and `reason` of an exception: a SQLGenerationError's own
|
|
358
|
+
reason, a parse error as such (its text says the line and column)."""
|
|
359
|
+
from prograph.plan.ir.sparql_front import QueryError
|
|
360
|
+
from prograph.utils.exceptions import is_statement_timeout
|
|
361
|
+
reason = getattr(error, "reason", None)
|
|
362
|
+
if is_statement_timeout(error):
|
|
363
|
+
reason = "timeout"
|
|
364
|
+
if reason is None:
|
|
365
|
+
reason = ("parse_error" if isinstance(error, QueryError)
|
|
366
|
+
else "not_built" if isinstance(error, NotImplementedError)
|
|
367
|
+
else type(error).__name__)
|
|
368
|
+
return {"error": f"{type(error).__name__}: {error}", "reason": reason}
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def error_response(engine: "ProGraph", query: str, error: Exception) -> Dict[str, Any]:
|
|
372
|
+
"""An error as data, with the declared IRIs nearest the query's
|
|
373
|
+
undeclared ones as `suggestions`."""
|
|
374
|
+
response = {"sparql": query, "sql": None, **_error(error)}
|
|
375
|
+
unknown = unknown_iri_warnings(engine, query)
|
|
376
|
+
suggestions = [s for w in unknown for s in w["nearest"]]
|
|
377
|
+
if suggestions:
|
|
378
|
+
response["suggestions"] = suggestions
|
|
379
|
+
if unknown:
|
|
380
|
+
response["hint"] = UNDECLARED_HINT
|
|
381
|
+
return response
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
# What a client writing its own SPARQL is told when a query names an IRI the schema does not declare.
|
|
385
|
+
UNDECLARED_HINT = "call get_schema and use its IRIs: the query names a class or predicate the schema does not declare"
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def unknown_iri_warnings(engine: "ProGraph", query: str) -> List[Dict[str, Any]]:
|
|
389
|
+
"""The query's undeclared classes and predicates with the nearest
|
|
390
|
+
declared ones (`prograph.schema.context.undeclared_iris`)."""
|
|
391
|
+
from prograph.schema.context import undeclared_iris
|
|
392
|
+
if engine.schema is None:
|
|
393
|
+
return []
|
|
394
|
+
return undeclared_iris(engine.schema, query, ontology=engine.ontology)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
__all__ = ["get_schema", "run_sparql", "run_cypher", "ask", "explain_sparql", "unknown_iri_warnings", "error_response"]
|