prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,992 @@
|
|
|
1
|
+
"""The engine's query path: a query's text, through its language's front
|
|
2
|
+
end, to one plan over the source maps, lowered to SQL and run on the
|
|
3
|
+
engine's backend (docs/architecture/one-plan.md).
|
|
4
|
+
|
|
5
|
+
What is not built yet raises NotImplementedError whose message starts
|
|
6
|
+
"one-plan:".
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import contextvars
|
|
12
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def source_maps(schema: Any) -> Any:
|
|
16
|
+
"""The schema's source maps: an R2RML mapping's compiled straight from
|
|
17
|
+
it (one-plan.md 2.6), kept on the schema; else the schema's own."""
|
|
18
|
+
r2rml = schema.__dict__.get("r2rml_source")
|
|
19
|
+
if r2rml is None:
|
|
20
|
+
return schema.source_maps()
|
|
21
|
+
from prograph.r2rml.schema import store_version, with_included
|
|
22
|
+
cached = schema.__dict__.get("_one_plan_maps")
|
|
23
|
+
version = (store_version(schema), getattr(schema, "decimal_text", None))
|
|
24
|
+
if cached is None or cached[0] is not r2rml or cached[1] != version:
|
|
25
|
+
from prograph.r2rml.compile import compile_r2rml
|
|
26
|
+
cached = (r2rml, version, with_included(schema, compile_r2rml(r2rml[0], r2rml[1])))
|
|
27
|
+
schema.__dict__["_one_plan_maps"] = cached
|
|
28
|
+
return cached[2]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def saturated(maps: Any, onto: Any) -> Any:
|
|
32
|
+
"""`maps` saturated by `onto` (T-mappings), kept on the maps for that ontology: the same maps
|
|
33
|
+
and ontology give the same saturated maps, and so the same property-graph view (`lpg_view.view_of`)."""
|
|
34
|
+
held = maps.__dict__.get("_saturated")
|
|
35
|
+
if held is None or held[0] is not onto:
|
|
36
|
+
from prograph.plan.ir.saturate import saturate
|
|
37
|
+
held = (onto, saturate(maps, onto))
|
|
38
|
+
maps.__dict__["_saturated"] = held
|
|
39
|
+
return held[1]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def mapped_tables(schema: Any) -> Any:
|
|
43
|
+
"""The logical tables the schema's source maps read (`SourceMaps.tables`): what an engine
|
|
44
|
+
loads and types, so they are the tables a plan reads."""
|
|
45
|
+
return source_maps(schema).tables()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _cypher_identity(engine: Any) -> Any:
|
|
49
|
+
"""The owl:sameAs components a Cypher query reads (one node per component), under profile rl; None else."""
|
|
50
|
+
if getattr(engine, "_profile", None) != "rl":
|
|
51
|
+
return None
|
|
52
|
+
from prograph.plan.ir.identity import for_engine
|
|
53
|
+
return for_engine(engine)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _cypher_stored(engine: Any) -> Any:
|
|
57
|
+
"""The engine's warnings of the node properties stored OWL RL facts give, under profile rl; None else."""
|
|
58
|
+
if getattr(engine, "_profile", None) != "rl":
|
|
59
|
+
return None
|
|
60
|
+
from prograph.plan.ir.cypher_ontology import stored_values_of
|
|
61
|
+
return stored_values_of(engine)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# The keys a running query (`run`) read beside stored OWL RL facts, gathered as it is planned (`_planned`): None
|
|
65
|
+
# outside `run`, so to_sql and explain gather none.
|
|
66
|
+
_STORED_READS: Any = contextvars.ContextVar("prograph_stored_reads", default=None)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _check_stored(engine: Any, reads: Any) -> None:
|
|
70
|
+
"""After a read-only Cypher query ran: whether the keys it read beside stored OWL RL facts have values that
|
|
71
|
+
differ, warned once per engine (`cypher_ontology.StoredValues.check`). Never from to_sql, explain or a write
|
|
72
|
+
(a write's translation gathers no keys)."""
|
|
73
|
+
checker = _cypher_stored(engine) if reads else None
|
|
74
|
+
if checker is not None:
|
|
75
|
+
for key, label, predicates in dict.fromkeys(reads):
|
|
76
|
+
checker.check(label, key, predicates)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def ontology_of(engine: Any) -> Any:
|
|
80
|
+
"""The engine's ontology as the plan reads it (`saturate.Ontology`), read once per ontology
|
|
81
|
+
content; None without one. The axioms its profile does not answer are refused, or dropped
|
|
82
|
+
(`profile_report`). The ontology's triples are read whole: a dropped axiom is not reasoned with (no OWL 2 RL
|
|
83
|
+
rule is compiled from it, `ProGraph._rl_rules`), and its triples stay in the default graph as stated."""
|
|
84
|
+
layer = getattr(engine, "_ontology", None)
|
|
85
|
+
if layer is None:
|
|
86
|
+
return None
|
|
87
|
+
from prograph.plan.ir.saturate import read_ontology
|
|
88
|
+
key, _, kept = profile_report(engine)
|
|
89
|
+
graph = layer.rdf_graph if not callable(layer.rdf_graph) else layer.rdf_graph()
|
|
90
|
+
cached = getattr(engine, "_saturation", None)
|
|
91
|
+
if cached is None or cached[0] != key:
|
|
92
|
+
# Under profile="rl" the axioms OWL RL's rules apply are applied, not ignored.
|
|
93
|
+
rl = getattr(engine, "_profile", None) == "rl"
|
|
94
|
+
# Under "ql" the QL axioms are read from the graph less those on_unsupported="ignore" dropped.
|
|
95
|
+
onto = read_ontology(graph, engine._rl_rules().consumed if rl else frozenset(), rl=rl, kept=kept)
|
|
96
|
+
cached = (key, onto)
|
|
97
|
+
engine._saturation = cached
|
|
98
|
+
return cached[1]
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def profile_report(engine: Any) -> Any:
|
|
102
|
+
"""(content key, `profiles.OntologyReport`, the graph OWL 2 RL's rules are compiled from) for the engine's ontology,
|
|
103
|
+
once per ontology content: the axioms its profile does not answer raise `OWLProfileError`, or under
|
|
104
|
+
`on_unsupported="ignore"` are dropped from the graph (`profiles.kept_graph`)."""
|
|
105
|
+
layer = engine._ontology
|
|
106
|
+
graph = layer.rdf_graph if not callable(layer.rdf_graph) else layer.rdf_graph()
|
|
107
|
+
key = (len(graph), hash(frozenset(graph)))
|
|
108
|
+
held = engine.__dict__.get("_profile_checked")
|
|
109
|
+
if held is None or held[0] != key:
|
|
110
|
+
from prograph.plan.ir import profiles
|
|
111
|
+
got = profiles.check(graph, engine._profile, getattr(layer, "on_unsupported", "error"))
|
|
112
|
+
held = (key, got, profiles.kept_graph(graph, got))
|
|
113
|
+
engine.__dict__["_profile_checked"] = held
|
|
114
|
+
return held
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _planned(engine: Any, query: str, language: str, hints: Any,
|
|
118
|
+
parameters: Optional[dict]):
|
|
119
|
+
"""The query's plan and form, and whether its answer is in full terms."""
|
|
120
|
+
if getattr(engine, "_profile", None) == "ql":
|
|
121
|
+
# The consistency verdict re-proven for this query, SPARQL or Cypher (`consistency.recheck`).
|
|
122
|
+
from prograph.plan.ir.consistency import recheck
|
|
123
|
+
from prograph.plan.ir.dataset import EXPLAINING
|
|
124
|
+
recheck(engine, explaining=EXPLAINING.get())
|
|
125
|
+
if language == "cypher":
|
|
126
|
+
# Cypher reads labels: refused over an R2RML mapping whatever the front end.
|
|
127
|
+
from prograph.r2rml.schema import require_property_graph
|
|
128
|
+
require_property_graph(engine._schema, "Cypher")
|
|
129
|
+
if language == "cypher":
|
|
130
|
+
from prograph.plan.ir import cypher_front
|
|
131
|
+
from prograph.plan.ir.keys import trusted_keys
|
|
132
|
+
from prograph.parameters import cypher_values
|
|
133
|
+
graph = getattr(getattr(engine, "executor", None), "graph", None)
|
|
134
|
+
# The ontology saturates the maps for Cypher (T-mappings): inferred labels,
|
|
135
|
+
# relationship types and property keys (one-plan 2.5). The owned store is read once for the planning
|
|
136
|
+
# (`store_now`: what another writer appended since, its labels, keys and types).
|
|
137
|
+
from prograph.plan.ir.dataset import maps_seen
|
|
138
|
+
onto = ontology_of(engine)
|
|
139
|
+
maps = maps_seen(engine, onto)
|
|
140
|
+
if onto is not None:
|
|
141
|
+
from prograph.plan.ir.cypher_ontology import warn_once
|
|
142
|
+
layer = engine._ontology
|
|
143
|
+
warn_once(engine, layer.rdf_graph if not callable(layer.rdf_graph) else layer.rdf_graph(),
|
|
144
|
+
[e.predicate for e in maps.edges if e.predicate],
|
|
145
|
+
cypher_front._view(maps, onto).typing_lost, cypher_front._view(maps, onto).derived_values)
|
|
146
|
+
translated = cypher_front.translate(query, maps, _held(engine._schema),
|
|
147
|
+
lambda m: trusted_keys(graph, m, maps),
|
|
148
|
+
procedures=engine._read_procedures(),
|
|
149
|
+
step=engine.__dict__.get("_write_step"),
|
|
150
|
+
dialect=graph.sql_dialect() if hasattr(graph, "sql_dialect") else None,
|
|
151
|
+
parameters=cypher_values(parameters),
|
|
152
|
+
superclasses=getattr(onto, "superclasses", None),
|
|
153
|
+
ontology=onto,
|
|
154
|
+
identity=_cypher_identity(engine),
|
|
155
|
+
stored=_cypher_stored(engine))
|
|
156
|
+
_warn_untrusted_keys(engine, translated)
|
|
157
|
+
gathering = _STORED_READS.get()
|
|
158
|
+
if gathering is not None: # a query `run` runs: its keys checked once it has answered
|
|
159
|
+
gathering.extend(getattr(translated, "stored_reads", ()))
|
|
160
|
+
# Cypher plans skip the plan rewrites (below, for SPARQL): lowered as translated
|
|
161
|
+
# (tests/plan/test_cypher_plans_skip_rewrites.py).
|
|
162
|
+
return translated, False, maps.settings.base_iri
|
|
163
|
+
if language != "sparql":
|
|
164
|
+
raise NotImplementedError(f"one-plan: the {language} front end")
|
|
165
|
+
from prograph.plan.ir import sparql_front
|
|
166
|
+
from prograph.parameters import bindings as bound_terms
|
|
167
|
+
schema = engine._schema
|
|
168
|
+
bindings = bound_terms(parameters, getattr(schema, "decimal_text", None))
|
|
169
|
+
graph = getattr(getattr(engine, "executor", None), "graph", None)
|
|
170
|
+
held = _held(schema, graph.sql_dialect() if hasattr(graph, "sql_dialect") else None, True)
|
|
171
|
+
# The dataset the query sees: the mapped data, the store's (read once for the planning, `store_now`: what
|
|
172
|
+
# another writer appended since, the record of the last run, the alias generation), saturated by the
|
|
173
|
+
# ontology (T-mappings).
|
|
174
|
+
from prograph.plan.ir.dataset import maps_seen, store_now
|
|
175
|
+
onto = ontology_of(engine)
|
|
176
|
+
now = store_now(engine, onto)
|
|
177
|
+
maps = maps_seen(engine, onto, now)
|
|
178
|
+
from prograph.plan.ir.keys import trusted_keys
|
|
179
|
+
graph = getattr(getattr(engine, "executor", None), "graph", None)
|
|
180
|
+
from prograph.plan.ir.identity import for_engine
|
|
181
|
+
translated = sparql_front.translate(query, maps, held, lambda m: trusted_keys(graph, m, maps),
|
|
182
|
+
union_default=getattr(engine, "default_graph", "union") == "union",
|
|
183
|
+
ontology=onto, bindings=bindings,
|
|
184
|
+
dataset=getattr(hints, "dataset", None), identity=for_engine(engine, now),
|
|
185
|
+
implied_individuals=getattr(engine, "_implied_individuals", False),
|
|
186
|
+
datatypes=getattr(engine, "_datatypes", "term"))
|
|
187
|
+
from dataclasses import replace
|
|
188
|
+
from prograph.plan.ir.rewrite import rewrite
|
|
189
|
+
engine_dialect = graph.sql_dialect() if hasattr(graph, "sql_dialect") else None
|
|
190
|
+
plan, certificates = rewrite(translated.plan, engine_dialect)
|
|
191
|
+
check_sources(engine, plan)
|
|
192
|
+
translated = replace(translated, plan=plan, certificates=certificates)
|
|
193
|
+
terms_all = getattr(hints, "terms", None) == "all"
|
|
194
|
+
return translated, terms_all, maps.settings.base_iri
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class UntrustedKeyWarning(UserWarning):
|
|
198
|
+
"""A node mapping whose key is not declared unique: Cypher merges its duplicate rows; a relationship
|
|
199
|
+
mapping with no key: Cypher numbers the rows of one start and end to tell them apart."""
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _warn_untrusted_keys(engine: Any, translated: Any) -> None:
|
|
203
|
+
"""Each node mapping a Cypher query grouped because its key is neither trusted nor checked, once per
|
|
204
|
+
engine, in property-graph words (its rows are grouped so a repeated key is one node; declaring the key
|
|
205
|
+
removes that work). Read off the plan: no catalog read."""
|
|
206
|
+
import warnings
|
|
207
|
+
warned = engine.__dict__.setdefault("_untrusted_keys_warned", set())
|
|
208
|
+
for label, columns, table in getattr(translated, "untrusted", ()):
|
|
209
|
+
if label in warned:
|
|
210
|
+
continue
|
|
211
|
+
warned.add(label)
|
|
212
|
+
warnings.warn(f"{label}'s key ({', '.join(columns)}) isn't declared unique, so queries merge duplicate "
|
|
213
|
+
f"rows of {table}. Declare keys: trusted, or keys: check on Delta.",
|
|
214
|
+
UntrustedKeyWarning, stacklevel=5)
|
|
215
|
+
for rel_type, table in getattr(translated, "untrusted_rels", ()):
|
|
216
|
+
if ("rel", rel_type) in warned:
|
|
217
|
+
continue
|
|
218
|
+
warned.add(("rel", rel_type))
|
|
219
|
+
warnings.warn(f"{rel_type}'s rows have no declared key, so queries number repeated rows of {table}. Declare "
|
|
220
|
+
f"id_column, or keys: trusted if {table} holds one row per start and end.",
|
|
221
|
+
UntrustedKeyWarning, stacklevel=5)
|
|
222
|
+
def check_sources(engine: Any, plan: Any) -> None:
|
|
223
|
+
"""Refuse a query that reads a source the engine could not read, naming the source, its table and the
|
|
224
|
+
database's message (`ProGraph._record_column_types`), or a column the mapping names that the table it
|
|
225
|
+
described lacks; else the cause would surface as a query that cannot be built."""
|
|
226
|
+
from prograph.plan.ir import ops as O
|
|
227
|
+
from prograph.utils.exceptions import SourceNotReadableError
|
|
228
|
+
unreadable = getattr(engine, "_unreadable_sources", None) or {}
|
|
229
|
+
schema = engine._schema
|
|
230
|
+
seen = set()
|
|
231
|
+
stack = [plan]
|
|
232
|
+
while stack:
|
|
233
|
+
op = stack.pop()
|
|
234
|
+
if id(op) in seen:
|
|
235
|
+
continue
|
|
236
|
+
seen.add(id(op))
|
|
237
|
+
stack.extend(op.children())
|
|
238
|
+
if not isinstance(op, O.MapScan):
|
|
239
|
+
continue
|
|
240
|
+
source = op.map.source
|
|
241
|
+
who = getattr(op.map, "label", None) or getattr(op.map, "rel_type", None) or op.map.id
|
|
242
|
+
join = getattr(op.map, "join", None)
|
|
243
|
+
named = [source] + ([join.parent] if join is not None else []) # a joint query's parent too
|
|
244
|
+
bad = next((x for x in named if x.table in unreadable), None)
|
|
245
|
+
if bad is not None:
|
|
246
|
+
source = bad
|
|
247
|
+
qualified, message = unreadable[source.table]
|
|
248
|
+
raise SourceNotReadableError(
|
|
249
|
+
f"the source {source.table!r} ({who}) could not be read{_where(engine)}: {message}. Name the table "
|
|
250
|
+
f"as the database knows it ({qualified!r} was read): qualify it (catalog.schema.table), or give "
|
|
251
|
+
f"the engine default_catalog= and default_schema=; a YAML mapping for a SQL backend takes the "
|
|
252
|
+
f"sources' tables with to_schema(by_table=True). A table created after the engine is read once "
|
|
253
|
+
f"update_schema() is called or the engine is built again")
|
|
254
|
+
described = (getattr(schema, "column_types", {}) or {}).get(source.table)
|
|
255
|
+
if described:
|
|
256
|
+
known = {str(c).lower() for c in described}
|
|
257
|
+
for r in op.reads:
|
|
258
|
+
if not r.parent and str(r.column).lower() not in known:
|
|
259
|
+
raise SourceNotReadableError(
|
|
260
|
+
f"the mapping {who} reads the column {r.column!r} of {source.table!r}, which the table "
|
|
261
|
+
f"does not have; its columns are {sorted(map(str, described))}")
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _where(engine: Any) -> str:
|
|
265
|
+
"""Where a bare table name resolves on the engine's connection (`on the warehouse, in main.default`), for a
|
|
266
|
+
message; empty where it cannot be read."""
|
|
267
|
+
conn = getattr(engine, "_connection", None)
|
|
268
|
+
backend = getattr(engine, "backend", None)
|
|
269
|
+
if conn is None:
|
|
270
|
+
return ""
|
|
271
|
+
sql = {"databricks_sql": "SELECT current_catalog(), current_schema()",
|
|
272
|
+
"duckdb": "SELECT current_database(), current_schema()",
|
|
273
|
+
"postgresql": "SELECT current_database(), current_schema()"}.get(backend)
|
|
274
|
+
if sql is None:
|
|
275
|
+
return f" on {backend}"
|
|
276
|
+
try:
|
|
277
|
+
from prograph.graph.drivers import fetch_rows
|
|
278
|
+
row = fetch_rows(conn, sql)[1][0]
|
|
279
|
+
return f" on {backend}, where a bare name is read in {row[0]}.{row[1]}"
|
|
280
|
+
except Exception: # noqa: BLE001 - the message stands without it
|
|
281
|
+
return f" on {backend}"
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _held(schema: Any, dialect: Optional[str] = None, text_of_untyped: bool = False):
|
|
285
|
+
"""How a table's column is held: (its datatype, zoned, digits, reader, bytewise), from its recorded type
|
|
286
|
+
(bytewise: a string that compares byte for byte, `collations.compares_bytes`);
|
|
287
|
+
`held.kind_of(table, column)` the recorded type itself (Cypher reads a TIME column as a local time).
|
|
288
|
+
On PostgreSQL (`dialect`) a NUMERIC column is read through `NUMERIC_VALUE`: a NaN or infinity, no
|
|
289
|
+
xsd:decimal, is a data error naming the column wherever the column is read, never a number PostgreSQL
|
|
290
|
+
ranks above every other (`lower._read`).
|
|
291
|
+
`text_of_untyped` (SPARQL): a column of a type with no natural datatype is read as the database's text
|
|
292
|
+
(rule `no-natural-datatype-text`); Cypher keeps its own values (lists, maps)."""
|
|
293
|
+
from prograph.schema import xsd
|
|
294
|
+
|
|
295
|
+
def held(table: str, column: str):
|
|
296
|
+
kind = schema.table_column_type(table, column)
|
|
297
|
+
dt = xsd.datatype_of_type(kind)
|
|
298
|
+
textual = text_of_untyped and (getattr(kind, "textual", False) or (dt is None and getattr(kind, "kind", None) in ("map", "list")))
|
|
299
|
+
if textual:
|
|
300
|
+
# R2RML 10.2: a value of a type with no natural datatype is a plain literal of the database's
|
|
301
|
+
# own text (a struct, a map, an array, jsonb, inet, interval, money, an enum, xml ...).
|
|
302
|
+
return xsd.STRING, False, None, "CAST_TEXT", False
|
|
303
|
+
digits = getattr(kind, "digits", None) if dt == xsd.DECIMAL else None
|
|
304
|
+
uuid = getattr(kind, "uuid", False)
|
|
305
|
+
reader = ("UUID_TEXT_UPPER" if dialect == "sqlserver" else "UUID_TEXT") if uuid else \
|
|
306
|
+
"CHAR_TEXT" if getattr(kind, "padded", False) else \
|
|
307
|
+
"FLOAT32_VALUE" if getattr(kind, "single", False) else \
|
|
308
|
+
"BINARY_TEXT" if getattr(kind, "collated", False) else \
|
|
309
|
+
"NUMERIC_VALUE" if dialect == "postgresql" and dt == xsd.DECIMAL else None
|
|
310
|
+
return dt, bool(getattr(kind, "zoned", False)), digits, reader, bool(getattr(kind, "bytewise", False)) or uuid
|
|
311
|
+
held.kind_of = schema.table_column_type
|
|
312
|
+
return held
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def node_keys(engine: Any, iri: str) -> List[Tuple[Any, Any, List[str]]]:
|
|
316
|
+
"""(node map, the database's answer, its key columns' names in it) for each node map, the
|
|
317
|
+
answer at most one row whose subject is `iri` (`ProGraph.resolve_iri`, which
|
|
318
|
+
reads it): over the source maps, so R2RML and YAML alike, each map's own rows (its filter or
|
|
319
|
+
query kept), the subject compared as a term, the key's values as its columns hold them."""
|
|
320
|
+
from prograph.plan.ir import expr as E, lower, ops as O, sparql_front
|
|
321
|
+
from prograph.plan.ir.terms import IRITerm
|
|
322
|
+
from prograph.schema.source_map import column_value
|
|
323
|
+
schema = engine._schema
|
|
324
|
+
maps = source_maps(schema)
|
|
325
|
+
translator = sparql_front.Translator(maps, _held(schema, None, True), lambda m: (), True, None)
|
|
326
|
+
run_sql, dialect = _runner(getattr(getattr(engine, "executor", None), "graph", None))
|
|
327
|
+
found = []
|
|
328
|
+
for m in maps.nodes:
|
|
329
|
+
key = list(m.key.columns)
|
|
330
|
+
values = [column_value(c) for c in key]
|
|
331
|
+
try:
|
|
332
|
+
scan, names = translator.scan(m, [m.subject] + values)
|
|
333
|
+
except Exception: # noqa: BLE001 (a map this cannot read: no node of it)
|
|
334
|
+
continue
|
|
335
|
+
x: O.Op = O.Extend(scan, "#s", E.MakeTerm(m.subject, tuple((c, E.Col(names[c]))
|
|
336
|
+
for c in m.subject.columns())))
|
|
337
|
+
x = O.Filter(x, E.Compare("=", E.Col("#s"), E.Const(IRITerm(iri)), E.TERM))
|
|
338
|
+
out = []
|
|
339
|
+
for i, (c, tm) in enumerate(zip(key, values)):
|
|
340
|
+
x = O.Extend(x, f"k{i}", E.MakeTerm(tm, ((c, E.Col(names[c])),)))
|
|
341
|
+
out.append(f"k{i}")
|
|
342
|
+
plan = O.Slice(O.Project(x, tuple(out)), 0, 1)
|
|
343
|
+
try:
|
|
344
|
+
sql_text = lower.to_sql(plan, dialect, base_iri=maps.settings.base_iri)
|
|
345
|
+
except NotImplementedError:
|
|
346
|
+
continue
|
|
347
|
+
found.append((m, run_sql(sql_text), [lower.sql_name(n) for n in out]))
|
|
348
|
+
return found
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def warn_written_constants(schema: Any) -> None:
|
|
352
|
+
"""At mapping load: a map's constant integer or decimal literal not in
|
|
353
|
+
canonical form (`rr:constant "5"^^xsd:decimal`) is a term of its own,
|
|
354
|
+
which no column's value of that datatype is (ProGraph writes those in
|
|
355
|
+
canonical form, D3), so a join or pattern meeting both never matches
|
|
356
|
+
(m2-functions.md 5)."""
|
|
357
|
+
import warnings
|
|
358
|
+
from prograph.plan.ir.slot import _canonical_text
|
|
359
|
+
from prograph.plan.ir.sparql_front import PatternNeverMatchesWarning
|
|
360
|
+
from prograph.schema import xsd
|
|
361
|
+
try:
|
|
362
|
+
maps = source_maps(schema)
|
|
363
|
+
except Exception: # noqa: BLE001 - a mapping that does not compile says so when planned
|
|
364
|
+
return
|
|
365
|
+
for m in getattr(maps, "nodes", ()):
|
|
366
|
+
for po in getattr(m, "predicate_objects", ()):
|
|
367
|
+
t = po.object
|
|
368
|
+
if t.constant is None or t.datatype not in (xsd.DECIMAL, xsd.INTEGER) \
|
|
369
|
+
or _canonical_text(t.constant, t.datatype):
|
|
370
|
+
continue
|
|
371
|
+
warnings.warn(
|
|
372
|
+
f"The mapping's constant \"{t.constant}\"^^{t.datatype.replace(xsd.XSD, 'xsd:')} for "
|
|
373
|
+
f"<{po.predicate}> is not in canonical form: it is a term of its own, which a "
|
|
374
|
+
f"column's value of that datatype never is, so a join between them never matches; "
|
|
375
|
+
f"FILTER(=) compares values.", PatternNeverMatchesWarning, stacklevel=3)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def run(engine: Any, query: str, language: str, **kw: Any) -> Any:
|
|
379
|
+
"""`_run` under the query's statement expectation (`execution.expect_long`); then, once it has answered, the
|
|
380
|
+
check of the stored OWL RL values its plan read (`_check_stored`)."""
|
|
381
|
+
from prograph.execution import statement_expectation
|
|
382
|
+
reads: List[Any] = []
|
|
383
|
+
token = _STORED_READS.set(reads)
|
|
384
|
+
try:
|
|
385
|
+
with statement_expectation():
|
|
386
|
+
out = _run(engine, query, language, **kw)
|
|
387
|
+
finally:
|
|
388
|
+
_STORED_READS.reset(token)
|
|
389
|
+
_check_stored(engine, reads)
|
|
390
|
+
return out
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def expects_long(plan: Any) -> bool:
|
|
394
|
+
"""Whether a plan is expected to run long: a property path's closure, or a shape the cost hints name
|
|
395
|
+
(`cost.SHAPES`: an unpinned join, an unseeded closure, the whole graph, a whole-IRI merge). Read off the
|
|
396
|
+
plan, no data: what decides how a statement is submitted on a SQL warehouse."""
|
|
397
|
+
from prograph.plan.ir import ops as O
|
|
398
|
+
from prograph.plan.ir.cost import SHAPES, cost_hints
|
|
399
|
+
stack, seen = [plan], set()
|
|
400
|
+
while stack:
|
|
401
|
+
op = stack.pop()
|
|
402
|
+
if id(op) in seen:
|
|
403
|
+
continue # a shared subtree is read once
|
|
404
|
+
seen.add(id(op))
|
|
405
|
+
if isinstance(op, O.Closure):
|
|
406
|
+
return True
|
|
407
|
+
stack.extend(op.children())
|
|
408
|
+
return any(h.shape in SHAPES for h in cost_hints(plan))
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def _run(engine: Any, query: str, language: str, *, hints: Any = None,
|
|
412
|
+
parameters: Optional[dict] = None, streaming: bool = False, step_key: Optional[str] = None) -> Any:
|
|
413
|
+
"""Answer `query`: front end, the plan, lowering to SQL, execution on
|
|
414
|
+
`engine`'s backend. Returns a pyarrow.Table of solutions (an ASK's: one
|
|
415
|
+
row, its column `boolean`), which `execute` converts to the asked
|
|
416
|
+
return type, or a CONSTRUCT's graph (an rdflib.Graph,
|
|
417
|
+
`construct.instantiate`). `query` is the text without its hint block, `hints` the
|
|
418
|
+
block's settings (`prograph.hints`)."""
|
|
419
|
+
graph = getattr(getattr(engine, "executor", None), "graph", None)
|
|
420
|
+
run_sql, dialect = _runner(graph)
|
|
421
|
+
from prograph.plan.ir import lower
|
|
422
|
+
# Spark: with no path_depth hint a closure walks first to the level form's depth; a
|
|
423
|
+
# longer walk raises in the statement, and it runs again to the default depth by the recursive form.
|
|
424
|
+
# Never a short answer. Which engines, and the depth: `lower.first_depth`, which to_sql shows too.
|
|
425
|
+
ladder = lower.closure_ladder(dialect, hints)
|
|
426
|
+
first = ladder[0] if ladder else None
|
|
427
|
+
retry = bool(ladder)
|
|
428
|
+
sql_text, names, translated, _pins, digits = _lowered(engine, query, language, dialect, hints,
|
|
429
|
+
parameters, full=True, depth=first,
|
|
430
|
+
step_key=step_key)
|
|
431
|
+
# A walk of a functional step (a parent pointer) skips the ladder's middle rungs (`closure_ladder`).
|
|
432
|
+
ladder = lower.closure_ladder(dialect, hints, translated.plan) if ladder else ladder
|
|
433
|
+
import time
|
|
434
|
+
from prograph.logging import get_plan_logger
|
|
435
|
+
# Each step below (a statement, a rung of the ladder, the RT form) starts only while the query is not
|
|
436
|
+
# cancelled (`execution.checkpoint`); a cancel between two never starts the next.
|
|
437
|
+
from prograph.execution import checkpoint, expect_long
|
|
438
|
+
checkpoint()
|
|
439
|
+
expect_long(expects_long(translated.plan))
|
|
440
|
+
# Streamed (return_type="batches"; decided 2026-11-11): a SELECT whose statement raises
|
|
441
|
+
# nowhere in the query, through the graph's own batch reader; a statement that may raise (a depth
|
|
442
|
+
# guard, a data error) is read whole, so no error follows rows already handed over.
|
|
443
|
+
# A write's answer, read in its step from the rows it wrote (`cypher_write.apply`); None for any other.
|
|
444
|
+
written = getattr(translated, "answer", None)
|
|
445
|
+
from prograph.plan.ir.cypher_mixed import holds_mixed, unwrap_table
|
|
446
|
+
# A column of values of several types answers as Arrow's dense union, rebuilt from the tagged
|
|
447
|
+
# struct (`cypher_mixed.unwrap_table`), so it is read whole, not streamed.
|
|
448
|
+
mixed_answer = lower.TAGGED_PROPERTIES.get() or any(holds_mixed(slot) for slot in translated.plan.slots.values())
|
|
449
|
+
if streaming and written is None and translated.form == "select" and names != ["__one"] \
|
|
450
|
+
and not mixed_answer and not lower.wrote_a_raise():
|
|
451
|
+
batches = _batch_runner(graph)
|
|
452
|
+
if batches is not None:
|
|
453
|
+
try:
|
|
454
|
+
opened = _with_digits(batches, digits)(sql_text, getattr(engine, "_batch_size", None) or 100_000)
|
|
455
|
+
except Exception as exc:
|
|
456
|
+
typed = _typed(exc, sql_text)
|
|
457
|
+
if typed is exc:
|
|
458
|
+
raise
|
|
459
|
+
raise typed from exc
|
|
460
|
+
reader = _renamed(opened, names, sql_text)
|
|
461
|
+
# The event every executed statement logs; its rows are not counted until they are read.
|
|
462
|
+
get_plan_logger().info("compositional_backend_executed", sql=sql_text, exec_time_ms=None,
|
|
463
|
+
result_count=None, sql_generator="one-plan", streamed=True)
|
|
464
|
+
return reader
|
|
465
|
+
started = time.time()
|
|
466
|
+
try:
|
|
467
|
+
try:
|
|
468
|
+
table = written if written is not None else _with_digits(run_sql, digits)(sql_text)
|
|
469
|
+
rung = first
|
|
470
|
+
except Exception as exc:
|
|
471
|
+
if not (retry and lower.depth_guard(str(exc))):
|
|
472
|
+
raise
|
|
473
|
+
# The ladder's next rungs (`lower.closure_ladder`), then the recursive form at the default
|
|
474
|
+
# depth: each runs only where the one before found a longer walk. Never a short answer.
|
|
475
|
+
table = None
|
|
476
|
+
for rung in tuple(ladder[1:]) + (None,):
|
|
477
|
+
checkpoint()
|
|
478
|
+
sql_text, names, translated, _pins, digits = _lowered(engine, query, language, dialect, hints,
|
|
479
|
+
parameters, full=True, costed=False,
|
|
480
|
+
depth=rung)
|
|
481
|
+
try:
|
|
482
|
+
table = _with_digits(run_sql, digits)(sql_text)
|
|
483
|
+
break
|
|
484
|
+
except Exception as again:
|
|
485
|
+
if rung is None or not lower.depth_guard(str(again)):
|
|
486
|
+
raise
|
|
487
|
+
except Exception as exc:
|
|
488
|
+
if lower.recursion_refused(str(exc)):
|
|
489
|
+
checkpoint()
|
|
490
|
+
# A warehouse that refuses WITH RECURSIVE (Lakehouse RT): the statement by levels to RT_LEVELS
|
|
491
|
+
# (or the path_depth hint, where smaller); a longer walk is a typed error naming the warehouse
|
|
492
|
+
# and path_depth. Never a short answer.
|
|
493
|
+
asked = getattr(hints, "path_depth", None) or lower.DEFAULT_PATH_DEPTH
|
|
494
|
+
depth = min(asked, lower.RT_LEVELS)
|
|
495
|
+
limit = lower.LEVEL_LIMIT.set(depth)
|
|
496
|
+
try:
|
|
497
|
+
sql_text, names, translated, _pins, digits = _lowered(engine, query, language, dialect, hints,
|
|
498
|
+
parameters, full=True, costed=False,
|
|
499
|
+
depth=depth)
|
|
500
|
+
finally:
|
|
501
|
+
lower.LEVEL_LIMIT.reset(limit)
|
|
502
|
+
rung = depth
|
|
503
|
+
try:
|
|
504
|
+
table = _with_digits(run_sql, digits)(sql_text)
|
|
505
|
+
except Exception as deeper:
|
|
506
|
+
if lower.depth_guard(str(deeper)):
|
|
507
|
+
from prograph.utils.exceptions import SQLRuntimeError
|
|
508
|
+
raise SQLRuntimeError(
|
|
509
|
+
f"A property path walks past {depth} steps, and this warehouse (a Databricks Lakehouse RT "
|
|
510
|
+
f"warehouse) refuses the recursive SQL that walks further: ProGraph walks it by levels to "
|
|
511
|
+
f"{lower.RT_LEVELS} steps there. Bound the path, or run the query on a SQL warehouse, "
|
|
512
|
+
f"where path_depth (up to 1000) sets the depth.", reason="path_too_deep",
|
|
513
|
+
sql=sql_text) from deeper
|
|
514
|
+
typed = _typed(deeper, sql_text)
|
|
515
|
+
if typed is deeper:
|
|
516
|
+
raise
|
|
517
|
+
if isinstance(typed, TwoValuesError):
|
|
518
|
+
typed = _two_values(engine, run_sql, dialect, typed)
|
|
519
|
+
raise typed from deeper
|
|
520
|
+
else:
|
|
521
|
+
typed = _typed(exc, sql_text)
|
|
522
|
+
if typed is exc:
|
|
523
|
+
raise
|
|
524
|
+
if isinstance(typed, TwoValuesError):
|
|
525
|
+
typed = _two_values(engine, run_sql, dialect, typed)
|
|
526
|
+
raise typed from exc
|
|
527
|
+
# The event every executed statement logs (`prograph.plan.run_record`).
|
|
528
|
+
get_plan_logger().info("compositional_backend_executed", sql=sql_text,
|
|
529
|
+
exec_time_ms=round((time.time() - started) * 1000, 2),
|
|
530
|
+
result_count=table.num_rows, sql_generator="one-plan",
|
|
531
|
+
closure_rung=(rung if rung is not None else "recursive") if retry else None)
|
|
532
|
+
# The answer's columns by the query's own names (`lower.sql_name`); a
|
|
533
|
+
# solution of no variable is a row of no column.
|
|
534
|
+
table = table.drop_columns(["__one"]) if names == ["__one"] else table.rename_columns(names)
|
|
535
|
+
if mixed_answer:
|
|
536
|
+
table = unwrap_table(table)
|
|
537
|
+
if translated.form == "construct":
|
|
538
|
+
from prograph.plan.ir.construct import instantiate
|
|
539
|
+
from prograph.plan.ir.sparql_front import HIDDEN
|
|
540
|
+
return instantiate(translated.template, table.to_pylist(), translated.prefixes, HIDDEN)
|
|
541
|
+
return table
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def _written(translated: Any, dialect: str, base: Optional[str], terms_all: bool) -> str:
|
|
546
|
+
"""The statement for a plan (`answer_sql`); a write's own answer is lowered in its step (`cypher_write.apply`)."""
|
|
547
|
+
return answer_sql(translated.plan, dialect, base, terms_all or translated.form == "construct",
|
|
548
|
+
parts=translated.form == "select" and getattr(translated, "write", None) is None)
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def answer_sql(plan: Any, dialect: str, base: Optional[str], terms_all: bool = False, parts: bool = True) -> str:
|
|
552
|
+
"""The statement of an answer: a read's, or a write's RETURN read in its step. On PostgreSQL it writes a whole
|
|
553
|
+
node, relationship, path or map, and a list of them, as flat typed columns the reader makes the Arrow struct of
|
|
554
|
+
(rule cypher-result-values). Where a step cannot write them and no answer column holds one (a quantifier over a
|
|
555
|
+
trail's nodes, a count), the statement is written without them, as it always was; where an answer column holds
|
|
556
|
+
one, the refusal stands."""
|
|
557
|
+
from prograph.plan.ir import lower
|
|
558
|
+
from prograph.plan.ir.cypher_answer import PartsRefused, answers_elements
|
|
559
|
+
try:
|
|
560
|
+
return lower.to_sql(plan, dialect, base_iri=base, terms_all=terms_all, split_terms=True, element_parts=parts)
|
|
561
|
+
except PartsRefused:
|
|
562
|
+
if not parts or any(answers_elements(plan.slots[c]) for c in plan.columns if c in plan.slots):
|
|
563
|
+
raise
|
|
564
|
+
return lower.to_sql(plan, dialect, base_iri=base, terms_all=terms_all, split_terms=True, element_parts=False)
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def _typed(exc: Exception, sql_text: str) -> Exception:
|
|
568
|
+
"""The database refused the statement (an overflow past D7's 38 digits, a limit): ProGraph's typed
|
|
569
|
+
error, to raise with the database's as its cause. ProGraph's own errors pass as they are."""
|
|
570
|
+
from prograph.execution import INTERRUPTS
|
|
571
|
+
if type(exc).__module__.split(".")[0] == "prograph" or isinstance(exc, INTERRUPTS):
|
|
572
|
+
# A cancel or the server's timeout (prograph.execution) is no refusal: it passes as it is.
|
|
573
|
+
return exc
|
|
574
|
+
from prograph.plan.ir.cypher_write import TWO_TYPES
|
|
575
|
+
if TWO_TYPES in str(exc):
|
|
576
|
+
# A read after a query's writes changed a property's type, where an element still holds the old one
|
|
577
|
+
# (`store_view._value`): named, never the database's raw error.
|
|
578
|
+
from prograph.plan.ir.facts import NotBuilt
|
|
579
|
+
return NotBuilt("a read of a property after this query's writes changed its type, where some element "
|
|
580
|
+
"still holds the old type (a column holds one type): give every element one type, or "
|
|
581
|
+
"read in a query of its own")
|
|
582
|
+
from prograph.plan.ir.lower import DATA_ERROR_MARK, TWO_VALUES, TWO_VALUES_SEP
|
|
583
|
+
if TWO_VALUES + TWO_VALUES_SEP in str(exc) and not _analysis_error(exc):
|
|
584
|
+
# TWO_VALUES ␟ label ␟ property ␟ one ␟ other ␟: the closing separator ends the payload, so what an
|
|
585
|
+
# engine wraps it in (PostgreSQL's quote, a newline and context) is not read as part of a value.
|
|
586
|
+
parts = str(exc)[str(exc).index(TWO_VALUES):].split(TWO_VALUES_SEP)[1:]
|
|
587
|
+
if len(parts) >= 5:
|
|
588
|
+
return TwoValuesError(*parts[:4])
|
|
589
|
+
if DATA_ERROR_MARK in str(exc).split("Query was:")[0] and not _analysis_error(exc):
|
|
590
|
+
# A row whose term R2RML cannot make (R2RML 11), raised by the statement; not the statement's
|
|
591
|
+
# own text, which a driver quotes after "Query was:" or an engine in an analysis error.
|
|
592
|
+
from prograph.r2rml.compile import R2RMLDataError
|
|
593
|
+
at = str(exc).index(DATA_ERROR_MARK)
|
|
594
|
+
message = str(exc)[at:].split("\n")[0]
|
|
595
|
+
if str(exc)[at - 1:at] == '"' and message.endswith('"'):
|
|
596
|
+
message = message[:-1] # PostgreSQL's quotes around the text its cast refused
|
|
597
|
+
return R2RMLDataError(message)
|
|
598
|
+
from prograph.plan.ir.cypher_runtime_errors import RUNTIME_ERROR_MARK
|
|
599
|
+
if RUNTIME_ERROR_MARK in str(exc).split("Query was:")[0] and not _analysis_error(exc):
|
|
600
|
+
# A Cypher TypeError a row raised (rule cypher-runtime-errors): its text from the mark, the first line,
|
|
601
|
+
# without the quote PostgreSQL puts around it.
|
|
602
|
+
from prograph.utils.exceptions import CypherExecutionError
|
|
603
|
+
message = str(exc)[str(exc).index(RUNTIME_ERROR_MARK) + len(RUNTIME_ERROR_MARK):].split("\n")[0]
|
|
604
|
+
return CypherExecutionError(message.rstrip('"'))
|
|
605
|
+
from prograph.utils.exceptions import SQLRuntimeError
|
|
606
|
+
return SQLRuntimeError(f"The database refused the statement: {exc}", reason="execution_error",
|
|
607
|
+
sql=sql_text)
|
|
608
|
+
|
|
609
|
+
def _analysis_error(exc: Exception) -> bool:
|
|
610
|
+
"""Whether the engine refused the statement before running it (Spark's analysis errors, a syntax
|
|
611
|
+
error): its message quotes the statement's text, the guard's raise among it, which is no raise."""
|
|
612
|
+
text = str(exc)
|
|
613
|
+
return "AnalysisException" in type(exc).__name__ or any(
|
|
614
|
+
mark in text for mark in ("DATATYPE_MISMATCH", "cannot resolve", "UNRESOLVED_", "syntax error at or near"))
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
class TwoValuesError(ValueError):
|
|
618
|
+
"""A node whose rows give one property two values, worded for a property-graph
|
|
619
|
+
user: its label, the property, both values and, where a diagnostic read finds them, the tables and keys
|
|
620
|
+
of the rows that hold them (rule cypher-node-scan). `detail` holds the node ids."""
|
|
621
|
+
|
|
622
|
+
def __init__(self, label: str, key: str, one: str, other: str, message: str = "", detail: str = ""):
|
|
623
|
+
self.label, self.key, self.one, self.other, self.detail = label, key, one, other, detail
|
|
624
|
+
super().__init__(message or f"{label or 'A'} node has two values of {key}: {one!r} and {other!r}")
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def _two_values(engine: Any, run_sql: Any, dialect: str, err: "TwoValuesError") -> "TwoValuesError":
|
|
628
|
+
"""The error worded from the rows that hold the two values: for each mapping of the label giving the
|
|
629
|
+
property, the rows whose property is one of the two (a small read, run only when the error fired); the
|
|
630
|
+
message names a node whose rows hold both, its table and key values ("Person id=7 has two values of
|
|
631
|
+
name: 'Ann' (people) and 'Anne' (staff)"). Any failure: the plain message."""
|
|
632
|
+
try:
|
|
633
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
634
|
+
from prograph.schema.lpg_view import view_of
|
|
635
|
+
d = get_dialect_instance(dialect)
|
|
636
|
+
quote = d.quote
|
|
637
|
+
view = view_of(source_maps(engine._schema))
|
|
638
|
+
found = [] # (node id, table, key text, value)
|
|
639
|
+
|
|
640
|
+
def literal(v: str) -> str:
|
|
641
|
+
"""A value's text from the error as the dialect's string literal (`dialect.literal`), which the
|
|
642
|
+
engines read as the column's type (a number, a date; a text column keeps '007' as it is)."""
|
|
643
|
+
return d.literal(v)
|
|
644
|
+
for m in view.node_maps(None): # every mapping giving the property: the node's rows may be any
|
|
645
|
+
props = view.property_of(m, err.key)
|
|
646
|
+
if len(props) != 1 or props[0].object.column is None:
|
|
647
|
+
continue
|
|
648
|
+
col, keys = props[0].object.column, list(m.subject.columns())
|
|
649
|
+
sql = (f"SELECT {', '.join(quote(c) for c in keys)}, {quote(col)} FROM {m.source.qualified} "
|
|
650
|
+
f"WHERE {quote(col)} IN ({literal(err.one)}, {literal(err.other)})")
|
|
651
|
+
# The property's own digits (a NUMERIC(10,2) shows 1.50), where its reader needs them.
|
|
652
|
+
digits = (None,) * len(keys) + (_held(engine._schema)(m.source.table, col)[2],)
|
|
653
|
+
for row in _with_digits(run_sql, digits)(sql).to_pylist():
|
|
654
|
+
key = ", ".join(f"{c}={row[c]}" for c in keys)
|
|
655
|
+
# The node's id as its mapping writes it: a template's text, or a column of whole ids' value.
|
|
656
|
+
node = "".join(str(row[t]) if is_col else t for t, is_col in m.subject.template) \
|
|
657
|
+
if m.subject.template is not None else \
|
|
658
|
+
str(row[m.subject.column]) if m.subject.column is not None else ""
|
|
659
|
+
found.append((node or key, m.source.table, key, str(row[col])))
|
|
660
|
+
by_node: Dict[str, List[Tuple[str, str, str]]] = {}
|
|
661
|
+
for node, table, key, value in found:
|
|
662
|
+
by_node.setdefault(node, []).append((table, key, value))
|
|
663
|
+
for node, held in by_node.items():
|
|
664
|
+
values = {v for _, _, v in held}
|
|
665
|
+
if len(values) > 1:
|
|
666
|
+
key = held[0][1]
|
|
667
|
+
held = [(t, v) for t, _, v in held]
|
|
668
|
+
shown = " and ".join(f"{v!r} ({t})" for t, v in sorted(set(held), key=lambda x: x[1]))
|
|
669
|
+
return TwoValuesError(err.label, err.key, err.one, err.other,
|
|
670
|
+
f"{err.label or 'A node'} {key} has two values of {err.key}: {shown}",
|
|
671
|
+
detail=f"node id {node}")
|
|
672
|
+
except Exception: # noqa: BLE001 - the diagnostic is a courtesy; the plain error stands
|
|
673
|
+
pass
|
|
674
|
+
return err
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def _with_digits(reader: Any, digits: Tuple[Any, ...]) -> Any:
|
|
678
|
+
"""`reader` told the answer columns' SQL decimal digits (`lower.ANSWER_DIGITS`), where it reads the
|
|
679
|
+
database's decimals as text and so needs them (`reads_answer_digits`: PostgreSQL); else as it is."""
|
|
680
|
+
if getattr(getattr(reader, "__self__", None), "reads_answer_digits", False):
|
|
681
|
+
import functools
|
|
682
|
+
return functools.partial(reader, digits=digits)
|
|
683
|
+
return reader
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def _batch_runner(graph: Any):
|
|
687
|
+
"""The graph's streaming read (sql, batch_size) -> RecordBatchReader: a database's cursor kept open
|
|
688
|
+
until the reader is read (`_execute_sql_batches`), DuckDB over frames on a cursor of its own; None
|
|
689
|
+
where the graph has none (Spark: its answer is read whole and sliced)."""
|
|
690
|
+
from prograph.graph.sql_graph import SQLGraph
|
|
691
|
+
if isinstance(graph, SQLGraph) and hasattr(graph, "_execute_sql_batches"):
|
|
692
|
+
return graph._execute_sql_batches
|
|
693
|
+
if hasattr(graph, "enable_local_sql") and hasattr(graph, "execute_local_sql_batches"):
|
|
694
|
+
if getattr(graph, "_local_sql", None) is None:
|
|
695
|
+
graph.enable_local_sql()
|
|
696
|
+
return graph.execute_local_sql_batches
|
|
697
|
+
return None
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
def _renamed(reader: Any, names: List[str], sql_text: str = "") -> Any:
|
|
701
|
+
"""`reader` under the answer's names (`lower.sql_name`'s inverse), schema only: each batch's
|
|
702
|
+
columns as they come, no cell touched. A database's error while the batches are read is
|
|
703
|
+
ProGraph's typed one, as a statement read whole gives it (`SQLRuntimeError`, the cause attached)."""
|
|
704
|
+
import pyarrow as pa
|
|
705
|
+
from prograph.utils.exceptions import SQLRuntimeError
|
|
706
|
+
schema = pa.schema([f.with_name(n) for f, n in zip(reader.schema, names)])
|
|
707
|
+
|
|
708
|
+
def batches():
|
|
709
|
+
try:
|
|
710
|
+
try:
|
|
711
|
+
for batch in reader:
|
|
712
|
+
yield pa.RecordBatch.from_arrays(batch.columns, schema=schema)
|
|
713
|
+
except Exception as exc:
|
|
714
|
+
from prograph.execution import INTERRUPTS
|
|
715
|
+
if type(exc).__module__.split(".")[0] == "prograph" or isinstance(exc, INTERRUPTS):
|
|
716
|
+
raise
|
|
717
|
+
raise SQLRuntimeError(f"The database refused the statement: {exc}",
|
|
718
|
+
reason="execution_error", sql=sql_text) from exc
|
|
719
|
+
finally:
|
|
720
|
+
# The driver's reader closes its cursor when it is closed: read through or abandoned, the
|
|
721
|
+
# statement does not outlive the stream (ADBC refuses to close a connection with one open).
|
|
722
|
+
reader.close()
|
|
723
|
+
return pa.RecordBatchReader.from_batches(schema, batches())
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
def _runner(graph: Any):
|
|
727
|
+
"""How the engine's graph runs SQL, and in which dialect: a database's
|
|
728
|
+
own connection; DuckDB over frames, the pandas backend's included
|
|
729
|
+
(one-plan.md 2.7: `memory` and `pandas` run on DuckDB)."""
|
|
730
|
+
from prograph.graph.sql_graph import SQLGraph
|
|
731
|
+
if isinstance(graph, SQLGraph) and hasattr(graph, "_execute_sql_arrow"):
|
|
732
|
+
return graph._execute_sql_arrow, graph.sql_dialect()
|
|
733
|
+
if hasattr(graph, "enable_local_sql") and hasattr(graph, "execute_local_sql_arrow"):
|
|
734
|
+
if getattr(graph, "_local_sql", None) is None:
|
|
735
|
+
graph.enable_local_sql()
|
|
736
|
+
return graph.execute_local_sql_arrow, "duckdb"
|
|
737
|
+
if type(graph).__name__ == "SparkGraph":
|
|
738
|
+
def run_on_spark(sql_text: str):
|
|
739
|
+
# The frames are temporary views under their table names; a
|
|
740
|
+
# statement routed to a SQL warehouse comes back as rows.
|
|
741
|
+
import pyarrow as pa
|
|
742
|
+
from pyspark.sql import SparkSession
|
|
743
|
+
graph._ensure_temp_views()
|
|
744
|
+
session = next((f.sparkSession for f in graph.dataframes.values()
|
|
745
|
+
if getattr(f, "sparkSession", None) is not None),
|
|
746
|
+
None) or SparkSession.getActiveSession()
|
|
747
|
+
result = graph._execute_sql(sql_text, session)
|
|
748
|
+
if isinstance(result, pa.Table):
|
|
749
|
+
return result
|
|
750
|
+
from prograph.results import rows_to_arrow
|
|
751
|
+
return rows_to_arrow(list(result or []))
|
|
752
|
+
return run_on_spark, graph.sql_dialect()
|
|
753
|
+
raise NotImplementedError(f"one-plan: running SQL on {type(graph).__name__}")
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def sql(engine: Any, query: str, language: str, dialect: Optional[str], *, hints: Any = None,
|
|
757
|
+
parameters: Optional[dict] = None) -> str:
|
|
758
|
+
"""The SQL `run` runs for `query`, in `dialect` (None: the backend's),
|
|
759
|
+
lowered and serialized, not executed."""
|
|
760
|
+
if dialect is None:
|
|
761
|
+
dialect = _runner(getattr(getattr(engine, "executor", None), "graph", None))[1]
|
|
762
|
+
from prograph.plan.ir import lower
|
|
763
|
+
return _lowered(engine, query, language, dialect, hints, parameters,
|
|
764
|
+
depth=lower.first_depth(dialect, hints))[0]
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
def _lowered(engine: Any, query: str, language: str, dialect: str, hints: Any,
|
|
768
|
+
parameters: Optional[dict], full: bool = False, depth: Optional[int] = None,
|
|
769
|
+
costed: bool = True, step_key: Optional[str] = None):
|
|
770
|
+
"""`_lowered_once`, planned again where the query's write step lost its race on Delta (`StepLostRace`: the
|
|
771
|
+
store changed after it was read, and a plan made on the old store may not see what the other writer wrote:
|
|
772
|
+
a MERGE of a label it created). Each time under the one step key; `Store.REDO_LIMIT` times, then
|
|
773
|
+
`StoreWriteConflictError`, nothing written."""
|
|
774
|
+
if full and costed and not step_key:
|
|
775
|
+
import uuid
|
|
776
|
+
step_key = uuid.uuid4().hex
|
|
777
|
+
from prograph.store import StepLostRace
|
|
778
|
+
tries = 0
|
|
779
|
+
while True:
|
|
780
|
+
try:
|
|
781
|
+
return _lowered_once(engine, query, language, dialect, hints, parameters, full, depth, costed, step_key)
|
|
782
|
+
except StepLostRace:
|
|
783
|
+
store = engine.__dict__.get("_store")
|
|
784
|
+
tries += 1
|
|
785
|
+
if tries >= getattr(store, "REDO_LIMIT", 20):
|
|
786
|
+
from prograph.utils.exceptions import StoreWriteConflictError
|
|
787
|
+
raise StoreWriteConflictError(
|
|
788
|
+
f"write contention: the write lost its race to other writers {tries} times (each redo plans the query again "
|
|
789
|
+
f"on the store as it is and runs the step whole); nothing was written. Call the query "
|
|
790
|
+
f"again, or write from fewer sessions at once") from None
|
|
791
|
+
import random
|
|
792
|
+
import time
|
|
793
|
+
time.sleep(random.uniform(0, 0.05 * tries))
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _lowered_once(engine: Any, query: str, language: str, dialect: str, hints: Any,
|
|
797
|
+
parameters: Optional[dict], full: bool = False, depth: Optional[int] = None,
|
|
798
|
+
costed: bool = True, step_key: Optional[str] = None):
|
|
799
|
+
"""The SQL for `query` in `dialect`, and its answer's column names (and,
|
|
800
|
+
when `full`, the translated query). A CONSTRUCT's solutions are in full
|
|
801
|
+
terms: its template needs each term's kind. When `full` (the statement is to run) and `costed`,
|
|
802
|
+
its costly shapes are warned of and an opted-in one refused before any SQL runs;
|
|
803
|
+
`explain` and `to_sql` only show them."""
|
|
804
|
+
from prograph.plan.ir import semantics
|
|
805
|
+
with semantics.planning_under(semantics.of_query(engine, hints)):
|
|
806
|
+
return _lowered_under(engine, query, language, dialect, hints, parameters, full, depth, costed, step_key)
|
|
807
|
+
|
|
808
|
+
|
|
809
|
+
def _lowered_under(engine: Any, query: str, language: str, dialect: str, hints: Any,
|
|
810
|
+
parameters: Optional[dict], full: bool, depth: Optional[int], costed: bool,
|
|
811
|
+
step_key: Optional[str]):
|
|
812
|
+
"""`_lowered_once` under the query's semantics profile."""
|
|
813
|
+
from prograph.plan.ir import lower
|
|
814
|
+
from prograph.plan.ir.keys import PLANNING_PINS
|
|
815
|
+
from prograph.plan.ir import key_checks as node_keys
|
|
816
|
+
# The keys checked while planning (keys: check) pin the tables they were checked at; the SQL
|
|
817
|
+
# reads them so (`key_checks.pinned_name` in the scans, `lower._pinned`).
|
|
818
|
+
pins = node_keys.Pins()
|
|
819
|
+
asked = PLANNING_PINS.set(pins)
|
|
820
|
+
if full and costed:
|
|
821
|
+
# A query that writes is one write step (Cypher's CREATE, one-plan 2.10): its key, the caller's or
|
|
822
|
+
# the run's; planned and run in the store's `writing` block (its reads after a write see the step's
|
|
823
|
+
# own rows, `Store.read_own`).
|
|
824
|
+
import uuid
|
|
825
|
+
engine.__dict__["_write_step"] = step_key or uuid.uuid4().hex
|
|
826
|
+
store = getattr(getattr(engine, "_schema", None), "__dict__", {}).get("owned_store") if full and costed else None
|
|
827
|
+
import contextlib
|
|
828
|
+
with store.writing() if store is not None else contextlib.nullcontext():
|
|
829
|
+
return _lowered_written(engine, query, language, dialect, hints, parameters, full, depth, costed, pins,
|
|
830
|
+
asked)
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def _lowered_written(engine: Any, query: str, language: str, dialect: str, hints: Any,
|
|
834
|
+
parameters: Optional[dict], full: bool, depth: Optional[int], costed: bool, pins: Any,
|
|
835
|
+
asked: Any):
|
|
836
|
+
"""`_lowered_once` from planning on."""
|
|
837
|
+
from prograph.plan.ir import lower
|
|
838
|
+
from prograph.plan.ir import key_checks as node_keys
|
|
839
|
+
from prograph.plan.ir.keys import PLANNING_PINS
|
|
840
|
+
try:
|
|
841
|
+
translated, terms_all, base = _planned(engine, query, language, hints, parameters)
|
|
842
|
+
finally:
|
|
843
|
+
PLANNING_PINS.reset(asked)
|
|
844
|
+
engine.__dict__.pop("_write_step", None)
|
|
845
|
+
if full and costed:
|
|
846
|
+
from prograph.plan.ir import cost
|
|
847
|
+
refuse = tuple(getattr(engine, "refuse_costly", ())) + (
|
|
848
|
+
(hints.refuse,) if getattr(hints, "refuse", None) else ())
|
|
849
|
+
cost.check(translated.plan, refuse)
|
|
850
|
+
pinned = node_keys._PINNED.set(pins or None)
|
|
851
|
+
depth = lower.PATH_DEPTH.set(depth or getattr(hints, "path_depth", None) or lower.DEFAULT_PATH_DEPTH)
|
|
852
|
+
from prograph.plan.ir import fresh
|
|
853
|
+
nonce = fresh.NONCE.set(fresh.new_nonce()) # one per lowering: each execution's values are new
|
|
854
|
+
try:
|
|
855
|
+
digits: Tuple[Any, ...] = ()
|
|
856
|
+
if translated.form == "ask":
|
|
857
|
+
sql_text, names = lower.to_sql_ask(translated.plan, dialect, base), ["boolean"]
|
|
858
|
+
else:
|
|
859
|
+
names = list(translated.plan.columns) or ["__one"]
|
|
860
|
+
sql_text = _written(translated, dialect, base, terms_all)
|
|
861
|
+
digits = lower.ANSWER_DIGITS.get()
|
|
862
|
+
write = getattr(translated, "write", None)
|
|
863
|
+
if full and costed and write is not None:
|
|
864
|
+
# A query that writes: its step appended once everything is lowered (a refused answer writes
|
|
865
|
+
# nothing), its rows read once, its answer read from them in the step's transaction (one that
|
|
866
|
+
# fails rolls it back) and kept for `run` (`cypher_write.apply`).
|
|
867
|
+
from prograph.plan.ir.cypher_write import apply
|
|
868
|
+
from prograph.store import Store
|
|
869
|
+
translated.answer = apply(engine, write, dialect, translated.plan if write.answers else None, base,
|
|
870
|
+
digest=Store.digest(query, parameters))
|
|
871
|
+
finally:
|
|
872
|
+
lower.PATH_DEPTH.reset(depth)
|
|
873
|
+
node_keys._PINNED.reset(pinned)
|
|
874
|
+
fresh.NONCE.reset(nonce)
|
|
875
|
+
return (sql_text, names, translated, pins, digits) if full else (sql_text, names)
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
def explain(engine: Any, query: str, language: str, dialect: Optional[str], *, hints: Any = None,
|
|
879
|
+
parameters: Optional[dict] = None) -> str:
|
|
880
|
+
"""What `run` would do for `query`, in words: the SQL, each rewrite of
|
|
881
|
+
the plan with what it relied on (its certificate), and each key the plan
|
|
882
|
+
may rely on with what makes it trusted. An asserted key (`keys:
|
|
883
|
+
trusted`) is shown as asserted, never as checked (m2-performance.md
|
|
884
|
+
2.4, decided 2026-11-09). Under profile='rl' it reports the stored facts' freshness check (how each table
|
|
885
|
+
was proven, its time; a refresh it would run, a refusal it would raise) and acts on none of it."""
|
|
886
|
+
if dialect is None:
|
|
887
|
+
dialect = _runner(getattr(getattr(engine, "executor", None), "graph", None))[1]
|
|
888
|
+
from prograph.plan.ir.dataset import EXPLAINING
|
|
889
|
+
explaining = EXPLAINING.set(True) # the freshness check reported, never acted on: nothing written or raised
|
|
890
|
+
try:
|
|
891
|
+
return _explained(engine, query, language, dialect, hints, parameters)
|
|
892
|
+
finally:
|
|
893
|
+
EXPLAINING.reset(explaining)
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def _explained(engine: Any, query: str, language: str, dialect: Optional[str], hints: Any,
|
|
897
|
+
parameters: Optional[dict]) -> str:
|
|
898
|
+
from prograph.plan.ir import ops as O
|
|
899
|
+
from prograph.plan.ir.keys import key_sources
|
|
900
|
+
import warnings
|
|
901
|
+
with warnings.catch_warnings(record=True) as said:
|
|
902
|
+
warnings.simplefilter("always")
|
|
903
|
+
from prograph.plan.ir import lower
|
|
904
|
+
first = lower.first_depth(dialect, hints)
|
|
905
|
+
text = _lowered(engine, query, language, dialect, hints, parameters, depth=first)[0]
|
|
906
|
+
from prograph.plan.ir import semantics
|
|
907
|
+
profile = semantics.of_query(engine, hints)
|
|
908
|
+
with semantics.planning_under(profile):
|
|
909
|
+
translated, _, _ = _planned(engine, query, language, hints, parameters)
|
|
910
|
+
notes = list(dict.fromkeys(str(w.message) for w in said))
|
|
911
|
+
if first is not None and "RECURSIVE" not in text and lower.DEPTH_MARK in text:
|
|
912
|
+
rungs = " then ".join(str(d) for d in lower.closure_ladder(dialect, hints, translated.plan))
|
|
913
|
+
notes.append(f"A property path is walked by levels to {rungs} steps in turn, each run only where the "
|
|
914
|
+
f"one before found a longer walk, then by the recursive form to path_depth "
|
|
915
|
+
f"{lower.DEFAULT_PATH_DEPTH} (the query hint path_depth sets one depth); the run's log "
|
|
916
|
+
f"names the rung that answered (closure_rung).")
|
|
917
|
+
lines = [f"SQL ({dialect}):", text, ""]
|
|
918
|
+
lines += [f"Semantics: {profile.name} (comparisons: {profile.comparisons})", ""]
|
|
919
|
+
if notes:
|
|
920
|
+
lines += ["Notes:"] + [f" {n}" for n in notes] + [""]
|
|
921
|
+
lines += ["Rewrites:"]
|
|
922
|
+
lines += [f" {c.rule}: {c.relied_on}" for c in translated.certificates] or [" none"]
|
|
923
|
+
from prograph.plan.ir.cost import WARNED_BY_DEFAULT, cost_hints
|
|
924
|
+
lines += ["", "Cost hints (refuse_costly / the refuse hint refuses a shape; a note is never refused):"]
|
|
925
|
+
lines += [f" {h}{' (warned)' if h.shape in WARNED_BY_DEFAULT else ''}"
|
|
926
|
+
for h in cost_hints(translated.plan)] or [" none"]
|
|
927
|
+
graph = getattr(getattr(engine, "executor", None), "graph", None)
|
|
928
|
+
seen, keys = set(), []
|
|
929
|
+
stack = [translated.plan]
|
|
930
|
+
while stack:
|
|
931
|
+
op = stack.pop()
|
|
932
|
+
stack.extend(op.children())
|
|
933
|
+
if isinstance(op, O.MapScan) and op.unique_keys and op.map.id not in seen:
|
|
934
|
+
seen.add(op.map.id)
|
|
935
|
+
for columns, how in key_sources(graph, op.map, source_maps(engine._schema)):
|
|
936
|
+
keys.append(f" {op.map.id} ({op.map.source.table}): "
|
|
937
|
+
f"({', '.join(columns)}), {how}")
|
|
938
|
+
lines += ["", "Keys the plan may rely on:"] + (sorted(keys) or [" none"])
|
|
939
|
+
onto = ontology_of(engine)
|
|
940
|
+
if getattr(engine, "_profile", None) == "rl" and getattr(engine, "_store", None) is not None:
|
|
941
|
+
from prograph.plan.ir.dataset import engine_key, store_now
|
|
942
|
+
now = store_now(engine, onto)
|
|
943
|
+
if now is not None and now.current:
|
|
944
|
+
said = (f" the facts of the last run, as of {now.ran_at}, under key {engine_key(engine, onto)}: "
|
|
945
|
+
f"{now.how}")
|
|
946
|
+
else:
|
|
947
|
+
said = " none: the store holds no run under this engine's key"
|
|
948
|
+
lines += ["", "Stored OWL RL facts:", said]
|
|
949
|
+
held = getattr(engine, "__dict__", {}).get("_consistency")
|
|
950
|
+
if onto is not None and held is not None:
|
|
951
|
+
# When the consistency verdict was computed (`consistency`): under 'ql' when the engine was built, under 'rl'
|
|
952
|
+
# by which run, and whether this engine reused it.
|
|
953
|
+
lines += ["", "Consistency verdict:", f" {held.summary()}"]
|
|
954
|
+
proofs = getattr(engine, "__dict__", {}).get("_ql_verdict")
|
|
955
|
+
if proofs is not None:
|
|
956
|
+
# Under 'ql': how each table the checks read was proven for this query (`consistency.recheck`).
|
|
957
|
+
for name, how in sorted(proofs.get("how", {}).items()):
|
|
958
|
+
lines.append(f" {name}: {how}")
|
|
959
|
+
if onto is not None:
|
|
960
|
+
# What the ontology added to the maps the plan reads, and from which axiom.
|
|
961
|
+
from prograph.plan.ir.saturate import origins
|
|
962
|
+
scanned, stack = set(), [translated.plan]
|
|
963
|
+
while stack:
|
|
964
|
+
op = stack.pop()
|
|
965
|
+
stack.extend(op.children())
|
|
966
|
+
if isinstance(op, O.MapScan):
|
|
967
|
+
scanned.add(op.map.id.split(" via ")[0])
|
|
968
|
+
notes = origins(source_maps(engine._schema), onto, scanned)
|
|
969
|
+
lines += ["", "Ontology (T-mappings):"] + ([f" {n}" for n in notes] or [" nothing the plan reads"])
|
|
970
|
+
return "\n".join(lines)
|
|
971
|
+
|
|
972
|
+
|
|
973
|
+
def check(engine: Any, query: str, language: str = "sparql") -> None:
|
|
974
|
+
"""Read `query` as `execute` would, up to its plan, and run nothing
|
|
975
|
+
(`engine` None: over an empty mapping, for the query alone):
|
|
976
|
+
a syntax error or a static error of the query (SPARQL 1.1 19.8, 18.2.1)
|
|
977
|
+
raises QueryError; an update is refused as read-only; what the plan does
|
|
978
|
+
not build raises NotImplementedError('one-plan: ...')."""
|
|
979
|
+
from prograph.hints import parse_hints
|
|
980
|
+
text, hints = parse_hints(query)
|
|
981
|
+
if language == "sparql":
|
|
982
|
+
from prograph.plan.ir.sparql_front import query_form
|
|
983
|
+
if query_form(text) == "update":
|
|
984
|
+
from prograph.read_only import refuse
|
|
985
|
+
refuse("a SPARQL Update")
|
|
986
|
+
if engine is None:
|
|
987
|
+
from types import SimpleNamespace
|
|
988
|
+
from prograph.schema import GraphSchema
|
|
989
|
+
engine = SimpleNamespace(_schema=GraphSchema())
|
|
990
|
+
from prograph.plan.ir import semantics
|
|
991
|
+
with semantics.planning_under(semantics.of_query(engine, hints)):
|
|
992
|
+
_planned(engine, text, language, hints, None)
|