prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/engine.py
ADDED
|
@@ -0,0 +1,2718 @@
|
|
|
1
|
+
"""ProGraph: SPARQL (and Cypher) over existing tables, as SQL.
|
|
2
|
+
|
|
3
|
+
A query is translated into one relational plan over the mapping's source
|
|
4
|
+
maps, lowered to SQL and run by the backend's database (DuckDB over
|
|
5
|
+
DataFrames, Spark, or a SQL connection); see docs/architecture/one-plan.md.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from typing import Any, Dict, List, Literal, Optional, Sequence, Set, Tuple, Union, TYPE_CHECKING
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
import logging
|
|
11
|
+
import re
|
|
12
|
+
import threading
|
|
13
|
+
import time
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _one_at_a_time(method):
|
|
20
|
+
"""Run an engine method under the engine's lock (see _execute_lock):
|
|
21
|
+
planning uses shared, stateful plan builders and schema changes alter
|
|
22
|
+
what a running query reads."""
|
|
23
|
+
import functools
|
|
24
|
+
|
|
25
|
+
@functools.wraps(method)
|
|
26
|
+
def locked(self, *args, **kwargs):
|
|
27
|
+
lock = getattr(self, "_execute_lock", None)
|
|
28
|
+
if lock is None: # during construction
|
|
29
|
+
return method(self, *args, **kwargs)
|
|
30
|
+
with lock:
|
|
31
|
+
return method(self, *args, **kwargs)
|
|
32
|
+
return locked
|
|
33
|
+
|
|
34
|
+
from prograph.executors.base import BaseExecutor
|
|
35
|
+
from prograph.hints import parse_hints, QueryHints
|
|
36
|
+
from prograph import read_only
|
|
37
|
+
from prograph.rules import home
|
|
38
|
+
from prograph.schema import GraphSchema, NodeMapping, RelationshipMapping, SchemaChangeSet
|
|
39
|
+
|
|
40
|
+
if TYPE_CHECKING:
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
import rdflib
|
|
43
|
+
from prograph.ontology import OntologyLayer
|
|
44
|
+
from prograph.execution import QueryContext
|
|
45
|
+
from prograph.shacl import ValidationReport
|
|
46
|
+
from prograph.plan.ir.semantics import Semantics
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
BackendType = Literal["spark", "databricks_sql", "duckdb", "mysql", "postgresql", "sqlserver", "oracle"]
|
|
50
|
+
LanguageType = Literal["auto", "cypher", "sparql"]
|
|
51
|
+
_UNSUPPORTED_LANGUAGES = {"gremlin": "Gremlin", "sqlpgq": "SQL/PGQ", "gql": "GQL"}
|
|
52
|
+
ReturnType = Literal["arrow", "batches", "list", "dataframe"]
|
|
53
|
+
|
|
54
|
+
# Backends that carry schema-mapped DataFrames the OWL materializer can
|
|
55
|
+
# write `_inferred` rows into.
|
|
56
|
+
def _require_duckdb() -> None:
|
|
57
|
+
"""DuckDB is the default engine for DataFrames; say how to get it."""
|
|
58
|
+
try:
|
|
59
|
+
import duckdb # noqa: F401
|
|
60
|
+
except ImportError as exc:
|
|
61
|
+
raise ImportError(
|
|
62
|
+
"Querying DataFrames runs SQL through DuckDB by default: "
|
|
63
|
+
"pip install 'prograph[duckdb]'."
|
|
64
|
+
) from exc
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _is_a_duckdb_connection(connection: Any) -> bool:
|
|
68
|
+
try:
|
|
69
|
+
import duckdb
|
|
70
|
+
except ImportError:
|
|
71
|
+
return False
|
|
72
|
+
return isinstance(connection, duckdb.DuckDBPyConnection)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _backend_when_none_is_named(dataframes: Any, spark: Any, connection: Any) -> str:
|
|
76
|
+
"""The backend ProGraph() uses when its caller names none.
|
|
77
|
+
|
|
78
|
+
A SparkSession means Spark (over the DataFrames given
|
|
79
|
+
beside it, if any); DataFrames alone mean DuckDB over them, SQL first; a
|
|
80
|
+
DuckDB connection means that database, SQL only, like a warehouse. A
|
|
81
|
+
connection of another kind, or a connection beside DataFrames or a
|
|
82
|
+
SparkSession, does not say which engine reads what, and the caller names
|
|
83
|
+
the backend. Nothing given (a schema alone, or not even that) is an
|
|
84
|
+
embedded DuckDB with no table: a query over the empty dataset (VALUES,
|
|
85
|
+
BIND) answers, and a triple pattern matches nothing.
|
|
86
|
+
"""
|
|
87
|
+
if connection is not None:
|
|
88
|
+
if dataframes is not None or spark is not None:
|
|
89
|
+
given = "spark=" if spark is not None else "dataframes="
|
|
90
|
+
raise ValueError(
|
|
91
|
+
f"ProGraph was given both connection= and {given} with no backend "
|
|
92
|
+
f"named, and reads one source of data: name the backend "
|
|
93
|
+
f"(backend='duckdb' with connection= alone runs SQL in that "
|
|
94
|
+
f"database; dataframes= alone runs SQL through an embedded DuckDB "
|
|
95
|
+
f"over them)."
|
|
96
|
+
)
|
|
97
|
+
if _is_a_duckdb_connection(connection):
|
|
98
|
+
return "duckdb"
|
|
99
|
+
raise ValueError(
|
|
100
|
+
f"ProGraph was given a connection of type {type(connection).__name__!r} "
|
|
101
|
+
f"with no backend named: name the database's backend "
|
|
102
|
+
f"('databricks_sql', 'postgresql', 'mysql', 'sqlserver' or 'oracle'). "
|
|
103
|
+
f"Only a DuckDB connection needs no backend name."
|
|
104
|
+
)
|
|
105
|
+
if spark is not None:
|
|
106
|
+
return "spark"
|
|
107
|
+
if dataframes is not None:
|
|
108
|
+
return "duckdb"
|
|
109
|
+
return "duckdb"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@home("yaml-declared-table", "Where a YAML source's declared table is read",
|
|
113
|
+
terms=("sources.table", "declared table", "to_schema", "by_table", "table_qualifications", "source name"))
|
|
114
|
+
def apply_declared_tables(schema: Any, dataframes: Optional[Dict[str, Any]]) -> None:
|
|
115
|
+
"""A YAML schema's declared tables (`GraphSchema.declared_tables`, `sources.<name>.table`) as the
|
|
116
|
+
engine's table qualifications: each source name read at its declared table (an unqualified one under the
|
|
117
|
+
default catalog and schema, as the source's name was), except a name the engine is given a frame of (the
|
|
118
|
+
frame is that source's rows) or one the user qualified (`table_qualifications`). The schema is shared by
|
|
119
|
+
the engines built over it, so a qualification a declared table gave an earlier engine is taken back for
|
|
120
|
+
an engine given that frame (`_declared_applied` marks them; the user's own are never touched)."""
|
|
121
|
+
declared = getattr(schema, "declared_tables", None) or {}
|
|
122
|
+
if not declared:
|
|
123
|
+
return
|
|
124
|
+
frames = set(dataframes or ())
|
|
125
|
+
held = schema.table_qualifications
|
|
126
|
+
applied = schema.__dict__.setdefault("_declared_applied", {})
|
|
127
|
+
before = dict(held)
|
|
128
|
+
for name, table in declared.items():
|
|
129
|
+
mine = name in applied and held.get(name) == applied[name]
|
|
130
|
+
if name in frames:
|
|
131
|
+
if mine:
|
|
132
|
+
del held[name]
|
|
133
|
+
del applied[name]
|
|
134
|
+
continue
|
|
135
|
+
if name in held and not mine:
|
|
136
|
+
continue # the user's own qualification decides
|
|
137
|
+
if "." not in table and schema.default_catalog and schema.default_schema:
|
|
138
|
+
table = f"{schema.default_catalog}.{schema.default_schema}.{table}"
|
|
139
|
+
elif "." not in table and schema.default_schema:
|
|
140
|
+
table = f"{schema.default_schema}.{table}"
|
|
141
|
+
held[name] = applied[name] = table
|
|
142
|
+
if held != before:
|
|
143
|
+
schema._schema_changed() # a new version only where a qualification changed (the maps recompile)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _shape_datatype(kind):
|
|
147
|
+
"""A column's `sh:datatype`: its type's datatype; xsd:string for one with no natural datatype (a map,
|
|
148
|
+
a list, a text-read type: rule `no-natural-datatype-text`)."""
|
|
149
|
+
from prograph.schema import xsd
|
|
150
|
+
return xsd.datatype_of_type(kind) or (xsd.STRING if getattr(kind, "kind", None) in ("map", "list") else None)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class ProGraph:
|
|
154
|
+
"""Main entry point for executing graph queries.
|
|
155
|
+
|
|
156
|
+
ProGraph provides a high-level interface for parsing and executing
|
|
157
|
+
graph queries against various backends. Supports multiple graph
|
|
158
|
+
query languages with automatic language detection.
|
|
159
|
+
|
|
160
|
+
Threads: an engine is safe to share, and runs one query at a time
|
|
161
|
+
(execute and schema updates take the engine's lock). The database still parallelises each query. For
|
|
162
|
+
queries running side by side, give each thread or worker its own
|
|
163
|
+
engine; on a SQL warehouse, each with its own connection.
|
|
164
|
+
|
|
165
|
+
Backends (the full list is in docs/guides/backends.md):
|
|
166
|
+
- 'pandas': pandas DataFrames, run as SQL on an embedded DuckDB.
|
|
167
|
+
- 'spark': Spark DataFrames, run as Spark SQL.
|
|
168
|
+
Every backend is read-only over the tables it maps (prograph/read_only.py), but for ``set_properties`` on a node
|
|
169
|
+
mapping marked ``mutable``.
|
|
170
|
+
- 'duckdb': The default when dataframes= is given with no backend (an
|
|
171
|
+
embedded DuckDB over the frames), and when connection= is a DuckDB connection (that
|
|
172
|
+
database, SQL only). See _backend_when_none_is_named.
|
|
173
|
+
- 'databricks_sql', 'postgresql', 'mysql', 'sqlserver', 'oracle': a SQL
|
|
174
|
+
database, reached through connection=.
|
|
175
|
+
|
|
176
|
+
Example (Pandas backend):
|
|
177
|
+
>>> import pandas as pd
|
|
178
|
+
>>> from prograph.schema import GraphSchema, NodeMapping
|
|
179
|
+
>>>
|
|
180
|
+
>>> movies_df = pd.DataFrame({'id': [1], 'title': ['The Matrix']})
|
|
181
|
+
>>> schema = GraphSchema()
|
|
182
|
+
>>> schema.add_node_mapping(NodeMapping(
|
|
183
|
+
... source_table='movies', label='Movie',
|
|
184
|
+
... id_column='id', property_columns={'title': 'title'}
|
|
185
|
+
... ))
|
|
186
|
+
>>>
|
|
187
|
+
>>> engine = ProGraph(
|
|
188
|
+
... schema=schema,
|
|
189
|
+
... dataframes={'movies': movies_df}
|
|
190
|
+
... )
|
|
191
|
+
>>> results = engine.execute("MATCH (m:Movie) RETURN m.title")
|
|
192
|
+
|
|
193
|
+
Example (Spark backend):
|
|
194
|
+
>>> from pyspark.sql import SparkSession
|
|
195
|
+
>>> from prograph.schema import GraphSchema, NodeMapping
|
|
196
|
+
>>>
|
|
197
|
+
>>> spark = SparkSession.builder.getOrCreate()
|
|
198
|
+
>>> movies_df = spark.createDataFrame([(1, 'The Matrix')], ['id', 'title'])
|
|
199
|
+
>>> schema = GraphSchema()
|
|
200
|
+
>>> schema.add_node_mapping(NodeMapping(
|
|
201
|
+
... source_table='movies', label='Movie',
|
|
202
|
+
... id_column='id', property_columns={'title': 'title'}
|
|
203
|
+
... ))
|
|
204
|
+
>>>
|
|
205
|
+
>>> engine = ProGraph(
|
|
206
|
+
... backend='spark',
|
|
207
|
+
... schema=schema,
|
|
208
|
+
... dataframes={'movies': movies_df}
|
|
209
|
+
... )
|
|
210
|
+
>>> results = engine.execute("MATCH (m:Movie) RETURN m.title")
|
|
211
|
+
"""
|
|
212
|
+
|
|
213
|
+
def __init__(
|
|
214
|
+
self,
|
|
215
|
+
backend: Optional[BackendType] = None,
|
|
216
|
+
schema: Optional[GraphSchema] = None,
|
|
217
|
+
dataframes: Optional[Dict[str, Any]] = None,
|
|
218
|
+
spark: Optional[Any] = None,
|
|
219
|
+
default_language: LanguageType = "auto",
|
|
220
|
+
default_return_type: ReturnType = "arrow",
|
|
221
|
+
ontology: Optional['OntologyLayer'] = None,
|
|
222
|
+
profile: Optional[str] = None,
|
|
223
|
+
default_shapes: Optional[str] = None,
|
|
224
|
+
sql_warehouse_id: Optional[str] = None,
|
|
225
|
+
warehouse_timeout: Optional[float] = None,
|
|
226
|
+
default_catalog: Optional[str] = None,
|
|
227
|
+
default_schema: Optional[str] = None,
|
|
228
|
+
table_qualifications: Optional[Dict[str, str]] = None,
|
|
229
|
+
connection: Optional[Any] = None,
|
|
230
|
+
default_graph: str = "unnamed",
|
|
231
|
+
ansi_session: bool = True,
|
|
232
|
+
refuse_costly: Sequence[str] = (),
|
|
233
|
+
store: Optional[Any] = None,
|
|
234
|
+
semantics: Optional[Union[str, 'Semantics']] = None,
|
|
235
|
+
sameas: str = "all",
|
|
236
|
+
implied_individuals: bool = False,
|
|
237
|
+
datatypes: Optional[str] = None,
|
|
238
|
+
freshness: Optional[str] = None,
|
|
239
|
+
version_columns: Optional[Dict[str, str]] = None,
|
|
240
|
+
inconsistency: str = "error",
|
|
241
|
+
):
|
|
242
|
+
"""Initialize the ProGraph engine.
|
|
243
|
+
|
|
244
|
+
Args:
|
|
245
|
+
backend: The execution backend to use. When omitted: 'spark'
|
|
246
|
+
if a SparkSession is given, 'duckdb' if dataframes are or
|
|
247
|
+
the connection is a DuckDB connection; with none of
|
|
248
|
+
them, ValueError. A connection of another kind, or one beside dataframes or a
|
|
249
|
+
SparkSession, raises ValueError: name the backend.
|
|
250
|
+
- 'duckdb': with dataframes, SQL through an embedded DuckDB
|
|
251
|
+
over them (the default local engine); with a connection,
|
|
252
|
+
that DuckDB database
|
|
253
|
+
- 'pandas': pandas DataFrames, on an embedded DuckDB
|
|
254
|
+
- 'spark': Spark DataFrames
|
|
255
|
+
- 'databricks_sql', 'postgresql', ...: a SQL database, SQL only
|
|
256
|
+
(requires schema and connection)
|
|
257
|
+
schema: GraphSchema defining node/relationship mappings
|
|
258
|
+
(required with dataframes or spark)
|
|
259
|
+
dataframes: Dictionary mapping table names to DataFrames.
|
|
260
|
+
If not provided but spark and schema are, tables are
|
|
261
|
+
loaded automatically from Unity Catalog using schema mappings.
|
|
262
|
+
spark: Optional SparkSession. If provided with schema but without
|
|
263
|
+
dataframes, tables are loaded automatically from Unity Catalog.
|
|
264
|
+
default_language: Default query language for execute() calls.
|
|
265
|
+
- 'auto': Detect language from query syntax (default)
|
|
266
|
+
- 'cypher': Parse all queries as Cypher
|
|
267
|
+
- 'sparql': Parse all queries as SPARQL
|
|
268
|
+
Can be overridden per-call via the language parameter.
|
|
269
|
+
default_return_type: Default return type for execute() calls.
|
|
270
|
+
- 'arrow': a pyarrow.Table (default): what the database
|
|
271
|
+
returned, with no per-row Python on the SQL path
|
|
272
|
+
- 'list': List[Dict], the table's to_pylist()
|
|
273
|
+
- 'dataframe': Return native DataFrame (Spark→Spark DF, else pandas)
|
|
274
|
+
Can be overridden per-call via the return_type parameter.
|
|
275
|
+
ontology: Optional OntologyLayer. Passing one turns reasoning
|
|
276
|
+
on: its triples are in the default graph and the RDFS core
|
|
277
|
+
(subClassOf, subPropertyOf, domain, range) is always
|
|
278
|
+
honoured, under ``profile``. None: a query matches simply.
|
|
279
|
+
profile: The OWL profile the ontology is reasoned under (needs
|
|
280
|
+
``ontology=``; passing it without one raises ValueError):
|
|
281
|
+
``'ql'`` (default with an ontology) answers by rewriting
|
|
282
|
+
alone, always current, nothing stored; ``'rl'`` also
|
|
283
|
+
materializes OWL 2 RL's facts into the owned store when the
|
|
284
|
+
engine is built (needs ``store=``), under the stored facts'
|
|
285
|
+
cache key, and refreshes them where the tables changed
|
|
286
|
+
(``freshness``). Any other value raises ValueError.
|
|
287
|
+
sql_warehouse_id: Optional Databricks SQL Warehouse ID for optimized
|
|
288
|
+
SQL execution. When provided (Spark backend only), generated
|
|
289
|
+
queries are routed to the SQL Warehouse for faster execution
|
|
290
|
+
via Photon. Falls back to Spark compute if unavailable.
|
|
291
|
+
warehouse_timeout: Optional timeout in seconds for SQL Warehouse
|
|
292
|
+
queries. When set, queries that exceed this time are cancelled
|
|
293
|
+
on the warehouse and raise TimeoutError. None (default) means
|
|
294
|
+
poll indefinitely until completion.
|
|
295
|
+
default_catalog: Optional Unity Catalog name for qualifying table
|
|
296
|
+
references. Sets schema.default_catalog for SQL generation.
|
|
297
|
+
default_schema: Optional Unity Catalog schema name. Sets
|
|
298
|
+
schema.default_schema for SQL generation.
|
|
299
|
+
table_qualifications: Optional explicit mapping of short table names
|
|
300
|
+
to fully qualified names. Takes precedence over default_catalog/
|
|
301
|
+
default_schema. Example: {"accounts": "bg.demo.my_accounts"}
|
|
302
|
+
default_graph: SPARQL's default graph, what a pattern outside
|
|
303
|
+
GRAPH reads. 'unnamed' (default, the standard: SPARQL 13,
|
|
304
|
+
R2RML 11): only the triples with no named graph (no graph_iri,
|
|
305
|
+
no graph map but rr:defaultGraph), so a named graph is read
|
|
306
|
+
through GRAPH alone. 'union': every triple, named graphs
|
|
307
|
+
included (Jena's unionDefaultGraph practice).
|
|
308
|
+
ansi_session: On Spark and Databricks, run the engine's SQL
|
|
309
|
+
with ANSI mode on, so that an integer overflow is an
|
|
310
|
+
error and not a wrapped number (default True). The session
|
|
311
|
+
or connection is the caller's: on a classic SparkSession
|
|
312
|
+
each statement runs in a clone and the session's own
|
|
313
|
+
setting is never changed; on Spark Connect the setting is
|
|
314
|
+
changed for the length of each statement; on a Databricks
|
|
315
|
+
SQL connection it is set once and put back by
|
|
316
|
+
:meth:`close`. On a classic SparkSession whose parser
|
|
317
|
+
drops :meth:`execute_sql`'s parameters (Delta's, on Spark
|
|
318
|
+
4.1) they are bound by Spark's analyzer for that statement.
|
|
319
|
+
False leaves every setting as it is. No effect on the
|
|
320
|
+
other backends.
|
|
321
|
+
store: An owned store (`prograph.store.Store`) on the engine's
|
|
322
|
+
own connection: the statements ProGraph writes, read beside
|
|
323
|
+
the mapped tables (SPARQL through its generated mapping,
|
|
324
|
+
Cypher through its property-graph view) and written by
|
|
325
|
+
Cypher's writes. None: the engine writes nothing. On DuckDB
|
|
326
|
+
and PostgreSQL (the ADBC connection the engine reads on).
|
|
327
|
+
semantics: The semantics profile queries are planned under: ``"default"`` (None) reads
|
|
328
|
+
17.3's operators plus XPath's for the other XSD types and the W3C optional features
|
|
329
|
+
KnownTypesDefault2Neq and LangTagAwareness; ``"core"`` is strict SPARQL 1.1 (only
|
|
330
|
+
17.3's operators, every other pair of literals RDFterm-equal's type error, YEAR,
|
|
331
|
+
MONTH and DAY of an xsd:date an error). A name or a
|
|
332
|
+
:class:`~prograph.plan.ir.semantics.Semantics`; a query's ``/*+ semantics=core */``
|
|
333
|
+
hint wins. An unknown name raises ValueError. Not entailment: it chooses operator
|
|
334
|
+
behaviour only.
|
|
335
|
+
sameas: Under profile='rl', how a pattern answers an
|
|
336
|
+
entity owl:sameAs gives several IRIs: 'all' (default, the
|
|
337
|
+
entailed answer) one solution per IRI of the entity;
|
|
338
|
+
'representative' one per entity, under its representative
|
|
339
|
+
(the least IRI by code point). FILTER, VALUES, BIND,
|
|
340
|
+
DISTINCT and GROUP BY compare RDF terms in both.
|
|
341
|
+
implied_individuals: Under profile='ql', whether a SPARQL query's blank
|
|
342
|
+
nodes and hidden variables (a variable not projected and read nowhere
|
|
343
|
+
outside its triples block) may be satisfied by individuals the ontology
|
|
344
|
+
implies but no data names (``B rdfs:subClassOf [owl:onProperty p;
|
|
345
|
+
owl:someValuesFrom C]``): the certain answers, by tree-witness
|
|
346
|
+
rewriting. False (default): the W3C reading, every term of a pattern a
|
|
347
|
+
named one. The answers with it on hold those with it off. Under
|
|
348
|
+
profile='rl' it raises ValueError (OWL 2 RL implies no unnamed
|
|
349
|
+
individual).
|
|
350
|
+
datatypes: How a SPARQL basic graph pattern matches literals: 'term'
|
|
351
|
+
(RDF term identity: `1` matches "1"^^xsd:integer only) or 'value'
|
|
352
|
+
(D-entailment and OWL 2 RL's dt-eq: a literal matches every literal
|
|
353
|
+
of the same value, so `1` also matches "1.00"^^xsd:decimal and
|
|
354
|
+
"1"^^xsd:short; xsd:float and xsd:double are value spaces of their
|
|
355
|
+
own). None (default): 'value' under profile='rl', else 'term'.
|
|
356
|
+
FILTER, sameTerm, STR, ORDER BY and BIND compare as before.
|
|
357
|
+
freshness: Under profile='ql', how the consistency verdict is kept
|
|
358
|
+
current: None (default) each table the checks read proven
|
|
359
|
+
unchanged per query as below, the checks run again where one
|
|
360
|
+
changed, and a table nothing proves keeps the build's verdict
|
|
361
|
+
with a ``ConsistencyVerdictWarning`` (once); ``'strict'``
|
|
362
|
+
refuses such a query (``ConsistencyVerdictNotCurrentError``);
|
|
363
|
+
``'last_run'`` keeps the build's verdict, nothing read.
|
|
364
|
+
Under profile='rl', when a query reads the stored
|
|
365
|
+
facts. None (default): only where they are provably
|
|
366
|
+
current: every mapped table unchanged since the run, by its
|
|
367
|
+
declared version column (``version_columns``), its Delta
|
|
368
|
+
version, or, for a table under its backend's size
|
|
369
|
+
threshold (``freshness.SMALL``), a checksum of all its
|
|
370
|
+
rows, read once per query, and the owned store unchanged;
|
|
371
|
+
where one changed, the engine refreshes them first (as
|
|
372
|
+
``materialize_entailments()``). A table none of them
|
|
373
|
+
proves is refused with ``StoredFactsNotCurrentError``, as
|
|
374
|
+
is a change the store cannot be written to refresh.
|
|
375
|
+
``explain()`` shows how each table was proven, and the time.
|
|
376
|
+
``'last_run'``: the facts of the last run, not checked;
|
|
377
|
+
``explain()`` says when that run was and under which key.
|
|
378
|
+
version_columns: Under profile='rl' or 'ql', a version column per mapped
|
|
379
|
+
table (``{table: column}``, the table as the mapping names
|
|
380
|
+
it): a column your writes keep increasing (each insert and
|
|
381
|
+
update sets a value above every one before). Its maximum and
|
|
382
|
+
the row count, compared with the run's, prove the table
|
|
383
|
+
unchanged. A table no map reads raises ValueError.
|
|
384
|
+
inconsistency: What an ontology and data that are inconsistent
|
|
385
|
+
under the profile do (an individual of two disjoint
|
|
386
|
+
classes, an irreflexive property's loop, two values of a
|
|
387
|
+
functional property that are not one value ...).
|
|
388
|
+
``'error'`` (default): ``InconsistentOntologyError``,
|
|
389
|
+
naming each axiom and a few of the facts, raised when the
|
|
390
|
+
engine is built or a materialization finds it.
|
|
391
|
+
``'report'``: queries answer as usual and
|
|
392
|
+
:meth:`consistency_report` lists what matched. Checked under
|
|
393
|
+
profile='rl' once per materialization (its verdict stored
|
|
394
|
+
with the facts), under profile='ql' once when the engine is
|
|
395
|
+
built. Any other value raises ValueError.
|
|
396
|
+
"""
|
|
397
|
+
|
|
398
|
+
if backend is None:
|
|
399
|
+
backend = _backend_when_none_is_named(dataframes, spark, connection)
|
|
400
|
+
if backend == "duckdb" and connection is None and dataframes is None:
|
|
401
|
+
# The empty dataset: no table, and a schema mapping none.
|
|
402
|
+
schema = schema if schema is not None else GraphSchema()
|
|
403
|
+
dataframes = {}
|
|
404
|
+
self.backend = backend
|
|
405
|
+
# True when queries run over in-memory frames (pandas, Spark, or
|
|
406
|
+
# DuckDB over DataFrames), which decides the W3C paths: reasoning
|
|
407
|
+
# writes `_inferred` rows back into frames rather than issuing
|
|
408
|
+
# INSERTs, which DuckDB's views over DataFrames would refuse. Set
|
|
409
|
+
# here for spark; the DuckDB-over-frames branch of the
|
|
410
|
+
# executor setup sets it True once it has built the frames' DuckDB.
|
|
411
|
+
self._frame_backed = backend == "spark"
|
|
412
|
+
# One query at a time per engine: a DuckDB connection is not safe
|
|
413
|
+
# across threads (four threads on one engine deadlocked).
|
|
414
|
+
self._execute_lock = threading.RLock()
|
|
415
|
+
self._connection = connection
|
|
416
|
+
|
|
417
|
+
# Set default catalog/schema/table_qualifications on the schema if provided
|
|
418
|
+
if schema is not None:
|
|
419
|
+
if default_catalog is not None:
|
|
420
|
+
schema.default_catalog = default_catalog
|
|
421
|
+
if default_schema is not None:
|
|
422
|
+
schema.default_schema = default_schema
|
|
423
|
+
if table_qualifications is not None:
|
|
424
|
+
schema.table_qualifications.update(table_qualifications)
|
|
425
|
+
apply_declared_tables(schema, dataframes)
|
|
426
|
+
self.default_language = default_language
|
|
427
|
+
self.default_return_type = default_return_type
|
|
428
|
+
self._schema = schema
|
|
429
|
+
self._spark = spark
|
|
430
|
+
self._store = store
|
|
431
|
+
if store is not None:
|
|
432
|
+
if schema is None:
|
|
433
|
+
raise ValueError("an owned store reads beside a schema: give the engine a schema too")
|
|
434
|
+
store.attach(schema)
|
|
435
|
+
if schema is not None:
|
|
436
|
+
# A schema whose node IRIs cannot name one node each is refused
|
|
437
|
+
# here, by name.
|
|
438
|
+
from prograph.schema.iri import check_identities, check_meetings
|
|
439
|
+
check_identities(schema)
|
|
440
|
+
check_meetings(schema)
|
|
441
|
+
# An asserted key is said so at load (m2-performance.md 2.4).
|
|
442
|
+
from prograph.plan.ir.keys import warn_asserted_keys
|
|
443
|
+
warn_asserted_keys(schema)
|
|
444
|
+
from prograph.plan.ir.entry import warn_written_constants
|
|
445
|
+
warn_written_constants(schema)
|
|
446
|
+
|
|
447
|
+
# Auto-load dataframes from Unity Catalog if spark provided without dataframes
|
|
448
|
+
if dataframes is None and spark is not None and schema is not None:
|
|
449
|
+
dataframes = self._load_dataframes_from_schema(spark, schema)
|
|
450
|
+
self._dataframes = dataframes
|
|
451
|
+
# An ontology saturates the source maps on the SPARQL path, over YAML and R2RML alike
|
|
452
|
+
# (prograph.plan.ir.saturate).
|
|
453
|
+
self._ontology = ontology
|
|
454
|
+
self._sql_warehouse_id = sql_warehouse_id
|
|
455
|
+
self._warehouse_timeout = warehouse_timeout
|
|
456
|
+
if ontology is not None and schema is not None and getattr(schema, "property_graph", True) is not False:
|
|
457
|
+
# Bind the ontology to the schema's labels (its class hierarchy is read by IRI); Cypher and SPARQL
|
|
458
|
+
# both read it as the saturated maps (`entry.ontology_of`).
|
|
459
|
+
ontology.bind_to_schema(schema, strict=False)
|
|
460
|
+
|
|
461
|
+
# The OWL profile (`dataset.engine_profile`), checked up front so misconfiguration surfaces at
|
|
462
|
+
# construction time rather than during materialization.
|
|
463
|
+
from prograph.plan.ir.dataset import engine_profile
|
|
464
|
+
profile = engine_profile(ontology, profile)
|
|
465
|
+
if profile == "rl" and store is None:
|
|
466
|
+
raise ValueError("profile='rl' materializes the ontology's OWL 2 RL entailments into the "
|
|
467
|
+
"owned store: give the engine a store= too")
|
|
468
|
+
self._profile = profile
|
|
469
|
+
if ontology is not None:
|
|
470
|
+
# The axioms the profile does not answer are refused here, by name, or dropped
|
|
471
|
+
# (on_unsupported="ignore"; `ontology_report`).
|
|
472
|
+
from prograph.plan.ir.entry import profile_report
|
|
473
|
+
profile_report(self)
|
|
474
|
+
# How the stored facts are trusted (`freshness`): checked before this engine's first run.
|
|
475
|
+
from prograph.plan.ir.freshness import engine_freshness
|
|
476
|
+
self._freshness = engine_freshness(freshness, profile)
|
|
477
|
+
if version_columns and profile not in ("rl", "ql"):
|
|
478
|
+
raise ValueError("version_columns= proves the stored OWL RL facts (profile='rl') or the consistency "
|
|
479
|
+
"verdict (profile='ql') current: build the engine with one of those profiles")
|
|
480
|
+
self._version_columns = dict(version_columns or {})
|
|
481
|
+
from prograph.plan.ir.consistency import engine_inconsistency
|
|
482
|
+
self._inconsistency = engine_inconsistency(inconsistency)
|
|
483
|
+
from prograph.plan.ir.identity import MODES as _SAMEAS_MODES
|
|
484
|
+
if sameas not in _SAMEAS_MODES:
|
|
485
|
+
raise ValueError(f"sameas is 'all' or 'representative', not {sameas!r}")
|
|
486
|
+
self._sameas = sameas
|
|
487
|
+
if implied_individuals and getattr(self, "_profile", None) == "rl":
|
|
488
|
+
raise ValueError("implied_individuals is a profile='ql' feature")
|
|
489
|
+
self._implied_individuals = bool(implied_individuals)
|
|
490
|
+
if datatypes not in (None, "term", "value"):
|
|
491
|
+
raise ValueError(f"datatypes is 'term' or 'value', not {datatypes!r}")
|
|
492
|
+
self._datatypes = datatypes or ("value" if getattr(self, "_profile", None) == "rl" else "term")
|
|
493
|
+
# Costly query shapes refused before they run (`prograph.plan.ir.cost`): opt-in.
|
|
494
|
+
from prograph.plan.ir.cost import SHAPES
|
|
495
|
+
unknown = set(refuse_costly) - set(SHAPES) - {"all"}
|
|
496
|
+
if unknown:
|
|
497
|
+
raise ValueError(f"refuse_costly takes {', '.join(SHAPES)} or all, not {sorted(unknown)}")
|
|
498
|
+
self.refuse_costly = tuple(refuse_costly)
|
|
499
|
+
from prograph.plan.ir.semantics import named
|
|
500
|
+
self.semantics = named(semantics)
|
|
501
|
+
# Operator-configured SHACL shapes for the runtime guardrail. When set,
|
|
502
|
+
# the MCP server exposes a zero-argument data-quality tool that agents'
|
|
503
|
+
# context passes auto-invoke, surfacing the conformance verdict into
|
|
504
|
+
# the agent's reasoning without it having to author shapes itself.
|
|
505
|
+
self._default_shapes = default_shapes
|
|
506
|
+
|
|
507
|
+
self._ansi_session = ansi_session
|
|
508
|
+
self.executor = self._create_executor(
|
|
509
|
+
backend, schema, dataframes, sql_warehouse_id,
|
|
510
|
+
warehouse_timeout,
|
|
511
|
+
connection=connection,
|
|
512
|
+
)
|
|
513
|
+
self._record_column_types()
|
|
514
|
+
if schema is not None:
|
|
515
|
+
schema.check_language_columns()
|
|
516
|
+
# keys: check on a source with no version is refused now, not at a first query.
|
|
517
|
+
from prograph.plan.ir.keys import refuse_unversioned_checks
|
|
518
|
+
refuse_unversioned_checks(self)
|
|
519
|
+
|
|
520
|
+
# Store config if created from YAML
|
|
521
|
+
self._config = None
|
|
522
|
+
|
|
523
|
+
# SPARQL's default graph (read by the plan's front end).
|
|
524
|
+
self.default_graph = default_graph
|
|
525
|
+
if profile == "rl":
|
|
526
|
+
# The ontology's kinds RL does not apply warn (or raise) now; the entailments are materialized
|
|
527
|
+
# once, where the store holds none for this engine's cache key (`dataset.cache_key`).
|
|
528
|
+
self._materialize_on_open()
|
|
529
|
+
elif profile == "ql":
|
|
530
|
+
# The negative axioms' checks, once (`consistency`): raised here under inconsistency='error'.
|
|
531
|
+
from prograph.plan.ir.consistency import check_on_open
|
|
532
|
+
check_on_open(self)
|
|
533
|
+
|
|
534
|
+
def close(self) -> None:
|
|
535
|
+
"""Give back what the engine changed on a session it was handed:
|
|
536
|
+
a Databricks SQL connection's ANSI mode is put back as it was
|
|
537
|
+
(``ansi_session``). The connection and the SparkSession are the
|
|
538
|
+
caller's and stay open. An engine never closed never restores."""
|
|
539
|
+
graph = getattr(getattr(self, 'executor', None), 'graph', None)
|
|
540
|
+
close = getattr(graph, 'close', None)
|
|
541
|
+
if callable(close):
|
|
542
|
+
close()
|
|
543
|
+
|
|
544
|
+
def __enter__(self) -> 'ProGraph':
|
|
545
|
+
return self
|
|
546
|
+
|
|
547
|
+
def __exit__(self, exc_type, exc, tb) -> None:
|
|
548
|
+
self.close()
|
|
549
|
+
|
|
550
|
+
@property
|
|
551
|
+
def spark(self) -> Optional[Any]:
|
|
552
|
+
"""The SparkSession this engine runs on, or None on another
|
|
553
|
+
backend: the one given at construction, else the session of the
|
|
554
|
+
DataFrames it was given."""
|
|
555
|
+
if self._spark is not None:
|
|
556
|
+
return self._spark
|
|
557
|
+
if self.backend != "spark":
|
|
558
|
+
return None
|
|
559
|
+
graph = getattr(self.executor, 'graph', None)
|
|
560
|
+
for frame in (getattr(graph, 'dataframes', None) or {}).values():
|
|
561
|
+
session = getattr(frame, 'sparkSession', None)
|
|
562
|
+
if session is not None:
|
|
563
|
+
return session
|
|
564
|
+
return None
|
|
565
|
+
|
|
566
|
+
def declare_procedure(self, name: str, inputs: Any, outputs: Any, rows: Any = None,
|
|
567
|
+
source: Optional[str] = None, columns: Optional[Dict[str, str]] = None) -> None:
|
|
568
|
+
"""Declare a Cypher procedure as data (rule cypher-procedures).
|
|
569
|
+
`inputs` and `outputs` are (name, type) pairs. Give `rows` (literal, each the inputs' values then
|
|
570
|
+
the outputs') or `source`, a table or view this engine reads (its columns named as the inputs then
|
|
571
|
+
outputs, or `columns` maps an input or output name to its column); a source's rows are read when a
|
|
572
|
+
query calling the procedure is planned. A mapping's `procedures:` section declares the same."""
|
|
573
|
+
from prograph.plan.ir.cypher_procedures import declare
|
|
574
|
+
proc = declare(name, inputs, outputs, rows, source, columns)
|
|
575
|
+
if not hasattr(self, "_procedures"):
|
|
576
|
+
self._procedures = {}
|
|
577
|
+
self._procedures[proc.name] = proc
|
|
578
|
+
|
|
579
|
+
def _declare_config_procedures(self, config: Any) -> None:
|
|
580
|
+
for proc in config.procedures:
|
|
581
|
+
self.declare_procedure(proc.name, proc.inputs, proc.outputs, proc.rows if proc.source is None else None,
|
|
582
|
+
proc.source, dict(proc.columns))
|
|
583
|
+
|
|
584
|
+
def _read_procedures(self) -> Optional[Any]:
|
|
585
|
+
"""The declared procedures for one query: a source's rows are read when a call looks it up."""
|
|
586
|
+
declared = getattr(self, "_procedures", None)
|
|
587
|
+
if not declared or all(p.source is None for p in declared.values()):
|
|
588
|
+
return declared
|
|
589
|
+
from prograph.plan.ir.cypher_procedures import Lazy
|
|
590
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
591
|
+
graph = getattr(self.executor, "graph", None)
|
|
592
|
+
dialect = get_dialect_instance((graph.sql_dialect() if hasattr(graph, "sql_dialect") else None)
|
|
593
|
+
or "spark")
|
|
594
|
+
|
|
595
|
+
def read(sql):
|
|
596
|
+
return [tuple(row.values()) for row in self.execute_sql(sql, return_type="list")]
|
|
597
|
+
return Lazy(declared, read, dialect)
|
|
598
|
+
|
|
599
|
+
def execute_sql(
|
|
600
|
+
self,
|
|
601
|
+
sql: str,
|
|
602
|
+
parameters: Optional[Dict[str, Any]] = None,
|
|
603
|
+
return_type: Optional[ReturnType] = None,
|
|
604
|
+
read_only: bool = False,
|
|
605
|
+
) -> Any:
|
|
606
|
+
"""Run SQL text on the backend's own connection, in its dialect,
|
|
607
|
+
and return the result as :meth:`execute` does.
|
|
608
|
+
|
|
609
|
+
For a layer above that needs the database the graph lives in: a
|
|
610
|
+
scoring query over the same tables, a table of its own. The SQL is
|
|
611
|
+
the backend's, not translated: Spark SQL on Spark, the database's
|
|
612
|
+
dialect on a SQL backend, DuckDB over DataFrames (each frame a
|
|
613
|
+
table under its name). The result is the database's own Arrow
|
|
614
|
+
result, converted to the return type and nothing else.
|
|
615
|
+
``parameters`` are bound by the backend, in its own placeholder
|
|
616
|
+
syntax (``:name`` on Spark, ``$name`` on DuckDB), never put into
|
|
617
|
+
the text. On a Spark session whose parser drops them (Delta's, on
|
|
618
|
+
Spark 4.1) they are bound in Spark's analyzer, where a value goes
|
|
619
|
+
where a constant goes (``prograph/graph/ansi_session.py``).
|
|
620
|
+
|
|
621
|
+
``read_only=True`` is for SQL the caller did not write (a policy
|
|
622
|
+
file's query): the text must be one query, and the backend itself
|
|
623
|
+
says so, not an inspection of the text. On DuckDB its own parser
|
|
624
|
+
must find exactly one statement, a SELECT; on Spark the text runs
|
|
625
|
+
as the subquery of one SELECT, and Spark takes one statement per
|
|
626
|
+
call. Text that is not one query raises ``NotAQueryError`` or the
|
|
627
|
+
backend's parse error, and nothing runs. The other SQL backends do
|
|
628
|
+
not enforce it yet and refuse the call.
|
|
629
|
+
|
|
630
|
+
Raises ``ValueError`` for ``parameters`` on a SQL backend other than DuckDB, whose
|
|
631
|
+
connection is given SQL text only, and for ``read_only`` on a
|
|
632
|
+
backend that cannot enforce it.
|
|
633
|
+
"""
|
|
634
|
+
import pyarrow as pa
|
|
635
|
+
|
|
636
|
+
wanted = return_type or self.default_return_type
|
|
637
|
+
graph = getattr(self.executor, 'graph', None)
|
|
638
|
+
|
|
639
|
+
if self.backend == "spark":
|
|
640
|
+
session = self.spark
|
|
641
|
+
if session is None:
|
|
642
|
+
raise ValueError("execute_sql needs a SparkSession, and this engine has none")
|
|
643
|
+
if read_only:
|
|
644
|
+
# One statement per call is Spark's own rule, so as the
|
|
645
|
+
# subquery of this SELECT the text can only be a query. A
|
|
646
|
+
# trailing semicolon ends a query everywhere else, and is
|
|
647
|
+
# dropped here so it does not end the wrapper early.
|
|
648
|
+
sql = f"SELECT * FROM (\n{sql.rstrip().rstrip(';').rstrip()}\n) AS _read_only"
|
|
649
|
+
if parameters:
|
|
650
|
+
from prograph.graph.ansi_session import parameter_session
|
|
651
|
+
with parameter_session(session, self._ansi_session) as bound:
|
|
652
|
+
frame = bound.sql(sql, args=dict(parameters))
|
|
653
|
+
else:
|
|
654
|
+
frame = session.sql(sql)
|
|
655
|
+
return frame if wanted == "dataframe" else self._convert_results(frame.toArrow(), wanted)
|
|
656
|
+
|
|
657
|
+
local = getattr(graph, '_local_sql', None) is not None # DuckDB over DataFrames
|
|
658
|
+
if not local and getattr(graph, '_connection', None) is None:
|
|
659
|
+
raise ValueError(
|
|
660
|
+
f"execute_sql needs a SQL connection, and the {self.backend} backend has none")
|
|
661
|
+
if read_only:
|
|
662
|
+
if self.backend != "duckdb":
|
|
663
|
+
raise ValueError(
|
|
664
|
+
f"execute_sql(read_only=True) is enforced on DuckDB and Spark; the "
|
|
665
|
+
f"{self.backend} backend does not enforce it yet")
|
|
666
|
+
from prograph.writes import require_one_query
|
|
667
|
+
require_one_query(graph._local_sql if local else graph._connection, sql)
|
|
668
|
+
if not parameters:
|
|
669
|
+
table = graph.execute_local_sql_arrow(sql) if local else graph._execute_sql_arrow(sql)
|
|
670
|
+
elif self.backend == "duckdb":
|
|
671
|
+
from prograph.graph.drivers import duckdb_arrow_table
|
|
672
|
+
connection = graph._prepare_local_sql() if local else graph._connection
|
|
673
|
+
table = duckdb_arrow_table(connection, sql, parameters)
|
|
674
|
+
else:
|
|
675
|
+
raise ValueError(
|
|
676
|
+
f"execute_sql binds parameters on Spark and DuckDB; the {self.backend} "
|
|
677
|
+
f"backend's connection is given SQL text only")
|
|
678
|
+
return self._convert_results(table, wanted)
|
|
679
|
+
|
|
680
|
+
@property
|
|
681
|
+
def schema(self) -> Optional[GraphSchema]:
|
|
682
|
+
"""The graph schema defining node and relationship mappings.
|
|
683
|
+
|
|
684
|
+
Returns the GraphSchema that maps tabular data sources to graph
|
|
685
|
+
concepts (nodes, relationships, predicates).
|
|
686
|
+
|
|
687
|
+
Returns:
|
|
688
|
+
The GraphSchema if one was provided, None otherwise.
|
|
689
|
+
"""
|
|
690
|
+
return self._schema
|
|
691
|
+
|
|
692
|
+
def _load_dataframes_from_schema(
|
|
693
|
+
self,
|
|
694
|
+
spark: Any,
|
|
695
|
+
schema: GraphSchema
|
|
696
|
+
) -> Dict[str, Any]:
|
|
697
|
+
"""Load DataFrames from Unity Catalog using schema mappings.
|
|
698
|
+
|
|
699
|
+
Args:
|
|
700
|
+
spark: SparkSession to use for loading tables
|
|
701
|
+
schema: GraphSchema (or an R2RML mapping's) whose source maps name the tables
|
|
702
|
+
|
|
703
|
+
Returns:
|
|
704
|
+
Dictionary mapping table names to Spark DataFrames.
|
|
705
|
+
Keys are the original (possibly unqualified) names from mappings.
|
|
706
|
+
"""
|
|
707
|
+
dataframes = {}
|
|
708
|
+
# The tables the source maps read (`entry.mapped_tables`), each by its name: a query
|
|
709
|
+
# source's rows are its query's.
|
|
710
|
+
from prograph.plan.ir.entry import mapped_tables
|
|
711
|
+
for source in mapped_tables(schema):
|
|
712
|
+
if source.table in dataframes:
|
|
713
|
+
continue
|
|
714
|
+
try:
|
|
715
|
+
# The name as Spark reads it (an R2RML delimited "Production"."Product" as
|
|
716
|
+
# `Production`.`Product`: rule source-table-names).
|
|
717
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
718
|
+
dataframes[source.table] = (spark.sql(source.query) if source.query
|
|
719
|
+
else spark.table(get_dialect_instance("spark").table_name(source.qualified)))
|
|
720
|
+
except Exception as e:
|
|
721
|
+
raise ValueError(f"Failed to load table '{source.qualified}' from Unity Catalog: {e}")
|
|
722
|
+
|
|
723
|
+
return dataframes
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
def _create_executor(
|
|
727
|
+
self,
|
|
728
|
+
backend: BackendType,
|
|
729
|
+
schema: Optional[GraphSchema],
|
|
730
|
+
dataframes: Optional[Dict[str, Any]],
|
|
731
|
+
sql_warehouse_id: Optional[str] = None,
|
|
732
|
+
warehouse_timeout: Optional[float] = None,
|
|
733
|
+
connection: Optional[Any] = None,
|
|
734
|
+
) -> BaseExecutor:
|
|
735
|
+
"""Create the appropriate executor for the backend."""
|
|
736
|
+
if backend == "spark":
|
|
737
|
+
if schema is None:
|
|
738
|
+
raise ValueError("Spark backend requires a schema parameter")
|
|
739
|
+
if dataframes is None:
|
|
740
|
+
raise ValueError(
|
|
741
|
+
"Spark backend requires either dataframes or spark parameter. "
|
|
742
|
+
"Provide dataframes dict directly, or pass spark to auto-load from Unity Catalog."
|
|
743
|
+
)
|
|
744
|
+
from prograph.executors.spark_executor import SparkExecutor
|
|
745
|
+
return SparkExecutor(
|
|
746
|
+
schema, dataframes,
|
|
747
|
+
sql_warehouse_id=sql_warehouse_id,
|
|
748
|
+
warehouse_timeout=warehouse_timeout,
|
|
749
|
+
ansi_session=self._ansi_session,
|
|
750
|
+
)
|
|
751
|
+
elif backend == "databricks_sql":
|
|
752
|
+
if schema is None:
|
|
753
|
+
raise ValueError("Databricks SQL backend requires a schema parameter")
|
|
754
|
+
if connection is None:
|
|
755
|
+
raise ValueError(
|
|
756
|
+
"Databricks SQL backend requires a connection parameter. "
|
|
757
|
+
"Pass a databricks.sql.connect() connection."
|
|
758
|
+
)
|
|
759
|
+
from prograph.executors.databricks_sql_executor import DatabricksSQLExecutor
|
|
760
|
+
return DatabricksSQLExecutor(
|
|
761
|
+
schema, connection,
|
|
762
|
+
catalog=self._schema.default_catalog if self._schema else None,
|
|
763
|
+
schema_name=self._schema.default_schema if self._schema else None,
|
|
764
|
+
ansi_session=self._ansi_session,
|
|
765
|
+
)
|
|
766
|
+
elif backend == "duckdb":
|
|
767
|
+
if schema is None:
|
|
768
|
+
raise ValueError("DuckDB backend requires a schema parameter")
|
|
769
|
+
frames = connection is None
|
|
770
|
+
if self._store is not None:
|
|
771
|
+
from prograph.store import engine_connection
|
|
772
|
+
# An owned store's connection is the one the frames are read on (rule `store-connection`).
|
|
773
|
+
shared = engine_connection(self._store, connection, frames)
|
|
774
|
+
if frames:
|
|
775
|
+
# DataFrames: SQL through an embedded DuckDB over the frames,
|
|
776
|
+
# the default local engine. Given a connection instead,
|
|
777
|
+
# DuckDB is a database like any warehouse.
|
|
778
|
+
# Neither: an engine over the empty dataset.
|
|
779
|
+
_require_duckdb()
|
|
780
|
+
from prograph.executors.pandas_executor import PandasExecutor
|
|
781
|
+
executor = PandasExecutor(schema, dataframes)
|
|
782
|
+
executor.graph.enable_local_sql(shared if self._store is not None else None)
|
|
783
|
+
self._connection = executor.graph._local_sql
|
|
784
|
+
self._frame_backed = True
|
|
785
|
+
return executor
|
|
786
|
+
from prograph.executors.duckdb_executor import DuckDBExecutor
|
|
787
|
+
return DuckDBExecutor(schema, connection)
|
|
788
|
+
elif backend == "mysql":
|
|
789
|
+
if schema is None:
|
|
790
|
+
raise ValueError("MySQL backend requires a schema parameter")
|
|
791
|
+
if connection is None:
|
|
792
|
+
raise ValueError(
|
|
793
|
+
"MySQL backend requires a connection parameter. "
|
|
794
|
+
"Pass a mysql.connector.connect() connection."
|
|
795
|
+
)
|
|
796
|
+
from prograph.executors.mysql_executor import MySQLExecutor
|
|
797
|
+
return MySQLExecutor(schema, connection)
|
|
798
|
+
elif backend == "postgresql":
|
|
799
|
+
if schema is None:
|
|
800
|
+
raise ValueError("PostgreSQL backend requires a schema parameter")
|
|
801
|
+
if connection is None:
|
|
802
|
+
raise ValueError("PostgreSQL backend requires a connection parameter")
|
|
803
|
+
from prograph.executors.postgresql_executor import PostgreSQLExecutor
|
|
804
|
+
executor = PostgreSQLExecutor(schema, connection)
|
|
805
|
+
# Given a URI, the connection is the one ProGraph opened.
|
|
806
|
+
self._connection = executor.graph._connection
|
|
807
|
+
return executor
|
|
808
|
+
elif backend == "sqlserver":
|
|
809
|
+
if schema is None:
|
|
810
|
+
raise ValueError("SQL Server backend requires a schema parameter")
|
|
811
|
+
if connection is None:
|
|
812
|
+
raise ValueError("SQL Server backend requires a connection parameter")
|
|
813
|
+
from prograph.executors.sqlserver_executor import SQLServerExecutor
|
|
814
|
+
return SQLServerExecutor(schema, connection)
|
|
815
|
+
elif backend == "oracle":
|
|
816
|
+
if schema is None:
|
|
817
|
+
raise ValueError("Oracle backend requires a schema parameter")
|
|
818
|
+
if connection is None:
|
|
819
|
+
raise ValueError("Oracle backend requires a connection parameter")
|
|
820
|
+
from prograph.executors.oracle_executor import OracleExecutor
|
|
821
|
+
return OracleExecutor(schema, connection)
|
|
822
|
+
else:
|
|
823
|
+
raise ValueError(f"Unknown backend: {backend}")
|
|
824
|
+
|
|
825
|
+
@classmethod
|
|
826
|
+
def _from_source_maps(cls, maps: Any, connection: Any = None, backend: str = "duckdb",
|
|
827
|
+
**kwargs: Any) -> "ProGraph":
|
|
828
|
+
"""An engine over source maps compiled elsewhere. Provisional and internal: the Cypher TCK
|
|
829
|
+
harness uses it for each scenario's typed tables, and a generated mapping will;
|
|
830
|
+
its public form is Brice's decision. The engine reads no authoring object:
|
|
831
|
+
column types come from the connection, as for an R2RML mapping."""
|
|
832
|
+
from prograph.schema.maps_schema import SourceMapsSchema
|
|
833
|
+
return cls(schema=SourceMapsSchema(maps), backend=backend, connection=connection, **kwargs)
|
|
834
|
+
|
|
835
|
+
@classmethod
|
|
836
|
+
def from_config(
|
|
837
|
+
cls,
|
|
838
|
+
config_path: str,
|
|
839
|
+
spark: Optional[Any] = None,
|
|
840
|
+
backend: Optional[BackendType] = None,
|
|
841
|
+
**kwargs
|
|
842
|
+
) -> "ProGraph":
|
|
843
|
+
"""Create a ProGraph engine from a YAML configuration file.
|
|
844
|
+
|
|
845
|
+
This factory method loads graph schema and data sources from a YAML
|
|
846
|
+
configuration file, following patterns from Databricks Metric Views
|
|
847
|
+
and Asset Bundles (DABs).
|
|
848
|
+
|
|
849
|
+
Args:
|
|
850
|
+
config_path: Path to YAML configuration file
|
|
851
|
+
spark: SparkSession (for the Spark backend; or pass connection= for table sources)
|
|
852
|
+
backend: Override backend type (default: Spark with spark=, the connection's with connection=,
|
|
853
|
+
DuckDB over the files otherwise)
|
|
854
|
+
**kwargs: Additional arguments passed to ProGraph constructor
|
|
855
|
+
|
|
856
|
+
Returns:
|
|
857
|
+
Configured ProGraph engine
|
|
858
|
+
|
|
859
|
+
Example YAML configuration:
|
|
860
|
+
|
|
861
|
+
version: "1.0"
|
|
862
|
+
|
|
863
|
+
graph:
|
|
864
|
+
name: social_network
|
|
865
|
+
comment: "LDBC SNB social network schema"
|
|
866
|
+
|
|
867
|
+
sources:
|
|
868
|
+
person:
|
|
869
|
+
table: catalog.schema.person
|
|
870
|
+
knows:
|
|
871
|
+
file: ./data/person_knows_person.csv
|
|
872
|
+
options:
|
|
873
|
+
delimiter: "|"
|
|
874
|
+
|
|
875
|
+
nodes:
|
|
876
|
+
- label: Person
|
|
877
|
+
source: person
|
|
878
|
+
id_column: id
|
|
879
|
+
properties:
|
|
880
|
+
firstName: firstName
|
|
881
|
+
lastName: lastName
|
|
882
|
+
|
|
883
|
+
relationships:
|
|
884
|
+
- type: KNOWS
|
|
885
|
+
source: knows
|
|
886
|
+
start_node:
|
|
887
|
+
label: Person
|
|
888
|
+
column: Person.id
|
|
889
|
+
end_node:
|
|
890
|
+
label: Person
|
|
891
|
+
column: Person.id.1
|
|
892
|
+
|
|
893
|
+
Example usage:
|
|
894
|
+
|
|
895
|
+
# With Spark (for table sources or large files)
|
|
896
|
+
engine = ProGraph.from_config('graph_schema.yaml', spark=spark)
|
|
897
|
+
|
|
898
|
+
# With pandas (for local files only)
|
|
899
|
+
engine = ProGraph.from_config('graph_schema.yaml')
|
|
900
|
+
|
|
901
|
+
# Over a database's tables (DuckDB, PostgreSQL, ...)
|
|
902
|
+
engine = ProGraph.from_config('graph_schema.yaml', connection=duckdb.connect('my.db'))
|
|
903
|
+
|
|
904
|
+
# Override backend explicitly
|
|
905
|
+
engine = ProGraph.from_config(
|
|
906
|
+
'graph_schema.yaml',
|
|
907
|
+
spark=spark,
|
|
908
|
+
backend='spark'
|
|
909
|
+
)
|
|
910
|
+
"""
|
|
911
|
+
from pathlib import Path
|
|
912
|
+
from prograph.config import load_config, validate_config, load_dataframes
|
|
913
|
+
|
|
914
|
+
# Load and validate configuration
|
|
915
|
+
config = load_config(config_path)
|
|
916
|
+
errors = validate_config(config)
|
|
917
|
+
if errors:
|
|
918
|
+
raise ValueError(f"Configuration errors: {'; '.join(errors)}")
|
|
919
|
+
|
|
920
|
+
return cls._from_graph_config(config, spark, backend, Path(config_path).parent, "from_config", kwargs)
|
|
921
|
+
|
|
922
|
+
@classmethod
|
|
923
|
+
def _from_graph_config(cls, config, spark, backend, base_path, factory: str, kwargs) -> "ProGraph":
|
|
924
|
+
"""The engine of a loaded YAML configuration (`from_config`, `from_yaml`). With `connection=` (a DuckDB,
|
|
925
|
+
PostgreSQL ... connection) every source is a table it holds, read where it is (a query or file source is
|
|
926
|
+
refused, by name); with `spark=` tables, queries and files are read by Spark; with neither, file sources
|
|
927
|
+
are read as frames by DuckDB."""
|
|
928
|
+
from prograph.config import load_dataframes
|
|
929
|
+
connection = kwargs.get("connection")
|
|
930
|
+
if spark is None and connection is not None:
|
|
931
|
+
# Over a database's tables: each source a table the connection holds, read where it is.
|
|
932
|
+
queries = [n for n, s in config.sources.items() if s.query]
|
|
933
|
+
files = [n for n, s in config.sources.items() if s.file]
|
|
934
|
+
if queries:
|
|
935
|
+
raise ValueError(
|
|
936
|
+
f"query sources with connection=: {', '.join(queries)}. Create a view of each query in the "
|
|
937
|
+
f"database and name it as a table source, or pass spark= to {factory}().")
|
|
938
|
+
if files:
|
|
939
|
+
raise ValueError(
|
|
940
|
+
f"file sources with connection=: {', '.join(files)}. Load the files into the database and "
|
|
941
|
+
f"name them as table sources, or leave out connection= to read the files with DuckDB.")
|
|
942
|
+
engine = cls(backend=backend, schema=config.to_schema(), spark=None, **kwargs)
|
|
943
|
+
else:
|
|
944
|
+
in_database = [n for n, s in config.sources.items() if s.table or s.query]
|
|
945
|
+
if in_database and spark is None:
|
|
946
|
+
raise ValueError(
|
|
947
|
+
f"these sources are tables or queries: {', '.join(in_database)}. Pass connection= (a "
|
|
948
|
+
f"database connection that holds the tables, e.g. duckdb.connect('my.db')) or spark= to "
|
|
949
|
+
f"{factory}().")
|
|
950
|
+
if backend is None:
|
|
951
|
+
backend = 'spark' if spark is not None else 'duckdb' # local files: DuckDB over the frames
|
|
952
|
+
dataframes = load_dataframes(config, spark=spark, base_path=base_path)
|
|
953
|
+
engine = cls(backend=backend, schema=config.to_schema(), dataframes=dataframes, spark=spark,
|
|
954
|
+
**kwargs)
|
|
955
|
+
engine._config = config # the configuration, for debugging
|
|
956
|
+
engine._declare_config_procedures(config)
|
|
957
|
+
return engine
|
|
958
|
+
|
|
959
|
+
@classmethod
|
|
960
|
+
def from_direct_mapping(
|
|
961
|
+
cls,
|
|
962
|
+
connection: Any,
|
|
963
|
+
base_iri: str,
|
|
964
|
+
schema: Optional[str] = None,
|
|
965
|
+
tables: Optional[Sequence[str]] = None,
|
|
966
|
+
backend: Optional[BackendType] = None,
|
|
967
|
+
**kwargs
|
|
968
|
+
) -> "ProGraph":
|
|
969
|
+
"""Create a ProGraph engine over a database's tables as the W3C Direct Mapping
|
|
970
|
+
(https://www.w3.org/TR/rdb-direct-mapping/) gives them as RDF, with no mapping to write.
|
|
971
|
+
|
|
972
|
+
The catalog (tables, columns, declared primary and foreign keys) is read once and the mapping
|
|
973
|
+
generated from it (`prograph.r2rml.direct`): a row of a table with a primary key is
|
|
974
|
+
`<base_iri>Table/pk=value`, a row of a table with none a blank node, a column a literal
|
|
975
|
+
`<base_iri>Table#Column`, a foreign key a reference `<base_iri>Table#ref-column`. SPARQL then
|
|
976
|
+
reads the live tables; nothing is copied. `ProGraph.direct_mapping` gives the same mapping as
|
|
977
|
+
R2RML Turtle, to edit.
|
|
978
|
+
|
|
979
|
+
Args:
|
|
980
|
+
connection: A DuckDB connection, or a PostgreSQL or Databricks SQL one with `backend` named.
|
|
981
|
+
base_iri: The absolute IRI the generated IRIs are relative to; required.
|
|
982
|
+
schema: The schema to map (`catalog.schema` on Databricks); the current one by default.
|
|
983
|
+
tables: Map only these tables; a foreign key to another is refused.
|
|
984
|
+
backend: 'duckdb' (the default for a DuckDB connection), 'postgresql' or 'databricks_sql'.
|
|
985
|
+
**kwargs: Passed to the ProGraph constructor.
|
|
986
|
+
|
|
987
|
+
Raises:
|
|
988
|
+
prograph.r2rml.catalog.DirectMappingError: A database with no catalog reader, a relative
|
|
989
|
+
base IRI, a table not there, or a foreign key to a table not mapped.
|
|
990
|
+
"""
|
|
991
|
+
from prograph.r2rml.direct import direct_mapping_r2rml
|
|
992
|
+
from prograph.r2rml.schema import R2RMLSchema
|
|
993
|
+
|
|
994
|
+
mapping = direct_mapping_r2rml(connection, base_iri, schema, tables, backend=backend)
|
|
995
|
+
return cls(backend=backend or 'duckdb', schema=R2RMLSchema(mapping, base_iri=base_iri),
|
|
996
|
+
connection=connection, **kwargs)
|
|
997
|
+
|
|
998
|
+
@classmethod
|
|
999
|
+
def direct_mapping(
|
|
1000
|
+
cls,
|
|
1001
|
+
connection: Any,
|
|
1002
|
+
base_iri: str,
|
|
1003
|
+
schema: Optional[str] = None,
|
|
1004
|
+
tables: Optional[Sequence[str]] = None,
|
|
1005
|
+
backend: Optional[BackendType] = None,
|
|
1006
|
+
format: str = "turtle",
|
|
1007
|
+
) -> Any:
|
|
1008
|
+
"""The Direct Mapping `from_direct_mapping` reads, as a mapping to edit and load (on the engine it was
|
|
1009
|
+
exported from: a keyless table's rows are numbered by a query in that engine's SQL).
|
|
1010
|
+
|
|
1011
|
+
Args:
|
|
1012
|
+
connection, base_iri, schema, tables, backend: As `from_direct_mapping`.
|
|
1013
|
+
format: "turtle" gives the R2RML Turtle text (write it to a file, edit it, load it with
|
|
1014
|
+
`from_r2rml`); "r2rml" gives the `R2RMLMapping` model. A YAML export is not offered:
|
|
1015
|
+
YAML source maps cannot state a keyless table's numbered blank nodes.
|
|
1016
|
+
|
|
1017
|
+
Raises:
|
|
1018
|
+
prograph.r2rml.catalog.DirectMappingError: As `from_direct_mapping`.
|
|
1019
|
+
ValueError: A format other than "turtle" or "r2rml".
|
|
1020
|
+
"""
|
|
1021
|
+
from prograph.r2rml.direct import direct_mapping_r2rml
|
|
1022
|
+
from prograph.r2rml.export import to_turtle
|
|
1023
|
+
|
|
1024
|
+
if format not in ("turtle", "r2rml"):
|
|
1025
|
+
raise ValueError(f"format must be 'turtle' or 'r2rml'; got {format!r}")
|
|
1026
|
+
mapping = direct_mapping_r2rml(connection, base_iri, schema, tables, backend=backend)
|
|
1027
|
+
return to_turtle(mapping) if format == "turtle" else mapping
|
|
1028
|
+
|
|
1029
|
+
@classmethod
|
|
1030
|
+
def from_r2rml(
|
|
1031
|
+
cls,
|
|
1032
|
+
r2rml_path: str,
|
|
1033
|
+
dataframes: Optional[Dict[str, Any]] = None,
|
|
1034
|
+
spark: Optional[Any] = None,
|
|
1035
|
+
backend: Optional[BackendType] = None,
|
|
1036
|
+
**kwargs
|
|
1037
|
+
) -> "ProGraph":
|
|
1038
|
+
"""Create a ProGraph engine from a W3C R2RML mapping file.
|
|
1039
|
+
|
|
1040
|
+
Parses the R2RML Turtle document, compiles it to a GraphSchema,
|
|
1041
|
+
and creates an engine ready for queries.
|
|
1042
|
+
|
|
1043
|
+
A TriplesMap whose logical table is an rr:sqlQuery (R2RML 5.2) is a
|
|
1044
|
+
mapping whose source is that query: SQL reads it as a derived table,
|
|
1045
|
+
on every backend, and a DataFrame backend computes its frame from
|
|
1046
|
+
the given frames (Spark: from the session's tables).
|
|
1047
|
+
|
|
1048
|
+
Args:
|
|
1049
|
+
r2rml_path: Path to R2RML Turtle file
|
|
1050
|
+
dataframes: Optional dict of source name -> DataFrame
|
|
1051
|
+
spark: SparkSession
|
|
1052
|
+
backend: Backend type (inferred from spark if not given)
|
|
1053
|
+
**kwargs: Additional arguments passed to ProGraph constructor
|
|
1054
|
+
|
|
1055
|
+
Returns:
|
|
1056
|
+
Configured ProGraph engine
|
|
1057
|
+
|
|
1058
|
+
Raises:
|
|
1059
|
+
FileNotFoundError: If the R2RML file does not exist
|
|
1060
|
+
ImportError: If rdflib is not installed
|
|
1061
|
+
|
|
1062
|
+
Example::
|
|
1063
|
+
|
|
1064
|
+
engine = ProGraph.from_r2rml(
|
|
1065
|
+
"mapping.ttl",
|
|
1066
|
+
spark=spark
|
|
1067
|
+
)
|
|
1068
|
+
results = engine.execute("MATCH (e:Employee) RETURN e.name")
|
|
1069
|
+
"""
|
|
1070
|
+
from prograph.r2rml.parser import R2RMLParser
|
|
1071
|
+
from prograph.r2rml.schema import R2RMLSchema
|
|
1072
|
+
|
|
1073
|
+
# The mapping's source maps, no property-graph view.
|
|
1074
|
+
schema = R2RMLSchema(R2RMLParser().parse_file(r2rml_path))
|
|
1075
|
+
|
|
1076
|
+
# Spark when a session is given; a connection alone is left to the
|
|
1077
|
+
# constructor (a DuckDB connection is DuckDB, SQL only; another kind
|
|
1078
|
+
# needs its backend named); otherwise DuckDB over the dataframes.
|
|
1079
|
+
resolved_backend = backend or (
|
|
1080
|
+
'spark' if spark else None if kwargs.get('connection') is not None
|
|
1081
|
+
and dataframes is None else 'duckdb')
|
|
1082
|
+
|
|
1083
|
+
return cls(
|
|
1084
|
+
backend=resolved_backend,
|
|
1085
|
+
schema=schema,
|
|
1086
|
+
dataframes=dataframes,
|
|
1087
|
+
spark=spark,
|
|
1088
|
+
**kwargs,
|
|
1089
|
+
)
|
|
1090
|
+
|
|
1091
|
+
@classmethod
|
|
1092
|
+
def from_yaml(
|
|
1093
|
+
cls,
|
|
1094
|
+
yaml_string: str,
|
|
1095
|
+
spark: Optional[Any] = None,
|
|
1096
|
+
backend: Optional[BackendType] = None,
|
|
1097
|
+
base_path: Optional[str] = None,
|
|
1098
|
+
**kwargs
|
|
1099
|
+
) -> "ProGraph":
|
|
1100
|
+
"""Create a ProGraph engine from a YAML string.
|
|
1101
|
+
|
|
1102
|
+
Similar to from_config() but accepts a YAML string directly.
|
|
1103
|
+
Useful for embedded configurations or testing.
|
|
1104
|
+
|
|
1105
|
+
Args:
|
|
1106
|
+
yaml_string: YAML configuration as string
|
|
1107
|
+
spark: SparkSession (for the Spark backend; or pass connection= for table sources)
|
|
1108
|
+
backend: Override backend type
|
|
1109
|
+
base_path: Base path for resolving relative file paths
|
|
1110
|
+
**kwargs: Additional arguments passed to ProGraph constructor
|
|
1111
|
+
|
|
1112
|
+
Returns:
|
|
1113
|
+
Configured ProGraph engine
|
|
1114
|
+
"""
|
|
1115
|
+
from pathlib import Path
|
|
1116
|
+
from prograph.config import load_config_from_string, validate_config, load_dataframes
|
|
1117
|
+
|
|
1118
|
+
# Load and validate configuration
|
|
1119
|
+
config = load_config_from_string(yaml_string)
|
|
1120
|
+
errors = validate_config(config)
|
|
1121
|
+
if errors:
|
|
1122
|
+
raise ValueError(f"Configuration errors: {'; '.join(errors)}")
|
|
1123
|
+
|
|
1124
|
+
return cls._from_graph_config(config, spark, backend, Path(base_path) if base_path else Path('.'),
|
|
1125
|
+
"from_yaml", kwargs)
|
|
1126
|
+
|
|
1127
|
+
|
|
1128
|
+
def _detect_language(self, query: str) -> str:
|
|
1129
|
+
"""Detect the query language from the query string.
|
|
1130
|
+
|
|
1131
|
+
Uses heuristics to determine which language the query is written in:
|
|
1132
|
+
- Starts with 'g.' -> Gremlin (refused: not supported)
|
|
1133
|
+
- Contains GRAPH_TABLE or PROPERTY GRAPH -> SQL/PGQ (refused: not supported)
|
|
1134
|
+
- Starts with PREFIX, SELECT, ... -> SPARQL
|
|
1135
|
+
- Otherwise Cypher
|
|
1136
|
+
|
|
1137
|
+
Args:
|
|
1138
|
+
query: The query string to analyze
|
|
1139
|
+
|
|
1140
|
+
Returns:
|
|
1141
|
+
Language identifier: "cypher", "sparql", or, for the two
|
|
1142
|
+
languages ProGraph does not support, "gremlin" or "sqlpgq"
|
|
1143
|
+
(which `_get_plan_builder` refuses).
|
|
1144
|
+
"""
|
|
1145
|
+
query_stripped = query.strip()
|
|
1146
|
+
|
|
1147
|
+
# Gremlin queries start with 'g.'
|
|
1148
|
+
if query_stripped.startswith('g.'):
|
|
1149
|
+
return "gremlin"
|
|
1150
|
+
|
|
1151
|
+
# SQL/PGQ: contains GRAPH_TABLE or PROPERTY GRAPH
|
|
1152
|
+
query_upper = query_stripped.upper()
|
|
1153
|
+
if 'GRAPH_TABLE' in query_upper or 'PROPERTY GRAPH' in query_upper:
|
|
1154
|
+
return "sqlpgq"
|
|
1155
|
+
|
|
1156
|
+
# SPARQL: starts with SELECT (without MATCH), PREFIX, BASE, CONSTRUCT, DESCRIBE, or ASK
|
|
1157
|
+
sparql_keywords = ('SELECT', 'PREFIX', 'BASE', 'CONSTRUCT', 'DESCRIBE', 'ASK')
|
|
1158
|
+
if any(query_upper.startswith(kw) for kw in sparql_keywords):
|
|
1159
|
+
# Distinguish from Cypher: SPARQL SELECT doesn't have MATCH
|
|
1160
|
+
if query_upper.startswith('SELECT') and 'MATCH' in query_upper:
|
|
1161
|
+
pass # This is likely Cypher, fall through
|
|
1162
|
+
else:
|
|
1163
|
+
return "sparql"
|
|
1164
|
+
|
|
1165
|
+
# Default to Cypher
|
|
1166
|
+
return "cypher"
|
|
1167
|
+
|
|
1168
|
+
|
|
1169
|
+
def _convert_results(self, table: Any, return_type: ReturnType) -> Any:
|
|
1170
|
+
"""The answer, a pyarrow.Table, in the asked return type."""
|
|
1171
|
+
import pyarrow as pa
|
|
1172
|
+
if isinstance(table, pa.RecordBatchReader):
|
|
1173
|
+
if return_type == "batches":
|
|
1174
|
+
return table
|
|
1175
|
+
table = table.read_all()
|
|
1176
|
+
if return_type == "batches":
|
|
1177
|
+
size = getattr(self, "_batch_size", None) or 100_000
|
|
1178
|
+
return pa.RecordBatchReader.from_batches(table.schema, table.to_batches(max_chunksize=size))
|
|
1179
|
+
if return_type == "arrow":
|
|
1180
|
+
return table
|
|
1181
|
+
if return_type == "list":
|
|
1182
|
+
return table.to_pylist(maps_as_pydicts="strict")
|
|
1183
|
+
if self.backend == "spark":
|
|
1184
|
+
from pyspark.sql import SparkSession
|
|
1185
|
+
spark = SparkSession.builder.getOrCreate()
|
|
1186
|
+
if table.num_columns == 0:
|
|
1187
|
+
# Rows with no column: a Spark DataFrame keeps the count.
|
|
1188
|
+
return spark.range(table.num_rows).select()
|
|
1189
|
+
return spark.createDataFrame(table)
|
|
1190
|
+
from prograph.results import table_to_pandas
|
|
1191
|
+
return table_to_pandas(table)
|
|
1192
|
+
|
|
1193
|
+
def _language_of(self, text: str, language: Optional[LanguageType]) -> str:
|
|
1194
|
+
"""The query's language: the call's, else the engine's default,
|
|
1195
|
+
detected from the text where that is 'auto'. Gremlin, SQL/PGQ and
|
|
1196
|
+
GQL are refused by name."""
|
|
1197
|
+
lang = language if language is not None else self.default_language
|
|
1198
|
+
if lang == "auto":
|
|
1199
|
+
lang = self._detect_language(text)
|
|
1200
|
+
if lang in _UNSUPPORTED_LANGUAGES:
|
|
1201
|
+
raise ValueError(f"{_UNSUPPORTED_LANGUAGES[lang]} is not supported in ProGraph 1.0. "
|
|
1202
|
+
f"Supported languages: sparql, cypher")
|
|
1203
|
+
if lang not in ("sparql", "cypher"):
|
|
1204
|
+
raise ValueError(f"Unsupported query language: {lang}. "
|
|
1205
|
+
f"Supported languages: sparql, cypher")
|
|
1206
|
+
return lang
|
|
1207
|
+
|
|
1208
|
+
def execute(
|
|
1209
|
+
self,
|
|
1210
|
+
query: str,
|
|
1211
|
+
language: Optional[LanguageType] = None,
|
|
1212
|
+
return_type: Optional[ReturnType] = None,
|
|
1213
|
+
context: Optional['QueryContext'] = None,
|
|
1214
|
+
batch_size: int = 100_000,
|
|
1215
|
+
parameters: Optional[Dict[str, Any]] = None,
|
|
1216
|
+
step_key: Optional[str] = None,
|
|
1217
|
+
dataset: Optional[Tuple[Sequence[str], Sequence[str]]] = None,
|
|
1218
|
+
) -> Any:
|
|
1219
|
+
"""Answer a query: it is translated into one relational plan over
|
|
1220
|
+
the mapping's source maps, lowered to SQL, and run on the backend
|
|
1221
|
+
(docs/architecture/one-plan.md).
|
|
1222
|
+
|
|
1223
|
+
Args:
|
|
1224
|
+
query: The query text, optionally after a hint block
|
|
1225
|
+
(``/*+ terms=all */``).
|
|
1226
|
+
language: "sparql", "cypher" or "auto" (detect from the text);
|
|
1227
|
+
default the engine's default_language.
|
|
1228
|
+
return_type: "arrow" (a pyarrow.Table, the default), "batches"
|
|
1229
|
+
(a pyarrow.RecordBatchReader of at most batch_size rows),
|
|
1230
|
+
"list" (one dict per row) or "dataframe" (a Spark DataFrame
|
|
1231
|
+
on Spark, else pandas).
|
|
1232
|
+
context: A QueryContext, to cancel the query from another thread:
|
|
1233
|
+
its cancel() stops the statement the database is running, and the
|
|
1234
|
+
query raises QueryCancelledException (a streamed answer is
|
|
1235
|
+
cancellable until its reader ends).
|
|
1236
|
+
parameters: Values for the query's parameters.
|
|
1237
|
+
dataset: A SPARQL query's dataset, (default graph IRIs, named
|
|
1238
|
+
graph IRIs), as the SPARQL Protocol's ``default-graph-uri``
|
|
1239
|
+
and ``named-graph-uri``: used in place of the query's own
|
|
1240
|
+
FROM and FROM NAMED clauses (SPARQL 1.1 Query 13.2).
|
|
1241
|
+
step_key: A query that writes (Cypher's CREATE, into the
|
|
1242
|
+
engine's owned store) is one write step; this is its
|
|
1243
|
+
idempotency key. A key the store applied already writes
|
|
1244
|
+
nothing, and the query answers as the first run did.
|
|
1245
|
+
None: a new key for each run.
|
|
1246
|
+
|
|
1247
|
+
Raises:
|
|
1248
|
+
NotImplementedError: For what the plan does not build yet; its
|
|
1249
|
+
message starts "one-plan:".
|
|
1250
|
+
ValueError: For a language ProGraph does not support.
|
|
1251
|
+
"""
|
|
1252
|
+
from prograph.plan.ir import entry
|
|
1253
|
+
with self._execute_lock:
|
|
1254
|
+
text, hints = parse_hints(query)
|
|
1255
|
+
if dataset is not None:
|
|
1256
|
+
hints.dataset = (tuple(dataset[0]), tuple(dataset[1]))
|
|
1257
|
+
lang = self._language_of(text, language)
|
|
1258
|
+
wanted = return_type if return_type is not None else self.default_return_type
|
|
1259
|
+
self._batch_size = batch_size
|
|
1260
|
+
from prograph.execution import running, until_read
|
|
1261
|
+
if context:
|
|
1262
|
+
context.mark_running()
|
|
1263
|
+
try:
|
|
1264
|
+
# Every statement the query starts registers with `context`, whose cancel()
|
|
1265
|
+
# stops it (`StatementGuard`); no later one starts.
|
|
1266
|
+
with running(context):
|
|
1267
|
+
if context:
|
|
1268
|
+
context.check_cancelled()
|
|
1269
|
+
table = entry.run(self, text, lang, hints=hints, parameters=parameters,
|
|
1270
|
+
streaming=wanted == "batches", step_key=step_key)
|
|
1271
|
+
import pyarrow as pa
|
|
1272
|
+
if context and isinstance(table, pa.RecordBatchReader):
|
|
1273
|
+
# Streamed: the query runs until its reader ends, and is cancellable until then.
|
|
1274
|
+
return until_read(table, context)
|
|
1275
|
+
if context:
|
|
1276
|
+
context.check_cancelled()
|
|
1277
|
+
context.mark_completed()
|
|
1278
|
+
if not isinstance(table, pa.Table):
|
|
1279
|
+
# A CONSTRUCT's answer is an RDF graph (16.2), which no
|
|
1280
|
+
# return type reshapes.
|
|
1281
|
+
return table
|
|
1282
|
+
return self._convert_results(table, wanted)
|
|
1283
|
+
except Exception as e:
|
|
1284
|
+
if context:
|
|
1285
|
+
context.mark_failed(str(e))
|
|
1286
|
+
raise
|
|
1287
|
+
|
|
1288
|
+
def describe_dataset(self, format: str = "graph", *, counts: bool = False, endpoint: Optional[str] = None,
|
|
1289
|
+
dataset_iri: Optional[str] = None, validation: Any = None,
|
|
1290
|
+
include_report: bool = False) -> Any:
|
|
1291
|
+
"""The mapped graph described as DCAT 3, VoID and a SPARQL 1.1 Service Description
|
|
1292
|
+
(`prograph.describe`), from the source maps alone.
|
|
1293
|
+
|
|
1294
|
+
Args:
|
|
1295
|
+
format: "graph" gives an rdflib Graph; an rdflib serialization name ("turtle", "xml",
|
|
1296
|
+
"nt", "json-ld") gives that text.
|
|
1297
|
+
counts: Add the VoID counts (`void:triples`, `void:entities`, per class and per property).
|
|
1298
|
+
Each is a SPARQL COUNT query run by this engine, so it reads the data.
|
|
1299
|
+
endpoint: The SPARQL endpoint's IRI; given, the description has the `sd:Service` too.
|
|
1300
|
+
dataset_iri: The dataset's IRI (default `<endpoint>#dataset`, else a blank node).
|
|
1301
|
+
validation: A `ValidationReport` (`validate_shapes`, or `shacl_last_report`): its DQV measurements
|
|
1302
|
+
(`dqv:hasQualityMeasurement`; conformance and violation count) are linked to the dataset.
|
|
1303
|
+
include_report: With `validation`, add the SHACL report graph too, each measurement's `dcterms:source`.
|
|
1304
|
+
"""
|
|
1305
|
+
import warnings
|
|
1306
|
+
from prograph.describe import RESULT_FORMAT_IRIS, describe
|
|
1307
|
+
from prograph.plan.ir.entry import source_maps
|
|
1308
|
+
from prograph.plan.ir.cost import ProGraphCostWarning
|
|
1309
|
+
|
|
1310
|
+
def count(query: str) -> int:
|
|
1311
|
+
with warnings.catch_warnings():
|
|
1312
|
+
warnings.simplefilter("ignore", ProGraphCostWarning)
|
|
1313
|
+
table = self.execute(query, language="sparql", return_type="arrow")
|
|
1314
|
+
return table.column(0)[0].as_py() or 0
|
|
1315
|
+
|
|
1316
|
+
graph = describe(source_maps(self.schema), default_graph=self.default_graph, endpoint=endpoint,
|
|
1317
|
+
dataset_iri=dataset_iri, formats=RESULT_FORMAT_IRIS if endpoint else (),
|
|
1318
|
+
count=count if counts else None, validation=validation, include_report=include_report)
|
|
1319
|
+
if format == "graph":
|
|
1320
|
+
return graph
|
|
1321
|
+
return graph.serialize(format=format)
|
|
1322
|
+
|
|
1323
|
+
def provenance(self, format: str = "graph", *, endpoint: Optional[str] = None,
|
|
1324
|
+
dataset_iri: Optional[str] = None, base: Optional[str] = None) -> Any:
|
|
1325
|
+
"""Where the mapped graph's triples come from, as PROV-O (`prograph.provenance`), from the source
|
|
1326
|
+
maps alone: each source (table or query), source map and triple pattern (a map's subject, predicate
|
|
1327
|
+
and object term) with the columns it reads, the dataset `prov:wasDerivedFrom` its sources, and the
|
|
1328
|
+
mapping's version (`dcterms:hasVersion`, a content hash of the compiled maps).
|
|
1329
|
+
|
|
1330
|
+
Args:
|
|
1331
|
+
format: "graph" gives an rdflib Graph; an rdflib serialization name gives that text.
|
|
1332
|
+
endpoint: As in `describe_dataset`: the dataset is `<endpoint>#dataset`, so the two graphs join.
|
|
1333
|
+
dataset_iri: The dataset's IRI; wins over `endpoint`.
|
|
1334
|
+
base: The prefix of the IRIs minted (default `urn:prograph:prov:`).
|
|
1335
|
+
"""
|
|
1336
|
+
from prograph.plan.ir.entry import source_maps
|
|
1337
|
+
from prograph.provenance import DEFAULT_BASE, provenance
|
|
1338
|
+
graph = provenance(source_maps(self.schema), base=base or DEFAULT_BASE,
|
|
1339
|
+
dataset_iri=dataset_iri or (endpoint + "#dataset" if endpoint else None))
|
|
1340
|
+
if format == "graph":
|
|
1341
|
+
return graph
|
|
1342
|
+
return graph.serialize(format=format)
|
|
1343
|
+
|
|
1344
|
+
def to_sql(
|
|
1345
|
+
self, query: str, language: Optional[LanguageType] = None,
|
|
1346
|
+
dialect: Optional[str] = None,
|
|
1347
|
+
parameters: Optional[Dict[str, Any]] = None,
|
|
1348
|
+
dataset: Optional[Tuple[Sequence[str], Sequence[str]]] = None,
|
|
1349
|
+
) -> str:
|
|
1350
|
+
"""The SQL `execute` runs for a query, in `dialect` (default: the
|
|
1351
|
+
dialect this engine's backend runs), without running it.
|
|
1352
|
+
|
|
1353
|
+
Over an owned store the SQL holds for the moment it was planned:
|
|
1354
|
+
which stored OWL RL facts and owl:sameAs components it reads was
|
|
1355
|
+
decided from the store's record then, and the SQL carries no
|
|
1356
|
+
condition on it, so run later, after a run under another cache
|
|
1357
|
+
key, it may read that key's facts. Plan again to run it later.
|
|
1358
|
+
|
|
1359
|
+
Raises:
|
|
1360
|
+
NotImplementedError: For what the plan does not build yet.
|
|
1361
|
+
ValueError: For an unknown dialect or language.
|
|
1362
|
+
"""
|
|
1363
|
+
from prograph.plan.ir import entry
|
|
1364
|
+
if dialect is not None:
|
|
1365
|
+
from prograph.plan.sql.dialect import get_dialect
|
|
1366
|
+
get_dialect(dialect) # an unknown dialect raises ValueError
|
|
1367
|
+
text, hints = parse_hints(query)
|
|
1368
|
+
if dataset is not None:
|
|
1369
|
+
hints.dataset = (tuple(dataset[0]), tuple(dataset[1]))
|
|
1370
|
+
lang = self._language_of(text, language)
|
|
1371
|
+
return entry.sql(self, text, lang, dialect, hints=hints, parameters=parameters)
|
|
1372
|
+
|
|
1373
|
+
def explain(
|
|
1374
|
+
self, query: str, language: Optional[LanguageType] = None,
|
|
1375
|
+
dialect: Optional[str] = None,
|
|
1376
|
+
parameters: Optional[Dict[str, Any]] = None,
|
|
1377
|
+
dataset: Optional[Tuple[Sequence[str], Sequence[str]]] = None,
|
|
1378
|
+
) -> str:
|
|
1379
|
+
"""What `execute` would do for a query, without running it: the SQL,
|
|
1380
|
+
each rewrite of the plan with what it relied on, and each key the
|
|
1381
|
+
plan may rely on with what makes it trusted (a constraint the
|
|
1382
|
+
database enforces, a Databricks RELY, or the mapping's own assertion,
|
|
1383
|
+
`keys: trusted`, which is never shown as checked).
|
|
1384
|
+
|
|
1385
|
+
Raises:
|
|
1386
|
+
NotImplementedError: For what the plan does not build yet.
|
|
1387
|
+
ValueError: For an unknown dialect or language.
|
|
1388
|
+
"""
|
|
1389
|
+
from prograph.plan.ir import entry
|
|
1390
|
+
if dialect is not None:
|
|
1391
|
+
from prograph.plan.sql.dialect import get_dialect
|
|
1392
|
+
get_dialect(dialect)
|
|
1393
|
+
text, hints = parse_hints(query)
|
|
1394
|
+
if dataset is not None:
|
|
1395
|
+
hints.dataset = (tuple(dataset[0]), tuple(dataset[1]))
|
|
1396
|
+
lang = self._language_of(text, language)
|
|
1397
|
+
return entry.explain(self, text, lang, dialect, hints=hints, parameters=parameters)
|
|
1398
|
+
|
|
1399
|
+
# ========== Live Schema Update ==========
|
|
1400
|
+
|
|
1401
|
+
def _record_column_types(self) -> None:
|
|
1402
|
+
"""Each mapped table's column value types, on the schema, for the
|
|
1403
|
+
SQL translator (prograph.schema.types): from the DataFrames' dtypes,
|
|
1404
|
+
or from the DuckDB connection's DESCRIBE. A table whose types
|
|
1405
|
+
cannot be read is left out, and its columns are UNKNOWN."""
|
|
1406
|
+
from prograph.schema import collations as C
|
|
1407
|
+
from prograph.schema import types as T
|
|
1408
|
+
schema = self._schema
|
|
1409
|
+
if schema is None:
|
|
1410
|
+
return
|
|
1411
|
+
# The graph's frames are the ones writes and entailment change.
|
|
1412
|
+
graph_frames = getattr(getattr(getattr(self, 'executor', None), 'graph', None), 'dataframes', None)
|
|
1413
|
+
frames = graph_frames if isinstance(graph_frames, dict) and graph_frames else (self._dataframes or {})
|
|
1414
|
+
for table, frame in frames.items():
|
|
1415
|
+
try:
|
|
1416
|
+
if hasattr(frame, 'dtypes') and hasattr(frame, 'columns') and hasattr(frame, 'iloc'):
|
|
1417
|
+
# A frame is read through DuckDB, whose strings compare byte for byte.
|
|
1418
|
+
schema.column_types[table] = C.mark_bytewise(T.of_frame(frame), 'duckdb', C.binary)
|
|
1419
|
+
elif hasattr(frame, 'dtypes'): # a Spark DataFrame: [(name, type)]
|
|
1420
|
+
kinds = dict(frame.dtypes)
|
|
1421
|
+
schema.column_types[table] = C.mark_bytewise(
|
|
1422
|
+
{c: T.of_sql_type(t, timestamp_is_an_instant=True) for c, t in frame.dtypes},
|
|
1423
|
+
'spark', lambda c: C.collation_in_type(kinds.get(c, "")))
|
|
1424
|
+
except Exception: # noqa: BLE001 - types are an optimisation of correctness, not required
|
|
1425
|
+
continue
|
|
1426
|
+
self._record_frame_decimal_digits(frames)
|
|
1427
|
+
self._record_query_column_types(schema)
|
|
1428
|
+
conn = getattr(self, '_connection', None)
|
|
1429
|
+
backend = getattr(self, 'backend', None)
|
|
1430
|
+
if conn is None or getattr(self, '_frame_backed', False):
|
|
1431
|
+
return
|
|
1432
|
+
from prograph.plan.ir.entry import mapped_tables
|
|
1433
|
+
sources = {s.table: s for s in mapped_tables(schema)}
|
|
1434
|
+
queries = {t: s.query for t, s in sources.items()}
|
|
1435
|
+
# Sources the engine could not read, by table: the database's message, raised by the first query that
|
|
1436
|
+
# reads one (`entry.check_sources`), so the cause is named where the query fails.
|
|
1437
|
+
self._unreadable_sources = {}
|
|
1438
|
+
files = set(getattr(schema, "file_sources", ()) or ())
|
|
1439
|
+
if backend == 'databricks_sql':
|
|
1440
|
+
# Every mapped table's columns in one catalog read, and which have a primary key in another; a table they do not cover is
|
|
1441
|
+
# described on its own below.
|
|
1442
|
+
self._warehouse_catalog_read(conn, schema, sources, queries)
|
|
1443
|
+
for table in [t for t in queries if t not in schema.column_types]:
|
|
1444
|
+
# A table whose types are not read has none, and every SQL form
|
|
1445
|
+
# chosen by a type takes the form for an unknown one: say so.
|
|
1446
|
+
# A whole backend was in that state unnoticed.
|
|
1447
|
+
try:
|
|
1448
|
+
rows = self._describe_table(conn, backend, table, query=queries[table],
|
|
1449
|
+
written=sources[table].qualified)
|
|
1450
|
+
except Exception as exc: # noqa: BLE001
|
|
1451
|
+
from prograph.logging import get_plan_logger
|
|
1452
|
+
get_plan_logger().warning("column_types_skip", reason="describe_failed",
|
|
1453
|
+
table=table, backend=backend, error=str(exc))
|
|
1454
|
+
if table in files:
|
|
1455
|
+
# A YAML file source the connection has no table or view for: refused by name now.
|
|
1456
|
+
from prograph.utils.exceptions import SourceNotReadableError
|
|
1457
|
+
raise SourceNotReadableError(
|
|
1458
|
+
f"the source {table!r} is a file (sources.{table}.file) and the {backend} connection has "
|
|
1459
|
+
f"no table or view {table!r}: load the file into a table and name it in the source "
|
|
1460
|
+
f"(sources.{table}.table), or build the engine with from_config/from_yaml, which read "
|
|
1461
|
+
f"files into frames") from exc
|
|
1462
|
+
if not isinstance(exc, NotImplementedError):
|
|
1463
|
+
# A source the database would not describe (missing, not readable); a source this backend
|
|
1464
|
+
# cannot describe (a query on MySQL, SQL Server, Oracle) keeps its columns untyped instead.
|
|
1465
|
+
self._unreadable_sources[table] = (sources[table].qualified, " ".join(str(exc).split())[:400])
|
|
1466
|
+
continue
|
|
1467
|
+
if rows:
|
|
1468
|
+
# Databricks' TIMESTAMP is an instant, as Spark's.
|
|
1469
|
+
instant = backend == 'databricks_sql'
|
|
1470
|
+
collations = C.string_collations(conn, backend, table, query=queries[table],
|
|
1471
|
+
written=sources[table].qualified, described=rows)
|
|
1472
|
+
schema.column_types[table] = C.mark_bytewise(
|
|
1473
|
+
{name: T.of_sql_type(kind, timestamp_is_an_instant=instant) for name, kind in rows},
|
|
1474
|
+
backend, collations)
|
|
1475
|
+
else:
|
|
1476
|
+
from prograph.logging import get_plan_logger
|
|
1477
|
+
get_plan_logger().warning("column_types_skip", reason="no_columns_described",
|
|
1478
|
+
table=table, backend=backend)
|
|
1479
|
+
self._unreadable_sources[table] = (sources[table].qualified, "the database described no columns")
|
|
1480
|
+
|
|
1481
|
+
def _warehouse_catalog_read(self, conn, schema, sources, queries) -> None:
|
|
1482
|
+
"""A Databricks SQL warehouse's column types for every mapped table (not a query source) from one read
|
|
1483
|
+
of `system.information_schema.columns` (`full_data_type`: DESCRIBE's vocabulary), and the tables that
|
|
1484
|
+
declare a primary key from one read of `table_constraints` (whether it is RELY is in DESCRIBE TABLE
|
|
1485
|
+
EXTENDED only: read per such table when a plan asks, `keys._constraint_keys`). A read that fails
|
|
1486
|
+
leaves the tables to be described one by one."""
|
|
1487
|
+
from prograph.graph.drivers import fetch_rows
|
|
1488
|
+
from prograph.schema import collations as C
|
|
1489
|
+
from prograph.schema import types as T
|
|
1490
|
+
|
|
1491
|
+
def parts(written):
|
|
1492
|
+
text = str(written)
|
|
1493
|
+
if text.count('`') and any('.' in seg for seg in text.split('`')[1::2]):
|
|
1494
|
+
return None # a delimited name holding a dot: described on its own
|
|
1495
|
+
names = [p.strip('`') for p in text.split('.')]
|
|
1496
|
+
if len(names) > 3:
|
|
1497
|
+
return None
|
|
1498
|
+
names = [None] * (3 - len(names)) + names
|
|
1499
|
+
return [n.lower() if n else None for n in names]
|
|
1500
|
+
wanted = {t: parts(sources[t].qualified) for t in queries if queries[t] is None}
|
|
1501
|
+
wanted = {t: spec for t, spec in wanted.items() if spec is not None}
|
|
1502
|
+
if not wanted:
|
|
1503
|
+
return
|
|
1504
|
+
|
|
1505
|
+
def where():
|
|
1506
|
+
ors = []
|
|
1507
|
+
for cat, sch, name in wanted.values():
|
|
1508
|
+
ors.append("(table_catalog = " + (f"'{cat}'" if cat else "current_catalog()") +
|
|
1509
|
+
" AND table_schema = " + (f"'{sch}'" if sch else "current_schema()") +
|
|
1510
|
+
f" AND table_name = '{name}')")
|
|
1511
|
+
return " OR ".join(ors)
|
|
1512
|
+
if any("'" in (n or "") for spec in wanted.values() for n in spec):
|
|
1513
|
+
return # a name with a quote: described one by one
|
|
1514
|
+
try:
|
|
1515
|
+
rows = fetch_rows(conn, "SELECT table_catalog, table_schema, table_name, column_name, full_data_type, "
|
|
1516
|
+
"current_catalog(), current_schema() "
|
|
1517
|
+
f"FROM system.information_schema.columns WHERE {where()} "
|
|
1518
|
+
"ORDER BY table_catalog, table_schema, table_name, ordinal_position")[1]
|
|
1519
|
+
keyed = fetch_rows(conn, "SELECT DISTINCT table_catalog, table_schema, table_name "
|
|
1520
|
+
"FROM system.information_schema.table_constraints "
|
|
1521
|
+
f"WHERE constraint_type = 'PRIMARY KEY' AND ({where()})")[1]
|
|
1522
|
+
except Exception as exc: # noqa: BLE001 - described one by one instead
|
|
1523
|
+
from prograph.logging import get_plan_logger
|
|
1524
|
+
get_plan_logger().info("warehouse_catalog_read_skipped", error=str(exc)[:200])
|
|
1525
|
+
return
|
|
1526
|
+
if not rows:
|
|
1527
|
+
return
|
|
1528
|
+
cur_cat, cur_sch = str(rows[0][5]).lower(), str(rows[0][6]).lower()
|
|
1529
|
+
|
|
1530
|
+
def full(spec):
|
|
1531
|
+
cat, sch, name = spec
|
|
1532
|
+
return (cat or cur_cat, sch or cur_sch, name)
|
|
1533
|
+
by_table: Dict[tuple, list] = {}
|
|
1534
|
+
for cat, sch, name, column, kind, _, _ in rows:
|
|
1535
|
+
by_table.setdefault((str(cat).lower(), str(sch).lower(), str(name).lower()), []).append(
|
|
1536
|
+
(str(column), str(kind)))
|
|
1537
|
+
for table, spec in wanted.items():
|
|
1538
|
+
columns = by_table.get(full(spec))
|
|
1539
|
+
if columns:
|
|
1540
|
+
kinds = dict(columns)
|
|
1541
|
+
schema.column_types[table] = C.mark_bytewise(
|
|
1542
|
+
{c: T.of_sql_type(k, timestamp_is_an_instant=True) for c, k in columns},
|
|
1543
|
+
'databricks_sql', lambda c, kinds=kinds: C.collation_in_type(kinds.get(c, "")))
|
|
1544
|
+
graph = getattr(getattr(self, 'executor', None), 'graph', None)
|
|
1545
|
+
if graph is not None:
|
|
1546
|
+
# Only a table the columns read found is known to have no primary key; one it missed (a view, a
|
|
1547
|
+
# name it did not match, no privilege) is asked on its own (DESCRIBE TABLE EXTENDED) as before.
|
|
1548
|
+
with_key = {tuple(str(v).lower() for v in r) for r in keyed}
|
|
1549
|
+
graph.__dict__["_warehouse_keyed"] = {sources[t].qualified: full(spec) in with_key
|
|
1550
|
+
for t, spec in wanted.items() if full(spec) in by_table}
|
|
1551
|
+
|
|
1552
|
+
def _record_query_column_types(self, schema) -> None:
|
|
1553
|
+
"""The column types of each rr:sqlQuery source over frames (`query_sources.describe`)."""
|
|
1554
|
+
from prograph.schema.query_sources import describe
|
|
1555
|
+
describe(self, schema)
|
|
1556
|
+
|
|
1557
|
+
def _record_frame_decimal_digits(self, frames) -> None:
|
|
1558
|
+
"""A pandas frame's DECIMAL columns' (precision, scale) as the
|
|
1559
|
+
graph's embedded DuckDB reads them, the type its SQL sees: a column
|
|
1560
|
+
of Decimals has no size of its own in pandas, and is read at the
|
|
1561
|
+
size all its values fit (`pandas_graph.frames_connection`). A decimal quotient's
|
|
1562
|
+
SQL form is chosen by it (ExprTranslator.decimal_digits). A column DuckDB does not read as a DECIMAL keeps none."""
|
|
1563
|
+
from dataclasses import replace
|
|
1564
|
+
from prograph.schema import types as T
|
|
1565
|
+
local = getattr(getattr(getattr(self, 'executor', None), 'graph', None), '_local_sql', None)
|
|
1566
|
+
schema = self._schema
|
|
1567
|
+
if local is None or schema is None:
|
|
1568
|
+
return
|
|
1569
|
+
for table, frame in frames.items():
|
|
1570
|
+
types = schema.column_types.get(table) or {}
|
|
1571
|
+
if not any(t.exact and t.digits is None for t in types.values()):
|
|
1572
|
+
continue
|
|
1573
|
+
cursor = None
|
|
1574
|
+
try:
|
|
1575
|
+
cursor = local.cursor()
|
|
1576
|
+
relation = cursor.from_df(frame)
|
|
1577
|
+
described = list(zip(relation.columns, (str(t) for t in relation.types)))
|
|
1578
|
+
except Exception: # noqa: BLE001 - a size is an optimisation, not required
|
|
1579
|
+
continue
|
|
1580
|
+
finally:
|
|
1581
|
+
if cursor is not None:
|
|
1582
|
+
cursor.close()
|
|
1583
|
+
for name, kind in described:
|
|
1584
|
+
held = types.get(name)
|
|
1585
|
+
if held is not None and held.exact and held.digits is None:
|
|
1586
|
+
read = T.of_sql_type(str(kind))
|
|
1587
|
+
if read.exact and read.digits is not None:
|
|
1588
|
+
types[name] = replace(held, digits=read.digits)
|
|
1589
|
+
|
|
1590
|
+
@staticmethod
|
|
1591
|
+
def _describe_table(conn, backend, table, query=None, written=None):
|
|
1592
|
+
"""(column, type name) for a table on a SQL connection; `query`:
|
|
1593
|
+
the SQL query that is the source instead (an R2RML view,
|
|
1594
|
+
`TableMapping.source_query`), described by the database where it
|
|
1595
|
+
describes a query (DuckDB's DESCRIBE, Databricks' DESCRIBE QUERY).
|
|
1596
|
+
`written`: the name as the statement reads it (`Source.qualified`); a
|
|
1597
|
+
part not delimited there is looked up as the database folds it
|
|
1598
|
+
(`Dialect.fold_identifier`: PostgreSQL's catalog holds `Person` as
|
|
1599
|
+
person)."""
|
|
1600
|
+
def fetch(sql, params=()):
|
|
1601
|
+
from prograph.graph.drivers import fetch_rows
|
|
1602
|
+
return [(str(r[0]), str(r[1])) for r in fetch_rows(conn, sql, params)[1]]
|
|
1603
|
+
|
|
1604
|
+
schema_name, _, name = table.rpartition('.')
|
|
1605
|
+
if query is not None and backend == 'postgresql':
|
|
1606
|
+
return _describe_postgresql_query(conn, query)
|
|
1607
|
+
if query is not None and backend not in ('duckdb', 'databricks_sql'):
|
|
1608
|
+
raise NotImplementedError(f"no description of a query on {backend}")
|
|
1609
|
+
if backend == 'duckdb':
|
|
1610
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
1611
|
+
# The name as the statement reads it where it differs (a qualification: `written` is SQL text
|
|
1612
|
+
# already), else the source's own name, quoted.
|
|
1613
|
+
quoted = query or (written if written and written != table
|
|
1614
|
+
else get_dialect_instance("duckdb").quote_qualified(table))
|
|
1615
|
+
return [(r[0], r[1]) for r in conn.execute(f"DESCRIBE {quoted}").fetchall()]
|
|
1616
|
+
if backend == 'oracle':
|
|
1617
|
+
# Which NUMBER holds integers is oracle_graph.number_is_integer's
|
|
1618
|
+
# rule, the one the result types follow; the table's name is kept
|
|
1619
|
+
# in upper case unless it was created quoted.
|
|
1620
|
+
from prograph.graph.drivers import fetch_rows
|
|
1621
|
+
from prograph.graph.oracle_graph import number_digits, number_is_integer
|
|
1622
|
+
rows = fetch_rows(conn, "SELECT column_name, data_type, data_scale, data_precision FROM user_tab_columns "
|
|
1623
|
+
"WHERE UPPER(table_name) = :1", (name.upper(),))[1]
|
|
1624
|
+
described = []
|
|
1625
|
+
for column, kind, scale, precision in rows:
|
|
1626
|
+
digits = number_digits(precision, scale) if kind == 'NUMBER' else None
|
|
1627
|
+
if kind == 'NUMBER' and number_is_integer(scale):
|
|
1628
|
+
kind = 'BIGINT'
|
|
1629
|
+
elif digits:
|
|
1630
|
+
# A NUMBER(p, s) is the exact decimal of its digits, as its answers are read
|
|
1631
|
+
# (`oracle_graph.number_digits`), so the plan knows them.
|
|
1632
|
+
kind = f'DECIMAL({digits[0]},{digits[1]})'
|
|
1633
|
+
described.append((str(column), str(kind)))
|
|
1634
|
+
return described
|
|
1635
|
+
if backend in ('postgresql', 'mysql', 'sqlserver'):
|
|
1636
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
1637
|
+
marker = get_dialect_instance(backend).param_placeholder
|
|
1638
|
+
kind = "data_type"
|
|
1639
|
+
if backend == 'postgresql':
|
|
1640
|
+
# An array column's data_type is the one word ARRAY, whatever
|
|
1641
|
+
# it holds, which names no type: every array column was
|
|
1642
|
+
# untyped, and two of them compared with `=` were compared as
|
|
1643
|
+
# the database compares arrays. The element
|
|
1644
|
+
# type is the array type's own: `integer[]`, `text[]`. A
|
|
1645
|
+
# domain over an array reports its base type here too.
|
|
1646
|
+
# PostgreSQL keeps no dimensions: an `integer[][]` column is
|
|
1647
|
+
# `integer[]` to its catalog and to this.
|
|
1648
|
+
kind = ("CASE WHEN data_type = 'ARRAY' THEN (SELECT pg_catalog.format_type("
|
|
1649
|
+
"t.typelem, NULL) FROM pg_catalog.pg_type t JOIN pg_catalog.pg_namespace n "
|
|
1650
|
+
"ON n.oid = t.typnamespace WHERE t.typname = udt_name "
|
|
1651
|
+
"AND n.nspname = udt_schema) || '[]' "
|
|
1652
|
+
# A NUMERIC(p, s) column's digits, which its answers are read back at
|
|
1653
|
+
# (postgresql_graph.numeric_digits); an unconstrained NUMERIC has none.
|
|
1654
|
+
"WHEN data_type = 'numeric' AND numeric_precision IS NOT NULL "
|
|
1655
|
+
"AND numeric_scale BETWEEN 0 AND numeric_precision THEN "
|
|
1656
|
+
"'numeric(' || numeric_precision || ',' || numeric_scale || ')' "
|
|
1657
|
+
# A bit string (bit(n), even bit(1): PostgreSQL has no boolean in it) is its text.
|
|
1658
|
+
"WHEN data_type = 'bit' THEN "
|
|
1659
|
+
"'bit(' || character_maximum_length || ')' "
|
|
1660
|
+
# The citext extension's type, which the catalog calls USER-DEFINED.
|
|
1661
|
+
"WHEN data_type = 'USER-DEFINED' AND udt_name = 'citext' THEN 'citext' "
|
|
1662
|
+
"ELSE data_type END")
|
|
1663
|
+
sql = (f"SELECT column_name, {kind} FROM information_schema.columns "
|
|
1664
|
+
f"WHERE table_name = {marker(1)}")
|
|
1665
|
+
if backend == 'postgresql':
|
|
1666
|
+
# The table the statement reads, resolved as PostgreSQL resolves the written
|
|
1667
|
+
# name (to_regclass: its search path for a bare name, its folding), so a
|
|
1668
|
+
# same-named table in another schema lends no column or type.
|
|
1669
|
+
sql = (f"SELECT column_name, {kind} FROM information_schema.columns "
|
|
1670
|
+
"WHERE (table_schema, table_name) = (SELECT n.nspname, c.relname "
|
|
1671
|
+
"FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n "
|
|
1672
|
+
f"ON n.oid = c.relnamespace WHERE c.oid = to_regclass({marker(1)})) "
|
|
1673
|
+
"ORDER BY ordinal_position")
|
|
1674
|
+
rows = fetch(sql, (str(written or table),))
|
|
1675
|
+
unconstrained = [c for c, k in rows if k == "numeric"]
|
|
1676
|
+
if unconstrained:
|
|
1677
|
+
warn_unknown_digits(f"The table {str(written or table)!r}", unconstrained,
|
|
1678
|
+
"that declare no precision and scale")
|
|
1679
|
+
return rows
|
|
1680
|
+
fold = get_dialect_instance(backend).fold_identifier
|
|
1681
|
+
delimited = [p[:1] == '"' for p in str(written or table).split('.')]
|
|
1682
|
+
params = [name if delimited[-1] else fold(name)]
|
|
1683
|
+
if schema_name:
|
|
1684
|
+
sql += f" AND table_schema = {marker(2)}"
|
|
1685
|
+
part = schema_name.split('.')[-1]
|
|
1686
|
+
params.append(part if len(delimited) > 1 and delimited[-2] else fold(part))
|
|
1687
|
+
return fetch(sql, tuple(params))
|
|
1688
|
+
if backend == 'databricks_sql':
|
|
1689
|
+
# DESCRIBE TABLE answers for a table and for a view, by a two-
|
|
1690
|
+
# or a three-part name, with the whole type (`array<bigint>`,
|
|
1691
|
+
# `decimal(10,2)`); information_schema needs the catalog and
|
|
1692
|
+
# gives another vocabulary (LONG, SHORT). After the columns it
|
|
1693
|
+
# lists a partitioned table's partition columns again, under
|
|
1694
|
+
# lines that start with `#`. Without this branch a warehouse
|
|
1695
|
+
# engine knew no column's type, and an integer column divided
|
|
1696
|
+
# by an integer was not an integer division.
|
|
1697
|
+
from prograph.graph.drivers import fetch_rows
|
|
1698
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
1699
|
+
described = (f"DESCRIBE QUERY {query}" if query is not None else
|
|
1700
|
+
f"DESCRIBE TABLE {get_dialect_instance('databricks_sql').quote_qualified(table)}")
|
|
1701
|
+
columns = []
|
|
1702
|
+
for row in fetch_rows(conn, described)[1]:
|
|
1703
|
+
column, kind = str(row[0] or ''), str(row[1] or '')
|
|
1704
|
+
if not column or column.startswith('#'):
|
|
1705
|
+
break
|
|
1706
|
+
columns.append((column, kind))
|
|
1707
|
+
return columns
|
|
1708
|
+
return []
|
|
1709
|
+
|
|
1710
|
+
@_one_at_a_time
|
|
1711
|
+
def update_schema(
|
|
1712
|
+
self,
|
|
1713
|
+
add_nodes: Optional[List[NodeMapping]] = None,
|
|
1714
|
+
remove_node_labels: Optional[List[str]] = None,
|
|
1715
|
+
replace_nodes: Optional[List[NodeMapping]] = None,
|
|
1716
|
+
add_relationships: Optional[List[RelationshipMapping]] = None,
|
|
1717
|
+
remove_rel_types: Optional[List[Union[str, Tuple[str, str, str]]]] = None,
|
|
1718
|
+
add_dataframes: Optional[Dict[str, Any]] = None,
|
|
1719
|
+
add_table_stats: Optional[Dict[str, int]] = None,
|
|
1720
|
+
) -> SchemaChangeSet:
|
|
1721
|
+
"""Update the schema on a live engine without recreating it.
|
|
1722
|
+
|
|
1723
|
+
Mutates the shared GraphSchema in place and selectively invalidates
|
|
1724
|
+
caches. Only rebuilds VL indexes if relationship mappings changed.
|
|
1725
|
+
|
|
1726
|
+
Ordering: removes relationships first, then nodes, then adds
|
|
1727
|
+
replacements, then adds new mappings (prevents FK constraint errors).
|
|
1728
|
+
|
|
1729
|
+
Args:
|
|
1730
|
+
add_nodes: Node mappings to add
|
|
1731
|
+
remove_node_labels: Node labels to remove
|
|
1732
|
+
replace_nodes: Node mappings to replace (matched by label)
|
|
1733
|
+
add_relationships: Relationship mappings to add
|
|
1734
|
+
remove_rel_types: Relationship types to remove. Each entry is
|
|
1735
|
+
either a string (removes all mappings for that type) or a
|
|
1736
|
+
tuple of (rel_type, start_label, end_label) for targeted
|
|
1737
|
+
removal.
|
|
1738
|
+
add_dataframes: New DataFrames to inject into the engine
|
|
1739
|
+
add_table_stats: Table name -> row count for new tables
|
|
1740
|
+
|
|
1741
|
+
Returns:
|
|
1742
|
+
SchemaChangeSet describing what changed
|
|
1743
|
+
|
|
1744
|
+
Raises:
|
|
1745
|
+
ValueError: If required DataFrames are missing or a node label
|
|
1746
|
+
has dependent relationships
|
|
1747
|
+
"""
|
|
1748
|
+
if self._schema is None:
|
|
1749
|
+
raise ValueError("Cannot update schema: engine has no schema")
|
|
1750
|
+
|
|
1751
|
+
changes = SchemaChangeSet()
|
|
1752
|
+
schema = self._schema
|
|
1753
|
+
add_dataframes = add_dataframes or {}
|
|
1754
|
+
|
|
1755
|
+
# Phase 1: Validate new mappings have DataFrames available
|
|
1756
|
+
graph = getattr(self.executor, 'graph', None)
|
|
1757
|
+
existing_dfs = getattr(graph, 'dataframes', {}) if graph else {}
|
|
1758
|
+
|
|
1759
|
+
for mapping in (add_nodes or []) + (replace_nodes or []) + (add_relationships or []):
|
|
1760
|
+
table = mapping.source_table
|
|
1761
|
+
if (
|
|
1762
|
+
table not in existing_dfs
|
|
1763
|
+
and table not in add_dataframes
|
|
1764
|
+
):
|
|
1765
|
+
# For Spark backend, try loading from catalog
|
|
1766
|
+
if self.backend == 'spark' and self._spark is not None:
|
|
1767
|
+
try:
|
|
1768
|
+
add_dataframes[table] = self._spark.table(
|
|
1769
|
+
mapping.qualified_source_table
|
|
1770
|
+
)
|
|
1771
|
+
except Exception as e:
|
|
1772
|
+
raise ValueError(
|
|
1773
|
+
f"Table '{table}' not available in existing "
|
|
1774
|
+
f"dataframes, add_dataframes, or catalog: {e}"
|
|
1775
|
+
)
|
|
1776
|
+
else:
|
|
1777
|
+
raise ValueError(
|
|
1778
|
+
f"Table '{table}' not in existing dataframes "
|
|
1779
|
+
f"and not provided in add_dataframes"
|
|
1780
|
+
)
|
|
1781
|
+
|
|
1782
|
+
# The node mappings as they will be: refused before anything
|
|
1783
|
+
# changes if their IRIs could not name one node each.
|
|
1784
|
+
from types import SimpleNamespace
|
|
1785
|
+
from prograph.schema.iri import check_identities
|
|
1786
|
+
going = set(remove_node_labels or []) | {m.label for m in (replace_nodes or [])}
|
|
1787
|
+
check_identities(SimpleNamespace(node_mappings=(
|
|
1788
|
+
[m for m in schema.node_mappings if m.label not in going]
|
|
1789
|
+
+ list(replace_nodes or []) + list(add_nodes or []))))
|
|
1790
|
+
|
|
1791
|
+
# Phase 2: Mutate schema (order matters)
|
|
1792
|
+
|
|
1793
|
+
# 2a. Remove relationships first
|
|
1794
|
+
for entry in (remove_rel_types or []):
|
|
1795
|
+
if isinstance(entry, tuple):
|
|
1796
|
+
rel_type, start_label, end_label = entry
|
|
1797
|
+
removed = schema.remove_relationship_mapping(
|
|
1798
|
+
rel_type, start_label=start_label, end_label=end_label
|
|
1799
|
+
)
|
|
1800
|
+
else:
|
|
1801
|
+
removed = schema.remove_relationship_mapping(entry)
|
|
1802
|
+
if removed:
|
|
1803
|
+
changes.removed_rel_types.add(
|
|
1804
|
+
entry if isinstance(entry, str) else entry[0]
|
|
1805
|
+
)
|
|
1806
|
+
for m in removed:
|
|
1807
|
+
changes.removed_tables.add(m.source_table)
|
|
1808
|
+
|
|
1809
|
+
# 2b. Remove nodes
|
|
1810
|
+
for label in (remove_node_labels or []):
|
|
1811
|
+
removed = schema.remove_node_mapping(label)
|
|
1812
|
+
if removed is not None:
|
|
1813
|
+
changes.removed_node_labels.add(label)
|
|
1814
|
+
changes.removed_tables.add(removed.source_table)
|
|
1815
|
+
|
|
1816
|
+
# 2c. Replace nodes
|
|
1817
|
+
for mapping in (replace_nodes or []):
|
|
1818
|
+
old = schema.replace_node_mapping(mapping)
|
|
1819
|
+
if old is not None:
|
|
1820
|
+
changes.replaced_node_labels.add(mapping.label)
|
|
1821
|
+
if old.source_table != mapping.source_table:
|
|
1822
|
+
changes.removed_tables.add(old.source_table)
|
|
1823
|
+
changes.added_tables.add(mapping.source_table)
|
|
1824
|
+
else:
|
|
1825
|
+
changes.added_node_labels.add(mapping.label)
|
|
1826
|
+
changes.added_tables.add(mapping.source_table)
|
|
1827
|
+
|
|
1828
|
+
# 2d. Add new nodes
|
|
1829
|
+
for mapping in (add_nodes or []):
|
|
1830
|
+
schema.add_node_mapping(mapping)
|
|
1831
|
+
changes.added_node_labels.add(mapping.label)
|
|
1832
|
+
changes.added_tables.add(mapping.source_table)
|
|
1833
|
+
|
|
1834
|
+
# 2e. Add new relationships
|
|
1835
|
+
for mapping in (add_relationships or []):
|
|
1836
|
+
schema.add_relationship_mapping(mapping)
|
|
1837
|
+
changes.added_rel_types.add(mapping.rel_type)
|
|
1838
|
+
changes.added_tables.add(mapping.source_table)
|
|
1839
|
+
|
|
1840
|
+
# Phase 3: Inject new DataFrames
|
|
1841
|
+
if add_dataframes and graph is not None:
|
|
1842
|
+
graph.dataframes.update(add_dataframes)
|
|
1843
|
+
# Also update executor-level dataframes if present
|
|
1844
|
+
if add_dataframes and hasattr(self.executor, 'dataframes'):
|
|
1845
|
+
self.executor.dataframes.update(add_dataframes)
|
|
1846
|
+
if add_dataframes:
|
|
1847
|
+
if self._dataframes is None:
|
|
1848
|
+
self._dataframes = {}
|
|
1849
|
+
self._dataframes.update(add_dataframes)
|
|
1850
|
+
|
|
1851
|
+
# Phase 4: Add table stats
|
|
1852
|
+
if add_table_stats:
|
|
1853
|
+
for table_name, row_count in add_table_stats.items():
|
|
1854
|
+
schema.set_table_stats(table_name, row_count=row_count)
|
|
1855
|
+
|
|
1856
|
+
# Phase 5: Invalidate caches
|
|
1857
|
+
if graph is not None and hasattr(graph, '_invalidate_caches'):
|
|
1858
|
+
graph._invalidate_caches(changes)
|
|
1859
|
+
|
|
1860
|
+
# Rebind ontology if present and any mapping changed
|
|
1861
|
+
if changes.has_changes and self._ontology is not None and schema is not None:
|
|
1862
|
+
self._ontology.bind_to_schema(schema, strict=False)
|
|
1863
|
+
|
|
1864
|
+
self._record_column_types()
|
|
1865
|
+
return changes
|
|
1866
|
+
|
|
1867
|
+
def add_mapping(
|
|
1868
|
+
self,
|
|
1869
|
+
mapping: Union[NodeMapping, RelationshipMapping],
|
|
1870
|
+
dataframe: Optional[Any] = None,
|
|
1871
|
+
row_count: Optional[int] = None,
|
|
1872
|
+
) -> None:
|
|
1873
|
+
"""Add a single node or relationship mapping.
|
|
1874
|
+
|
|
1875
|
+
Convenience wrapper around update_schema().
|
|
1876
|
+
|
|
1877
|
+
Args:
|
|
1878
|
+
mapping: A NodeMapping or RelationshipMapping to add
|
|
1879
|
+
dataframe: Optional DataFrame for the mapping's source table
|
|
1880
|
+
row_count: Optional row count for broadcast join decisions
|
|
1881
|
+
"""
|
|
1882
|
+
add_dfs = {mapping.source_table: dataframe} if dataframe is not None else None
|
|
1883
|
+
add_stats = {mapping.source_table: row_count} if row_count is not None else None
|
|
1884
|
+
|
|
1885
|
+
if isinstance(mapping, NodeMapping):
|
|
1886
|
+
self.update_schema(
|
|
1887
|
+
add_nodes=[mapping],
|
|
1888
|
+
add_dataframes=add_dfs,
|
|
1889
|
+
add_table_stats=add_stats,
|
|
1890
|
+
)
|
|
1891
|
+
elif isinstance(mapping, RelationshipMapping):
|
|
1892
|
+
self.update_schema(
|
|
1893
|
+
add_relationships=[mapping],
|
|
1894
|
+
add_dataframes=add_dfs,
|
|
1895
|
+
add_table_stats=add_stats,
|
|
1896
|
+
)
|
|
1897
|
+
else:
|
|
1898
|
+
raise TypeError(
|
|
1899
|
+
f"Expected NodeMapping or RelationshipMapping, got {type(mapping).__name__}"
|
|
1900
|
+
)
|
|
1901
|
+
|
|
1902
|
+
def remove_mapping(
|
|
1903
|
+
self,
|
|
1904
|
+
label_or_type: str,
|
|
1905
|
+
kind: str = 'node',
|
|
1906
|
+
) -> None:
|
|
1907
|
+
"""Remove a single mapping by label (node) or type (relationship).
|
|
1908
|
+
|
|
1909
|
+
Convenience wrapper around update_schema().
|
|
1910
|
+
|
|
1911
|
+
Args:
|
|
1912
|
+
label_or_type: The node label or relationship type to remove
|
|
1913
|
+
kind: 'node' or 'relationship'
|
|
1914
|
+
|
|
1915
|
+
Raises:
|
|
1916
|
+
ValueError: If kind is not 'node' or 'relationship', or if
|
|
1917
|
+
the node has dependent relationships
|
|
1918
|
+
"""
|
|
1919
|
+
if kind == 'node':
|
|
1920
|
+
self.update_schema(remove_node_labels=[label_or_type])
|
|
1921
|
+
elif kind == 'relationship':
|
|
1922
|
+
self.update_schema(remove_rel_types=[label_or_type])
|
|
1923
|
+
else:
|
|
1924
|
+
raise ValueError(f"kind must be 'node' or 'relationship', got '{kind}'")
|
|
1925
|
+
|
|
1926
|
+
|
|
1927
|
+
@property
|
|
1928
|
+
def ontology(self) -> Optional['OntologyLayer']:
|
|
1929
|
+
"""Get the ontology layer, if configured."""
|
|
1930
|
+
return self._ontology
|
|
1931
|
+
|
|
1932
|
+
@property
|
|
1933
|
+
def profile(self) -> Optional[str]:
|
|
1934
|
+
"""The OWL profile the ontology is reasoned under (``'ql'`` or ``'rl'``); None with no
|
|
1935
|
+
ontology, where a query matches simply."""
|
|
1936
|
+
return self._profile
|
|
1937
|
+
|
|
1938
|
+
def ontology_report(self):
|
|
1939
|
+
"""Which W3C OWL 2 profiles the engine's ontology fits and which of its axioms the engine answers
|
|
1940
|
+
(``prograph.plan.ir.profiles.OntologyReport``): every axiom with the profiles it belongs to
|
|
1941
|
+
(``profiles``: of ``'ql'``, ``'rl'``; ``why`` it is outside the others) and those ProGraph answers it
|
|
1942
|
+
under today (``answered``); ``fits`` ('both', 'ql', 'rl' or 'neither'); ``refused``, the axioms the
|
|
1943
|
+
engine's profile does not answer, and ``dropped``, those ``on_unsupported="ignore"`` dropped.
|
|
1944
|
+
None with no ontology."""
|
|
1945
|
+
if self._ontology is None:
|
|
1946
|
+
return None
|
|
1947
|
+
from prograph.plan.ir.entry import profile_report
|
|
1948
|
+
return profile_report(self)[1]
|
|
1949
|
+
|
|
1950
|
+
@property
|
|
1951
|
+
def default_shapes(self) -> Optional[str]:
|
|
1952
|
+
"""Operator-configured SHACL shapes for the runtime guardrail.
|
|
1953
|
+
|
|
1954
|
+
A Turtle string or path to a ``.ttl`` file, or ``None``. When set,
|
|
1955
|
+
the MCP server exposes a zero-argument ``check_data_quality`` tool
|
|
1956
|
+
that validates the engine's current data against these shapes, the
|
|
1957
|
+
deterministic honesty boundary, surfaced to agents automatically.
|
|
1958
|
+
"""
|
|
1959
|
+
return self._default_shapes
|
|
1960
|
+
|
|
1961
|
+
def iri_of(self, label: str, key: Any) -> str:
|
|
1962
|
+
"""The IRI of the node of `label` with this key: a value, or a
|
|
1963
|
+
tuple in ``id_columns`` order for a composite key. The one way a
|
|
1964
|
+
caller holding a key gets the identity the semantic layer uses
|
|
1965
|
+
(SHACL focus nodes, SPARQL results, provenance); the inverse of
|
|
1966
|
+
:meth:`resolve_iri`. The node is not looked up.
|
|
1967
|
+
|
|
1968
|
+
Raises ``ValueError`` for an unknown label, a key of the wrong
|
|
1969
|
+
arity or a missing key part, and ``UnstableIdentityError`` where
|
|
1970
|
+
the label's ``iri_template`` reads columns other than its key.
|
|
1971
|
+
"""
|
|
1972
|
+
return self.iris_of(label, [key])[0]
|
|
1973
|
+
|
|
1974
|
+
def iris_of(self, label: str, keys: Sequence[Any]) -> List[str]:
|
|
1975
|
+
""":meth:`iri_of` for many keys of one label: one IRI per key, in
|
|
1976
|
+
order."""
|
|
1977
|
+
from prograph.schema.iri import node_iri_from_id, require_key_template
|
|
1978
|
+
from prograph.schema.types import is_missing
|
|
1979
|
+
|
|
1980
|
+
if self._schema is None:
|
|
1981
|
+
raise ValueError("iri_of needs a schema")
|
|
1982
|
+
mapping = self._schema.get_node_mapping(self._schema.require_name('label', label))
|
|
1983
|
+
require_key_template(mapping)
|
|
1984
|
+
arity = len(mapping.id_columns)
|
|
1985
|
+
iris = []
|
|
1986
|
+
for key in keys:
|
|
1987
|
+
parts = tuple(key) if isinstance(key, (tuple, list)) else (key,)
|
|
1988
|
+
if len(parts) != arity:
|
|
1989
|
+
raise ValueError(
|
|
1990
|
+
f"{label} is keyed by {', '.join(mapping.id_columns)}: "
|
|
1991
|
+
f"a key has {arity} part(s), got {key!r}")
|
|
1992
|
+
if any(is_missing(part) for part in parts):
|
|
1993
|
+
raise ValueError(f"{key!r} is no key of {label}: a part of it is missing")
|
|
1994
|
+
iris.append(node_iri_from_id(mapping, parts if arity > 1 else parts[0]))
|
|
1995
|
+
return iris
|
|
1996
|
+
|
|
1997
|
+
def set_properties(
|
|
1998
|
+
self,
|
|
1999
|
+
label: str,
|
|
2000
|
+
identities: Sequence[Any],
|
|
2001
|
+
values: Dict[str, Any],
|
|
2002
|
+
*,
|
|
2003
|
+
by: str = "key",
|
|
2004
|
+
) -> int:
|
|
2005
|
+
"""Set properties on the nodes of `label` named by `identities`,
|
|
2006
|
+
and return how many rows were written.
|
|
2007
|
+
|
|
2008
|
+
Args:
|
|
2009
|
+
label: The nodes' label. Its mapping must be ``mutable``.
|
|
2010
|
+
identities: Keys (a value, or a tuple in ``id_columns`` order
|
|
2011
|
+
for a composite key), or with ``by="iri"`` the nodes' IRIs
|
|
2012
|
+
(:meth:`iri_of`). An IRI that names no node of the label
|
|
2013
|
+
writes nothing.
|
|
2014
|
+
values: Property name to literal value (None, bool, int, float
|
|
2015
|
+
or str). Names are checked against the schema.
|
|
2016
|
+
by: ``"key"`` or ``"iri"``: what `identities` holds. It is
|
|
2017
|
+
stated, not guessed, because a key may be text that looks
|
|
2018
|
+
like an IRI.
|
|
2019
|
+
|
|
2020
|
+
Only rows the label's mapping owns are written (its
|
|
2021
|
+
``filter_predicate``, on a table labels share), and never a row a
|
|
2022
|
+
reasoner materialized (``_inferred``): the rule of every write by
|
|
2023
|
+
identity (:mod:`prograph.writes`). The key and the columns of
|
|
2024
|
+
the ``filter_predicate`` cannot be set. On a SQL database it is one
|
|
2025
|
+
``UPDATE``; on Spark, a Delta ``UPDATE``; on DataFrames, the frame
|
|
2026
|
+
is changed in place.
|
|
2027
|
+
|
|
2028
|
+
It is the one write ProGraph makes to a table it maps on request,
|
|
2029
|
+
and only to a mapping marked ``mutable`` (decided 2026-11-04); a
|
|
2030
|
+
query's CREATE, SET or DELETE and a SPARQL Update stay refused
|
|
2031
|
+
(:mod:`prograph.read_only`).
|
|
2032
|
+
|
|
2033
|
+
Raises ``ReadOnlyGraphError`` for a mapping that is not mutable and
|
|
2034
|
+
on the memory backend (which maps no table), ``ValueError`` for an
|
|
2035
|
+
unknown label or property, a key of the wrong arity, or a value
|
|
2036
|
+
that is no literal.
|
|
2037
|
+
"""
|
|
2038
|
+
from prograph.r2rml.schema import require_property_graph
|
|
2039
|
+
require_property_graph(self._schema, "set_properties")
|
|
2040
|
+
from prograph.writes import set_properties
|
|
2041
|
+
return set_properties(self, label, identities, values, by=by)
|
|
2042
|
+
|
|
2043
|
+
def own_rows_sql(self, label: str, columns: Optional[Sequence[str]] = None) -> str:
|
|
2044
|
+
"""SQL text, in the backend's dialect, selecting the rows that are
|
|
2045
|
+
this label's nodes: its table, the rows its mapping owns (its
|
|
2046
|
+
``filter_predicate``, on a table labels share), asserted (no row a
|
|
2047
|
+
reasoner materialized). ``columns`` are column names of the table
|
|
2048
|
+
and default to the label's key columns.
|
|
2049
|
+
|
|
2050
|
+
For a caller that writes its own SQL for :meth:`execute_sql` and
|
|
2051
|
+
has to stay on one label's nodes: join against it as a subquery,
|
|
2052
|
+
``... JOIN (<own_rows_sql>) o ON o.id = q.key``. It is the read
|
|
2053
|
+
side of the rule every write by identity follows (``prograph/writes.py``), so what such a query scores is what
|
|
2054
|
+
:meth:`set_properties` can write.
|
|
2055
|
+
|
|
2056
|
+
Raises ``ValueError`` for an unknown label, for a column the
|
|
2057
|
+
label's table does not have.
|
|
2058
|
+
"""
|
|
2059
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
2060
|
+
from prograph.writes import INFERRED_COLUMN, own_rows_sql
|
|
2061
|
+
|
|
2062
|
+
if self._schema is None:
|
|
2063
|
+
raise ValueError("own_rows_sql needs a schema")
|
|
2064
|
+
mapping = self._schema.get_node_mapping(self._schema.require_name('label', label))
|
|
2065
|
+
graph = getattr(self.executor, 'graph', None)
|
|
2066
|
+
dialect_name = graph.sql_dialect() if hasattr(graph, 'sql_dialect') else None
|
|
2067
|
+
if dialect_name is None:
|
|
2068
|
+
raise ValueError(
|
|
2069
|
+
f"own_rows_sql needs a SQL connection, and the {self.backend} backend has none")
|
|
2070
|
+
frames = getattr(graph, 'dataframes', None) if self._frame_backed else None
|
|
2071
|
+
if frames is not None and mapping.source_table in frames:
|
|
2072
|
+
present = list(frames[mapping.source_table].columns)
|
|
2073
|
+
else:
|
|
2074
|
+
table = get_dialect_instance(dialect_name).quote_qualified(
|
|
2075
|
+
mapping.qualified_source_table)
|
|
2076
|
+
present = graph._execute_sql_arrow(f"SELECT * FROM {table} WHERE 1 = 0").column_names
|
|
2077
|
+
unknown = [c for c in (columns or ()) if c not in present]
|
|
2078
|
+
if unknown:
|
|
2079
|
+
raise ValueError(
|
|
2080
|
+
f"{', '.join(unknown)} is not a column of {mapping.source_table}, "
|
|
2081
|
+
f"the table of {label}")
|
|
2082
|
+
return own_rows_sql(mapping, get_dialect_instance(dialect_name),
|
|
2083
|
+
INFERRED_COLUMN in present, columns)
|
|
2084
|
+
|
|
2085
|
+
def resolve_iri(self, iri: str) -> Optional[Dict[str, Any]]:
|
|
2086
|
+
"""The node an instance IRI names, or None if there is none.
|
|
2087
|
+
|
|
2088
|
+
The inverse of the IRIs the semantic layer mints (SHACL focus nodes,
|
|
2089
|
+
SPARQL results, provenance keys; :mod:`prograph.schema.iri`): an
|
|
2090
|
+
action layer holding a violation's focus node gets back what a Cypher
|
|
2091
|
+
template needs, ``{"label": str, "labels": [str, ...], "id": value}``
|
|
2092
|
+
(``id`` a tuple for a composite key). The node is looked up, so the id
|
|
2093
|
+
comes back with its column's type and a well-formed IRI of a missing
|
|
2094
|
+
node resolves to None. ``labels`` lists every label the row carries,
|
|
2095
|
+
for mappings sharing a table. Synthesized IRIs are read under the
|
|
2096
|
+
schema's ``base_iri``.
|
|
2097
|
+
"""
|
|
2098
|
+
from prograph.schema.iri import parse_node_iri
|
|
2099
|
+
|
|
2100
|
+
from prograph.plan.ir.entry import node_keys
|
|
2101
|
+
found: Dict[str, Any] = {}
|
|
2102
|
+
labels: List[str] = []
|
|
2103
|
+
# Over the source maps: each node map's rows whose subject is the IRI, R2RML's
|
|
2104
|
+
# (labelled by the map's id) and YAML's alike, the key as its columns hold it.
|
|
2105
|
+
for m, answer, names in node_keys(self, iri):
|
|
2106
|
+
rows = self._convert_results(answer, "list") # the answer's one conversion
|
|
2107
|
+
if not rows:
|
|
2108
|
+
continue
|
|
2109
|
+
values = tuple(rows[0][n] for n in names)
|
|
2110
|
+
label = m.label or m.id
|
|
2111
|
+
labels.append(label)
|
|
2112
|
+
found.setdefault("label", label)
|
|
2113
|
+
found.setdefault("id", values[0] if len(values) == 1 else values)
|
|
2114
|
+
if not labels:
|
|
2115
|
+
return None
|
|
2116
|
+
found["labels"] = labels
|
|
2117
|
+
return found
|
|
2118
|
+
|
|
2119
|
+
# =========================================================================
|
|
2120
|
+
# OWL entailment
|
|
2121
|
+
# =========================================================================
|
|
2122
|
+
|
|
2123
|
+
@_one_at_a_time
|
|
2124
|
+
def materialize_entailments(self, max_rounds: int = 64):
|
|
2125
|
+
"""Materialize the ontology's OWL 2 RL entailments into the owned store's RL graph
|
|
2126
|
+
(``urn:prograph:store:rl``: in the default graph of an engine of this cache key, never a named
|
|
2127
|
+
graph), as one write step: the rules' closure over the
|
|
2128
|
+
graph SPARQL answers now, written as its difference from what the RL graph holds (asserts for
|
|
2129
|
+
new facts, retracts for facts no longer derived), with each mapped table's version as the run
|
|
2130
|
+
read it (its declared version column, its Delta version, or a small table's checksum). A query
|
|
2131
|
+
refreshes the facts itself where a table proves a change (``freshness``); under
|
|
2132
|
+
``freshness='last_run'`` call this after the tables change. Rules applied: prp-inv1/2, prp-symp, prp-trp, prp-spo2,
|
|
2133
|
+
cls-hv1/2, cls-svf1/2, cls-avf, cls-int1/2, cls-uni, cls-oo (subClassOf, subPropertyOf,
|
|
2134
|
+
equivalence, domain and range stay query rewrites). owl:sameAs: the graph's links and those
|
|
2135
|
+
prp-fp, prp-ifp, prp-key, cls-maxc2 and cls-maxqc3/4 derive are closed into components, written
|
|
2136
|
+
as a new generation of the store's alias table, through which queries match (``sameas``).
|
|
2137
|
+
``max_rounds``: the rounds of the fixpoint at most; more raise ``MaterializationBudgetExceeded``
|
|
2138
|
+
with nothing written.
|
|
2139
|
+
|
|
2140
|
+
Returns a ``MaterializationReport``: statements asserted and retracted per rule, the rounds,
|
|
2141
|
+
the consistency rules (cax-dw, prp-irp, prp-asyp, cls-nothing2, cls-maxc1 ...) that
|
|
2142
|
+
matched, each with a sample (``report.consistent``), and the owl:sameAs components merged and
|
|
2143
|
+
left unmerged (``report.unmerged``).
|
|
2144
|
+
|
|
2145
|
+
Where the store already holds a run under this engine's key (its rules, ontology and mapping) and every
|
|
2146
|
+
mapped table is proven unchanged since it (a declared version column, a Delta version, or a small
|
|
2147
|
+
table's checksum), nothing is recomputed: the call returns that run's record (``seq`` None, ``rounds``
|
|
2148
|
+
0, nothing asserted or retracted, its ``digest``, ``ran_at``, ``sources`` and consistency verdict) and
|
|
2149
|
+
settles the stored verdict as the engine's, raising under ``inconsistency='error'`` if it is
|
|
2150
|
+
inconsistent. A table no version proves counts as changed, so the closure is computed. ``max_rounds`` is
|
|
2151
|
+
checked against the recorded run's rounds: a budget below them (or a run that recorded none) computes the
|
|
2152
|
+
closure, so the budget error is raised as before."""
|
|
2153
|
+
from prograph.plan.ir.dataset import provably_current
|
|
2154
|
+
if self._store is not None and self._ontology is not None and self._profile == "rl" and max_rounds >= 1 \
|
|
2155
|
+
and provably_current(self):
|
|
2156
|
+
from prograph.plan.ir.consistency import recorded, settle
|
|
2157
|
+
from prograph.plan.ir.owl_rl import reused_report
|
|
2158
|
+
held = recorded(self, "by this call to materialize_entailments()")
|
|
2159
|
+
report = reused_report(self, self._rl_rules(), held, max_rounds)
|
|
2160
|
+
if report is not None:
|
|
2161
|
+
settle(self, held)
|
|
2162
|
+
return report
|
|
2163
|
+
return self._materialize_run(max_rounds)
|
|
2164
|
+
|
|
2165
|
+
@_one_at_a_time
|
|
2166
|
+
def _materialize_run(self, max_rounds: int = 64):
|
|
2167
|
+
"""`materialize_entailments`' run: the closure computed and its difference written, whether or not the
|
|
2168
|
+
store's facts are current."""
|
|
2169
|
+
if self._store is None or self._ontology is None:
|
|
2170
|
+
raise ValueError("materialize_entailments() writes the ontology's OWL 2 RL entailments into the "
|
|
2171
|
+
"owned store: give the engine an ontology= and a store=")
|
|
2172
|
+
if self._profile != "rl":
|
|
2173
|
+
raise ValueError("materialize_entailments() stores OWL 2 RL's facts for an engine that reads them: "
|
|
2174
|
+
"build the engine with profile='rl' (it materializes when it is built)")
|
|
2175
|
+
from prograph.plan.ir.consistency import rl_report, settle
|
|
2176
|
+
from prograph.plan.ir.entry import ontology_of
|
|
2177
|
+
from prograph.plan.ir.owl_rl import materialize
|
|
2178
|
+
ontology_of(self) # the ontology read (its profile checked at construction) before anything runs
|
|
2179
|
+
report = materialize(self, self._rl_rules(), max_rounds=max_rounds)
|
|
2180
|
+
# Written with the facts; raised here under inconsistency='error'.
|
|
2181
|
+
settle(self, rl_report(report.inconsistencies, report.ran_at, report.not_run))
|
|
2182
|
+
return report
|
|
2183
|
+
|
|
2184
|
+
def consistency_report(self):
|
|
2185
|
+
"""Whether the ontology and the data are consistent under the engine's profile
|
|
2186
|
+
(``prograph.plan.ir.consistency.ConsistencyReport``): ``consistent``, and each rule or check that
|
|
2187
|
+
matched (``inconsistencies``: the OWL 2 RL rule id or the QL check, the axiom in short Turtle, how many
|
|
2188
|
+
facts and a few of them), with ``text()`` the whole in words. Under profile='rl' the verdict of the
|
|
2189
|
+
last materialization (stored with its facts), under profile='ql' the check made when the engine was
|
|
2190
|
+
built. Under ``inconsistency='error'`` (the default) an engine with an inconsistent one is never built,
|
|
2191
|
+
so it is read under ``inconsistency='report'``. None with no ontology."""
|
|
2192
|
+
if self._ontology is None:
|
|
2193
|
+
return None
|
|
2194
|
+
held = self.__dict__.get("_consistency")
|
|
2195
|
+
if held is None and self._profile == "rl":
|
|
2196
|
+
from prograph.plan.ir.consistency import recorded
|
|
2197
|
+
held = recorded(self)
|
|
2198
|
+
return held
|
|
2199
|
+
|
|
2200
|
+
def explain_identity(self, a: str, b: str):
|
|
2201
|
+
"""Why two IRIs are one entity under owl:sameAs, or not (``profile='rl'``): an
|
|
2202
|
+
``IdentityExplanation`` with a shortest chain of links from ``a`` to ``b`` (each a sameAs triple of
|
|
2203
|
+
the graph, by the maps that may give it, or a link an OWL 2 RL rule derived: prp-fp, prp-ifp,
|
|
2204
|
+
prp-key, cls-maxc2, cls-maxqc3/4, with its axiom), each IRI's representative and component size,
|
|
2205
|
+
and, where a component holding either was left unmerged (inconsistent, or past the cap), why.
|
|
2206
|
+
Read from the last materialization's components, in the default graph."""
|
|
2207
|
+
from prograph.plan.ir.identity import explain
|
|
2208
|
+
return explain(self, a, b)
|
|
2209
|
+
|
|
2210
|
+
def _rl_rules(self):
|
|
2211
|
+
"""The ontology's OWL 2 RL rules (`owl_rl_rules.compile_rules`), compiled once per ontology content."""
|
|
2212
|
+
from prograph.plan.ir.owl_rl_rules import compile_rules
|
|
2213
|
+
from prograph.plan.ir.entry import profile_report
|
|
2214
|
+
# The graph the profile check kept: the axioms on_unsupported="ignore" dropped are not compiled.
|
|
2215
|
+
key, _, graph = profile_report(self)
|
|
2216
|
+
held = self.__dict__.get("_rl_compiled")
|
|
2217
|
+
if held is None or held[0] != key:
|
|
2218
|
+
held = (key, compile_rules(graph))
|
|
2219
|
+
self.__dict__["_rl_compiled"] = held
|
|
2220
|
+
return held[1]
|
|
2221
|
+
|
|
2222
|
+
def _materialize_on_open(self):
|
|
2223
|
+
"""The stored facts made current when the engine is built: a run where the store holds none under the
|
|
2224
|
+
engine's key, or a source or the store changed since that run (`dataset.stale`).
|
|
2225
|
+
Under freshness='last_run' a store that cannot be written keeps its last run."""
|
|
2226
|
+
from prograph.plan.ir.dataset import engine_key, recorded_key, stale_sources, stored_sources
|
|
2227
|
+
from prograph.plan.ir.entry import ontology_of
|
|
2228
|
+
from prograph.plan.ir.freshness import refresh
|
|
2229
|
+
from prograph.utils.exceptions import StoredFactsNotCurrentError
|
|
2230
|
+
from prograph.plan.ir.consistency import recorded, settle
|
|
2231
|
+
ontology_of(self)
|
|
2232
|
+
stored_sources(self) # a version column on a table no map reads is refused first
|
|
2233
|
+
changed = stale_sources(self)
|
|
2234
|
+
if changed:
|
|
2235
|
+
if recorded_key(self._store) != engine_key(self):
|
|
2236
|
+
self._materialize_run()
|
|
2237
|
+
return
|
|
2238
|
+
try:
|
|
2239
|
+
refresh(self, changed)
|
|
2240
|
+
return
|
|
2241
|
+
except StoredFactsNotCurrentError:
|
|
2242
|
+
if self._freshness != "last_run":
|
|
2243
|
+
raise
|
|
2244
|
+
# The facts are current (or read as the last run's): that run's verdict, not checked again.
|
|
2245
|
+
settle(self, recorded(self))
|
|
2246
|
+
|
|
2247
|
+
def shapes(self, format: str = "graph", *, shape_base: Optional[str] = None) -> Any:
|
|
2248
|
+
"""SHACL shapes generated from the mapping and the database's catalog (`prograph.shacl.generate`):
|
|
2249
|
+
one `sh:NodeShape` per class, a property shape per predicate with its datatype, node kind or class,
|
|
2250
|
+
and `sh:minCount 1` / `sh:maxCount 1` only where the mapping or the catalog proves them. They hold
|
|
2251
|
+
on the graph SPARQL answers, so `validate_shapes(engine.shapes())` conforms; add constraints to them to
|
|
2252
|
+
check more. Over frames or Spark (no catalog) no NOT NULL is known and no `sh:minCount` is written.
|
|
2253
|
+
|
|
2254
|
+
Args:
|
|
2255
|
+
format: "graph" gives an rdflib Graph; an rdflib serialization name ("turtle", ...) the text.
|
|
2256
|
+
shape_base: The prefix of the node shapes' IRIs (default `urn:prograph:shape:`).
|
|
2257
|
+
"""
|
|
2258
|
+
from prograph.plan.ir.entry import source_maps
|
|
2259
|
+
from prograph.plan.ir.keys import trusted_keys
|
|
2260
|
+
from prograph.shacl.generate import DEFAULT_SHAPE_BASE, generate_shapes
|
|
2261
|
+
from prograph.r2rml import catalog
|
|
2262
|
+
from prograph.schema import xsd
|
|
2263
|
+
|
|
2264
|
+
schema = self.schema
|
|
2265
|
+
maps = source_maps(schema)
|
|
2266
|
+
graph = getattr(getattr(self, "executor", None), "graph", None)
|
|
2267
|
+
read: Dict[Any, Any] = {}
|
|
2268
|
+
|
|
2269
|
+
def not_null(source: Any) -> Any:
|
|
2270
|
+
if self._connection is None or self.backend not in catalog.DIALECTS or source.query is not None:
|
|
2271
|
+
return None
|
|
2272
|
+
qualifier, _, table = source.table.rpartition(".")
|
|
2273
|
+
if qualifier not in read:
|
|
2274
|
+
try:
|
|
2275
|
+
read[qualifier] = catalog.read_not_null(self._connection, self.backend, qualifier or None)
|
|
2276
|
+
except Exception: # noqa: BLE001 - a catalog that cannot be read proves nothing
|
|
2277
|
+
read[qualifier] = None
|
|
2278
|
+
found = read[qualifier]
|
|
2279
|
+
if found is None:
|
|
2280
|
+
return None
|
|
2281
|
+
if table in found:
|
|
2282
|
+
return found[table]
|
|
2283
|
+
folded = [v for k, v in found.items() if k.lower() == table.lower()]
|
|
2284
|
+
return folded[0] if len(folded) == 1 else frozenset()
|
|
2285
|
+
|
|
2286
|
+
try:
|
|
2287
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
2288
|
+
fold = get_dialect_instance(self.backend).fold_identifier
|
|
2289
|
+
except Exception: # noqa: BLE001 - a backend with no dialect keeps names as written
|
|
2290
|
+
fold = lambda name: name # noqa: E731
|
|
2291
|
+
|
|
2292
|
+
graph_out = generate_shapes(
|
|
2293
|
+
maps, shape_base=shape_base or DEFAULT_SHAPE_BASE, not_null=not_null,
|
|
2294
|
+
column_datatype=lambda t, c: _shape_datatype(schema.table_column_type(t, c)),
|
|
2295
|
+
keys=lambda m: trusted_keys(graph, m, maps), fold=fold)
|
|
2296
|
+
return graph_out if format == "graph" else graph_out.serialize(format=format)
|
|
2297
|
+
|
|
2298
|
+
def validate_shapes(
|
|
2299
|
+
self,
|
|
2300
|
+
shapes: Union[str, "Path", "rdflib.Graph"],
|
|
2301
|
+
*,
|
|
2302
|
+
inference: str = "none",
|
|
2303
|
+
) -> "ValidationReport":
|
|
2304
|
+
"""Validate the RDF graph SPARQL answers against SHACL shapes.
|
|
2305
|
+
|
|
2306
|
+
The data graph is ``CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }`` over the default graph, the
|
|
2307
|
+
graph :meth:`shapes` holds on, for any mapping (an R2RML one too). It is what a SPARQL query
|
|
2308
|
+
sees: the mapping's joins, its quoted-case columns and its typed values are all in it.
|
|
2309
|
+
|
|
2310
|
+
Args:
|
|
2311
|
+
shapes: Path to a ``.ttl`` file, an ``rdflib.Graph``, or a Turtle string of SHACL shapes.
|
|
2312
|
+
inference: ``pyshacl``'s pre-validation entailment profile, ``'none'`` | ``'rdfs'`` |
|
|
2313
|
+
``'owlrl'`` | ``'both'``, applied to that graph.
|
|
2314
|
+
|
|
2315
|
+
Returns:
|
|
2316
|
+
A :class:`prograph.shacl.ValidationReport` with structured violations plus the canonical
|
|
2317
|
+
SHACL report graph as Turtle.
|
|
2318
|
+
"""
|
|
2319
|
+
import warnings
|
|
2320
|
+
from prograph.plan.ir.cost import ProGraphCostWarning
|
|
2321
|
+
from prograph.shacl import validate
|
|
2322
|
+
from prograph.shacl.validator import _coerce_shapes_graph
|
|
2323
|
+
with warnings.catch_warnings():
|
|
2324
|
+
warnings.simplefilter("ignore", ProGraphCostWarning)
|
|
2325
|
+
try:
|
|
2326
|
+
data_graph = self.execute("CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o }", language="sparql")
|
|
2327
|
+
except Exception as error:
|
|
2328
|
+
error.add_note("validate_shapes reads the mapped graph with CONSTRUCT { ?s ?p ?o } WHERE "
|
|
2329
|
+
"{ ?s ?p ?o }; the error above is that query's on this engine.")
|
|
2330
|
+
raise
|
|
2331
|
+
report = validate(data_graph, shapes=_coerce_shapes_graph(shapes), inference=inference)
|
|
2332
|
+
self._shacl_last_report = report
|
|
2333
|
+
return report
|
|
2334
|
+
|
|
2335
|
+
@property
|
|
2336
|
+
def shacl_last_report(self) -> Optional["ValidationReport"]:
|
|
2337
|
+
"""The :class:`ValidationReport` from the most recent ``validate_shapes`` call.
|
|
2338
|
+
|
|
2339
|
+
``None`` if validation has never been run on this engine.
|
|
2340
|
+
"""
|
|
2341
|
+
return getattr(self, "_shacl_last_report", None)
|
|
2342
|
+
|
|
2343
|
+
def generate_views(
|
|
2344
|
+
self,
|
|
2345
|
+
target_catalog: Optional[str] = None,
|
|
2346
|
+
target_schema: str = "prograph_views",
|
|
2347
|
+
include_labels: Optional[List[str]] = None,
|
|
2348
|
+
exclude_labels: Optional[List[str]] = None,
|
|
2349
|
+
dialect: Optional[Any] = None,
|
|
2350
|
+
) -> List[Any]:
|
|
2351
|
+
"""Generate Genie-consumable SQL views from this engine's schema.
|
|
2352
|
+
|
|
2353
|
+
Wraps :func:`prograph.genie.generate_views`. Returns a list of
|
|
2354
|
+
:class:`prograph.genie.ViewDefinition` instances ready to be
|
|
2355
|
+
rendered as ``CREATE OR REPLACE VIEW`` statements. The engine's
|
|
2356
|
+
ontology (if any) drives subclass-closure and OWL-defined-class
|
|
2357
|
+
view emission.
|
|
2358
|
+
|
|
2359
|
+
This method only *builds* view definitions; it does **not**
|
|
2360
|
+
execute any DDL against the backend. To create the views on the
|
|
2361
|
+
warehouse, use :meth:`write_views` (which calls this method
|
|
2362
|
+
internally) or hand the result to
|
|
2363
|
+
:func:`prograph.genie.view_writer.write_views`.
|
|
2364
|
+
|
|
2365
|
+
Args:
|
|
2366
|
+
target_catalog / target_schema: Naming convention metadata
|
|
2367
|
+
passed through to the writer if you later call
|
|
2368
|
+
:meth:`write_views`. Ignored at definition-build time;
|
|
2369
|
+
provided here so callers can construct the writer
|
|
2370
|
+
arguments without re-deriving them.
|
|
2371
|
+
include_labels / exclude_labels: Filter node mappings by
|
|
2372
|
+
label before generating views.
|
|
2373
|
+
dialect: Optional :class:`DialectConfig` override.
|
|
2374
|
+
|
|
2375
|
+
Returns:
|
|
2376
|
+
List of :class:`ViewDefinition`.
|
|
2377
|
+
"""
|
|
2378
|
+
from prograph.genie import generate_views as _gen
|
|
2379
|
+
return _gen(
|
|
2380
|
+
self._schema,
|
|
2381
|
+
ontology=self._ontology,
|
|
2382
|
+
dialect=self._view_dialect(dialect),
|
|
2383
|
+
include_labels=include_labels,
|
|
2384
|
+
exclude_labels=exclude_labels,
|
|
2385
|
+
)
|
|
2386
|
+
|
|
2387
|
+
def write_views(
|
|
2388
|
+
self,
|
|
2389
|
+
target_catalog: Optional[str] = None,
|
|
2390
|
+
target_schema: str = "prograph_views",
|
|
2391
|
+
include_labels: Optional[List[str]] = None,
|
|
2392
|
+
exclude_labels: Optional[List[str]] = None,
|
|
2393
|
+
dialect: Optional[Any] = None,
|
|
2394
|
+
ensure_schema: bool = True,
|
|
2395
|
+
) -> List[Tuple[str, str]]:
|
|
2396
|
+
"""Generate Genie views and execute ``CREATE OR REPLACE VIEW`` against this engine.
|
|
2397
|
+
|
|
2398
|
+
Only available on SQL backends that own a connection
|
|
2399
|
+
(``databricks_sql``, ``duckdb``, ``postgresql``, ``sqlserver``,
|
|
2400
|
+
``mysql``, ``oracle``). Other backends raise :class:`ValueError`
|
|
2401
|
+
because there's no SQL connection to write views against.
|
|
2402
|
+
|
|
2403
|
+
When ``dialect`` is left ``None`` it defaults to the dialect of
|
|
2404
|
+
this engine's backend, so the emitted DDL uses the backend's own
|
|
2405
|
+
identifier quoting (e.g. double quotes on DuckDB rather than
|
|
2406
|
+
Spark backticks).
|
|
2407
|
+
|
|
2408
|
+
Args:
|
|
2409
|
+
target_catalog / target_schema: Where the views live.
|
|
2410
|
+
Convention is ``<catalog>.prograph_views.*`` on
|
|
2411
|
+
three-part backends; ``prograph_views.*`` on two-part
|
|
2412
|
+
backends.
|
|
2413
|
+
ensure_schema: Issue ``CREATE SCHEMA IF NOT EXISTS`` before
|
|
2414
|
+
creating views. Set ``False`` if the schema is managed
|
|
2415
|
+
externally.
|
|
2416
|
+
(others): See :meth:`generate_views`.
|
|
2417
|
+
|
|
2418
|
+
Returns:
|
|
2419
|
+
List of ``(view_name, fully_qualified_name)`` tuples for
|
|
2420
|
+
the views actually created. Empty list if the schema has
|
|
2421
|
+
no mappings; raises if the backend doesn't support DDL.
|
|
2422
|
+
"""
|
|
2423
|
+
from prograph.genie.view_writer import write_views as _write
|
|
2424
|
+
|
|
2425
|
+
connection = self._resolve_sql_connection()
|
|
2426
|
+
if connection is None and self.backend == "duckdb":
|
|
2427
|
+
raise ValueError(
|
|
2428
|
+
"write_views needs a DuckDB database to write the views "
|
|
2429
|
+
"into: construct the engine with connection= (a DuckDB "
|
|
2430
|
+
"connection) instead of dataframes=. Over DataFrames, call "
|
|
2431
|
+
"generate_views() to get the SQL."
|
|
2432
|
+
)
|
|
2433
|
+
if connection is None:
|
|
2434
|
+
raise ValueError(
|
|
2435
|
+
f"write_views is not supported on the {self.backend!r} "
|
|
2436
|
+
"backend. It requires an open SQL connection; use "
|
|
2437
|
+
"'databricks_sql', 'duckdb', 'postgresql', 'sqlserver', "
|
|
2438
|
+
"'mysql', or 'oracle'. On 'spark', or 'duckdb' over DataFrames, call generate_views() to get the SQL and execute "
|
|
2439
|
+
"it through your own client."
|
|
2440
|
+
)
|
|
2441
|
+
|
|
2442
|
+
dialect = self._view_dialect(dialect)
|
|
2443
|
+
views = self.generate_views(
|
|
2444
|
+
target_catalog=target_catalog,
|
|
2445
|
+
target_schema=target_schema,
|
|
2446
|
+
include_labels=include_labels,
|
|
2447
|
+
exclude_labels=exclude_labels,
|
|
2448
|
+
dialect=dialect,
|
|
2449
|
+
)
|
|
2450
|
+
return _write(
|
|
2451
|
+
views,
|
|
2452
|
+
connection,
|
|
2453
|
+
target_catalog=target_catalog,
|
|
2454
|
+
target_schema=target_schema,
|
|
2455
|
+
dialect=dialect,
|
|
2456
|
+
ensure_schema=ensure_schema,
|
|
2457
|
+
)
|
|
2458
|
+
|
|
2459
|
+
def emit_genie_space(
|
|
2460
|
+
self,
|
|
2461
|
+
name: Optional[str] = None,
|
|
2462
|
+
description: str = "",
|
|
2463
|
+
target_catalog: Optional[str] = None,
|
|
2464
|
+
target_schema: str = "prograph_views",
|
|
2465
|
+
dialect: Optional[Any] = None,
|
|
2466
|
+
include_labels: Optional[List[str]] = None,
|
|
2467
|
+
exclude_labels: Optional[List[str]] = None,
|
|
2468
|
+
mcp_servers: Optional[List[Any]] = None,
|
|
2469
|
+
summary_instructions: bool = False,
|
|
2470
|
+
max_sample_queries: int = 25,
|
|
2471
|
+
) -> Any:
|
|
2472
|
+
"""Build a complete :class:`GenieSpace` from this engine's schema.
|
|
2473
|
+
|
|
2474
|
+
Bundles :meth:`generate_views`, instruction generation, and
|
|
2475
|
+
sample-query generation into one :class:`GenieSpace` ready to
|
|
2476
|
+
serialize as YAML. Pure function of the engine's schema +
|
|
2477
|
+
ontology; no DDL or network I/O.
|
|
2478
|
+
|
|
2479
|
+
Pair with :func:`prograph.genie.render_space_yaml` /
|
|
2480
|
+
:func:`prograph.genie.write_space_yaml` to write the result to
|
|
2481
|
+
disk for import into Databricks Genie.
|
|
2482
|
+
|
|
2483
|
+
Args:
|
|
2484
|
+
name: Display name for the Space. Defaults to the schema's
|
|
2485
|
+
``name`` attribute when set.
|
|
2486
|
+
description: Free-form description.
|
|
2487
|
+
target_catalog / target_schema: Where generated views live
|
|
2488
|
+
in the warehouse, the Space references views under
|
|
2489
|
+
this prefix.
|
|
2490
|
+
dialect: SQL dialect for view bodies. Defaults to
|
|
2491
|
+
Spark/Databricks SQL backticks.
|
|
2492
|
+
include_labels / exclude_labels: Filter on NodeMapping
|
|
2493
|
+
labels.
|
|
2494
|
+
mcp_servers: Optional list of
|
|
2495
|
+
:class:`McpServerRegistration` for the Path B escape
|
|
2496
|
+
hatch.
|
|
2497
|
+
summary_instructions: When ``True``, emit just the
|
|
2498
|
+
overview section.
|
|
2499
|
+
max_sample_queries: Cap on sample queries.
|
|
2500
|
+
|
|
2501
|
+
Returns:
|
|
2502
|
+
A :class:`prograph.genie.GenieSpace`.
|
|
2503
|
+
"""
|
|
2504
|
+
from prograph.genie import emit_genie_space as _emit
|
|
2505
|
+
return _emit(
|
|
2506
|
+
schema=self._schema,
|
|
2507
|
+
ontology=self._ontology,
|
|
2508
|
+
name=name,
|
|
2509
|
+
description=description,
|
|
2510
|
+
target_catalog=target_catalog,
|
|
2511
|
+
target_schema=target_schema,
|
|
2512
|
+
dialect=self._view_dialect(dialect),
|
|
2513
|
+
include_labels=include_labels,
|
|
2514
|
+
exclude_labels=exclude_labels,
|
|
2515
|
+
mcp_servers=mcp_servers,
|
|
2516
|
+
summary_instructions=summary_instructions,
|
|
2517
|
+
max_sample_queries=max_sample_queries,
|
|
2518
|
+
)
|
|
2519
|
+
|
|
2520
|
+
def _view_dialect(self, dialect: Optional[Any]) -> Any:
|
|
2521
|
+
"""Resolve the SQL dialect to use for view generation/writing.
|
|
2522
|
+
|
|
2523
|
+
When the caller passes an explicit ``dialect``, use it. Otherwise
|
|
2524
|
+
default to the dialect that matches this engine's backend, so
|
|
2525
|
+
``generate_views`` / ``write_views`` emit valid SQL for the
|
|
2526
|
+
backend they run against (e.g. double-quoted identifiers on
|
|
2527
|
+
DuckDB, not Spark backticks). A backend with no dialect of its own
|
|
2528
|
+
falls back to the Spark dialect.
|
|
2529
|
+
"""
|
|
2530
|
+
if dialect is not None:
|
|
2531
|
+
return dialect
|
|
2532
|
+
from prograph.plan.sql.dialect import get_dialect, SPARK_DIALECT
|
|
2533
|
+
try:
|
|
2534
|
+
return get_dialect(self.backend)
|
|
2535
|
+
except ValueError:
|
|
2536
|
+
return SPARK_DIALECT
|
|
2537
|
+
|
|
2538
|
+
def _resolve_sql_connection(self) -> Optional[Any]:
|
|
2539
|
+
"""Return the engine's SQL connection if any; ``None`` otherwise.
|
|
2540
|
+
|
|
2541
|
+
Different SQL backends store the connection in different
|
|
2542
|
+
places. Centralize the lookup here so :meth:`write_views` and
|
|
2543
|
+
any future SQL-DDL methods stay consistent.
|
|
2544
|
+
"""
|
|
2545
|
+
# postgresql / sqlserver backends, connection on the executor's graph.
|
|
2546
|
+
graph = getattr(self.executor, "graph", None)
|
|
2547
|
+
for attr in ("_connection", "connection", "_conn"):
|
|
2548
|
+
conn = getattr(graph, attr, None)
|
|
2549
|
+
if conn is not None:
|
|
2550
|
+
return conn
|
|
2551
|
+
# databricks_sql backend, likely on the executor directly.
|
|
2552
|
+
for attr in ("_connection", "connection", "_conn"):
|
|
2553
|
+
conn = getattr(self.executor, attr, None)
|
|
2554
|
+
if conn is not None:
|
|
2555
|
+
return conn
|
|
2556
|
+
return None
|
|
2557
|
+
|
|
2558
|
+
def _collect_dataframes(self) -> Dict[str, Any]:
|
|
2559
|
+
"""Gather the dataframes visible to schema mappings.
|
|
2560
|
+
|
|
2561
|
+
Prefers the engine's ``_dataframes`` dict (the constructor input)
|
|
2562
|
+
but falls back to ``executor.graph.dataframes`` if the engine
|
|
2563
|
+
was constructed without dataframes but later had them injected.
|
|
2564
|
+
"""
|
|
2565
|
+
if self._dataframes:
|
|
2566
|
+
return dict(self._dataframes)
|
|
2567
|
+
graph = getattr(self.executor, "graph", None)
|
|
2568
|
+
if graph is not None and hasattr(graph, "dataframes"):
|
|
2569
|
+
return dict(graph.dataframes)
|
|
2570
|
+
return {}
|
|
2571
|
+
|
|
2572
|
+
|
|
2573
|
+
def __repr__(self) -> str:
|
|
2574
|
+
return f"ProGraph(backend='{self.backend}')"
|
|
2575
|
+
|
|
2576
|
+
|
|
2577
|
+
# Backward compatibility alias
|
|
2578
|
+
CypherEngine = ProGraph
|
|
2579
|
+
|
|
2580
|
+
|
|
2581
|
+
def _describe_postgresql_query(conn, query: str):
|
|
2582
|
+
"""(column, type name) of an R2RML view (rr:sqlQuery) on PostgreSQL, from the database's
|
|
2583
|
+
description of the statement, run on no row: the result's names from `LIMIT 0`, its types from
|
|
2584
|
+
a prepared statement's `result_types` (pg_prepared_statements, PostgreSQL 14+), deallocated
|
|
2585
|
+
after. Nothing is written (D6): without it a query source's literals were not built."""
|
|
2586
|
+
import uuid
|
|
2587
|
+
from prograph.graph.drivers import fetch_rows
|
|
2588
|
+
names, _ = fetch_rows(conn, f"SELECT * FROM ({query}\n) AS q LIMIT 0")
|
|
2589
|
+
statement = "prograph_describe_" + uuid.uuid4().hex[:12]
|
|
2590
|
+
fetch_rows(conn, f"PREPARE {statement} AS SELECT * FROM ({query}\n) AS q")
|
|
2591
|
+
try:
|
|
2592
|
+
_, rows = fetch_rows(conn, "SELECT unnest(result_types)::text FROM pg_prepared_statements "
|
|
2593
|
+
f"WHERE name = '{statement}'")
|
|
2594
|
+
finally:
|
|
2595
|
+
fetch_rows(conn, f"DEALLOCATE {statement}")
|
|
2596
|
+
kinds = [str(r[0]) for r in rows]
|
|
2597
|
+
if len(kinds) != len(names):
|
|
2598
|
+
return []
|
|
2599
|
+
described = list(zip([str(n) for n in names], kinds))
|
|
2600
|
+
numerics = [name for name, kind in described if kind == "numeric"]
|
|
2601
|
+
if numerics:
|
|
2602
|
+
digits = _postgresql_query_numeric_digits(conn, query, numerics)
|
|
2603
|
+
described = [(name, digits.get(name, kind) if kind == "numeric" else kind) for name, kind in described]
|
|
2604
|
+
return described
|
|
2605
|
+
|
|
2606
|
+
|
|
2607
|
+
class UnknownNumericDigitsWarning(UserWarning):
|
|
2608
|
+
"""A PostgreSQL source's NUMERIC columns of no known precision and scale (a table's unconstrained
|
|
2609
|
+
NUMERIC, or an R2RML view's columns where no temporary view could be made), read as an unconstrained
|
|
2610
|
+
NUMERIC is: once per source and engine, naming why and the way to their exact types."""
|
|
2611
|
+
|
|
2612
|
+
|
|
2613
|
+
# The CREATE TEMPORARY VIEW failures that mean the session may not make one, not that the query is wrong:
|
|
2614
|
+
# a read-only transaction (a hot standby or read-only endpoint gives the same, PostgreSQL's
|
|
2615
|
+
# PreventCommandIfReadOnly and PreventCommandDuringRecovery) and no TEMP privilege.
|
|
2616
|
+
_NO_TEMPORARY_VIEW = {"25006", "42501"}
|
|
2617
|
+
|
|
2618
|
+
|
|
2619
|
+
@home("postgresql-column-digits", "A PostgreSQL source's NUMERIC column digits (a table's, an R2RML view's)",
|
|
2620
|
+
terms=("rr:sqlQuery", "R2RML view", "temporary view", "unconstrained NUMERIC", "precision", "scale",
|
|
2621
|
+
"UnknownNumericDigitsWarning", "read-only", "PostgreSQL"))
|
|
2622
|
+
def _postgresql_query_numeric_digits(conn, query: str, numerics: Sequence[str]) -> Dict[str, str]:
|
|
2623
|
+
"""Each `numeric` column of an R2RML view (rr:sqlQuery) named in `numerics`, as `numeric(p,s)`.
|
|
2624
|
+
PostgreSQL's prepared statements and the driver's schema name a result column's type but never its
|
|
2625
|
+
modifier, so this runs `CREATE TEMPORARY VIEW <unique name> AS <the query>`, reads that view's rows of
|
|
2626
|
+
`information_schema.columns` and runs `DROP VIEW`, in a `finally`. The view lives in the session's own
|
|
2627
|
+
temporary schema: no table and no row of the user's is written (D6). A column the catalog gives no
|
|
2628
|
+
precision (an unconstrained NUMERIC) keeps `numeric`.
|
|
2629
|
+
|
|
2630
|
+
Where the session may not create the view (SQLSTATE 25006, a read-only transaction, a hot standby or
|
|
2631
|
+
read-only endpoint; 42501, no TEMP privilege; or a passed connection whose open transaction is the
|
|
2632
|
+
caller's: autocommit off, where a failed statement would abort the caller's work) nothing is created,
|
|
2633
|
+
the columns keep plain `numeric`, read as an unconstrained NUMERIC is (`postgresql_graph.numeric_digits`:
|
|
2634
|
+
rounded past 18 places, a whole part past 20 digits an error naming the column; arithmetic the
|
|
2635
|
+
database's own, `facts.arithmetic_slot`), and `warn_unknown_digits` says so once. Any other failure
|
|
2636
|
+
(a query no view can hold, such as one with two output columns of one name) reads the columns the
|
|
2637
|
+
same way and warns too, naming the database's reason, so the query can be fixed; where the query
|
|
2638
|
+
itself is wrong (a column reference it cannot resolve), its own error is the database's when it runs."""
|
|
2639
|
+
import uuid
|
|
2640
|
+
from prograph.graph.drivers import fetch_rows
|
|
2641
|
+
view = "prograph_describe_" + uuid.uuid4().hex[:12]
|
|
2642
|
+
# Inside a transaction the caller holds open (a passed connection with autocommit off) a failed CREATE
|
|
2643
|
+
# would leave the caller's transaction aborted, and the driver then refuses even a ROLLBACK TO SAVEPOINT:
|
|
2644
|
+
# the view is made only in autocommit, where a failed statement aborts nothing.
|
|
2645
|
+
if not _autocommit(conn):
|
|
2646
|
+
warn_unknown_digits(_shown_query(query), numerics, "whose precision and scale ProGraph could not "
|
|
2647
|
+
"read: the connection passed in has a transaction of yours open (autocommit off), "
|
|
2648
|
+
"in which ProGraph creates no temporary view")
|
|
2649
|
+
return {}
|
|
2650
|
+
try:
|
|
2651
|
+
fetch_rows(conn, f"CREATE TEMPORARY VIEW {view} AS {query}")
|
|
2652
|
+
except Exception as error: # noqa: BLE001 - a read-only session, or names a view cannot hold: not an error
|
|
2653
|
+
state = getattr(error, "sqlstate", None) or getattr(error.__cause__, "sqlstate", None)
|
|
2654
|
+
if state in _NO_TEMPORARY_VIEW:
|
|
2655
|
+
warn_unknown_digits(_shown_query(query), numerics, "whose precision and scale ProGraph "
|
|
2656
|
+
f"could not read: the session may not create a temporary view of it "
|
|
2657
|
+
f"({_primary_message(error)}, SQLSTATE {state})")
|
|
2658
|
+
else: # the database will not hold the query as a view: its reason, so the query can be fixed
|
|
2659
|
+
warn_unknown_digits(_shown_query(query), numerics, "whose precision and scale ProGraph "
|
|
2660
|
+
f"could not read: the database would not create a view of the query "
|
|
2661
|
+
f"({_primary_message(error)}" + (f", SQLSTATE {state}" if state else "") + ")")
|
|
2662
|
+
return {}
|
|
2663
|
+
try:
|
|
2664
|
+
_, rows = fetch_rows(conn, "SELECT column_name, numeric_precision, numeric_scale "
|
|
2665
|
+
"FROM information_schema.columns WHERE data_type = 'numeric' "
|
|
2666
|
+
f"AND numeric_precision IS NOT NULL AND table_name = '{view}'")
|
|
2667
|
+
found = {str(c): f"numeric({int(p)},{int(sc)})" for c, p, sc in rows if 0 <= int(sc) <= int(p)}
|
|
2668
|
+
finally:
|
|
2669
|
+
try:
|
|
2670
|
+
fetch_rows(conn, f"DROP VIEW IF EXISTS pg_temp.{view}")
|
|
2671
|
+
except Exception: # noqa: BLE001 - the view dies with the session
|
|
2672
|
+
pass
|
|
2673
|
+
unconstrained = [name for name in numerics if name not in found]
|
|
2674
|
+
if unconstrained:
|
|
2675
|
+
warn_unknown_digits(_shown_query(query), unconstrained, "whose precision and scale the query does not "
|
|
2676
|
+
"state (an unconstrained NUMERIC)")
|
|
2677
|
+
return found
|
|
2678
|
+
|
|
2679
|
+
|
|
2680
|
+
def warn_unknown_digits(source: str, numerics: Sequence[str], why: str) -> None:
|
|
2681
|
+
"""`postgresql-column-digits`' one warning for a source whose `numerics` have no known digits (each
|
|
2682
|
+
engine describes a source once). Given with a registry of its own, so Python's once-per-text filter
|
|
2683
|
+
does not swallow a second engine's first sight of the same source."""
|
|
2684
|
+
import sys
|
|
2685
|
+
import warnings
|
|
2686
|
+
from prograph.graph.postgresql_graph import numeric_digits
|
|
2687
|
+
(p, s), _ = numeric_digits(None)
|
|
2688
|
+
text = (f"{source} has NUMERIC columns ({', '.join(numerics)}) {why}. They are read as an unconstrained "
|
|
2689
|
+
f"NUMERIC is: as DECIMAL({p}, {s}), rounded past {s} places after the point, a value whose whole "
|
|
2690
|
+
f"part passes {p - s} digits an error naming the column. For their exact types, map a table, or a "
|
|
2691
|
+
f"view created in the database, whose columns declare NUMERIC(p, s), by rr:tableName (or a YAML "
|
|
2692
|
+
f"source's table).")
|
|
2693
|
+
caller = sys._getframe(1)
|
|
2694
|
+
warnings.warn_explicit(text, UnknownNumericDigitsWarning, caller.f_code.co_filename, caller.f_lineno,
|
|
2695
|
+
module=caller.f_globals.get("__name__"), registry={})
|
|
2696
|
+
|
|
2697
|
+
|
|
2698
|
+
def _shown_query(query: str) -> str:
|
|
2699
|
+
shown = " ".join(query.split())
|
|
2700
|
+
return "The R2RML view (rr:sqlQuery) " + repr(shown if len(shown) <= 120 else shown[:117] + "...")
|
|
2701
|
+
|
|
2702
|
+
|
|
2703
|
+
def _primary_message(error: BaseException) -> str:
|
|
2704
|
+
for key, value in getattr(error, "details", None) or ():
|
|
2705
|
+
if key == b"PG_DIAG_MESSAGE_PRIMARY":
|
|
2706
|
+
return value.decode("utf-8", "replace")
|
|
2707
|
+
said = " ".join(str(error).split())
|
|
2708
|
+
return said.rpartition("ERROR:")[2].split(" SQLSTATE")[0].strip(" .")[:200]
|
|
2709
|
+
|
|
2710
|
+
|
|
2711
|
+
def _autocommit(conn) -> bool:
|
|
2712
|
+
"""Whether an ADBC connection runs each statement in a transaction of its own."""
|
|
2713
|
+
try:
|
|
2714
|
+
return str(conn.adbc_connection.get_option("adbc.connection.autocommit")).lower() == "true"
|
|
2715
|
+
except Exception: # noqa: BLE001 - a driver that cannot say: treat it as the caller's transaction
|
|
2716
|
+
return False
|
|
2717
|
+
|
|
2718
|
+
|