prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
"""SHACL → SQL view generator for the easily-translatable constraint subset.
|
|
2
|
+
|
|
3
|
+
For each SHACL ``NodeShape``, this module emits one SQL view named
|
|
4
|
+
``<target_label>_violations`` whose rows are the data-quality failures
|
|
5
|
+
for that shape: one row per (focus_node, constraint_kind) pair.
|
|
6
|
+
|
|
7
|
+
Supported PropertyShape constraint components:
|
|
8
|
+
|
|
9
|
+
- ``sh:minCount 1``: the property is missing (NULL).
|
|
10
|
+
- ``sh:minInclusive`` / ``sh:maxInclusive``, numeric value out of range.
|
|
11
|
+
- ``sh:in (...)``, value not in the declared set.
|
|
12
|
+
|
|
13
|
+
Unsupported shapes fall through silently. They remain enforced by
|
|
14
|
+
:meth:`prograph.engine.ProGraph.validate_shapes`, which runs the full
|
|
15
|
+
pyshacl machinery and returns a structured :class:`ValidationReport`.
|
|
16
|
+
The view layer is a Genie-/BI-friendly *fast path* over the cheap
|
|
17
|
+
constraints, not a SHACL replacement.
|
|
18
|
+
|
|
19
|
+
See docs/guides/genie.md "Path A: Generated SQL views" for
|
|
20
|
+
where these views fit in the stack.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from dataclasses import dataclass
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import TYPE_CHECKING, Any, List, Optional, Union
|
|
28
|
+
|
|
29
|
+
if TYPE_CHECKING:
|
|
30
|
+
from rdflib import Graph
|
|
31
|
+
from prograph.plan.sql.dialect import DialectConfig
|
|
32
|
+
from prograph.schema.mapping import GraphSchema, NodeMapping
|
|
33
|
+
|
|
34
|
+
from prograph.genie.formats import ViewDefinition, ViewSource
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
# SHACL namespace shorthand
|
|
38
|
+
_SH = "http://www.w3.org/ns/shacl#"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class _PropertyConstraint:
|
|
43
|
+
"""One translated constraint on a SHACL PropertyShape."""
|
|
44
|
+
|
|
45
|
+
property_name: str # OWL local name (matches NodeMapping property_columns keys)
|
|
46
|
+
min_count: Optional[int] = None # sh:minCount
|
|
47
|
+
min_inclusive: Optional[float] = None # sh:minInclusive
|
|
48
|
+
max_inclusive: Optional[float] = None # sh:maxInclusive
|
|
49
|
+
in_values: Optional[List[Any]] = None # sh:in
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def generate_shacl_violation_views(
|
|
53
|
+
schema: "GraphSchema",
|
|
54
|
+
shapes: Union[str, Path, "Graph"],
|
|
55
|
+
*,
|
|
56
|
+
dialect: Optional["DialectConfig"] = None,
|
|
57
|
+
) -> List[ViewDefinition]:
|
|
58
|
+
"""Walk every ``sh:NodeShape`` and emit one violation view per shape.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
schema: The :class:`GraphSchema` whose NodeMappings the shapes
|
|
62
|
+
target. Views are only emitted for shapes whose
|
|
63
|
+
``sh:targetClass`` resolves to a known label in the schema.
|
|
64
|
+
shapes: Path to a ``.ttl`` file, an :class:`rdflib.Graph`, or a
|
|
65
|
+
Turtle string of SHACL shapes.
|
|
66
|
+
dialect: SQL dialect for identifier quoting. Defaults to the
|
|
67
|
+
Spark backtick form.
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
List of :class:`ViewDefinition` (``source=SHACL_VIOLATION``).
|
|
71
|
+
Empty list if no shapes are translatable.
|
|
72
|
+
"""
|
|
73
|
+
if dialect is None:
|
|
74
|
+
from prograph.plan.sql.dialect import SPARK_DIALECT
|
|
75
|
+
dialect = SPARK_DIALECT
|
|
76
|
+
|
|
77
|
+
graph = _load_shapes_graph(shapes)
|
|
78
|
+
views: List[ViewDefinition] = []
|
|
79
|
+
|
|
80
|
+
for node_shape, target_label, property_constraints in _walk_shapes(graph):
|
|
81
|
+
if not property_constraints:
|
|
82
|
+
continue
|
|
83
|
+
mapping = schema.get_node_mapping(target_label)
|
|
84
|
+
if mapping is None:
|
|
85
|
+
continue
|
|
86
|
+
view = _build_view(
|
|
87
|
+
shape_name=node_shape, target_label=target_label,
|
|
88
|
+
constraints=property_constraints, mapping=mapping, dialect=dialect,
|
|
89
|
+
)
|
|
90
|
+
if view is not None:
|
|
91
|
+
views.append(view)
|
|
92
|
+
return views
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
# ─── Parsing ─────────────────────────────────────────────────────────
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _load_shapes_graph(shapes: Union[str, Path, "Graph"]) -> "Graph":
|
|
99
|
+
"""Coerce shapes input into an rdflib.Graph."""
|
|
100
|
+
try:
|
|
101
|
+
from rdflib import Graph as _Graph
|
|
102
|
+
except ImportError as e:
|
|
103
|
+
raise ImportError(
|
|
104
|
+
"SHACL view generation requires rdflib. Install with: "
|
|
105
|
+
"pip install 'prograph[ontology]'"
|
|
106
|
+
) from e
|
|
107
|
+
|
|
108
|
+
if hasattr(shapes, "subjects"):
|
|
109
|
+
return shapes # already a Graph
|
|
110
|
+
g = _Graph()
|
|
111
|
+
if isinstance(shapes, (str, Path)) and Path(str(shapes)).exists():
|
|
112
|
+
g.parse(str(shapes), format="turtle")
|
|
113
|
+
else:
|
|
114
|
+
g.parse(data=str(shapes), format="turtle")
|
|
115
|
+
return g
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _walk_shapes(graph: "Graph"):
|
|
119
|
+
"""Yield ``(shape_local_name, target_label, [constraints])`` per NodeShape.
|
|
120
|
+
|
|
121
|
+
NodeShapes without sh:targetClass are skipped (we can't bind them
|
|
122
|
+
to a NodeMapping). PropertyShapes whose only constraints are
|
|
123
|
+
unsupported don't yield a constraint; the caller drops the shape
|
|
124
|
+
if the list is empty.
|
|
125
|
+
"""
|
|
126
|
+
from rdflib import RDF, URIRef
|
|
127
|
+
SH = lambda local: URIRef(_SH + local) # noqa: E731
|
|
128
|
+
|
|
129
|
+
for node_shape in graph.subjects(RDF.type, SH("NodeShape")):
|
|
130
|
+
targets = list(graph.objects(node_shape, SH("targetClass")))
|
|
131
|
+
if not targets:
|
|
132
|
+
continue
|
|
133
|
+
target_label = _local(str(targets[0]))
|
|
134
|
+
|
|
135
|
+
constraints: List[_PropertyConstraint] = []
|
|
136
|
+
for prop_shape in graph.objects(node_shape, SH("property")):
|
|
137
|
+
paths = list(graph.objects(prop_shape, SH("path")))
|
|
138
|
+
if not paths:
|
|
139
|
+
continue
|
|
140
|
+
prop_name = _local(str(paths[0]))
|
|
141
|
+
|
|
142
|
+
c = _PropertyConstraint(property_name=prop_name)
|
|
143
|
+
min_count = _first_int(graph, prop_shape, SH("minCount"))
|
|
144
|
+
if min_count is not None and min_count >= 1:
|
|
145
|
+
c.min_count = min_count
|
|
146
|
+
c.min_inclusive = _first_number(graph, prop_shape, SH("minInclusive"))
|
|
147
|
+
c.max_inclusive = _first_number(graph, prop_shape, SH("maxInclusive"))
|
|
148
|
+
c.in_values = _parse_in_list(graph, prop_shape, SH("in"))
|
|
149
|
+
|
|
150
|
+
if any([c.min_count, c.min_inclusive is not None,
|
|
151
|
+
c.max_inclusive is not None, c.in_values]):
|
|
152
|
+
constraints.append(c)
|
|
153
|
+
|
|
154
|
+
yield _local(str(node_shape)), target_label, constraints
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _local(iri: str) -> str:
|
|
158
|
+
"""IRI → local name (after '#' or last '/')."""
|
|
159
|
+
if "#" in iri:
|
|
160
|
+
return iri.rsplit("#", 1)[-1]
|
|
161
|
+
return iri.rsplit("/", 1)[-1]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _first_int(graph: "Graph", subj, pred) -> Optional[int]:
|
|
165
|
+
for o in graph.objects(subj, pred):
|
|
166
|
+
try:
|
|
167
|
+
return int(o)
|
|
168
|
+
except (TypeError, ValueError):
|
|
169
|
+
return None
|
|
170
|
+
return None
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _first_number(graph: "Graph", subj, pred) -> Optional[float]:
|
|
174
|
+
for o in graph.objects(subj, pred):
|
|
175
|
+
try:
|
|
176
|
+
return float(o)
|
|
177
|
+
except (TypeError, ValueError):
|
|
178
|
+
return None
|
|
179
|
+
return None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _parse_in_list(graph: "Graph", subj, pred) -> Optional[List[Any]]:
|
|
183
|
+
"""sh:in points to an RDF list. Walk it."""
|
|
184
|
+
from rdflib import RDF
|
|
185
|
+
head = next(graph.objects(subj, pred), None)
|
|
186
|
+
if head is None:
|
|
187
|
+
return None
|
|
188
|
+
values: List[Any] = []
|
|
189
|
+
while head is not None and str(head) != str(RDF.nil):
|
|
190
|
+
first = next(graph.objects(head, RDF.first), None)
|
|
191
|
+
if first is None:
|
|
192
|
+
break
|
|
193
|
+
try:
|
|
194
|
+
values.append(first.toPython())
|
|
195
|
+
except AttributeError:
|
|
196
|
+
values.append(str(first))
|
|
197
|
+
head = next(graph.objects(head, RDF.rest), None)
|
|
198
|
+
return values or None
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
# ─── SQL emission ────────────────────────────────────────────────────
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _build_view(
|
|
205
|
+
*, shape_name: str, target_label: str, constraints: List[_PropertyConstraint],
|
|
206
|
+
mapping: "NodeMapping", dialect: "DialectConfig",
|
|
207
|
+
) -> Optional[ViewDefinition]:
|
|
208
|
+
"""Emit a single shape's violation view."""
|
|
209
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
210
|
+
d = get_dialect_instance(dialect.name)
|
|
211
|
+
q = d.quote
|
|
212
|
+
lit = d.literal
|
|
213
|
+
|
|
214
|
+
# Resolve property names against the mapping. Skip constraints whose
|
|
215
|
+
# property isn't materialized. We can't query what isn't there.
|
|
216
|
+
resolved: List[tuple[_PropertyConstraint, str]] = []
|
|
217
|
+
for c in constraints:
|
|
218
|
+
col = mapping.property_columns.get(c.property_name)
|
|
219
|
+
if col is None:
|
|
220
|
+
from prograph.genie.view_generator import _snake_case # local
|
|
221
|
+
col = mapping.property_columns.get(_snake_case(c.property_name))
|
|
222
|
+
if col is not None:
|
|
223
|
+
resolved.append((c, col))
|
|
224
|
+
if not resolved:
|
|
225
|
+
return None
|
|
226
|
+
|
|
227
|
+
where_clauses: List[str] = []
|
|
228
|
+
case_branches: List[str] = []
|
|
229
|
+
for c, col in resolved:
|
|
230
|
+
col_quoted = q(col)
|
|
231
|
+
if c.min_count and c.min_count >= 1:
|
|
232
|
+
where_clauses.append(f"{col_quoted} IS NULL")
|
|
233
|
+
case_branches.append(
|
|
234
|
+
f"WHEN {col_quoted} IS NULL THEN "
|
|
235
|
+
f"{lit(f'minCount: {c.property_name} is null')}"
|
|
236
|
+
)
|
|
237
|
+
if c.min_inclusive is not None:
|
|
238
|
+
where_clauses.append(f"{col_quoted} < {lit(c.min_inclusive)}")
|
|
239
|
+
case_branches.append(
|
|
240
|
+
f"WHEN {col_quoted} < {lit(c.min_inclusive)} THEN "
|
|
241
|
+
f"{lit(f'minInclusive: {c.property_name} < {c.min_inclusive}')}"
|
|
242
|
+
)
|
|
243
|
+
if c.max_inclusive is not None:
|
|
244
|
+
where_clauses.append(f"{col_quoted} > {lit(c.max_inclusive)}")
|
|
245
|
+
case_branches.append(
|
|
246
|
+
f"WHEN {col_quoted} > {lit(c.max_inclusive)} THEN "
|
|
247
|
+
f"{lit(f'maxInclusive: {c.property_name} > {c.max_inclusive}')}"
|
|
248
|
+
)
|
|
249
|
+
if c.in_values:
|
|
250
|
+
in_list = ", ".join(lit(v) for v in c.in_values)
|
|
251
|
+
where_clauses.append(
|
|
252
|
+
f"({col_quoted} IS NOT NULL AND {col_quoted} NOT IN ({in_list}))"
|
|
253
|
+
)
|
|
254
|
+
case_branches.append(
|
|
255
|
+
f"WHEN {col_quoted} IS NOT NULL AND "
|
|
256
|
+
f"{col_quoted} NOT IN ({in_list}) THEN "
|
|
257
|
+
f"{lit(f'in: {c.property_name} not in allowed set')}"
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
id_col = mapping.id_columns[0] if len(mapping.id_columns) == 1 else "id"
|
|
261
|
+
select_terms = [
|
|
262
|
+
q(id_col) + (f" AS {q('id')}" if id_col != "id" else ""),
|
|
263
|
+
f"{lit(shape_name)} AS {q('shape')}",
|
|
264
|
+
]
|
|
265
|
+
# Include the property columns in the SELECT so the user can see
|
|
266
|
+
# the offending values alongside the shape name.
|
|
267
|
+
for c, col in resolved:
|
|
268
|
+
select_terms.append(
|
|
269
|
+
q(col) if col == c.property_name
|
|
270
|
+
else f"{q(col)} AS {q(c.property_name)}"
|
|
271
|
+
)
|
|
272
|
+
select_terms.append(
|
|
273
|
+
f"CASE {' '.join(case_branches)} END AS {q('violation_message')}"
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
sql_parts = [
|
|
277
|
+
"SELECT " + ", ".join(select_terms),
|
|
278
|
+
f"FROM {d.quote_qualified(mapping.source_table)}",
|
|
279
|
+
]
|
|
280
|
+
if mapping.filter_predicate:
|
|
281
|
+
# Combine the filter_predicate with the violation predicate.
|
|
282
|
+
from prograph.schema.rows import column_condition
|
|
283
|
+
filter_clauses = [
|
|
284
|
+
column_condition(q(col), val, lit)
|
|
285
|
+
for col, val in mapping.filter_predicate.items()
|
|
286
|
+
]
|
|
287
|
+
sql_parts.append(
|
|
288
|
+
"WHERE (" + " OR ".join(where_clauses) + ") AND "
|
|
289
|
+
+ " AND ".join(filter_clauses)
|
|
290
|
+
)
|
|
291
|
+
else:
|
|
292
|
+
sql_parts.append("WHERE " + " OR ".join(where_clauses))
|
|
293
|
+
|
|
294
|
+
sql = "\n".join(sql_parts)
|
|
295
|
+
|
|
296
|
+
cols = [
|
|
297
|
+
"id", "shape",
|
|
298
|
+
*(c.property_name for c, _ in resolved),
|
|
299
|
+
"violation_message",
|
|
300
|
+
]
|
|
301
|
+
from prograph.genie.view_generator import _label_to_view_name
|
|
302
|
+
view_name = f"{_label_to_view_name(target_label)}_violations"
|
|
303
|
+
|
|
304
|
+
return ViewDefinition(
|
|
305
|
+
name=view_name,
|
|
306
|
+
sql=sql,
|
|
307
|
+
source=ViewSource.SHACL_VIOLATION,
|
|
308
|
+
source_label=target_label,
|
|
309
|
+
description=(
|
|
310
|
+
f"SHACL {shape_name}: rows in {mapping.source_table} that "
|
|
311
|
+
f"violate {', '.join(c.property_name for c, _ in resolved)} "
|
|
312
|
+
f"constraints (minCount / range / in)."
|
|
313
|
+
),
|
|
314
|
+
columns=cols,
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
__all__ = ["generate_shacl_violation_views"]
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
"""Assemble a complete :class:`GenieSpace` from schema state.
|
|
2
|
+
|
|
3
|
+
W5's deliverable. Wraps the W1+W2 view generator, W4's instruction
|
|
4
|
+
and sample-query generators, plus optional MCP server registration,
|
|
5
|
+
into one :class:`GenieSpace` instance and serializes it to YAML.
|
|
6
|
+
|
|
7
|
+
The output YAML is **ProGraph's portable Space description format**,
|
|
8
|
+
a documented superset of the fields Databricks Genie's API accepts.
|
|
9
|
+
Users import it into Databricks one of two ways:
|
|
10
|
+
|
|
11
|
+
1. Manually: paste the table list / instructions / sample queries
|
|
12
|
+
into the Genie Space UI form.
|
|
13
|
+
2. Programmatically: pipe the YAML into a script that calls
|
|
14
|
+
Databricks' Genie REST API (planned for W6+).
|
|
15
|
+
|
|
16
|
+
Either way the YAML stays the source of truth; Databricks renders
|
|
17
|
+
it; we own it.
|
|
18
|
+
|
|
19
|
+
YAML shape:
|
|
20
|
+
|
|
21
|
+
.. code-block:: yaml
|
|
22
|
+
|
|
23
|
+
name: Turbofan Predictive Maintenance
|
|
24
|
+
description: Engine health monitoring with OWL + SHACL
|
|
25
|
+
tables:
|
|
26
|
+
- workspace.maintenance.engines
|
|
27
|
+
- workspace.prograph_views.high_wear_component
|
|
28
|
+
- ...
|
|
29
|
+
instructions:
|
|
30
|
+
sections:
|
|
31
|
+
- heading: Overview
|
|
32
|
+
body: |
|
|
33
|
+
This Space queries ...
|
|
34
|
+
order: 10
|
|
35
|
+
- ...
|
|
36
|
+
rendered: |
|
|
37
|
+
## Overview
|
|
38
|
+
|
|
39
|
+
This Space queries ...
|
|
40
|
+
sample_queries:
|
|
41
|
+
- question: Which components are at high wear risk?
|
|
42
|
+
sql: SELECT * FROM workspace.prograph_views.high_wear_component
|
|
43
|
+
tags: [count, component]
|
|
44
|
+
mcp_servers:
|
|
45
|
+
- name: prograph
|
|
46
|
+
endpoint: https://...
|
|
47
|
+
auth_method: databricks_pat
|
|
48
|
+
views:
|
|
49
|
+
- name: high_wear_component
|
|
50
|
+
sql: SELECT id ... WHERE wear_index >= 0.7
|
|
51
|
+
source: owl_defined_class
|
|
52
|
+
...
|
|
53
|
+
metadata:
|
|
54
|
+
generated_by: prograph
|
|
55
|
+
generated_at: '2026-05-16T12:34:56Z'
|
|
56
|
+
prograph_version: 0.4.21
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
from __future__ import annotations
|
|
60
|
+
|
|
61
|
+
from datetime import datetime, timezone
|
|
62
|
+
from pathlib import Path
|
|
63
|
+
from typing import TYPE_CHECKING, Any, List, Optional
|
|
64
|
+
|
|
65
|
+
from prograph.genie.formats import (
|
|
66
|
+
GenieSpace,
|
|
67
|
+
McpServerRegistration,
|
|
68
|
+
)
|
|
69
|
+
from prograph.genie.instruction_generator import (
|
|
70
|
+
DEFAULT_CHAR_BUDGET,
|
|
71
|
+
generate_instructions,
|
|
72
|
+
)
|
|
73
|
+
from prograph.genie.sample_query_generator import (
|
|
74
|
+
DEFAULT_MAX_QUERIES,
|
|
75
|
+
generate_sample_queries,
|
|
76
|
+
)
|
|
77
|
+
from prograph.genie.view_generator import generate_views
|
|
78
|
+
|
|
79
|
+
if TYPE_CHECKING:
|
|
80
|
+
from prograph.genie.formats import ViewDefinition
|
|
81
|
+
from prograph.ontology.layer import OntologyLayer
|
|
82
|
+
from prograph.plan.sql.dialect import DialectConfig
|
|
83
|
+
from prograph.schema import GraphSchema
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def emit_genie_space(
|
|
87
|
+
schema: "GraphSchema",
|
|
88
|
+
ontology: Optional["OntologyLayer"] = None,
|
|
89
|
+
*,
|
|
90
|
+
name: Optional[str] = None,
|
|
91
|
+
description: str = "",
|
|
92
|
+
target_catalog: Optional[str] = None,
|
|
93
|
+
target_schema: str = "prograph_views",
|
|
94
|
+
dialect: Optional["DialectConfig"] = None,
|
|
95
|
+
include_labels: Optional[List[str]] = None,
|
|
96
|
+
exclude_labels: Optional[List[str]] = None,
|
|
97
|
+
mcp_servers: Optional[List[McpServerRegistration]] = None,
|
|
98
|
+
instructions_char_budget: int = DEFAULT_CHAR_BUDGET,
|
|
99
|
+
summary_instructions: bool = False,
|
|
100
|
+
max_sample_queries: int = DEFAULT_MAX_QUERIES,
|
|
101
|
+
source_config: Optional[str] = None,
|
|
102
|
+
) -> GenieSpace:
|
|
103
|
+
"""Assemble a complete :class:`GenieSpace` from schema state.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
schema: The :class:`GraphSchema`.
|
|
107
|
+
ontology: Optional :class:`OntologyLayer`. Unlocks subclass
|
|
108
|
+
closure views, OWL-defined class views, type-hierarchy
|
|
109
|
+
instructions, and defined-class sample queries.
|
|
110
|
+
name: Display name for the Space. Defaults to the schema's
|
|
111
|
+
``name`` attribute, or ``"ProGraph Generated Space"`` if
|
|
112
|
+
unset.
|
|
113
|
+
description: Free-form Space description. Lands in the YAML
|
|
114
|
+
verbatim.
|
|
115
|
+
target_catalog / target_schema: Where the views live in the
|
|
116
|
+
warehouse. The Space's ``tables`` list references views
|
|
117
|
+
using these prefixes so Genie picks them up natively.
|
|
118
|
+
dialect: SQL dialect for the view bodies. Defaults to Spark
|
|
119
|
+
backtick form.
|
|
120
|
+
include_labels / exclude_labels: NodeMapping label filters
|
|
121
|
+
applied to view + instruction + sample-query generation
|
|
122
|
+
consistently.
|
|
123
|
+
mcp_servers: Optional :class:`McpServerRegistration` list.
|
|
124
|
+
When given, the resulting Space registers the ProGraph
|
|
125
|
+
MCP server for graph-shaped queries Genie can't answer
|
|
126
|
+
via SQL alone.
|
|
127
|
+
instructions_char_budget: Token-friendly cap on the assembled
|
|
128
|
+
instructions body.
|
|
129
|
+
summary_instructions: When ``True``, emit just the overview
|
|
130
|
+
section.
|
|
131
|
+
max_sample_queries: Cap on the number of sample queries.
|
|
132
|
+
source_config: Optional path to the source YAML config; lands
|
|
133
|
+
in the ``metadata`` field as provenance.
|
|
134
|
+
|
|
135
|
+
Returns:
|
|
136
|
+
A fully-populated :class:`GenieSpace`.
|
|
137
|
+
"""
|
|
138
|
+
from prograph.r2rml.schema import require_property_graph
|
|
139
|
+
require_property_graph(schema, "a Genie space")
|
|
140
|
+
views = generate_views(
|
|
141
|
+
schema=schema,
|
|
142
|
+
ontology=ontology,
|
|
143
|
+
dialect=dialect,
|
|
144
|
+
include_labels=include_labels,
|
|
145
|
+
exclude_labels=exclude_labels,
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
instructions = generate_instructions(
|
|
149
|
+
schema=schema,
|
|
150
|
+
ontology=ontology,
|
|
151
|
+
views=views,
|
|
152
|
+
char_budget=instructions_char_budget,
|
|
153
|
+
summary_only=summary_instructions,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
samples = generate_sample_queries(
|
|
157
|
+
schema=schema,
|
|
158
|
+
ontology=ontology,
|
|
159
|
+
views=views,
|
|
160
|
+
max_queries=max_sample_queries,
|
|
161
|
+
view_namespace=_namespace_string(target_catalog, target_schema),
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
tables = _assemble_table_list(
|
|
165
|
+
schema=schema,
|
|
166
|
+
views=views,
|
|
167
|
+
target_catalog=target_catalog,
|
|
168
|
+
target_schema=target_schema,
|
|
169
|
+
include_labels=include_labels,
|
|
170
|
+
exclude_labels=exclude_labels,
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
metadata = _build_metadata(source_config=source_config)
|
|
174
|
+
|
|
175
|
+
return GenieSpace(
|
|
176
|
+
name=name or _default_space_name(schema),
|
|
177
|
+
description=description,
|
|
178
|
+
tables=tables,
|
|
179
|
+
instructions=instructions,
|
|
180
|
+
sample_queries=samples,
|
|
181
|
+
mcp_servers=mcp_servers or [],
|
|
182
|
+
views=views,
|
|
183
|
+
metadata=metadata,
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def render_space_yaml(space: GenieSpace) -> str:
|
|
188
|
+
"""Serialize a :class:`GenieSpace` to YAML.
|
|
189
|
+
|
|
190
|
+
Output is human-readable and stable across runs (deterministic key
|
|
191
|
+
order, no Python type tags). Importable by any YAML reader; safe
|
|
192
|
+
to commit to git.
|
|
193
|
+
"""
|
|
194
|
+
import yaml
|
|
195
|
+
|
|
196
|
+
return yaml.safe_dump(
|
|
197
|
+
space.to_dict(),
|
|
198
|
+
default_flow_style=False,
|
|
199
|
+
sort_keys=False,
|
|
200
|
+
allow_unicode=True,
|
|
201
|
+
width=100,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def write_space_yaml(space: GenieSpace, output_path: str | Path) -> Path:
|
|
206
|
+
"""Write a :class:`GenieSpace` to a YAML file. Returns the path."""
|
|
207
|
+
path = Path(output_path)
|
|
208
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
209
|
+
path.write_text(render_space_yaml(space), encoding="utf-8")
|
|
210
|
+
return path
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def load_space_yaml(path: str | Path) -> GenieSpace:
|
|
214
|
+
"""Read a :class:`GenieSpace` back from YAML. Roundtrips with
|
|
215
|
+
:func:`render_space_yaml`."""
|
|
216
|
+
import yaml
|
|
217
|
+
|
|
218
|
+
raw = Path(path).read_text(encoding="utf-8")
|
|
219
|
+
data = yaml.safe_load(raw)
|
|
220
|
+
return GenieSpace.from_dict(data)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
# ─── Helpers ─────────────────────────────────────────────────────────
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _default_space_name(schema: "GraphSchema") -> str:
|
|
227
|
+
n = getattr(schema, "name", None)
|
|
228
|
+
if n:
|
|
229
|
+
return n.replace("_", " ").title() + ": ProGraph Space"
|
|
230
|
+
return "ProGraph Generated Space"
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _namespace_string(
|
|
234
|
+
target_catalog: Optional[str], target_schema: str
|
|
235
|
+
) -> str:
|
|
236
|
+
if target_catalog:
|
|
237
|
+
return f"{target_catalog}.{target_schema}"
|
|
238
|
+
return target_schema
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _assemble_table_list(
|
|
242
|
+
schema: "GraphSchema",
|
|
243
|
+
views: List["ViewDefinition"],
|
|
244
|
+
target_catalog: Optional[str],
|
|
245
|
+
target_schema: str,
|
|
246
|
+
include_labels: Optional[List[str]] = None,
|
|
247
|
+
exclude_labels: Optional[List[str]] = None,
|
|
248
|
+
) -> List[str]:
|
|
249
|
+
"""Build the Space's table list: user tables first, then generated views.
|
|
250
|
+
|
|
251
|
+
User tables are deduplicated (NodeMappings + RelationshipMappings
|
|
252
|
+
may reference the same source). Views land under the configured
|
|
253
|
+
catalog/schema prefix.
|
|
254
|
+
|
|
255
|
+
When ``include_labels`` / ``exclude_labels`` are set, only emit
|
|
256
|
+
source tables backing kept node mappings, and only emit relationship
|
|
257
|
+
source tables whose start AND end labels survive the filter. Without
|
|
258
|
+
this, the table list grows past Genie's 30-table-per-Space cap on
|
|
259
|
+
schemas with many shared-table labels.
|
|
260
|
+
"""
|
|
261
|
+
include_set = set(include_labels) if include_labels is not None else None
|
|
262
|
+
exclude_set = set(exclude_labels) if exclude_labels else set()
|
|
263
|
+
|
|
264
|
+
def _label_kept(label: str) -> bool:
|
|
265
|
+
if include_set is not None and label not in include_set:
|
|
266
|
+
return False
|
|
267
|
+
if label in exclude_set:
|
|
268
|
+
return False
|
|
269
|
+
return True
|
|
270
|
+
|
|
271
|
+
kept_node_tables: list[tuple[str, "NodeMapping"]] = [
|
|
272
|
+
(m.source_table, m) for m in schema.node_mappings if _label_kept(m.label)
|
|
273
|
+
]
|
|
274
|
+
kept_labels = {m.label for _, m in kept_node_tables if m.label is not None}
|
|
275
|
+
|
|
276
|
+
user_tables: List[str] = []
|
|
277
|
+
seen: set[str] = set()
|
|
278
|
+
for table, _ in kept_node_tables:
|
|
279
|
+
if table in seen:
|
|
280
|
+
continue
|
|
281
|
+
seen.add(table)
|
|
282
|
+
user_tables.append(table)
|
|
283
|
+
|
|
284
|
+
for rel in schema.relationship_mappings:
|
|
285
|
+
# Skip rel mappings whose endpoint labels were filtered out.
|
|
286
|
+
# When no filters are set, kept_labels is the full set so this
|
|
287
|
+
# is a no-op.
|
|
288
|
+
if include_set is not None or exclude_set:
|
|
289
|
+
if (
|
|
290
|
+
rel.start_node_label not in kept_labels
|
|
291
|
+
or rel.end_node_label not in kept_labels
|
|
292
|
+
):
|
|
293
|
+
continue
|
|
294
|
+
if rel.source_table in seen:
|
|
295
|
+
continue
|
|
296
|
+
seen.add(rel.source_table)
|
|
297
|
+
user_tables.append(rel.source_table)
|
|
298
|
+
|
|
299
|
+
view_prefix = _namespace_string(target_catalog, target_schema)
|
|
300
|
+
view_tables = [f"{view_prefix}.{v.name}" for v in views]
|
|
301
|
+
|
|
302
|
+
return user_tables + view_tables
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _build_metadata(source_config: Optional[str]) -> dict[str, Any]:
|
|
306
|
+
"""Provenance metadata for the Space.
|
|
307
|
+
|
|
308
|
+
Stable across runs in field ordering; the timestamp is the only
|
|
309
|
+
non-deterministic piece, which is fine, it's information, not
|
|
310
|
+
something tests compare on.
|
|
311
|
+
"""
|
|
312
|
+
from prograph import __version__ as prograph_version
|
|
313
|
+
|
|
314
|
+
meta: dict[str, Any] = {
|
|
315
|
+
"generated_by": "prograph",
|
|
316
|
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
317
|
+
"prograph_version": prograph_version,
|
|
318
|
+
}
|
|
319
|
+
if source_config:
|
|
320
|
+
meta["source_config"] = source_config
|
|
321
|
+
return meta
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
__all__ = [
|
|
325
|
+
"emit_genie_space",
|
|
326
|
+
"render_space_yaml",
|
|
327
|
+
"write_space_yaml",
|
|
328
|
+
"load_space_yaml",
|
|
329
|
+
]
|