prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
"""SHACL shapes generated from a mapping and its database schema (https://www.w3.org/TR/shacl/).
|
|
2
|
+
|
|
3
|
+
One `sh:NodeShape` per class (`sh:targetClass`) of the source maps; a property shape for each predicate
|
|
4
|
+
that class's nodes are given. Every constraint is a fact of the maps or of the database catalog, so the
|
|
5
|
+
shapes hold on the mapped data by construction, and each property shape's `sh:description` says which fact:
|
|
6
|
+
|
|
7
|
+
- `sh:datatype`: a literal's own datatype (R2RML 10.2 natural mapping of the column's recorded type, a
|
|
8
|
+
declared `rr:datatype`, `xsd:string` for a template); `rdf:langString` (and `sh:languageIn` for a fixed
|
|
9
|
+
tag) for a language-tagged literal. A column whose type is not recorded gets none.
|
|
10
|
+
- `sh:nodeKind` and `sh:class`: a reference (a relationship, an R2RML referencing object map) has an IRI
|
|
11
|
+
or blank node object, of the end map's classes where the end's row must exist.
|
|
12
|
+
- `sh:minCount 1`: every map that gives the class has the predicate from a NOT NULL column of its own
|
|
13
|
+
table (read from the catalog), or a constant. A map over a query, and a database with no catalog
|
|
14
|
+
reader (frames, Spark), prove nothing here. A row filter keeps it (a subset of rows). Not claimed for a
|
|
15
|
+
reference: the end's row must exist too, which only an enforced foreign key proves.
|
|
16
|
+
- `sh:maxCount 1`: the predicate has exactly one giver in the whole mapping (so no second map, nor an
|
|
17
|
+
overlapping subject, adds a value), and one row gives one subject: a trusted key (`keys.trusted_keys`)
|
|
18
|
+
inside the subject's columns, the subject written one term per values. For a reference, the join to the
|
|
19
|
+
parent row matches the parent's trusted key too.
|
|
20
|
+
|
|
21
|
+
No constraint is claimed where a map has a predicate made from the row, or gives `rdf:type` itself
|
|
22
|
+
(class membership is then open), for `sh:minCount`; for `sh:maxCount` where a predicate is made from a row.
|
|
23
|
+
Triples in named graphs are shaped like the default graph's. Inferred rows are not the mapping's.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
from typing import Any, Callable, Dict, FrozenSet, List, Optional, Tuple
|
|
29
|
+
|
|
30
|
+
from prograph.rules import module_home
|
|
31
|
+
from prograph.schema import xsd
|
|
32
|
+
from prograph.schema.source_map import BLANK, IRI
|
|
33
|
+
|
|
34
|
+
module_home("shapes-from-schema", "SHACL shapes generated from a mapping and its database catalog",
|
|
35
|
+
terms=("generate shapes", "shapes from schema", "sh:minCount", "sh:maxCount", "NOT NULL", "sh:NodeShape",
|
|
36
|
+
"engine.shapes", "shape generation"))
|
|
37
|
+
|
|
38
|
+
SH = "http://www.w3.org/ns/shacl#"
|
|
39
|
+
RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
|
|
40
|
+
DEFAULT_SHAPE_BASE = "urn:prograph:shape:"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def generate_shapes(maps: Any, *, column_datatype: Callable[[str, str], Optional[str]] = lambda t, c: None,
|
|
44
|
+
not_null: Callable[[Any], Optional[FrozenSet[str]]] = lambda source: None,
|
|
45
|
+
keys: Callable[[Any], Tuple[Tuple[str, ...], ...]] = lambda m: (),
|
|
46
|
+
fold: Callable[[str], str] = lambda name: name,
|
|
47
|
+
shape_base: str = DEFAULT_SHAPE_BASE) -> Any:
|
|
48
|
+
"""The shapes, an rdflib Graph.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
maps: The `SourceMaps`.
|
|
52
|
+
column_datatype: (table, column) -> the datatype IRI of the column's recorded type, or None.
|
|
53
|
+
not_null: A source -> its table's NOT NULL columns, None where the catalog was not read.
|
|
54
|
+
keys: A source map -> the column sets a plan may rely on as unique (`keys.trusted_keys`).
|
|
55
|
+
fold: The database's folding of an undelimited name (`Dialect.fold_identifier`).
|
|
56
|
+
shape_base: The prefix of the node shapes' IRIs.
|
|
57
|
+
"""
|
|
58
|
+
from rdflib import BNode, Graph, Literal, Namespace, URIRef
|
|
59
|
+
from rdflib.namespace import RDF, RDFS, XSD
|
|
60
|
+
sh = Namespace(SH)
|
|
61
|
+
g = Graph()
|
|
62
|
+
g.bind("sh", sh)
|
|
63
|
+
g.bind("xsd", XSD)
|
|
64
|
+
|
|
65
|
+
open_predicates = bool(maps.open_predicates)
|
|
66
|
+
types_given = RDF_TYPE in maps.by_predicate
|
|
67
|
+
# Gives per class: predicate -> [(map, predicate_object or None)], in map order.
|
|
68
|
+
per_class: Dict[str, Dict[str, List[Tuple[Any, Any]]]] = {}
|
|
69
|
+
for node in maps.nodes:
|
|
70
|
+
for cls in node.classes:
|
|
71
|
+
slot = per_class.setdefault(cls, {})
|
|
72
|
+
for po in node.predicate_objects:
|
|
73
|
+
if po.predicate and po.predicate != RDF_TYPE:
|
|
74
|
+
slot.setdefault(po.predicate, []).append((node, po))
|
|
75
|
+
for edge in maps.edges:
|
|
76
|
+
start = maps.node(edge.start.map_id) if edge.start.map_id else None
|
|
77
|
+
if start is None or not edge.predicate or edge.predicate == RDF_TYPE:
|
|
78
|
+
continue
|
|
79
|
+
for cls in start.classes:
|
|
80
|
+
per_class.setdefault(cls, {}).setdefault(edge.predicate, []).append((edge, None))
|
|
81
|
+
|
|
82
|
+
def key_of(m: Any) -> Tuple[Tuple[str, ...], ...]:
|
|
83
|
+
try:
|
|
84
|
+
return keys(m)
|
|
85
|
+
except Exception: # noqa: BLE001 - no catalog to read: no key to rely on
|
|
86
|
+
return ()
|
|
87
|
+
|
|
88
|
+
def one_giver_is_unique(predicate: str) -> bool:
|
|
89
|
+
givers = maps.by_predicate.get(predicate, [])
|
|
90
|
+
if len(givers) != 1 or open_predicates:
|
|
91
|
+
return False
|
|
92
|
+
m, po = givers[0]
|
|
93
|
+
if po is not None: # a node map's literal
|
|
94
|
+
subject = m.subject
|
|
95
|
+
return po.predicate_map is None and subject.one_term_per_values() and any(
|
|
96
|
+
set(k) <= set(subject.columns()) for k in key_of(m))
|
|
97
|
+
start = m.start.term
|
|
98
|
+
if start is None or not start.one_term_per_values() or not any(
|
|
99
|
+
set(k) <= set(m.start.columns) for k in key_of(m)):
|
|
100
|
+
return False
|
|
101
|
+
if m.join is not None or m.end.parent_columns:
|
|
102
|
+
parent = maps.node(m.end.map_id) if m.end.map_id else None
|
|
103
|
+
matched = {p for _, p in m.join.conditions} if m.join is not None else set(m.end.parent_columns)
|
|
104
|
+
return parent is not None and any(set(k) <= matched for k in key_of(parent))
|
|
105
|
+
return True
|
|
106
|
+
|
|
107
|
+
def held_name(source: Any, column: str) -> str:
|
|
108
|
+
"""The name the catalog holds a column under: an undelimited name as the database folds it
|
|
109
|
+
(`Dialect.fold_identifier`), every other name exactly."""
|
|
110
|
+
if column in source.regular_columns:
|
|
111
|
+
return fold(column)
|
|
112
|
+
return column
|
|
113
|
+
|
|
114
|
+
def nonnull(node: Any, po: Any) -> bool:
|
|
115
|
+
obj = po.object
|
|
116
|
+
if obj.constant is not None:
|
|
117
|
+
return True
|
|
118
|
+
if node.source.query is not None:
|
|
119
|
+
return False
|
|
120
|
+
held = not_null(node.source)
|
|
121
|
+
cols = obj.columns()
|
|
122
|
+
if held is None or not cols:
|
|
123
|
+
return False
|
|
124
|
+
return all(held_name(node.source, c) in held for c in cols)
|
|
125
|
+
|
|
126
|
+
def has_min(cls: str, predicate: str) -> bool:
|
|
127
|
+
if open_predicates or types_given:
|
|
128
|
+
return False
|
|
129
|
+
nodes = [n for n in maps.by_class.get(cls, [])]
|
|
130
|
+
return bool(nodes) and all(
|
|
131
|
+
any(po.predicate == predicate and nonnull(n, po) for po in n.predicate_objects) for n in nodes)
|
|
132
|
+
|
|
133
|
+
def literal_claim(node: Any, po: Any) -> Dict[str, Any]:
|
|
134
|
+
obj = po.object
|
|
135
|
+
claim: Dict[str, Any] = {}
|
|
136
|
+
facts = []
|
|
137
|
+
if obj.language_column:
|
|
138
|
+
# A row's own tag wins; a row with none has the fixed tag, else a plain string. A NOT NULL
|
|
139
|
+
# column can hold '' (an untagged string): only a fixed non-empty tag proves every value tagged.
|
|
140
|
+
tagged = bool(obj.language)
|
|
141
|
+
claim["datatype"] = xsd.RDF_LANGSTRING if tagged else None
|
|
142
|
+
claim["or_string"] = not tagged
|
|
143
|
+
facts.append(f"language tag from column {obj.language_column}")
|
|
144
|
+
elif obj.language:
|
|
145
|
+
claim["datatype"] = xsd.RDF_LANGSTRING
|
|
146
|
+
claim["languages"] = (obj.language,)
|
|
147
|
+
facts.append(f"fixed language tag {obj.language!r}")
|
|
148
|
+
elif obj.datatype:
|
|
149
|
+
claim["datatype"] = obj.datatype
|
|
150
|
+
facts.append("declared datatype")
|
|
151
|
+
elif obj.column is not None:
|
|
152
|
+
dt = column_datatype(node.source.table, obj.column)
|
|
153
|
+
if dt is not None:
|
|
154
|
+
claim["datatype"] = dt
|
|
155
|
+
facts.append(f"type of column {_table_name(node.source)}.{obj.column}")
|
|
156
|
+
elif obj.template is not None:
|
|
157
|
+
claim["datatype"] = xsd.STRING
|
|
158
|
+
facts.append("a template's text")
|
|
159
|
+
claim["facts"] = facts
|
|
160
|
+
return claim
|
|
161
|
+
|
|
162
|
+
def reference_claim(edge: Any) -> Dict[str, Any]:
|
|
163
|
+
end = edge.end
|
|
164
|
+
claim: Dict[str, Any] = {"facts": []}
|
|
165
|
+
if end.term is not None:
|
|
166
|
+
claim["nodekind"] = {IRI: "IRI", BLANK: "BlankNode"}.get(end.term.term_type)
|
|
167
|
+
claim["facts"].append(f"the end's term is a {end.term.term_type}")
|
|
168
|
+
if end.exists and end.map_id:
|
|
169
|
+
maps_of_end = [maps.node(i) for i in (end.map_id,) + tuple(end.peers)]
|
|
170
|
+
sets = {tuple(sorted(n.classes)) for n in maps_of_end if n is not None}
|
|
171
|
+
if len(sets) == 1 and next(iter(sets)):
|
|
172
|
+
claim["classes"] = next(iter(sets))
|
|
173
|
+
claim["facts"].append("the end's row must exist")
|
|
174
|
+
return claim
|
|
175
|
+
|
|
176
|
+
def emit_claim(shape: Any, claim: Dict[str, Any]) -> None:
|
|
177
|
+
if claim.get("datatype"):
|
|
178
|
+
g.add((shape, sh.datatype, URIRef(claim["datatype"])))
|
|
179
|
+
if claim.get("languages"):
|
|
180
|
+
lst = BNode()
|
|
181
|
+
from rdflib.collection import Collection
|
|
182
|
+
Collection(g, lst, [Literal(x) for x in claim["languages"]])
|
|
183
|
+
g.add((shape, sh.languageIn, lst))
|
|
184
|
+
if claim.get("nodekind"):
|
|
185
|
+
g.add((shape, sh.nodeKind, sh[claim["nodekind"]]))
|
|
186
|
+
for c in claim.get("classes", ()):
|
|
187
|
+
g.add((shape, getattr(sh, "class"), URIRef(c)))
|
|
188
|
+
|
|
189
|
+
used: Dict[str, str] = {}
|
|
190
|
+
for cls in sorted(per_class):
|
|
191
|
+
local = cls.rstrip("/#").rpartition("/")[2].rpartition("#")[2] or "Class"
|
|
192
|
+
iri = shape_base + local + "Shape"
|
|
193
|
+
while iri in used:
|
|
194
|
+
iri += "_"
|
|
195
|
+
used[iri] = cls
|
|
196
|
+
node_shape = URIRef(iri)
|
|
197
|
+
g.add((node_shape, RDF.type, sh.NodeShape))
|
|
198
|
+
g.add((node_shape, sh.targetClass, URIRef(cls)))
|
|
199
|
+
for predicate in sorted(per_class[cls]):
|
|
200
|
+
givers = per_class[cls][predicate]
|
|
201
|
+
claims, facts = [], []
|
|
202
|
+
for m, po in givers:
|
|
203
|
+
c = literal_claim(m, po) if po is not None else reference_claim(m)
|
|
204
|
+
facts += c.pop("facts", [])
|
|
205
|
+
claims.append(c)
|
|
206
|
+
# Any value of the predicate, on any node, is one some giver of it gives.
|
|
207
|
+
for other in maps.by_predicate.get(predicate, []):
|
|
208
|
+
if all(other.map is not m for m, _ in givers):
|
|
209
|
+
c = literal_claim(other.map, other.predicate_object) if other.predicate_object is not None \
|
|
210
|
+
else reference_claim(other.map)
|
|
211
|
+
c.pop("facts", None)
|
|
212
|
+
claims.append(c)
|
|
213
|
+
prop = BNode()
|
|
214
|
+
g.add((node_shape, sh.property, prop))
|
|
215
|
+
g.add((prop, sh.path, URIRef(predicate)))
|
|
216
|
+
uniq = []
|
|
217
|
+
for c in claims:
|
|
218
|
+
if c not in uniq:
|
|
219
|
+
uniq.append(c)
|
|
220
|
+
usable = [u for u in uniq if any(u.get(k) for k in ("datatype", "nodekind", "classes"))
|
|
221
|
+
or u.get("or_string")]
|
|
222
|
+
if len(usable) == len(uniq):
|
|
223
|
+
alts = []
|
|
224
|
+
for u in uniq:
|
|
225
|
+
if u.get("or_string"):
|
|
226
|
+
alts.append({"datatype": xsd.STRING})
|
|
227
|
+
alts.append({"datatype": xsd.RDF_LANGSTRING})
|
|
228
|
+
else:
|
|
229
|
+
alts.append(u)
|
|
230
|
+
if len(alts) == 1:
|
|
231
|
+
emit_claim(prop, alts[0])
|
|
232
|
+
else:
|
|
233
|
+
from rdflib.collection import Collection
|
|
234
|
+
members = []
|
|
235
|
+
for a in alts:
|
|
236
|
+
b = BNode()
|
|
237
|
+
emit_claim(b, a)
|
|
238
|
+
members.append(b)
|
|
239
|
+
lst = BNode()
|
|
240
|
+
Collection(g, lst, members)
|
|
241
|
+
g.add((prop, sh["or"], lst))
|
|
242
|
+
if has_min(cls, predicate):
|
|
243
|
+
g.add((prop, sh.minCount, Literal(1)))
|
|
244
|
+
facts.append("NOT NULL in every map of the class")
|
|
245
|
+
if one_giver_is_unique(predicate):
|
|
246
|
+
g.add((prop, sh.maxCount, Literal(1)))
|
|
247
|
+
facts.append("one giver, one row per subject by a trusted key")
|
|
248
|
+
if facts:
|
|
249
|
+
g.add((prop, sh.description, Literal("; ".join(dict.fromkeys(facts)))))
|
|
250
|
+
return g
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _table_name(source: Any) -> str:
|
|
254
|
+
"""The table a fact is about, for a description: a query source has no table of its own."""
|
|
255
|
+
return "(query source)" if source.query is not None else source.table
|
prograph/shacl/report.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Typed validation report.
|
|
2
|
+
|
|
3
|
+
Wraps the RDF graph that pyshacl produces in a Pythonic shape so callers
|
|
4
|
+
don't have to parse RDF to understand failures. The original Turtle is
|
|
5
|
+
preserved as :attr:`ValidationReport.raw_rdf` for downstream tooling
|
|
6
|
+
(audit pipelines, compliance reports) that expects canonical SHACL output.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections import Counter
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from typing import Any, Dict, List, Optional
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class ViolationDetail:
|
|
18
|
+
"""A single SHACL validation result.
|
|
19
|
+
|
|
20
|
+
Mirrors the fields of an ``sh:ValidationResult`` node but flattened
|
|
21
|
+
to a Python-native shape. All identifiers (``focus_node``,
|
|
22
|
+
``source_shape``, etc.) are full IRIs as strings.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
focus_node: str
|
|
26
|
+
"""IRI of the entity that violated the shape."""
|
|
27
|
+
|
|
28
|
+
severity: str
|
|
29
|
+
"""One of ``"Violation"``, ``"Warning"``, ``"Info"``.
|
|
30
|
+
|
|
31
|
+
Stored without the ``sh:`` prefix for ergonomic comparison.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
source_shape: str
|
|
35
|
+
"""IRI of the shape that produced this result."""
|
|
36
|
+
|
|
37
|
+
source_constraint_component: str
|
|
38
|
+
"""Local name of the constraint component (e.g. ``"MinCountConstraintComponent"``)."""
|
|
39
|
+
|
|
40
|
+
result_path: Optional[str] = None
|
|
41
|
+
"""Property path involved, if any (None for shape-level violations)."""
|
|
42
|
+
|
|
43
|
+
value: Optional[Any] = None
|
|
44
|
+
"""Offending value, where applicable (e.g. the literal that
|
|
45
|
+
failed a pattern check). ``None`` for cardinality / shape-level
|
|
46
|
+
violations."""
|
|
47
|
+
|
|
48
|
+
message: str = ""
|
|
49
|
+
"""Human-readable message. Either pyshacl's default or the value of
|
|
50
|
+
``sh:message`` on the originating shape."""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass
|
|
54
|
+
class ValidationReport:
|
|
55
|
+
"""Result of running SHACL validation against an engine's state.
|
|
56
|
+
|
|
57
|
+
Construct via :func:`prograph.shacl.validate` or
|
|
58
|
+
:meth:`ProGraph.validate_shapes`; not intended to be built manually
|
|
59
|
+
except in tests.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
conforms: bool
|
|
63
|
+
violations: List[ViolationDetail]
|
|
64
|
+
raw_rdf: str = ""
|
|
65
|
+
"""The full ``sh:ValidationReport`` graph serialized as Turtle.
|
|
66
|
+
|
|
67
|
+
Useful for callers who need to forward canonical SHACL output
|
|
68
|
+
downstream (audit pipelines, compliance systems, RDF logs).
|
|
69
|
+
Most user code can ignore this and work with :attr:`violations`.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
by_severity: Dict[str, int] = field(default_factory=dict)
|
|
73
|
+
by_shape: Dict[str, int] = field(default_factory=dict)
|
|
74
|
+
by_focus_node: Dict[str, int] = field(default_factory=dict)
|
|
75
|
+
|
|
76
|
+
# ─── Factory ─────────────────────────────────────────────────────
|
|
77
|
+
|
|
78
|
+
@classmethod
|
|
79
|
+
def from_violations(
|
|
80
|
+
cls,
|
|
81
|
+
violations: List[ViolationDetail],
|
|
82
|
+
raw_rdf: str = "",
|
|
83
|
+
) -> "ValidationReport":
|
|
84
|
+
"""Build a report from a violation list. Aggregates auto-computed.
|
|
85
|
+
|
|
86
|
+
``conforms`` follows strict SHACL semantics: data is considered
|
|
87
|
+
conforming when no results have severity ``Violation``. Warnings
|
|
88
|
+
and Infos surface in :attr:`violations` and the aggregates but
|
|
89
|
+
don't flip ``conforms`` to ``False``, they're advisory signals,
|
|
90
|
+
not hard failures.
|
|
91
|
+
"""
|
|
92
|
+
conforms = not any(v.severity == "Violation" for v in violations)
|
|
93
|
+
return cls(
|
|
94
|
+
conforms=conforms,
|
|
95
|
+
violations=violations,
|
|
96
|
+
raw_rdf=raw_rdf,
|
|
97
|
+
by_severity=dict(Counter(v.severity for v in violations)),
|
|
98
|
+
by_shape=dict(Counter(v.source_shape for v in violations)),
|
|
99
|
+
by_focus_node=dict(Counter(v.focus_node for v in violations)),
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
# ─── Convenience accessors ───────────────────────────────────────
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def violation_count(self) -> int:
|
|
106
|
+
return len(self.violations)
|
|
107
|
+
|
|
108
|
+
def violations_of_severity(self, severity: str) -> List[ViolationDetail]:
|
|
109
|
+
"""Filter to violations of a specific severity level."""
|
|
110
|
+
return [v for v in self.violations if v.severity == severity]
|
|
111
|
+
|
|
112
|
+
def violations_for_focus(self, focus_node: str) -> List[ViolationDetail]:
|
|
113
|
+
"""All violations involving a given entity IRI."""
|
|
114
|
+
return [v for v in self.violations if v.focus_node == focus_node]
|
|
115
|
+
|
|
116
|
+
def violations_from_shape(self, shape_iri: str) -> List[ViolationDetail]:
|
|
117
|
+
"""All violations produced by a given shape IRI."""
|
|
118
|
+
return [v for v in self.violations if v.source_shape == shape_iri]
|
|
119
|
+
|
|
120
|
+
def summary(self) -> str:
|
|
121
|
+
"""One-line summary suitable for logs.
|
|
122
|
+
|
|
123
|
+
Shape: ``"Data conforms to all shapes"`` if there are no results
|
|
124
|
+
at all; ``"Conforms; N advisory result(s) (...)"`` if every
|
|
125
|
+
result is advisory severity (Warning / Info); otherwise
|
|
126
|
+
``"N violations (...)"`` listing the severity breakdown.
|
|
127
|
+
"""
|
|
128
|
+
if not self.violations:
|
|
129
|
+
return "Data conforms to all shapes"
|
|
130
|
+
sev = ", ".join(
|
|
131
|
+
f"{count} {name}" for name, count in sorted(self.by_severity.items())
|
|
132
|
+
)
|
|
133
|
+
if self.conforms:
|
|
134
|
+
return f"Conforms; {self.violation_count} advisory result(s) ({sev})"
|
|
135
|
+
return f"{self.violation_count} violations ({sev})"
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
__all__ = ["ValidationReport", "ViolationDetail"]
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""SHACL validation: a pyshacl wrapper that produces a typed report.
|
|
2
|
+
|
|
3
|
+
Takes an RDF data graph and a SHACL shapes graph, runs pyshacl, and
|
|
4
|
+
returns a :class:`ValidationReport` with structured violation details
|
|
5
|
+
plus the raw RDF report.
|
|
6
|
+
|
|
7
|
+
The public entry point users will most often touch is
|
|
8
|
+
:meth:`ProGraph.validate_shapes`, which constructs the data graph with a
|
|
9
|
+
SPARQL CONSTRUCT and delegates here. This module exposes :func:`validate` directly for
|
|
10
|
+
callers that already have an rdflib data graph or want to validate
|
|
11
|
+
against an arbitrary triple set.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any, Optional, Union
|
|
18
|
+
|
|
19
|
+
from prograph.shacl.report import ValidationReport, ViolationDetail
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def validate(
|
|
23
|
+
data_graph: Any,
|
|
24
|
+
shapes: Union[str, Path, Any],
|
|
25
|
+
inference: str = "none",
|
|
26
|
+
advanced: bool = True,
|
|
27
|
+
) -> ValidationReport:
|
|
28
|
+
"""Validate ``data_graph`` against ``shapes``.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
data_graph: An ``rdflib.Graph`` of data to validate.
|
|
32
|
+
shapes: SHACL shapes, accepted as a file path, a Turtle
|
|
33
|
+
string, or an ``rdflib.Graph``.
|
|
34
|
+
inference: pyshacl pre-validation inference profile.
|
|
35
|
+
``'none'`` | ``'rdfs'`` | ``'owlrl'`` | ``'both'``.
|
|
36
|
+
Mostly orthogonal to ProGraph's own OWL 2 RL+ reasoner,
|
|
37
|
+
if your engine already reasons (``ontology=``, ``profile=``),
|
|
38
|
+
keep this ``'none'`` to avoid redundant work.
|
|
39
|
+
advanced: When ``True`` (default), enables pyshacl's
|
|
40
|
+
SHACL-AF advanced features (SPARQL-based constraints,
|
|
41
|
+
SHACL functions, etc.).
|
|
42
|
+
|
|
43
|
+
Returns:
|
|
44
|
+
:class:`ValidationReport`.
|
|
45
|
+
"""
|
|
46
|
+
import pyshacl
|
|
47
|
+
import rdflib
|
|
48
|
+
|
|
49
|
+
shapes_graph = _coerce_shapes_graph(shapes)
|
|
50
|
+
|
|
51
|
+
conforms, results_graph, results_text = pyshacl.validate(
|
|
52
|
+
data_graph=data_graph,
|
|
53
|
+
shacl_graph=shapes_graph,
|
|
54
|
+
inference=inference,
|
|
55
|
+
advanced=advanced,
|
|
56
|
+
debug=False,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
violations = _extract_violations(results_graph) if not conforms else []
|
|
60
|
+
raw_rdf = results_graph.serialize(format="turtle")
|
|
61
|
+
if isinstance(raw_rdf, bytes): # pragma: no cover
|
|
62
|
+
raw_rdf = raw_rdf.decode("utf-8")
|
|
63
|
+
|
|
64
|
+
return ValidationReport.from_violations(
|
|
65
|
+
violations=violations,
|
|
66
|
+
raw_rdf=raw_rdf,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _coerce_shapes_graph(shapes: Union[str, Path, Any]) -> Any:
|
|
71
|
+
"""Accept shapes as path, string, or rdflib.Graph."""
|
|
72
|
+
import rdflib
|
|
73
|
+
|
|
74
|
+
if isinstance(shapes, rdflib.Graph):
|
|
75
|
+
return shapes
|
|
76
|
+
|
|
77
|
+
g = rdflib.Graph()
|
|
78
|
+
if isinstance(shapes, Path):
|
|
79
|
+
g.parse(str(shapes), format=_guess_format(shapes))
|
|
80
|
+
return g
|
|
81
|
+
if isinstance(shapes, str):
|
|
82
|
+
# Disambiguate: existing file path vs raw Turtle.
|
|
83
|
+
try:
|
|
84
|
+
p = Path(shapes)
|
|
85
|
+
if p.exists() and p.is_file():
|
|
86
|
+
g.parse(str(p), format=_guess_format(p))
|
|
87
|
+
return g
|
|
88
|
+
except OSError:
|
|
89
|
+
pass
|
|
90
|
+
g.parse(data=shapes, format="turtle")
|
|
91
|
+
return g
|
|
92
|
+
|
|
93
|
+
raise TypeError(
|
|
94
|
+
f"shapes must be a path, Turtle string, or rdflib.Graph; "
|
|
95
|
+
f"got {type(shapes).__name__}"
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _guess_format(path: Path) -> str:
|
|
100
|
+
suffix = path.suffix.lower()
|
|
101
|
+
return {
|
|
102
|
+
".ttl": "turtle",
|
|
103
|
+
".n3": "n3",
|
|
104
|
+
".nt": "nt",
|
|
105
|
+
".rdf": "xml",
|
|
106
|
+
".xml": "xml",
|
|
107
|
+
".jsonld": "json-ld",
|
|
108
|
+
}.get(suffix, "turtle")
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _extract_violations(results_graph: Any) -> list:
|
|
112
|
+
"""Parse a SHACL Validation Report graph into ViolationDetail list."""
|
|
113
|
+
import rdflib
|
|
114
|
+
from rdflib.namespace import RDF, SH
|
|
115
|
+
|
|
116
|
+
violations = []
|
|
117
|
+
for result_node in results_graph.subjects(RDF.type, SH.ValidationResult):
|
|
118
|
+
focus_node = results_graph.value(result_node, SH.focusNode)
|
|
119
|
+
severity_uri = results_graph.value(result_node, SH.resultSeverity)
|
|
120
|
+
source_shape = results_graph.value(result_node, SH.sourceShape)
|
|
121
|
+
constraint_uri = results_graph.value(
|
|
122
|
+
result_node, SH.sourceConstraintComponent
|
|
123
|
+
)
|
|
124
|
+
result_path = results_graph.value(result_node, SH.resultPath)
|
|
125
|
+
value = results_graph.value(result_node, SH.value)
|
|
126
|
+
message_node = results_graph.value(result_node, SH.resultMessage)
|
|
127
|
+
|
|
128
|
+
violations.append(ViolationDetail(
|
|
129
|
+
focus_node=str(focus_node) if focus_node else "",
|
|
130
|
+
severity=_local_name(severity_uri) if severity_uri else "Violation",
|
|
131
|
+
source_shape=str(source_shape) if source_shape else "",
|
|
132
|
+
source_constraint_component=(
|
|
133
|
+
_local_name(constraint_uri) if constraint_uri else ""
|
|
134
|
+
),
|
|
135
|
+
result_path=str(result_path) if result_path else None,
|
|
136
|
+
value=_coerce_value(value),
|
|
137
|
+
message=str(message_node) if message_node else "",
|
|
138
|
+
))
|
|
139
|
+
|
|
140
|
+
return violations
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _local_name(uri: Any) -> str:
|
|
144
|
+
s = str(uri)
|
|
145
|
+
for sep in ("#", "/"):
|
|
146
|
+
if sep in s:
|
|
147
|
+
return s.rsplit(sep, 1)[1]
|
|
148
|
+
return s
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _coerce_value(node: Any):
|
|
152
|
+
"""Convert an rdflib node to a Python-native value when possible."""
|
|
153
|
+
if node is None:
|
|
154
|
+
return None
|
|
155
|
+
import rdflib
|
|
156
|
+
if isinstance(node, rdflib.Literal):
|
|
157
|
+
try:
|
|
158
|
+
return node.toPython()
|
|
159
|
+
except Exception: # pragma: no cover
|
|
160
|
+
return str(node)
|
|
161
|
+
return str(node)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
__all__ = ["validate"]
|