prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
"""Inconsistency: an ontology and the data it reads that have no model, found and then raised or reported.
|
|
2
|
+
|
|
3
|
+
Under Direct semantics an inconsistent ontology entails every axiom (OWL 2 Direct Semantics §2.4, Entailment Regimes
|
|
4
|
+
§7 requires a consistent one), and OWL 2 RL's rules that conclude `false` (Profiles §4.3, tables 4 to 9) mark the same
|
|
5
|
+
under the RDF-Based semantics. An answer drawn from such a graph means nothing, so by default the engine refuses
|
|
6
|
+
(`inconsistency='error'`: `InconsistentOntologyError`, raised where the inconsistency is found); `inconsistency=
|
|
7
|
+
'report'` answers as usual and keeps the report (`ProGraph.consistency_report()`).
|
|
8
|
+
|
|
9
|
+
- **What is checked.** profile='rl': the RL rules that conclude false, over the closure the run computed
|
|
10
|
+
(`owl_rl._Run.consistency`): cax-dw, cax-adc, cls-com, cls-nothing2, cls-maxc1, cls-maxqc1, cls-maxqc2, prp-irp, prp-asyp,
|
|
11
|
+
prp-pdw, prp-adp, prp-npa1, prp-npa2; eq-diff1/2/3 (a stated `x owl:differentFrom x`, owl:AllDifferent naming one
|
|
12
|
+
IRI twice, and an owl:sameAs component holding two different IRIs, which the run leaves unmerged); two values of a
|
|
13
|
+
property that has at most one (prp-fp, cls-maxc2, cls-maxqc3/4) that are not one value; dt-not-type (a literal of
|
|
14
|
+
a property whose range is a datatype outside that datatype's value space, and a numeric or boolean literal whose
|
|
15
|
+
lexical form is not one of its datatype). profile='ql': the QL negative axioms' checks (`ql.violations`).
|
|
16
|
+
- **Values, not texts.** Two literals are one value where their value spaces nest (`"1"^^xsd:integer` and
|
|
17
|
+
`"1.0"^^xsd:decimal` are one, never a clash); xsd:float and xsd:double each by its own value (+0.0 and -0.0 two).
|
|
18
|
+
- **When.** profile='rl': once per materialization, over the run's staged closure before its step is written, the
|
|
19
|
+
verdict written in that step beside the run's record (`owl_rl.RUN_CONSISTENCY`), so the stored facts' freshness
|
|
20
|
+
proof covers it: an engine built over current facts reads the verdict and checks nothing again; a run (on open, or
|
|
21
|
+
a query's refresh where a table changed) checks again. profile='ql' (nothing stored, answered by rewriting): once,
|
|
22
|
+
when the engine is built; the data are not watched after, so a change made later is checked when an engine is
|
|
23
|
+
built again.
|
|
24
|
+
- **What is said.** Each rule or check that matched: its id, the axiom in short Turtle, how many solutions, and up to
|
|
25
|
+
`SAMPLE` of them as the facts that clash (subject, property, values).
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import json
|
|
31
|
+
import re
|
|
32
|
+
from dataclasses import dataclass, replace
|
|
33
|
+
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
|
34
|
+
|
|
35
|
+
from prograph.rules import home, module_home
|
|
36
|
+
from prograph.schema import xsd
|
|
37
|
+
|
|
38
|
+
module_home("owl-inconsistency", "An ontology and its data with no model: checked once per run or engine, raised "
|
|
39
|
+
"by default or reported", terms=("inconsistency", "inconsistent", "consistency report",
|
|
40
|
+
"consistency_report", "InconsistentOntologyError", "inconsistency=",
|
|
41
|
+
"eq-diff1", "dt-not-type", "clash", "rules that conclude false",
|
|
42
|
+
"cax-dw", "prp-irp", "prp-npa1", "cls-nothing2"))
|
|
43
|
+
|
|
44
|
+
MODES = ("error", "report")
|
|
45
|
+
SAMPLE = 3
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class Inconsistency:
|
|
50
|
+
"""One rule (an OWL 2 RL rule id) or QL check (its kind) that matched: the axiom in short Turtle, how many
|
|
51
|
+
solutions (`count`; 0 where the ontology alone is inconsistent), up to `SAMPLE` of them (`sample`: each
|
|
52
|
+
variable's text) and those as facts (`facts`: subject, property, values, in short Turtle)."""
|
|
53
|
+
rule: str
|
|
54
|
+
axiom: str
|
|
55
|
+
sample: Tuple[Dict[str, Any], ...] = ()
|
|
56
|
+
count: int = 0
|
|
57
|
+
facts: Tuple[str, ...] = ()
|
|
58
|
+
# What to do, where the cause is known from the mapping (a data range: the column whose datatype is outside).
|
|
59
|
+
hint: str = ""
|
|
60
|
+
|
|
61
|
+
def text(self) -> str:
|
|
62
|
+
head = f"{self.rule}: {short(self.axiom)}"
|
|
63
|
+
if not self.count and not self.facts:
|
|
64
|
+
return head + " (the ontology alone)"
|
|
65
|
+
n = self.count or len(self.facts)
|
|
66
|
+
lines = [f"{head}: {n} {'violation' if n == 1 else 'violations'}" + (", e.g." if self.facts else "")]
|
|
67
|
+
lines += [f" {short(f)}" for f in self.facts]
|
|
68
|
+
if self.hint:
|
|
69
|
+
lines.append(f" {self.hint}")
|
|
70
|
+
return "\n".join(lines)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class ConsistencyReport:
|
|
75
|
+
"""Whether the engine's ontology and data are consistent under its profile (`profile`), the rules and checks
|
|
76
|
+
that matched (`inconsistencies`), and when they were checked (`checked`)."""
|
|
77
|
+
profile: str
|
|
78
|
+
inconsistencies: Tuple[Inconsistency, ...] = ()
|
|
79
|
+
checked: str = ""
|
|
80
|
+
# (rule or check, axiom, why) of each check that could not run: a data error its query met (R2RML 11). No
|
|
81
|
+
# verdict for it; the queries that read that data raise the error as before.
|
|
82
|
+
not_run: Tuple[Tuple[str, str, str], ...] = ()
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def consistent(self) -> bool:
|
|
86
|
+
return not self.inconsistencies
|
|
87
|
+
|
|
88
|
+
def summary(self) -> str:
|
|
89
|
+
if self.consistent:
|
|
90
|
+
return f"consistent under profile '{self.profile}' ({self.checked})"
|
|
91
|
+
rules = ", ".join(dict.fromkeys(i.rule for i in self.inconsistencies))
|
|
92
|
+
return f"inconsistent under profile '{self.profile}': {rules} ({self.checked})"
|
|
93
|
+
|
|
94
|
+
def text(self) -> str:
|
|
95
|
+
"""The report in words: each rule or check that matched, its axiom, count and facts; each check not run."""
|
|
96
|
+
lines = [self.summary()] + [" " + i.text().replace("\n", "\n ") for i in self.inconsistencies]
|
|
97
|
+
lines += [f" {rule}: {short(axiom)}: check not run: {why}" for rule, axiom, why in self.not_run]
|
|
98
|
+
return "\n".join(lines)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def engine_inconsistency(value: Optional[str]) -> str:
|
|
102
|
+
"""The engine's `inconsistency=` checked: 'error' (default, also for None) or 'report'."""
|
|
103
|
+
if value is None:
|
|
104
|
+
return "error"
|
|
105
|
+
if value not in MODES:
|
|
106
|
+
raise ValueError(f"inconsistency is 'error' or 'report', not {value!r}")
|
|
107
|
+
return value
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def settle(engine: Any, report: ConsistencyReport) -> ConsistencyReport:
|
|
111
|
+
"""Keep `report` as the engine's (`ProGraph.consistency_report`); under inconsistency='error' an
|
|
112
|
+
inconsistent one raises `InconsistentOntologyError`."""
|
|
113
|
+
engine.__dict__["_consistency"] = report
|
|
114
|
+
refuse_if_inconsistent(engine)
|
|
115
|
+
return report
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def refuse_if_inconsistent(engine: Any) -> None:
|
|
119
|
+
"""Under inconsistency='error', the engine's verdict (`settle`'s) refuses a query where it is inconsistent: a
|
|
120
|
+
query after a refresh that found the inconsistency (and raised) is refused too."""
|
|
121
|
+
report = engine.__dict__.get("_consistency")
|
|
122
|
+
if report is not None and not report.consistent and getattr(engine, "_inconsistency", "error") == "error":
|
|
123
|
+
from prograph.utils.exceptions import InconsistentOntologyError
|
|
124
|
+
raise InconsistentOntologyError(message(report), report)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def message(report: ConsistencyReport) -> str:
|
|
128
|
+
rules = len(report.inconsistencies)
|
|
129
|
+
what = ("OWL 2 RL's rules that conclude false (OWL 2 Profiles §4.3)" if report.profile == "rl" else
|
|
130
|
+
"OWL 2 QL's negative axioms (OWL 2 Direct Semantics)")
|
|
131
|
+
lines = [f"the ontology and the data are inconsistent under profile '{report.profile}': {rules} of "
|
|
132
|
+
f"{what} matched, so no answer would mean anything"]
|
|
133
|
+
lines += [" " + i.text().replace("\n", "\n ") for i in report.inconsistencies]
|
|
134
|
+
lines.append("Correct the data or the axiom; or build the engine with inconsistency='report' to answer anyway "
|
|
135
|
+
"and read engine.consistency_report()")
|
|
136
|
+
return "\n".join(lines)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
# -- the facts ------------------------------------------------------------------------------------------------
|
|
140
|
+
_VAR = re.compile(r"\?([A-Za-z_][A-Za-z0-9_]*)")
|
|
141
|
+
_PREFIXES = (("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"), ("rdfs", "http://www.w3.org/2000/01/rdf-schema#"),
|
|
142
|
+
("owl", "http://www.w3.org/2002/07/owl#"), ("xsd", xsd.XSD))
|
|
143
|
+
_W3C = re.compile("<(" + "|".join(re.escape(ns) for _, ns in _PREFIXES) + r")([A-Za-z][A-Za-z0-9_-]*)>")
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def short(text: str) -> str:
|
|
147
|
+
"""Turtle-ish text with the W3C vocabularies' IRIs as prefixed names (rdf:type as `a`)."""
|
|
148
|
+
names = {ns: prefix for prefix, ns in _PREFIXES}
|
|
149
|
+
text = text.replace("<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>", "a").replace(" rdf:type ", " a ")
|
|
150
|
+
return _W3C.sub(lambda m: f"{names[m.group(1)]}:{m.group(2)}", text)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def variables(template: str) -> List[str]:
|
|
154
|
+
"""The variables of a facts template, in order of first use."""
|
|
155
|
+
return list(dict.fromkeys(_VAR.findall(template)))
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def term(text: Optional[str], kind: str = "i", datatype: Optional[str] = None, lang: Optional[str] = None) -> str:
|
|
159
|
+
"""A term in short Turtle: `kind` 'i' an IRI, 'b' a blank node, 'l' a literal (`datatype`, `lang`)."""
|
|
160
|
+
if kind == "b":
|
|
161
|
+
return f"_:{text}" if text else "[]"
|
|
162
|
+
if text is None:
|
|
163
|
+
return "?"
|
|
164
|
+
if kind == "i":
|
|
165
|
+
return f"<{text}>"
|
|
166
|
+
quoted = '"' + str(text).replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
167
|
+
if lang:
|
|
168
|
+
return f"{quoted}@{lang}"
|
|
169
|
+
if datatype and datatype != xsd.STRING:
|
|
170
|
+
return f"{quoted}^^{datatype.replace(xsd.XSD, 'xsd:')}" if datatype.startswith(xsd.XSD) else \
|
|
171
|
+
f"{quoted}^^<{datatype}>"
|
|
172
|
+
return quoted
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def fill(template: str, terms: Dict[str, str]) -> str:
|
|
176
|
+
"""`template` (a group pattern in short Turtle) with each variable replaced by its term (`term`), the
|
|
177
|
+
full stops between triples kept as ' . ' and the last one dropped."""
|
|
178
|
+
text = _VAR.sub(lambda m: terms.get(m.group(1), m.group(0)), template)
|
|
179
|
+
text = text.replace("<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>", "a")
|
|
180
|
+
return re.sub(r"\s+", " ", text).strip().rstrip(".").strip()
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def rdflib_term(value: Any) -> str:
|
|
184
|
+
"""An answer's cell (`execute(..., return_type='list')`: a str IRI, a number, a dict for a typed literal or a
|
|
185
|
+
blank node) in short Turtle."""
|
|
186
|
+
if isinstance(value, dict):
|
|
187
|
+
from prograph.plan.ir.lower import ANSWER_TYPE
|
|
188
|
+
from prograph.schema.source_map import BLANK, IRI
|
|
189
|
+
kind = {ANSWER_TYPE[IRI]: "i", ANSWER_TYPE[BLANK]: "b"}.get(value.get("type"), "l")
|
|
190
|
+
return term(value.get("value"), kind, value.get("datatype"), value.get("lang"))
|
|
191
|
+
if isinstance(value, str):
|
|
192
|
+
return term(value, "i") if ":" in value and " " not in value else term(value, "l")
|
|
193
|
+
if isinstance(value, bool):
|
|
194
|
+
return term("true" if value else "false", "l", xsd.BOOLEAN)
|
|
195
|
+
if isinstance(value, int):
|
|
196
|
+
return str(value)
|
|
197
|
+
return term(str(value), "l")
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
# -- the stored verdict (profile='rl') -------------------------------------------------------------------------
|
|
201
|
+
def as_record(found: Iterable[Inconsistency], not_run: Iterable[Tuple[str, str, str]] = ()) -> str:
|
|
202
|
+
return json.dumps({"found": [{"rule": i.rule, "axiom": i.axiom, "count": i.count, "facts": list(i.facts),
|
|
203
|
+
"sample": [dict(s) for s in i.sample], "hint": i.hint} for i in found],
|
|
204
|
+
"not_run": [list(x) for x in not_run]}, sort_keys=True, default=str)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def from_record(text: Optional[str]) -> Tuple[Tuple[Inconsistency, ...], Tuple[Tuple[str, str, str], ...]]:
|
|
208
|
+
if not text:
|
|
209
|
+
return (), ()
|
|
210
|
+
held = json.loads(text)
|
|
211
|
+
found = tuple(Inconsistency(r["rule"], r["axiom"], tuple(r.get("sample", ())), int(r.get("count", 0)),
|
|
212
|
+
tuple(r.get("facts", ())), r.get("hint", "")) for r in held.get("found", ()))
|
|
213
|
+
return found, tuple(tuple(x) for x in held.get("not_run", ()))
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def rl_report(found: Iterable[Inconsistency], ran_at: str,
|
|
217
|
+
not_run: Iterable[Tuple[str, str, str]] = (), reused: bool = False,
|
|
218
|
+
when: str = "when this engine was built") -> ConsistencyReport:
|
|
219
|
+
"""profile='rl''s verdict: computed by the run of `ran_at` this engine made, or (`reused`) that run's verdict read
|
|
220
|
+
from the store, its facts proven current (`dataset.stale_sources`) or read by freshness='last_run'."""
|
|
221
|
+
run = f"the OWL RL run of {ran_at}" if ran_at else "the last OWL RL run"
|
|
222
|
+
checked = (f"computed by {run}; reused {when}, the stored facts current" if reused else
|
|
223
|
+
f"computed by {run}, made by this engine")
|
|
224
|
+
return ConsistencyReport("rl", tuple(found), checked, tuple(tuple(x) for x in not_run))
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def recorded(engine: Any, when: str = "when this engine was built") -> ConsistencyReport:
|
|
228
|
+
"""The verdict the store's last OWL RL run wrote (profile='rl'), read from the store now
|
|
229
|
+
(`dataset.recorded_run`); `when` says when it was reused, in the report's `checked`."""
|
|
230
|
+
from prograph.plan.ir.dataset import recorded_run
|
|
231
|
+
from prograph.plan.ir.owl_rl import RUN_AT, RUN_CONSISTENCY
|
|
232
|
+
held = recorded_run(engine._store, (RUN_CONSISTENCY, RUN_AT))
|
|
233
|
+
found, not_run = from_record(held.get(RUN_CONSISTENCY))
|
|
234
|
+
return rl_report(found, held.get(RUN_AT) or "", not_run, reused=True, when=when)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
# -- profile='ql' ------------------------------------------------------------------------------------------------
|
|
238
|
+
def ql_report(engine: Any) -> ConsistencyReport:
|
|
239
|
+
"""The QL checks run once (`ql.violations`), each a sample of `SAMPLE` and a count."""
|
|
240
|
+
from prograph.plan.ir import ql
|
|
241
|
+
from prograph.plan.ir.dataset import maps_seen
|
|
242
|
+
from prograph.plan.ir.entry import _held, ontology_of
|
|
243
|
+
found: List[Inconsistency] = []
|
|
244
|
+
not_run: List[Tuple[str, str, str]] = []
|
|
245
|
+
violated = ql.violations(engine, limit=SAMPLE)
|
|
246
|
+
maps = maps_seen(engine, ontology_of(engine)) if any(v.check.range for v in violated) else None
|
|
247
|
+
for v in violated:
|
|
248
|
+
if v.error:
|
|
249
|
+
not_run.append((v.check.kind, v.check.axiom, v.error))
|
|
250
|
+
continue
|
|
251
|
+
hint = ql.range_hint(maps, _held(engine._schema, None, True), *v.check.range) if v.check.range else ""
|
|
252
|
+
names = variables(v.check.facts)
|
|
253
|
+
facts = tuple(fill(v.check.facts, {n: rdflib_term(r.get(n)) for n in names}) for r in v.rows) \
|
|
254
|
+
if v.check.facts else ()
|
|
255
|
+
sample = tuple({k: (x.get("value") if isinstance(x, dict) else x) for k, x in r.items()} for r in v.rows)
|
|
256
|
+
found.append(Inconsistency(v.check.kind, v.check.axiom, sample, v.count, facts, hint))
|
|
257
|
+
import datetime as _dt
|
|
258
|
+
at = _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
|
|
259
|
+
return ConsistencyReport("ql", tuple(found), f"computed when the engine was built, at {at}", tuple(not_run))
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
import contextvars # noqa: E402
|
|
263
|
+
|
|
264
|
+
# Set while the ql checks themselves are planned and run: their queries read no verdict (`recheck`); per thread.
|
|
265
|
+
RECHECKING: contextvars.ContextVar = contextvars.ContextVar("prograph_ql_rechecking", default=False)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
class ConsistencyVerdictWarning(UserWarning):
|
|
269
|
+
"""Under profile='ql': a table the consistency checks read that nothing proves unchanged since they ran, so
|
|
270
|
+
the engine keeps the verdict of when it was built for it (`recheck`). Once per engine."""
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def check_on_open(engine: Any) -> None:
|
|
274
|
+
"""profile='ql': the checks run when the engine is built, the verdict settled (`settle`), and each table they
|
|
275
|
+
read recorded by its version or checksum (`freshness.read_now`), so a query proves it current (`recheck`)."""
|
|
276
|
+
settle(engine, ql_report(engine))
|
|
277
|
+
if getattr(engine, "_freshness", None) != "last_run":
|
|
278
|
+
_record(engine)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
@home("ql-verdict-freshness", "The ql consistency verdict, re-proven per query under the freshness rule",
|
|
282
|
+
terms=("consistency verdict", "ql", "freshness", "strict", "re-proven", "ConsistencyVerdictWarning",
|
|
283
|
+
"ConsistencyVerdictNotCurrentError", "version_columns", "stale verdict"))
|
|
284
|
+
def _record(engine: Any) -> None:
|
|
285
|
+
"""The tables the ql checks read (the scans of each check's plan), each with its token now (a declared
|
|
286
|
+
version column, a Delta version, or a small table's checksum; None where none proves it)."""
|
|
287
|
+
from prograph.plan.ir import freshness as F
|
|
288
|
+
from prograph.plan.ir import ops as O
|
|
289
|
+
from prograph.plan.ir.dataset import own_maps
|
|
290
|
+
from prograph.plan.ir.entry import _planned, ontology_of
|
|
291
|
+
from prograph.plan.ir import ql as Q
|
|
292
|
+
from prograph.plan.ir.dataset import maps_seen
|
|
293
|
+
onto = ontology_of(engine)
|
|
294
|
+
tbox = getattr(onto, "ql", None)
|
|
295
|
+
checks = [c for c in (getattr(tbox, "checks", None) or ()) if c.query]
|
|
296
|
+
if checks and Q.DIFFERENT in maps_seen(engine, onto).by_predicate:
|
|
297
|
+
checks.append(Q.SELF_DIFFERENT) # as `ql.violations` adds it
|
|
298
|
+
if not checks:
|
|
299
|
+
return
|
|
300
|
+
read: Dict[str, Tuple[str, ...]] = {} # a table: the checks that read it
|
|
301
|
+
token = RECHECKING.set(True) # a check's planning reads no verdict
|
|
302
|
+
try:
|
|
303
|
+
for c in checks:
|
|
304
|
+
plan = _planned(engine, c.query, "sparql", None, None)[0].plan
|
|
305
|
+
stack, seen = [plan], set()
|
|
306
|
+
while stack:
|
|
307
|
+
op = stack.pop()
|
|
308
|
+
if id(op) in seen:
|
|
309
|
+
continue
|
|
310
|
+
seen.add(id(op))
|
|
311
|
+
stack.extend(op.children())
|
|
312
|
+
if isinstance(op, O.MapScan):
|
|
313
|
+
named = [op.map.source] + ([op.map.join.parent] if getattr(op.map, "join", None) else [])
|
|
314
|
+
for src in named:
|
|
315
|
+
read[src.table] = tuple(dict.fromkeys(read.get(src.table, ()) + (f"{c.kind}: {short(c.axiom)}",)))
|
|
316
|
+
finally:
|
|
317
|
+
RECHECKING.reset(token)
|
|
318
|
+
srcs = tuple(x for x in F.sources(engine, own_maps(engine)) if x.name in read)
|
|
319
|
+
import datetime as _dt
|
|
320
|
+
import threading
|
|
321
|
+
engine.__dict__["_ql_verdict"] = {"sources": srcs, "read": read, "tokens": F.read_now(engine, srcs),
|
|
322
|
+
"at": _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat(),
|
|
323
|
+
"warned": False, "how": {}, "lock": threading.RLock()}
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def recheck(engine: Any, explaining: bool = False) -> None:
|
|
327
|
+
"""profile='ql', per query: each table the checks read proven unchanged since they ran (`_record`), else the
|
|
328
|
+
checks run again (and under inconsistency='error' an inconsistent verdict raises). A table nothing proves:
|
|
329
|
+
by default the verdict of when the engine was built is kept for it and `ConsistencyVerdictWarning` names it
|
|
330
|
+
and its checks, once per engine; under freshness='strict' the query is refused
|
|
331
|
+
(`ConsistencyVerdictNotCurrentError`); under freshness='last_run' nothing is read. `explaining`: how each
|
|
332
|
+
table was proven is kept for explain(), nothing raised, warned or run."""
|
|
333
|
+
held = engine.__dict__.get("_ql_verdict")
|
|
334
|
+
if held is None or RECHECKING.get() or getattr(engine, "_freshness", None) == "last_run":
|
|
335
|
+
return
|
|
336
|
+
with held["lock"]: # one recheck at a time per engine; the verdict and tokens are its
|
|
337
|
+
_recheck(engine, held, explaining)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _recheck(engine: Any, held: Dict[str, Any], explaining: bool) -> None:
|
|
341
|
+
from prograph.plan.ir import freshness as F
|
|
342
|
+
checks: Dict[str, Any] = {}
|
|
343
|
+
now = F.read_now(engine, held["sources"], checks)
|
|
344
|
+
unproven = [s for s in held["sources"] if now.get(s.name) is None]
|
|
345
|
+
changed = [s for s in held["sources"] if now.get(s.name) is not None and now[s.name] != held["tokens"].get(s.name)]
|
|
346
|
+
strict = getattr(engine, "_freshness", None) == "strict"
|
|
347
|
+
held["how"] = {
|
|
348
|
+
s.name: checks[s.name].how + (
|
|
349
|
+
("; this query is refused (freshness='strict')" if strict else
|
|
350
|
+
f"; the verdict of the build ({held['at']}) is kept") if s in unproven else
|
|
351
|
+
"; changed since the checks ran: they run again" if s in changed else "; unchanged")
|
|
352
|
+
for s in held["sources"]}
|
|
353
|
+
if explaining:
|
|
354
|
+
return
|
|
355
|
+
if unproven:
|
|
356
|
+
what = "; ".join(f"{s.label or s.name} (read by {', '.join(held['read'][s.name])})" for s in unproven)
|
|
357
|
+
if getattr(engine, "_freshness", None) == "strict":
|
|
358
|
+
from prograph.utils.exceptions import ConsistencyVerdictNotCurrentError
|
|
359
|
+
raise ConsistencyVerdictNotCurrentError(
|
|
360
|
+
f"under freshness='strict' the consistency verdict must be proven current for each table its "
|
|
361
|
+
f"checks read, and nothing proves these unchanged since the engine was built: {what}. Declare a "
|
|
362
|
+
f"version column (version_columns={{table: column}}), keep the table in Delta, or keep it under "
|
|
363
|
+
f"the size a checksum per query covers; or build the engine with the default freshness, which "
|
|
364
|
+
f"keeps the verdict of the build for them and warns")
|
|
365
|
+
if not held["warned"]:
|
|
366
|
+
held["warned"] = True
|
|
367
|
+
import warnings
|
|
368
|
+
warnings.warn(
|
|
369
|
+
f"the consistency verdict for these tables is the one computed when the engine was built, at "
|
|
370
|
+
f"{held['at']}: nothing proves them unchanged since (no version column, not Delta, too big for "
|
|
371
|
+
f"a checksum per query): {what}. Declare version_columns= to have it re-proven per query, or "
|
|
372
|
+
f"freshness='strict' to refuse such a query", ConsistencyVerdictWarning, stacklevel=_user_level())
|
|
373
|
+
if changed:
|
|
374
|
+
token = RECHECKING.set(True)
|
|
375
|
+
try:
|
|
376
|
+
report = ql_report(engine)
|
|
377
|
+
finally:
|
|
378
|
+
RECHECKING.reset(token)
|
|
379
|
+
held["tokens"] = dict(now)
|
|
380
|
+
import datetime as _dt
|
|
381
|
+
held["at"] = _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
|
|
382
|
+
engine.__dict__["_consistency"] = replace(report, checked=f"computed again at {held['at']}: "
|
|
383
|
+
f"{', '.join(s.label or s.name for s in changed)} changed")
|
|
384
|
+
# Under inconsistency='error' an inconsistent verdict refuses every query, as it refused the build's.
|
|
385
|
+
refuse_if_inconsistent(engine)
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _user_level() -> int:
|
|
389
|
+
"""The `stacklevel` of the first frame outside ProGraph: a warning names the user's call."""
|
|
390
|
+
import inspect
|
|
391
|
+
import os
|
|
392
|
+
root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
393
|
+
for level, frame in enumerate(inspect.stack()[1:], start=2):
|
|
394
|
+
if not os.path.abspath(frame.filename).startswith(root + os.sep):
|
|
395
|
+
return level
|
|
396
|
+
return 2
|
|
397
|
+
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""A CONSTRUCT's graph (SPARQL 1.1 16.2): its template instantiated with
|
|
2
|
+
each solution, above the plan (one-plan.md 5.10).
|
|
3
|
+
|
|
4
|
+
The answer is an RDF graph, an `rdflib.Graph`: iterable over its triples,
|
|
5
|
+
with `.serialize()` for Turtle and the rest. What 16.2 says, and this does:
|
|
6
|
+
|
|
7
|
+
- A template triple with an unbound variable, or one that would put a
|
|
8
|
+
literal as subject or anything but an IRI as predicate, is left out
|
|
9
|
+
(16.2: "not a valid RDF triple").
|
|
10
|
+
- A graph is a set: duplicate triples are one.
|
|
11
|
+
- A template's blank node is fresh for each solution; a blank node of the
|
|
12
|
+
data is one node across the solutions that bind it (its label names it).
|
|
13
|
+
- A literal keeps the text ProGraph wrote (no rdflib normalization).
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
|
|
20
|
+
from typing import Any, Dict, Iterable
|
|
21
|
+
|
|
22
|
+
from prograph.rules import module_home
|
|
23
|
+
|
|
24
|
+
module_home("construct-instantiation", "A CONSTRUCT template instantiated with solutions",
|
|
25
|
+
terms=("CONSTRUCT", "template", "instantiate", "rdflib.Graph", "fresh blank node"))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def instantiate(template: tuple, rows: Iterable[Dict[str, Any]],
|
|
29
|
+
prefixes: Dict[str, str], hidden: str) -> Any:
|
|
30
|
+
"""The graph of `template` over `rows`, each a solution of full answer
|
|
31
|
+
terms (`lower.answer` with terms_all: dicts of value, type, datatype,
|
|
32
|
+
lang; None where unbound). A `Var` whose name starts with `hidden` is a
|
|
33
|
+
template blank node."""
|
|
34
|
+
import rdflib
|
|
35
|
+
from rdflib import BNode, Literal, URIRef
|
|
36
|
+
from prograph.plan.ir.lower import ANSWER_TYPE
|
|
37
|
+
from prograph.plan.ir.terms import BlankTerm, IRITerm, LiteralTerm
|
|
38
|
+
from prograph.schema.source_map import BLANK, IRI
|
|
39
|
+
from prograph.schema import xsd
|
|
40
|
+
|
|
41
|
+
def literal(text: str, datatype, language) -> Any:
|
|
42
|
+
if language:
|
|
43
|
+
return Literal(text, lang=language)
|
|
44
|
+
if datatype in (None, xsd.STRING):
|
|
45
|
+
return Literal(text)
|
|
46
|
+
return Literal(text, datatype=URIRef(datatype), normalize=False)
|
|
47
|
+
|
|
48
|
+
graph = rdflib.Graph()
|
|
49
|
+
for prefix, iri in prefixes.items():
|
|
50
|
+
try:
|
|
51
|
+
graph.bind(prefix, iri)
|
|
52
|
+
except Exception: # noqa: BLE001 - a prefix rdflib rejects is cosmetic
|
|
53
|
+
pass
|
|
54
|
+
data_blanks: Dict[str, Any] = {}
|
|
55
|
+
for row in rows:
|
|
56
|
+
fresh: Dict[str, Any] = {}
|
|
57
|
+
|
|
58
|
+
def term(t: Any) -> Any:
|
|
59
|
+
if isinstance(t, IRITerm):
|
|
60
|
+
return URIRef(t.text)
|
|
61
|
+
if isinstance(t, LiteralTerm):
|
|
62
|
+
return literal(t.lexical, t.datatype, t.language)
|
|
63
|
+
if isinstance(t, BlankTerm):
|
|
64
|
+
return fresh.setdefault(t.label, BNode())
|
|
65
|
+
if t.name.startswith(hidden):
|
|
66
|
+
return fresh.setdefault(t.name, BNode())
|
|
67
|
+
bound = row.get(t.name)
|
|
68
|
+
if bound is None:
|
|
69
|
+
return None
|
|
70
|
+
if isinstance(bound, str):
|
|
71
|
+
# PostgreSQL's full term is a jsonb object, read as its JSON text (D8).
|
|
72
|
+
bound = json.loads(bound)
|
|
73
|
+
kind = bound["type"]
|
|
74
|
+
if kind == ANSWER_TYPE[IRI]:
|
|
75
|
+
return URIRef(bound["value"])
|
|
76
|
+
if kind == ANSWER_TYPE[BLANK]:
|
|
77
|
+
return data_blanks.setdefault(bound["value"], BNode())
|
|
78
|
+
return literal(bound["value"], bound.get("datatype"), bound.get("lang"))
|
|
79
|
+
|
|
80
|
+
for s, p, o in template:
|
|
81
|
+
s, p, o = term(s), term(p), term(o)
|
|
82
|
+
if s is None or p is None or o is None:
|
|
83
|
+
continue
|
|
84
|
+
if isinstance(s, Literal) or not isinstance(p, URIRef):
|
|
85
|
+
continue
|
|
86
|
+
graph.add((s, p, o))
|
|
87
|
+
return graph
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
__all__ = ["instantiate"]
|