prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,397 @@
1
+ """Inconsistency: an ontology and the data it reads that have no model, found and then raised or reported.
2
+
3
+ Under Direct semantics an inconsistent ontology entails every axiom (OWL 2 Direct Semantics §2.4, Entailment Regimes
4
+ §7 requires a consistent one), and OWL 2 RL's rules that conclude `false` (Profiles §4.3, tables 4 to 9) mark the same
5
+ under the RDF-Based semantics. An answer drawn from such a graph means nothing, so by default the engine refuses
6
+ (`inconsistency='error'`: `InconsistentOntologyError`, raised where the inconsistency is found); `inconsistency=
7
+ 'report'` answers as usual and keeps the report (`ProGraph.consistency_report()`).
8
+
9
+ - **What is checked.** profile='rl': the RL rules that conclude false, over the closure the run computed
10
+ (`owl_rl._Run.consistency`): cax-dw, cax-adc, cls-com, cls-nothing2, cls-maxc1, cls-maxqc1, cls-maxqc2, prp-irp, prp-asyp,
11
+ prp-pdw, prp-adp, prp-npa1, prp-npa2; eq-diff1/2/3 (a stated `x owl:differentFrom x`, owl:AllDifferent naming one
12
+ IRI twice, and an owl:sameAs component holding two different IRIs, which the run leaves unmerged); two values of a
13
+ property that has at most one (prp-fp, cls-maxc2, cls-maxqc3/4) that are not one value; dt-not-type (a literal of
14
+ a property whose range is a datatype outside that datatype's value space, and a numeric or boolean literal whose
15
+ lexical form is not one of its datatype). profile='ql': the QL negative axioms' checks (`ql.violations`).
16
+ - **Values, not texts.** Two literals are one value where their value spaces nest (`"1"^^xsd:integer` and
17
+ `"1.0"^^xsd:decimal` are one, never a clash); xsd:float and xsd:double each by its own value (+0.0 and -0.0 two).
18
+ - **When.** profile='rl': once per materialization, over the run's staged closure before its step is written, the
19
+ verdict written in that step beside the run's record (`owl_rl.RUN_CONSISTENCY`), so the stored facts' freshness
20
+ proof covers it: an engine built over current facts reads the verdict and checks nothing again; a run (on open, or
21
+ a query's refresh where a table changed) checks again. profile='ql' (nothing stored, answered by rewriting): once,
22
+ when the engine is built; the data are not watched after, so a change made later is checked when an engine is
23
+ built again.
24
+ - **What is said.** Each rule or check that matched: its id, the axiom in short Turtle, how many solutions, and up to
25
+ `SAMPLE` of them as the facts that clash (subject, property, values).
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import json
31
+ import re
32
+ from dataclasses import dataclass, replace
33
+ from typing import Any, Dict, Iterable, List, Optional, Tuple
34
+
35
+ from prograph.rules import home, module_home
36
+ from prograph.schema import xsd
37
+
38
+ module_home("owl-inconsistency", "An ontology and its data with no model: checked once per run or engine, raised "
39
+ "by default or reported", terms=("inconsistency", "inconsistent", "consistency report",
40
+ "consistency_report", "InconsistentOntologyError", "inconsistency=",
41
+ "eq-diff1", "dt-not-type", "clash", "rules that conclude false",
42
+ "cax-dw", "prp-irp", "prp-npa1", "cls-nothing2"))
43
+
44
+ MODES = ("error", "report")
45
+ SAMPLE = 3
46
+
47
+
48
+ @dataclass(frozen=True)
49
+ class Inconsistency:
50
+ """One rule (an OWL 2 RL rule id) or QL check (its kind) that matched: the axiom in short Turtle, how many
51
+ solutions (`count`; 0 where the ontology alone is inconsistent), up to `SAMPLE` of them (`sample`: each
52
+ variable's text) and those as facts (`facts`: subject, property, values, in short Turtle)."""
53
+ rule: str
54
+ axiom: str
55
+ sample: Tuple[Dict[str, Any], ...] = ()
56
+ count: int = 0
57
+ facts: Tuple[str, ...] = ()
58
+ # What to do, where the cause is known from the mapping (a data range: the column whose datatype is outside).
59
+ hint: str = ""
60
+
61
+ def text(self) -> str:
62
+ head = f"{self.rule}: {short(self.axiom)}"
63
+ if not self.count and not self.facts:
64
+ return head + " (the ontology alone)"
65
+ n = self.count or len(self.facts)
66
+ lines = [f"{head}: {n} {'violation' if n == 1 else 'violations'}" + (", e.g." if self.facts else "")]
67
+ lines += [f" {short(f)}" for f in self.facts]
68
+ if self.hint:
69
+ lines.append(f" {self.hint}")
70
+ return "\n".join(lines)
71
+
72
+
73
+ @dataclass(frozen=True)
74
+ class ConsistencyReport:
75
+ """Whether the engine's ontology and data are consistent under its profile (`profile`), the rules and checks
76
+ that matched (`inconsistencies`), and when they were checked (`checked`)."""
77
+ profile: str
78
+ inconsistencies: Tuple[Inconsistency, ...] = ()
79
+ checked: str = ""
80
+ # (rule or check, axiom, why) of each check that could not run: a data error its query met (R2RML 11). No
81
+ # verdict for it; the queries that read that data raise the error as before.
82
+ not_run: Tuple[Tuple[str, str, str], ...] = ()
83
+
84
+ @property
85
+ def consistent(self) -> bool:
86
+ return not self.inconsistencies
87
+
88
+ def summary(self) -> str:
89
+ if self.consistent:
90
+ return f"consistent under profile '{self.profile}' ({self.checked})"
91
+ rules = ", ".join(dict.fromkeys(i.rule for i in self.inconsistencies))
92
+ return f"inconsistent under profile '{self.profile}': {rules} ({self.checked})"
93
+
94
+ def text(self) -> str:
95
+ """The report in words: each rule or check that matched, its axiom, count and facts; each check not run."""
96
+ lines = [self.summary()] + [" " + i.text().replace("\n", "\n ") for i in self.inconsistencies]
97
+ lines += [f" {rule}: {short(axiom)}: check not run: {why}" for rule, axiom, why in self.not_run]
98
+ return "\n".join(lines)
99
+
100
+
101
+ def engine_inconsistency(value: Optional[str]) -> str:
102
+ """The engine's `inconsistency=` checked: 'error' (default, also for None) or 'report'."""
103
+ if value is None:
104
+ return "error"
105
+ if value not in MODES:
106
+ raise ValueError(f"inconsistency is 'error' or 'report', not {value!r}")
107
+ return value
108
+
109
+
110
+ def settle(engine: Any, report: ConsistencyReport) -> ConsistencyReport:
111
+ """Keep `report` as the engine's (`ProGraph.consistency_report`); under inconsistency='error' an
112
+ inconsistent one raises `InconsistentOntologyError`."""
113
+ engine.__dict__["_consistency"] = report
114
+ refuse_if_inconsistent(engine)
115
+ return report
116
+
117
+
118
+ def refuse_if_inconsistent(engine: Any) -> None:
119
+ """Under inconsistency='error', the engine's verdict (`settle`'s) refuses a query where it is inconsistent: a
120
+ query after a refresh that found the inconsistency (and raised) is refused too."""
121
+ report = engine.__dict__.get("_consistency")
122
+ if report is not None and not report.consistent and getattr(engine, "_inconsistency", "error") == "error":
123
+ from prograph.utils.exceptions import InconsistentOntologyError
124
+ raise InconsistentOntologyError(message(report), report)
125
+
126
+
127
+ def message(report: ConsistencyReport) -> str:
128
+ rules = len(report.inconsistencies)
129
+ what = ("OWL 2 RL's rules that conclude false (OWL 2 Profiles §4.3)" if report.profile == "rl" else
130
+ "OWL 2 QL's negative axioms (OWL 2 Direct Semantics)")
131
+ lines = [f"the ontology and the data are inconsistent under profile '{report.profile}': {rules} of "
132
+ f"{what} matched, so no answer would mean anything"]
133
+ lines += [" " + i.text().replace("\n", "\n ") for i in report.inconsistencies]
134
+ lines.append("Correct the data or the axiom; or build the engine with inconsistency='report' to answer anyway "
135
+ "and read engine.consistency_report()")
136
+ return "\n".join(lines)
137
+
138
+
139
+ # -- the facts ------------------------------------------------------------------------------------------------
140
+ _VAR = re.compile(r"\?([A-Za-z_][A-Za-z0-9_]*)")
141
+ _PREFIXES = (("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"), ("rdfs", "http://www.w3.org/2000/01/rdf-schema#"),
142
+ ("owl", "http://www.w3.org/2002/07/owl#"), ("xsd", xsd.XSD))
143
+ _W3C = re.compile("<(" + "|".join(re.escape(ns) for _, ns in _PREFIXES) + r")([A-Za-z][A-Za-z0-9_-]*)>")
144
+
145
+
146
+ def short(text: str) -> str:
147
+ """Turtle-ish text with the W3C vocabularies' IRIs as prefixed names (rdf:type as `a`)."""
148
+ names = {ns: prefix for prefix, ns in _PREFIXES}
149
+ text = text.replace("<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>", "a").replace(" rdf:type ", " a ")
150
+ return _W3C.sub(lambda m: f"{names[m.group(1)]}:{m.group(2)}", text)
151
+
152
+
153
+ def variables(template: str) -> List[str]:
154
+ """The variables of a facts template, in order of first use."""
155
+ return list(dict.fromkeys(_VAR.findall(template)))
156
+
157
+
158
+ def term(text: Optional[str], kind: str = "i", datatype: Optional[str] = None, lang: Optional[str] = None) -> str:
159
+ """A term in short Turtle: `kind` 'i' an IRI, 'b' a blank node, 'l' a literal (`datatype`, `lang`)."""
160
+ if kind == "b":
161
+ return f"_:{text}" if text else "[]"
162
+ if text is None:
163
+ return "?"
164
+ if kind == "i":
165
+ return f"<{text}>"
166
+ quoted = '"' + str(text).replace("\\", "\\\\").replace('"', '\\"') + '"'
167
+ if lang:
168
+ return f"{quoted}@{lang}"
169
+ if datatype and datatype != xsd.STRING:
170
+ return f"{quoted}^^{datatype.replace(xsd.XSD, 'xsd:')}" if datatype.startswith(xsd.XSD) else \
171
+ f"{quoted}^^<{datatype}>"
172
+ return quoted
173
+
174
+
175
+ def fill(template: str, terms: Dict[str, str]) -> str:
176
+ """`template` (a group pattern in short Turtle) with each variable replaced by its term (`term`), the
177
+ full stops between triples kept as ' . ' and the last one dropped."""
178
+ text = _VAR.sub(lambda m: terms.get(m.group(1), m.group(0)), template)
179
+ text = text.replace("<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>", "a")
180
+ return re.sub(r"\s+", " ", text).strip().rstrip(".").strip()
181
+
182
+
183
+ def rdflib_term(value: Any) -> str:
184
+ """An answer's cell (`execute(..., return_type='list')`: a str IRI, a number, a dict for a typed literal or a
185
+ blank node) in short Turtle."""
186
+ if isinstance(value, dict):
187
+ from prograph.plan.ir.lower import ANSWER_TYPE
188
+ from prograph.schema.source_map import BLANK, IRI
189
+ kind = {ANSWER_TYPE[IRI]: "i", ANSWER_TYPE[BLANK]: "b"}.get(value.get("type"), "l")
190
+ return term(value.get("value"), kind, value.get("datatype"), value.get("lang"))
191
+ if isinstance(value, str):
192
+ return term(value, "i") if ":" in value and " " not in value else term(value, "l")
193
+ if isinstance(value, bool):
194
+ return term("true" if value else "false", "l", xsd.BOOLEAN)
195
+ if isinstance(value, int):
196
+ return str(value)
197
+ return term(str(value), "l")
198
+
199
+
200
+ # -- the stored verdict (profile='rl') -------------------------------------------------------------------------
201
+ def as_record(found: Iterable[Inconsistency], not_run: Iterable[Tuple[str, str, str]] = ()) -> str:
202
+ return json.dumps({"found": [{"rule": i.rule, "axiom": i.axiom, "count": i.count, "facts": list(i.facts),
203
+ "sample": [dict(s) for s in i.sample], "hint": i.hint} for i in found],
204
+ "not_run": [list(x) for x in not_run]}, sort_keys=True, default=str)
205
+
206
+
207
+ def from_record(text: Optional[str]) -> Tuple[Tuple[Inconsistency, ...], Tuple[Tuple[str, str, str], ...]]:
208
+ if not text:
209
+ return (), ()
210
+ held = json.loads(text)
211
+ found = tuple(Inconsistency(r["rule"], r["axiom"], tuple(r.get("sample", ())), int(r.get("count", 0)),
212
+ tuple(r.get("facts", ())), r.get("hint", "")) for r in held.get("found", ()))
213
+ return found, tuple(tuple(x) for x in held.get("not_run", ()))
214
+
215
+
216
+ def rl_report(found: Iterable[Inconsistency], ran_at: str,
217
+ not_run: Iterable[Tuple[str, str, str]] = (), reused: bool = False,
218
+ when: str = "when this engine was built") -> ConsistencyReport:
219
+ """profile='rl''s verdict: computed by the run of `ran_at` this engine made, or (`reused`) that run's verdict read
220
+ from the store, its facts proven current (`dataset.stale_sources`) or read by freshness='last_run'."""
221
+ run = f"the OWL RL run of {ran_at}" if ran_at else "the last OWL RL run"
222
+ checked = (f"computed by {run}; reused {when}, the stored facts current" if reused else
223
+ f"computed by {run}, made by this engine")
224
+ return ConsistencyReport("rl", tuple(found), checked, tuple(tuple(x) for x in not_run))
225
+
226
+
227
+ def recorded(engine: Any, when: str = "when this engine was built") -> ConsistencyReport:
228
+ """The verdict the store's last OWL RL run wrote (profile='rl'), read from the store now
229
+ (`dataset.recorded_run`); `when` says when it was reused, in the report's `checked`."""
230
+ from prograph.plan.ir.dataset import recorded_run
231
+ from prograph.plan.ir.owl_rl import RUN_AT, RUN_CONSISTENCY
232
+ held = recorded_run(engine._store, (RUN_CONSISTENCY, RUN_AT))
233
+ found, not_run = from_record(held.get(RUN_CONSISTENCY))
234
+ return rl_report(found, held.get(RUN_AT) or "", not_run, reused=True, when=when)
235
+
236
+
237
+ # -- profile='ql' ------------------------------------------------------------------------------------------------
238
+ def ql_report(engine: Any) -> ConsistencyReport:
239
+ """The QL checks run once (`ql.violations`), each a sample of `SAMPLE` and a count."""
240
+ from prograph.plan.ir import ql
241
+ from prograph.plan.ir.dataset import maps_seen
242
+ from prograph.plan.ir.entry import _held, ontology_of
243
+ found: List[Inconsistency] = []
244
+ not_run: List[Tuple[str, str, str]] = []
245
+ violated = ql.violations(engine, limit=SAMPLE)
246
+ maps = maps_seen(engine, ontology_of(engine)) if any(v.check.range for v in violated) else None
247
+ for v in violated:
248
+ if v.error:
249
+ not_run.append((v.check.kind, v.check.axiom, v.error))
250
+ continue
251
+ hint = ql.range_hint(maps, _held(engine._schema, None, True), *v.check.range) if v.check.range else ""
252
+ names = variables(v.check.facts)
253
+ facts = tuple(fill(v.check.facts, {n: rdflib_term(r.get(n)) for n in names}) for r in v.rows) \
254
+ if v.check.facts else ()
255
+ sample = tuple({k: (x.get("value") if isinstance(x, dict) else x) for k, x in r.items()} for r in v.rows)
256
+ found.append(Inconsistency(v.check.kind, v.check.axiom, sample, v.count, facts, hint))
257
+ import datetime as _dt
258
+ at = _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
259
+ return ConsistencyReport("ql", tuple(found), f"computed when the engine was built, at {at}", tuple(not_run))
260
+
261
+
262
+ import contextvars # noqa: E402
263
+
264
+ # Set while the ql checks themselves are planned and run: their queries read no verdict (`recheck`); per thread.
265
+ RECHECKING: contextvars.ContextVar = contextvars.ContextVar("prograph_ql_rechecking", default=False)
266
+
267
+
268
+ class ConsistencyVerdictWarning(UserWarning):
269
+ """Under profile='ql': a table the consistency checks read that nothing proves unchanged since they ran, so
270
+ the engine keeps the verdict of when it was built for it (`recheck`). Once per engine."""
271
+
272
+
273
+ def check_on_open(engine: Any) -> None:
274
+ """profile='ql': the checks run when the engine is built, the verdict settled (`settle`), and each table they
275
+ read recorded by its version or checksum (`freshness.read_now`), so a query proves it current (`recheck`)."""
276
+ settle(engine, ql_report(engine))
277
+ if getattr(engine, "_freshness", None) != "last_run":
278
+ _record(engine)
279
+
280
+
281
+ @home("ql-verdict-freshness", "The ql consistency verdict, re-proven per query under the freshness rule",
282
+ terms=("consistency verdict", "ql", "freshness", "strict", "re-proven", "ConsistencyVerdictWarning",
283
+ "ConsistencyVerdictNotCurrentError", "version_columns", "stale verdict"))
284
+ def _record(engine: Any) -> None:
285
+ """The tables the ql checks read (the scans of each check's plan), each with its token now (a declared
286
+ version column, a Delta version, or a small table's checksum; None where none proves it)."""
287
+ from prograph.plan.ir import freshness as F
288
+ from prograph.plan.ir import ops as O
289
+ from prograph.plan.ir.dataset import own_maps
290
+ from prograph.plan.ir.entry import _planned, ontology_of
291
+ from prograph.plan.ir import ql as Q
292
+ from prograph.plan.ir.dataset import maps_seen
293
+ onto = ontology_of(engine)
294
+ tbox = getattr(onto, "ql", None)
295
+ checks = [c for c in (getattr(tbox, "checks", None) or ()) if c.query]
296
+ if checks and Q.DIFFERENT in maps_seen(engine, onto).by_predicate:
297
+ checks.append(Q.SELF_DIFFERENT) # as `ql.violations` adds it
298
+ if not checks:
299
+ return
300
+ read: Dict[str, Tuple[str, ...]] = {} # a table: the checks that read it
301
+ token = RECHECKING.set(True) # a check's planning reads no verdict
302
+ try:
303
+ for c in checks:
304
+ plan = _planned(engine, c.query, "sparql", None, None)[0].plan
305
+ stack, seen = [plan], set()
306
+ while stack:
307
+ op = stack.pop()
308
+ if id(op) in seen:
309
+ continue
310
+ seen.add(id(op))
311
+ stack.extend(op.children())
312
+ if isinstance(op, O.MapScan):
313
+ named = [op.map.source] + ([op.map.join.parent] if getattr(op.map, "join", None) else [])
314
+ for src in named:
315
+ read[src.table] = tuple(dict.fromkeys(read.get(src.table, ()) + (f"{c.kind}: {short(c.axiom)}",)))
316
+ finally:
317
+ RECHECKING.reset(token)
318
+ srcs = tuple(x for x in F.sources(engine, own_maps(engine)) if x.name in read)
319
+ import datetime as _dt
320
+ import threading
321
+ engine.__dict__["_ql_verdict"] = {"sources": srcs, "read": read, "tokens": F.read_now(engine, srcs),
322
+ "at": _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat(),
323
+ "warned": False, "how": {}, "lock": threading.RLock()}
324
+
325
+
326
+ def recheck(engine: Any, explaining: bool = False) -> None:
327
+ """profile='ql', per query: each table the checks read proven unchanged since they ran (`_record`), else the
328
+ checks run again (and under inconsistency='error' an inconsistent verdict raises). A table nothing proves:
329
+ by default the verdict of when the engine was built is kept for it and `ConsistencyVerdictWarning` names it
330
+ and its checks, once per engine; under freshness='strict' the query is refused
331
+ (`ConsistencyVerdictNotCurrentError`); under freshness='last_run' nothing is read. `explaining`: how each
332
+ table was proven is kept for explain(), nothing raised, warned or run."""
333
+ held = engine.__dict__.get("_ql_verdict")
334
+ if held is None or RECHECKING.get() or getattr(engine, "_freshness", None) == "last_run":
335
+ return
336
+ with held["lock"]: # one recheck at a time per engine; the verdict and tokens are its
337
+ _recheck(engine, held, explaining)
338
+
339
+
340
+ def _recheck(engine: Any, held: Dict[str, Any], explaining: bool) -> None:
341
+ from prograph.plan.ir import freshness as F
342
+ checks: Dict[str, Any] = {}
343
+ now = F.read_now(engine, held["sources"], checks)
344
+ unproven = [s for s in held["sources"] if now.get(s.name) is None]
345
+ changed = [s for s in held["sources"] if now.get(s.name) is not None and now[s.name] != held["tokens"].get(s.name)]
346
+ strict = getattr(engine, "_freshness", None) == "strict"
347
+ held["how"] = {
348
+ s.name: checks[s.name].how + (
349
+ ("; this query is refused (freshness='strict')" if strict else
350
+ f"; the verdict of the build ({held['at']}) is kept") if s in unproven else
351
+ "; changed since the checks ran: they run again" if s in changed else "; unchanged")
352
+ for s in held["sources"]}
353
+ if explaining:
354
+ return
355
+ if unproven:
356
+ what = "; ".join(f"{s.label or s.name} (read by {', '.join(held['read'][s.name])})" for s in unproven)
357
+ if getattr(engine, "_freshness", None) == "strict":
358
+ from prograph.utils.exceptions import ConsistencyVerdictNotCurrentError
359
+ raise ConsistencyVerdictNotCurrentError(
360
+ f"under freshness='strict' the consistency verdict must be proven current for each table its "
361
+ f"checks read, and nothing proves these unchanged since the engine was built: {what}. Declare a "
362
+ f"version column (version_columns={{table: column}}), keep the table in Delta, or keep it under "
363
+ f"the size a checksum per query covers; or build the engine with the default freshness, which "
364
+ f"keeps the verdict of the build for them and warns")
365
+ if not held["warned"]:
366
+ held["warned"] = True
367
+ import warnings
368
+ warnings.warn(
369
+ f"the consistency verdict for these tables is the one computed when the engine was built, at "
370
+ f"{held['at']}: nothing proves them unchanged since (no version column, not Delta, too big for "
371
+ f"a checksum per query): {what}. Declare version_columns= to have it re-proven per query, or "
372
+ f"freshness='strict' to refuse such a query", ConsistencyVerdictWarning, stacklevel=_user_level())
373
+ if changed:
374
+ token = RECHECKING.set(True)
375
+ try:
376
+ report = ql_report(engine)
377
+ finally:
378
+ RECHECKING.reset(token)
379
+ held["tokens"] = dict(now)
380
+ import datetime as _dt
381
+ held["at"] = _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
382
+ engine.__dict__["_consistency"] = replace(report, checked=f"computed again at {held['at']}: "
383
+ f"{', '.join(s.label or s.name for s in changed)} changed")
384
+ # Under inconsistency='error' an inconsistent verdict refuses every query, as it refused the build's.
385
+ refuse_if_inconsistent(engine)
386
+
387
+
388
+ def _user_level() -> int:
389
+ """The `stacklevel` of the first frame outside ProGraph: a warning names the user's call."""
390
+ import inspect
391
+ import os
392
+ root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
393
+ for level, frame in enumerate(inspect.stack()[1:], start=2):
394
+ if not os.path.abspath(frame.filename).startswith(root + os.sep):
395
+ return level
396
+ return 2
397
+
@@ -0,0 +1,90 @@
1
+ """A CONSTRUCT's graph (SPARQL 1.1 16.2): its template instantiated with
2
+ each solution, above the plan (one-plan.md 5.10).
3
+
4
+ The answer is an RDF graph, an `rdflib.Graph`: iterable over its triples,
5
+ with `.serialize()` for Turtle and the rest. What 16.2 says, and this does:
6
+
7
+ - A template triple with an unbound variable, or one that would put a
8
+ literal as subject or anything but an IRI as predicate, is left out
9
+ (16.2: "not a valid RDF triple").
10
+ - A graph is a set: duplicate triples are one.
11
+ - A template's blank node is fresh for each solution; a blank node of the
12
+ data is one node across the solutions that bind it (its label names it).
13
+ - A literal keeps the text ProGraph wrote (no rdflib normalization).
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import json
19
+
20
+ from typing import Any, Dict, Iterable
21
+
22
+ from prograph.rules import module_home
23
+
24
+ module_home("construct-instantiation", "A CONSTRUCT template instantiated with solutions",
25
+ terms=("CONSTRUCT", "template", "instantiate", "rdflib.Graph", "fresh blank node"))
26
+
27
+
28
+ def instantiate(template: tuple, rows: Iterable[Dict[str, Any]],
29
+ prefixes: Dict[str, str], hidden: str) -> Any:
30
+ """The graph of `template` over `rows`, each a solution of full answer
31
+ terms (`lower.answer` with terms_all: dicts of value, type, datatype,
32
+ lang; None where unbound). A `Var` whose name starts with `hidden` is a
33
+ template blank node."""
34
+ import rdflib
35
+ from rdflib import BNode, Literal, URIRef
36
+ from prograph.plan.ir.lower import ANSWER_TYPE
37
+ from prograph.plan.ir.terms import BlankTerm, IRITerm, LiteralTerm
38
+ from prograph.schema.source_map import BLANK, IRI
39
+ from prograph.schema import xsd
40
+
41
+ def literal(text: str, datatype, language) -> Any:
42
+ if language:
43
+ return Literal(text, lang=language)
44
+ if datatype in (None, xsd.STRING):
45
+ return Literal(text)
46
+ return Literal(text, datatype=URIRef(datatype), normalize=False)
47
+
48
+ graph = rdflib.Graph()
49
+ for prefix, iri in prefixes.items():
50
+ try:
51
+ graph.bind(prefix, iri)
52
+ except Exception: # noqa: BLE001 - a prefix rdflib rejects is cosmetic
53
+ pass
54
+ data_blanks: Dict[str, Any] = {}
55
+ for row in rows:
56
+ fresh: Dict[str, Any] = {}
57
+
58
+ def term(t: Any) -> Any:
59
+ if isinstance(t, IRITerm):
60
+ return URIRef(t.text)
61
+ if isinstance(t, LiteralTerm):
62
+ return literal(t.lexical, t.datatype, t.language)
63
+ if isinstance(t, BlankTerm):
64
+ return fresh.setdefault(t.label, BNode())
65
+ if t.name.startswith(hidden):
66
+ return fresh.setdefault(t.name, BNode())
67
+ bound = row.get(t.name)
68
+ if bound is None:
69
+ return None
70
+ if isinstance(bound, str):
71
+ # PostgreSQL's full term is a jsonb object, read as its JSON text (D8).
72
+ bound = json.loads(bound)
73
+ kind = bound["type"]
74
+ if kind == ANSWER_TYPE[IRI]:
75
+ return URIRef(bound["value"])
76
+ if kind == ANSWER_TYPE[BLANK]:
77
+ return data_blanks.setdefault(bound["value"], BNode())
78
+ return literal(bound["value"], bound.get("datatype"), bound.get("lang"))
79
+
80
+ for s, p, o in template:
81
+ s, p, o = term(s), term(p), term(o)
82
+ if s is None or p is None or o is None:
83
+ continue
84
+ if isinstance(s, Literal) or not isinstance(p, URIRef):
85
+ continue
86
+ graph.add((s, p, o))
87
+ return graph
88
+
89
+
90
+ __all__ = ["instantiate"]