prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1119 @@
1
+ """OWL 2 RL materialized into the owned store's RL graph (one-plan 2.7 and 2.10; docs/reference/owl-rl.md).
2
+
3
+ `materialize(engine)` computes the closure of the ontology's RL rules (`owl_rl_rules`) over the graph SPARQL
4
+ answers, then writes the difference from the RL graph (`store.RL_GRAPH`) as one write step.
5
+
6
+ - **Premises through the plan.** Each round, every rule's pattern is planned over the saturated maps, the
7
+ store's statements of every other writer, and the facts derived so far in this run (`Store.premises`:
8
+ the store's generated maps read that relation for the round), then lowered, so a rule reads exactly the
9
+ default graph a query reads, on every engine, R2RML and YAML mappings alike. Never a table by name. The RL
10
+ graph as stored is not a premise: a stale fact never supports itself.
11
+ - **Naive fixpoint.** Every rule runs each round; a round keeps the triples the default graph does not hold
12
+ yet (a `NOT EXISTS`, saturation's virtual facts included), staged in the engine (`StoreConnection.stage`).
13
+ It stops when a round derives nothing; `max_rounds` rounds without that raise
14
+ `MaterializationBudgetExceeded`, nothing written. A transitive property is the plan's closure `p+`
15
+ (`Closure`, on Spark and Databricks by the level ladder), re-run each round.
16
+ - **One set-level diff.** The closure is compared with the RL graph as a set of triples: asserts for triples
17
+ newly derived, retracts for triples no longer derived; a triple derived twice, or held by the source too,
18
+ is kept while one derivation stands (a source triple is never stored). Each statement's writer is
19
+ `rl:<rule id>`. The run's ontology digest and time are written in the same step, as the store's own
20
+ statements (`last_run`). On Delta a step that lost its race recomputes from the start.
21
+ - **Consistency** (`consistency`): the rules that conclude false derive nothing; they run once, after the
22
+ fixpoint, over the closure the run staged (`_Run.consistency`), and their verdict is written in the run's step
23
+ (`RUN_CONSISTENCY`), so an engine over current facts reads it and checks nothing again.
24
+ - **owl:sameAs** (`identity`): each round first closes the links (the graph's own sameAs triples, an
25
+ ontology's, and those prp-fp, prp-ifp, prp-key, cls-maxc2 and cls-maxqc3/4 derived so far) into
26
+ components, then leaves unmerged each component that is inconsistent (owl:differentFrom or
27
+ owl:AllDifferent between its members; a functional property's two values that are not one value) or past
28
+ the cap, then runs every rule with its patterns matched through the components, each solution under its
29
+ representative: a derived fact is stored once, on the representative. A component found inconsistent
30
+ after rules have read it runs the fixpoint again from its first round. The components are written as a
31
+ new generation of the alias table with the run's step.
32
+ """
33
+
34
+ from __future__ import annotations
35
+
36
+ import datetime as _dt
37
+ import uuid
38
+ from dataclasses import dataclass, field, replace
39
+ from typing import Any, Dict, List, Optional, Tuple
40
+
41
+ from prograph.rules import module_home
42
+ from prograph.schema import xsd
43
+
44
+ module_home("owl-rl-materialization", "OWL 2 RL materialized into the owned store: fixpoint, diff, one step",
45
+ terms=("materialize_entailments", "profile='rl'", "materialization", "fixpoint", "naive fixpoint",
46
+ "RL graph", "derived fact", "max_rounds", "MaterializationBudgetExceeded", "set-level diff",
47
+ "retract derived", "ontology digest", "last run", "consistency report", "rl:"))
48
+
49
+ RUN_SUBJECT = "urn:prograph:store:rl"
50
+ RUN_DIGEST = "urn:prograph:store:rl-ontology"
51
+ RUN_AT = "urn:prograph:store:rl-at"
52
+ # The versions of the sources the run read (`freshness.read_now`), as JSON: what a planning compares.
53
+ RUN_SOURCES = "urn:prograph:store:rl-sources"
54
+ # The rules that concluded false in the run (`consistency.as_record`), as JSON: the stored verdict.
55
+ RUN_CONSISTENCY = "urn:prograph:store:rl-consistency"
56
+ # The rounds the run took, as text: a call reusing the run checks its `max_rounds` against it.
57
+ RUN_ROUNDS = "urn:prograph:store:rl-rounds"
58
+ MAX_ROUNDS = 64
59
+ # The deepest transitive walk a round's closure takes after the engine's own ladder (the path_depth hint's
60
+ # ceiling); a longer walk is a typed error naming the depth.
61
+ DEEPEST = 1000
62
+ SAMPLE = 3
63
+ STATEMENT = ("subject", "predicate", "o_kind", "o_text", "o_datatype", "o_lang")
64
+
65
+
66
+ from prograph.plan.ir.consistency import Inconsistency # noqa: E402 - the report's record (`consistency`)
67
+
68
+
69
+ @dataclass(frozen=True)
70
+ class Unmerged:
71
+ """A component of owl:sameAs left unmerged: `reason` 'inconsistent' (the rule that found it: eq-diff1
72
+ for owl:differentFrom, eq-diff2/3 for owl:AllDifferent, prp-fp or cls-maxc2 for two values; `axiom` its axiom)
73
+ or 'cap' (more members than the cap), its graph ('' the default graph), its representative, size and a sample
74
+ of members."""
75
+ reason: str
76
+ rule: str
77
+ graph: str
78
+ representative: str
79
+ size: int
80
+ sample: Tuple[str, ...]
81
+ axiom: str = ""
82
+
83
+
84
+ @dataclass
85
+ class MaterializationReport:
86
+ """What `materialize_entailments()` did: the statements asserted and retracted per rule (`rl:<rule>`
87
+ writers), the rounds run, the consistency rules that matched (`inconsistencies`), the axioms skipped
88
+ (sameAs-deriving and outside OWL 2 RL), the run's cache key (`dataset.cache_key`)
89
+ and the step's seq (None where nothing
90
+ changed but the run's record).
91
+
92
+ A call that found the store's last run current for the engine's key and tables returns that run's record
93
+ instead of computing again: ``seq`` None, ``rounds`` 0, nothing asserted or retracted, and ``digest``,
94
+ ``ran_at``, ``sources``, ``inconsistencies``, ``not_run``, ``components``, ``aliases``, ``largest`` and
95
+ ``unmerged`` those the
96
+ last run stored. The call's ``max_rounds`` is checked against the rounds the last run recorded: a smaller
97
+ budget computes the closure and raises ``MaterializationBudgetExceeded``."""
98
+ asserted: Dict[str, int] = field(default_factory=dict)
99
+ retracted: Dict[str, int] = field(default_factory=dict)
100
+ rounds: int = 0
101
+ inconsistencies: List[Inconsistency] = field(default_factory=list)
102
+ skipped: List[Tuple[str, str, str]] = field(default_factory=list)
103
+ digest: str = ""
104
+ seq: Optional[int] = None
105
+ ran_at: str = ""
106
+ # Each source table's versions as the run read it (`freshness.read_run`): a declared column's maximum and
107
+ # count or a Delta version, and a checksum of the whole table.
108
+ sources: Dict[str, List[str]] = field(default_factory=dict)
109
+ # owl:sameAs: the components merged (of two IRIs or more), their aliases (IRIs other than the
110
+ # representative), the largest component's size, and the components left unmerged.
111
+ components: int = 0
112
+ aliases: int = 0
113
+ largest: int = 0
114
+ unmerged: List[Unmerged] = field(default_factory=list)
115
+ # The consistency checks that could not run (rule, axiom, why: a data error their query met).
116
+ not_run: List[Tuple[str, str, str]] = field(default_factory=list)
117
+
118
+ @property
119
+ def consistent(self) -> bool:
120
+ """No consistency rule matched and no owl:sameAs component was left unmerged as inconsistent."""
121
+ return not self.inconsistencies and not any(u.reason == "inconsistent" for u in self.unmerged)
122
+
123
+ def summary(self) -> str:
124
+ a, r = sum(self.asserted.values()), sum(self.retracted.values())
125
+ same = f"; {self.components} sameAs components ({self.aliases} aliases)" if self.components else ""
126
+ apart = f"; {len(self.unmerged)} left unmerged" if self.unmerged else ""
127
+ return (f"{a} asserted, {r} retracted in {self.rounds} rounds{same}{apart}; "
128
+ f"{'consistent' if self.consistent else 'inconsistent: ' + ', '.join(dict.fromkeys([i.rule for i in self.inconsistencies] + [u.rule for u in self.unmerged if u.reason == 'inconsistent']))}")
129
+
130
+
131
+ def last_run(store: Any) -> Optional[Tuple[str, str]]:
132
+ """The (ontology digest, time) the store's RL graph was last materialized for; None where never."""
133
+ from prograph.store import RL_GRAPH
134
+ rows = store.db.execute(
135
+ f"SELECT predicate, o_text FROM ({store.effective_sql([RL_GRAPH])}) AS _e WHERE subject = ?",
136
+ [RUN_SUBJECT]).fetchall()
137
+ held = dict(rows)
138
+ if RUN_DIGEST not in held:
139
+ return None
140
+ return held[RUN_DIGEST], held.get(RUN_AT, "")
141
+
142
+
143
+ def _unmerged_order(u: Unmerged) -> Tuple[Any, ...]:
144
+ return (u.graph, u.representative, u.rule, u.reason)
145
+
146
+
147
+ def reused_report(engine: Any, rules: Any, held: Any, max_rounds: int) -> Optional[MaterializationReport]:
148
+ """The report of a call that reused the store's last run (`ProGraph.materialize_entailments`): that run's
149
+ key, time, source versions and consistency verdict `held` (`consistency.recorded`); no round was run. None
150
+ where the run recorded no rounds, or more than `max_rounds` (the call computes, and the budget is checked)."""
151
+ from prograph.plan.ir.dataset import recorded_run
152
+ rounds = recorded_run(engine._store, (RUN_ROUNDS,))[RUN_ROUNDS]
153
+ if rounds is None or int(rounds) > max_rounds:
154
+ return None
155
+ from prograph.plan.ir import freshness
156
+ from prograph.plan.ir.dataset import _first, _record_parts, cache_key
157
+ store = engine._store
158
+ rows = store.db.execute(" UNION ALL ".join(_record_parts(store, True))).fetchall()
159
+ report = MaterializationReport(skipped=list(rules.skipped), digest=cache_key(engine, rules),
160
+ ran_at=_first(rows, "at") or "", sources=freshness.recorded(_first(rows, "sources")),
161
+ inconsistencies=list(held.inconsistencies), not_run=list(held.not_run))
162
+ # owl:sameAs: the latest generation's marker and unmerged components, where it was written under this key.
163
+ import json
164
+ from prograph.plan.ir import identity as I
165
+ meta = I.meta_table(store)
166
+ if store.db.has_table(meta):
167
+ marker = store.db.execute(
168
+ f"SELECT a, note, generation FROM {meta} WHERE kind = 'generation' AND generation = "
169
+ f"(SELECT max(generation) FROM {meta} WHERE kind = 'generation')").fetchall()
170
+ if marker and marker[0][0] == report.digest:
171
+ note = json.loads(marker[0][1] or "{}")
172
+ report.components, report.aliases, report.largest = (int(note.get(k, 0)) for k in
173
+ ("components", "aliases", "largest"))
174
+ for graph, rep, rule, text in store.db.execute(
175
+ f"SELECT graph, a, b, note FROM {meta} WHERE kind = 'unmerged' AND generation = {int(marker[0][2])}"
176
+ ).fetchall():
177
+ found = json.loads(text or "{}")
178
+ report.unmerged.append(Unmerged(found.get("reason", ""), rule or "", graph or "", rep,
179
+ int(found.get("size", 0)), tuple(found.get("sample", ())),
180
+ found.get("axiom", "")))
181
+ report.unmerged.sort(key=_unmerged_order)
182
+ return report
183
+
184
+
185
+ def materialize(engine: Any, rules: Any, max_rounds: int = MAX_ROUNDS) -> MaterializationReport:
186
+ """Materialize `rules` (`owl_rl_rules.RuleSet`) into the engine's owned store (see the module)."""
187
+ from prograph.store import StepLostRace
188
+ store = engine._store
189
+ if max_rounds < 1:
190
+ raise ValueError(f"max_rounds is at least 1, not {max_rounds}")
191
+ graph = getattr(getattr(engine, "executor", None), "graph", None)
192
+ if getattr(graph, "_shared", False):
193
+ graph._prepare_local_sql() # the frames' views on the store's connection are this engine's (not another's)
194
+ running = engine.__dict__.get("_rl_running")
195
+ engine.__dict__["_rl_running"] = True # the run's own planning reads the store without a freshness check
196
+ try:
197
+ for _ in range(getattr(store, "REDO_LIMIT", 20)):
198
+ store.sync()
199
+ run = _Run(engine, store, rules, max_rounds)
200
+ try:
201
+ return run.go()
202
+ except StepLostRace:
203
+ continue # another writer's step landed: the closure is computed again from the start
204
+ finally:
205
+ run.cleanup()
206
+ finally:
207
+ engine.__dict__["_rl_running"] = running
208
+ from prograph.utils.exceptions import StoreWriteConflictError
209
+ raise StoreWriteConflictError("write contention: OWL RL materialization lost its race to other writers "
210
+ f"{store.REDO_LIMIT} times; nothing was written. Call it again")
211
+
212
+
213
+ class _Run:
214
+ def __init__(self, engine: Any, store: Any, rules: Any, max_rounds: int) -> None:
215
+ from prograph.plan.sql.dialect import get_dialect_instance
216
+ self.engine, self.store, self.rules, self.max_rounds = engine, store, rules, max_rounds
217
+ self.db = store.db
218
+ self.dialect = store.dialect
219
+ self.d = get_dialect_instance(self.dialect)
220
+ self.key = uuid.uuid4().hex[:16]
221
+ self.stages: List[Tuple[str, str]] = [] # (read, cleanup) per round with rows
222
+ self.cleanups: List[str] = []
223
+ self.kinds: set = set()
224
+ self.base: Optional[str] = None
225
+ self.types_before = set(engine._schema.column_types) if hasattr(engine._schema, "column_types") else set()
226
+ # owl:sameAs: the graph's own links (staged once), the links derived in this pass, the IRIs left
227
+ # unmerged, the components of the round (`identity.components`) and the identity rules read under.
228
+ self.stated: Optional[str] = None
229
+ self.link_stages: List[str] = []
230
+ self.poison_stages: List[str] = []
231
+ self.unmerged_reps: set = set()
232
+ self.comps: Any = None
233
+ self.identity: Any = None
234
+ self.disagree: Optional[str] = None # the staged (class, predicate) pairs of `stage_disagreeing`
235
+ self.component_index = 0
236
+
237
+ # -- SQL helpers -----------------------------------------------------------------------------------------
238
+ def lit(self, v: Optional[str]) -> str:
239
+ return f"CAST(NULL AS {self.d.type_name('STRING')})" if v is None else self.d.literal(v)
240
+
241
+ def text(self, sql: str) -> str:
242
+ return f"CAST({sql} AS {self.d.type_name('STRING')})"
243
+
244
+ def derived(self) -> str:
245
+ """The rows derived so far in this run, as statements of the RL graph."""
246
+ from prograph.store import RL_GRAPH
247
+ cols = ", ".join(STATEMENT)
248
+ parts = [f"SELECT {self.lit(RL_GRAPH)} AS graph, {cols} FROM {read} AS _d" for read, _ in self.stages]
249
+ return " UNION ALL ".join(parts)
250
+
251
+ def premise_source(self) -> str:
252
+ """The store's statements of every other graph (staged once per run, `stage_base`) and this run's
253
+ derivations: what the generated maps read in a round."""
254
+ cols = "graph, " + ", ".join(STATEMENT)
255
+ base = f"SELECT {cols} FROM {self.base} AS _o"
256
+ return base + (" UNION ALL " + self.derived() if self.stages else "")
257
+
258
+ def stage(self, index: int, name: str, sql: str) -> Tuple[str, str]:
259
+ """Stage `sql`'s rows (`StoreConnection.stage`); (how a statement reads them, how they are removed)."""
260
+ read, write, drop = self.db.stage(self.store.table, f"rl-{self.key}", index, name, sql)
261
+ self.db.execute(write)
262
+ if drop not in self.cleanups:
263
+ self.cleanups.append(drop)
264
+ return read, drop
265
+
266
+ def stage_base(self) -> None:
267
+ """The store's effective statements outside the RL graph, staged once: each round's maps read this
268
+ table, never the effective view's text again in every scan (Spark's analyzer runs out of memory on a
269
+ closure's levels each holding it)."""
270
+ from prograph.plan.ir.dataset import premise_statements
271
+ cols = "graph, " + ", ".join(STATEMENT)
272
+ self.base, _ = self.stage(0, f"_pg_rl_{self.key}_base",
273
+ f"SELECT {cols} FROM ({premise_statements(self.store)}) AS _o")
274
+
275
+ def stage_disagreeing(self) -> None:
276
+ """What Cypher reads of this run's owl:sameAs components (`cypher_front`, one node per component), staged for
277
+ the generation's meta rows (`alias_statements`): each class a member of a merged component has ('merged',
278
+ so a Cypher pattern of another class needs no alias join), and each (class, Cypher property key) whose
279
+ values differ across a component's members ('disagree': the first query naming it warns). Values are
280
+ compared as values, type-tolerantly: `1` and `1.0`, or "10504" and 10504, are one value; a key the view
281
+ gives by several predicates is one key. One small query per key: a variable predicate over every map is
282
+ past the engines' statement limits."""
283
+ from prograph.plan.ir.facts import NotBuilt
284
+ from prograph.plan.ir.identity import SAME_AS
285
+ from prograph.schema.lpg_view import view_of
286
+ if self.comps is None or not self.comps.aliases:
287
+ return
288
+ every = {"http://www.w3.org/2000/01/rdf-schema#Resource", "http://www.w3.org/2002/07/owl#Thing"}
289
+
290
+ def plan(maps: Any, onto: Any) -> List[Tuple[str, str]]:
291
+ q = self.d.quote
292
+ out = []
293
+ # Under the run's identity owl:sameAs binds representatives, (rep, rep): a merged entity's; an IRI
294
+ # with no alias has no such triple.
295
+ merged = f"SELECT DISTINCT (STR(?c) AS ?k) WHERE {{ ?a <{SAME_AS}> ?b . ?a a ?c FILTER(isIRI(?c)) }}"
296
+ sql, _ = self.lowered(merged, maps, onto, self.identity)
297
+ out.append(("merged", f"SELECT {self.text('_r.' + q('k'))} AS c, {self.lit(None)} AS p FROM ({sql}) AS _r"))
298
+ by_key: Dict[str, set] = {}
299
+ view = view_of(maps)
300
+ for props in view.properties.values():
301
+ for key, pos in props.items():
302
+ by_key.setdefault(key, set()).update(po.predicate for po in pos if po.predicate is not None)
303
+ for key, preds in sorted(by_key.items()):
304
+ listed = " ".join(f"<{p}>" for p in sorted(preds))
305
+ query = (f"SELECT DISTINCT (STR(?c) AS ?k) WHERE {{ ?a <{SAME_AS}> ?b . VALUES ?p {{ {listed} }} "
306
+ f"VALUES ?r {{ {listed} }} ?a ?p ?v . ?a ?r ?w . ?a a ?c FILTER(isLiteral(?v) && isLiteral(?w) "
307
+ f"&& STR(?v) != STR(?w) && !COALESCE(?v = ?w, false) && isIRI(?c)) }}")
308
+ try:
309
+ sql, _ = self.lowered(query, maps, onto, self.identity)
310
+ except NotBuilt:
311
+ continue
312
+ out.append(("disagree", f"SELECT {self.text('_r.' + q('k'))} AS c, {self.lit(key)} AS p "
313
+ f"FROM ({sql}) AS _r"))
314
+ return out
315
+ rows = sorted({(kind, r[0], r[1]) for kind, sql in self.with_maps(plan)
316
+ for r in self.db.execute(sql).fetchall() if r[0] is not None and r[0] not in every})
317
+ if rows:
318
+ sql = " UNION ALL ".join(f"SELECT {self.lit(k)} AS kind, {self.lit(c)} AS c, {self.lit(p)} AS p"
319
+ for k, c, p in rows)
320
+ self.disagree, _ = self.stage(999_999_998, f"_pg_rl_{self.key}_disagree", sql)
321
+
322
+ # -- planning one rule --------------------------------------------------------------------------------
323
+ def rule_query(self, r: Any) -> str:
324
+ """The rule's SPARQL: its derived statements' subject (and object) as text, `STR` of an IRI (a variable
325
+ the plan carries as a whole term, a property of IRI and literal objects, reads as its text too); only
326
+ IRI subjects, and an IRI object where the conclusion needs one (a literal is no premise there)."""
327
+ s = r.subject
328
+ if r.cls is not None:
329
+ return (f"SELECT DISTINCT (STR({s}) AS ?s) WHERE {{ {r.where} FILTER(isIRI({s})) "
330
+ f"FILTER NOT EXISTS {{ {s} a <{r.cls}> }} }}")
331
+ o = r.object
332
+ if r.value is not None: # a constant object (cls-hv1)
333
+ return (f"SELECT DISTINCT (STR({s}) AS ?s) WHERE {{ {r.where} FILTER(isIRI({s})) "
334
+ f"FILTER NOT EXISTS {{ {s} <{r.predicate}> {o} }} }}")
335
+ if r.literal:
336
+ return (f"SELECT DISTINCT (STR({s}) AS ?s) (STR({o}) AS ?t) (STR(DATATYPE({o})) AS ?d) (LANG({o}) AS ?l) "
337
+ f"WHERE {{ {r.where} FILTER(isIRI({s}) && !isBlank({o})) "
338
+ f"FILTER NOT EXISTS {{ {s} <{r.predicate}> {o} }} }}")
339
+ return (f"SELECT DISTINCT (STR({s}) AS ?s) (STR({o}) AS ?t) WHERE {{ {r.where} FILTER(isIRI({s}) && isIRI({o})) "
340
+ f"FILTER NOT EXISTS {{ {s} <{r.predicate}> {o} }} }}")
341
+
342
+ def lowered(self, query: str, maps: Any, onto: Any, identity: Any = None) -> str:
343
+ from prograph.plan.ir import lower, sparql_front
344
+ from prograph.plan.ir.entry import _held
345
+ from prograph.plan.ir.keys import trusted_keys
346
+ from prograph.plan.ir.rewrite import rewrite
347
+ graph = getattr(getattr(self.engine, "executor", None), "graph", None)
348
+ held = _held(self.engine._schema, self.dialect, True)
349
+ t = sparql_front.translate(query, maps, held, lambda m: trusted_keys(graph, m, maps),
350
+ union_default=getattr(self.engine, "default_graph", "union") == "union",
351
+ ontology=onto, identity=identity)
352
+ plan, _ = rewrite(t.plan, self.dialect)
353
+ from prograph.plan.ir.slot import Bound
354
+ # A column never bound (a DATATYPE of IRIs only) is written as a term's null: read as a null text.
355
+ never = frozenset(c for c in plan.columns if c in plan.slots and plan.slots[c].present is Bound.NEVER)
356
+ return lower.to_sql(plan, self.dialect, base_iri=maps.settings.base_iri), never
357
+
358
+ def wrapped(self, r: Any, lowered: Tuple[str, frozenset]) -> str:
359
+ from prograph.store import IRI, LITERAL
360
+ sql, never = lowered
361
+ q = self.d.quote
362
+
363
+ def col(name: str) -> str:
364
+ return self.lit(None) if name in never else f"_r.{q(name)}"
365
+ s = col("s")
366
+ if r.cls is not None:
367
+ obj = (self.lit(IRI), self.lit(r.cls), self.lit(None), self.lit(None))
368
+ elif r.value is not None:
369
+ from rdflib import Literal
370
+ v = r.value
371
+ if isinstance(v, Literal):
372
+ lang = getattr(v, "language", None)
373
+ dt = None if lang else str(v.datatype or xsd.STRING)
374
+ obj = (self.lit(LITERAL), self.lit(str(v)), self.lit(dt), self.lit(lang))
375
+ else:
376
+ obj = (self.lit(IRI), self.lit(str(v)), self.lit(None), self.lit(None))
377
+ elif r.literal:
378
+ t, dt, lang = col("t"), col("d"), col("l")
379
+ obj = (f"CASE WHEN {dt} IS NULL THEN {self.lit(IRI)} ELSE {self.lit(LITERAL)} END", self.text(t),
380
+ f"CASE WHEN {dt} = {self.lit(xsd.RDF_LANGSTRING)} "
381
+ f"THEN {self.lit(None)} ELSE {self.text(dt)} END",
382
+ f"CASE WHEN {lang} = '' THEN {self.lit(None)} ELSE {self.text(lang)} END")
383
+ else:
384
+ obj = (self.lit(IRI), self.text(col("t")), self.lit(None), self.lit(None))
385
+ return (f"SELECT {self.text(s)} AS subject, {self.lit(r.predicate)} AS predicate, {obj[0]} AS o_kind, "
386
+ f"{obj[1]} AS o_text, {obj[2]} AS o_datatype, {obj[3]} AS o_lang, {self.lit(r.rule)} AS rule "
387
+ f"FROM ({sql}) AS _r")
388
+
389
+ def planned(self, which: List[int], depth: int) -> Dict[int, str]:
390
+ """The SQL of the rules `which` (indexes into `all_rules`: the rules, then the link rules), each its
391
+ derived statements (a link rule's, its links), planned over this round's premises with closures
392
+ walked to `depth`, matched through this round's components (`identity`)."""
393
+ rules = self.all_rules
394
+
395
+ def sql(j: int, maps: Any, onto: Any) -> str:
396
+ r = rules[j]
397
+ if j < len(self.rules.rules):
398
+ return self.wrapped(r, self.lowered(self.rule_query(r), maps, onto, self.identity))
399
+ return self.link_wrapped(r, self.lowered(self.link_query(r), maps, onto, self.identity))
400
+ return self.with_maps(lambda maps, onto: {j: sql(j, maps, onto) for j in which}, depth)
401
+
402
+ @property
403
+ def all_rules(self) -> List[Any]:
404
+ return list(self.rules.rules) + list(self.rules.links)
405
+
406
+ def with_maps(self, plan: Any, depth: Optional[int] = None) -> Any:
407
+ """`plan(maps, onto)` over this round's premises (the saturated maps reading the store's other graphs
408
+ and this run's derivations), closures walked to `depth`."""
409
+ from prograph.plan.ir import lower
410
+ from prograph.plan.ir.dataset import maps_seen
411
+ from prograph.plan.ir.entry import ontology_of
412
+ onto = ontology_of(self.engine)
413
+ store = self.store
414
+ token = lower.PATH_DEPTH.set(depth or lower.DEFAULT_PATH_DEPTH)
415
+ try:
416
+ with store.premises(self.premise_source(), store._profile.kinds, self.kinds):
417
+ return plan(maps_seen(self.engine, onto), onto)
418
+ finally:
419
+ lower.PATH_DEPTH.reset(token)
420
+
421
+ # -- owl:sameAs (`identity`) -----------------------------------------------------------------------------
422
+ def link_query(self, r: Any) -> str:
423
+ """A link rule's SPARQL: its two IRIs' texts, two IRIs that are not one representative."""
424
+ s, o = r.subject, r.object
425
+ return (f"SELECT DISTINCT (STR({s}) AS ?s) (STR({o}) AS ?t) WHERE {{ {r.where} "
426
+ f"FILTER(isIRI({s}) && isIRI({o}) && !sameTerm({s}, {o})) }}")
427
+
428
+ def link_wrapped(self, r: Any, lowered: Tuple[str, frozenset]) -> str:
429
+ """A link rule's rows as links (g, a, b, source), the default graph's, none with an IRI left
430
+ unmerged."""
431
+ sql, _ = lowered
432
+ q = self.d.quote
433
+ rows = (f"SELECT {self.lit('')} AS g, {self.text('_r.' + q('s'))} AS a, {self.text('_r.' + q('t'))} AS b, "
434
+ f"{self.lit(f'rl:{r.rule} {r.axiom}')} AS source FROM ({sql}) AS _r")
435
+ if self.comps is not None and self.comps.aliases:
436
+ # Only a link between two components is new (a pattern's ends are representatives already, a
437
+ # VALUES's constants not).
438
+ alias = f"SELECT iri, rep FROM {self.comps.rows} AS _a WHERE g = {self.lit('')}"
439
+ rows = (f"SELECT _k.g, _k.a, _k.b, _k.source FROM ({rows}) AS _k LEFT JOIN ({alias}) AS _ra ON "
440
+ f"_ra.iri = _k.a LEFT JOIN ({alias}) AS _rb ON _rb.iri = _k.b "
441
+ f"WHERE coalesce(_ra.rep, _k.a) <> coalesce(_rb.rep, _k.b)")
442
+ return self.unpoisoned(rows)
443
+
444
+ def unpoisoned(self, links: str) -> str:
445
+ """The links `links` with neither end an IRI left unmerged."""
446
+ poisoned = self.poisoned()
447
+ if poisoned is None:
448
+ return links
449
+ return (f"SELECT _l.g, _l.a, _l.b, _l.source FROM ({links}) AS _l LEFT JOIN ({poisoned}) AS _pa ON _pa.g = _l.g "
450
+ f"AND _pa.iri = _l.a LEFT JOIN ({poisoned}) AS _pb ON _pb.g = _l.g AND _pb.iri = _l.b "
451
+ f"WHERE _pa.iri IS NULL AND _pb.iri IS NULL")
452
+
453
+ def poisoned(self) -> Optional[str]:
454
+ """SQL of the IRIs (g, iri) left unmerged so far, None where none is."""
455
+ return " UNION ALL ".join(f"SELECT g, iri FROM {p} AS _p" for p in self.poison_stages) or None
456
+
457
+ def links(self) -> Optional[str]:
458
+ """SQL of every link (g, a, b, source): the graph's own and those derived in this pass; None where
459
+ there is none."""
460
+ parts = ([f"SELECT g, a, b, source FROM {self.stated} AS _s"] if self.stated else []) + \
461
+ [f"SELECT g, a, b, source FROM {x} AS _d" for x in self.link_stages]
462
+ return " UNION ALL ".join(parts) or None
463
+
464
+ def stage_component(self, name: str, sql: str) -> Tuple[str, str]:
465
+ """Stage one statement of the components' computation (`identity.components`)."""
466
+ self.component_index += 1
467
+ return self.stage(1_000_000_000 + self.component_index, f"_pg_rl_{self.key}_c{self.component_index}", sql)
468
+
469
+ def stage_stated(self) -> None:
470
+ """The graph's own sameAs links between two IRIs (g '' the default graph, else each named graph's),
471
+ read as the rules' premises are, owl:sameAs as stated; staged once, where some map or the store
472
+ gives owl:sameAs."""
473
+ from prograph.plan.ir.identity import SAME_AS
474
+
475
+ def plan(maps: Any, onto: Any) -> Optional[str]:
476
+ if SAME_AS not in maps.by_predicate:
477
+ return None
478
+ test = "FILTER(isIRI(?a) && isIRI(?b) && !sameTerm(?a, ?b))"
479
+ default, _ = self.lowered(f"SELECT DISTINCT (STR(?a) AS ?x) (STR(?b) AS ?y) WHERE {{ ?a <{SAME_AS}> ?b {test} }}",
480
+ maps, onto)
481
+ named, _ = self.lowered(f"SELECT DISTINCT (STR(?g) AS ?h) (STR(?a) AS ?x) (STR(?b) AS ?y) WHERE "
482
+ f"{{ GRAPH ?g {{ ?a <{SAME_AS}> ?b {test} }} }}", maps, onto)
483
+ q = self.d.quote
484
+ x, y, h = (self.text(f"_r.{q(n)}") for n in ("x", "y", "h"))
485
+ stated = self.lit("stated")
486
+ return (f"SELECT {self.lit('')} AS g, {x} AS a, {y} AS b, {stated} AS source FROM ({default}) AS _r "
487
+ f"UNION ALL SELECT {h} AS g, {x} AS a, {y} AS b, {stated} AS source FROM ({named}) AS _r")
488
+ sql = self.with_maps(plan)
489
+ if sql is not None:
490
+ self.stated, _ = self.stage(999_999_999, f"_pg_rl_{self.key}_stated", sql)
491
+
492
+ def close_links(self, report: MaterializationReport, first: bool) -> bool:
493
+ """This round's components (`identity.components`), each inconsistent one or one past the cap left
494
+ unmerged (named in `report.unmerged`), and the identity the round's rules read under. True where a
495
+ component rules already read is left unmerged now: the pass runs again from its first round."""
496
+ from prograph.plan.ir import identity as I
497
+ links = self.links()
498
+ if links is None:
499
+ self.comps, self.identity = None, None
500
+ return False
501
+ self.comps = I.components(self, links, self.poisoned())
502
+ self.poison_capped(report)
503
+ bad = self.inconsistent(report)
504
+ if bad:
505
+ self.comps = I.components(self, links, self.poisoned())
506
+ if not first:
507
+ return True
508
+ self.identity = I.Identity(f"SELECT g, iri, rep FROM {self.comps.rows} AS _a", None, self.comps.largest,
509
+ "representative", run=lambda sql: self.db.execute(sql).fetchall(),
510
+ literal=self.d.literal, tag=f"{self.key}{self.component_index}") \
511
+ if self.comps.aliases else None
512
+ return False
513
+
514
+ def value_key(self, text: str, datatype: str, lang: str) -> str:
515
+ """SQL of a literal's value as text, one text per value (a clash's test; `owl_rl_rules.same_value` is the
516
+ rule): an xsd:decimal or any integer type by its number, xsd:float and xsd:double each by its own number
517
+ (with its sign: +0.0 and -0.0 are two), an xsd:boolean by its canonical form ("1" is "true"), an
518
+ xsd:dateTime with a timezone by its instant (`Z` is `+00:00`) and one without by its local time
519
+ (`value_group` keeps the two apart: XML Schema 1.1 §3.3.7 leaves them incomparable), any other by its
520
+ datatype, language and lexical form."""
521
+ decimal = ", ".join(self.lit(xsd.XSD + t) for t in (
522
+ "decimal", "integer", "long", "int", "short", "byte", "nonNegativeInteger", "positiveInteger",
523
+ "negativeInteger", "nonPositiveInteger", "unsignedLong", "unsignedInt", "unsignedShort", "unsignedByte"))
524
+ cast = "CAST" if self.dialect == "postgresql" else "TRY_CAST"
525
+ double = "DOUBLE PRECISION" if self.dialect == "postgresql" else "DOUBLE"
526
+ if self.dialect == "postgresql":
527
+ instant = f"EXTRACT(EPOCH FROM CAST({text} AS TIMESTAMPTZ))"
528
+ elif self.dialect == "duckdb":
529
+ instant = f"epoch_us(TRY_CAST({text} AS TIMESTAMPTZ))"
530
+ else: # Spark SQL, Databricks SQL: a TIMESTAMP read with its offset is an instant
531
+ instant = f"unix_micros(TRY_CAST({text} AS TIMESTAMP))"
532
+ local = f"{cast}({text} AS TIMESTAMP)"
533
+ boolean = self.lit(xsd.BOOLEAN)
534
+ return (f"CASE WHEN {datatype} IN ({decimal}) THEN {self.lit('n ')} || "
535
+ f"{self.text(f'{cast}({text} AS DECIMAL(38, 10))')} "
536
+ f"WHEN {datatype} IN ({self.lit(xsd.XSD + 'float')}, {self.lit(xsd.XSD + 'double')}) THEN "
537
+ f"{datatype} || {self.text(f'{cast}({text} AS {double})')} "
538
+ f"WHEN {datatype} = {boolean} THEN CASE WHEN {text} IN ('true', '1') THEN 'b true' "
539
+ f"WHEN {text} IN ('false', '0') THEN 'b false' ELSE 'b ' || {text} END "
540
+ f"WHEN {self.value_group(text, datatype)} = 'z' THEN 'z ' || {self.text(instant)} "
541
+ f"WHEN {self.value_group(text, datatype)} = 'u' THEN 'u ' || {self.text(local)} "
542
+ f"ELSE coalesce({datatype}, '') || ' ' || coalesce({lang}, '') || ' ' || {text} END")
543
+
544
+ def value_group(self, text: str, datatype: str) -> str:
545
+ """SQL: 'z' for an xsd:dateTime with a timezone, 'u' for one without, '' for any other literal."""
546
+ stamps = f"{self.lit(xsd.XSD + 'dateTime')}, {self.lit(xsd.XSD + 'dateTimeStamp')}"
547
+ return (f"CASE WHEN {datatype} IN ({stamps}) THEN CASE WHEN {text} LIKE '%Z' OR {text} LIKE '%+__:__' OR "
548
+ f"{text} LIKE '%-__:__' THEN 'z' ELSE 'u' END ELSE '' END")
549
+
550
+ @staticmethod
551
+ def two_values(key: str = "k", group: str = "g") -> str:
552
+ """A HAVING of two values that are not one value: two keys among the values comparable with each other
553
+ (a dateTime with a timezone and one without are not)."""
554
+ return (f"count(DISTINCT CASE WHEN {group} <> 'u' THEN {key} END) > 1 OR "
555
+ f"count(DISTINCT CASE WHEN {group} <> 'z' THEN {key} END) > 1")
556
+
557
+ def poison_capped(self, report: MaterializationReport) -> None:
558
+ for g, rep, size in self.comps.capped:
559
+ if (g, rep) in self.unmerged_reps:
560
+ continue
561
+ self.unmerged_reps.add((g, rep))
562
+ sample = tuple(r[0] for r in self.db.execute(
563
+ f"SELECT x FROM {self.comps.labels} AS _l WHERE g = {self.lit(g)} AND l = {self.lit(rep)} "
564
+ f"ORDER BY x LIMIT {SAMPLE}").fetchall())
565
+ report.unmerged.append(Unmerged("cap", "", g, rep, size, sample))
566
+ read, _ = self.stage_component("capped", f"SELECT g, x AS iri FROM {self.comps.labels} AS _l "
567
+ f"WHERE g = {self.lit(g)} AND l = {self.lit(rep)}")
568
+ self.poison_stages.append(read)
569
+
570
+ def inconsistent(self, report: MaterializationReport) -> bool:
571
+ """Leave unmerged each component of the default graph that owl:differentFrom or owl:AllDifferent
572
+ (eq-diff2, eq-diff3) or two values of a functional property (prp-fp, cls-maxc2: two values that are
573
+ not one value, dt-diff's reading) make inconsistent; a lone IRI's clash is an inconsistency of the
574
+ report. True where one was left unmerged now."""
575
+ from prograph.plan.ir import identity as I
576
+ if not self.comps.aliases:
577
+ return False
578
+ ident = I.Identity(f"SELECT g, iri, rep FROM {self.comps.rows} AS _a", None, self.comps.largest,
579
+ "representative", run=lambda sql: self.db.execute(sql).fetchall(),
580
+ literal=self.d.literal, tag=f"{self.key}{self.component_index}x")
581
+ found: Dict[str, Tuple[str, str]] = {} # rep -> (rule, axiom)
582
+ different = f"<{I.DIFFERENT_FROM}>"
583
+ checks = [(c.rule, c.axiom, True,
584
+ f"SELECT DISTINCT (STR(?x) AS ?r) (STR(?v) AS ?t) (STR(DATATYPE(?v)) AS ?d) (LANG(?v) AS ?l) "
585
+ f"WHERE {{ {c.where} FILTER(isIRI(?x) && isLiteral(?v)) }}") for c in self.rules.clashes]
586
+
587
+ def plan(maps: Any, onto: Any) -> List[Tuple[str, str, bool, str]]:
588
+ out = []
589
+ wanted = checks + ([("eq-diff1", different, False,
590
+ f"SELECT DISTINCT (STR(?a) AS ?r) WHERE {{ ?a <{I.DIFFERENT_FROM}> ?b "
591
+ f"FILTER(isIRI(?a) && sameTerm(?a, ?b)) }}")]
592
+ if I.DIFFERENT_FROM in maps.by_predicate else [])
593
+ for rule, axiom, values, query in wanted:
594
+ out.append((rule, axiom, values, self.lowered(query, maps, onto, ident)))
595
+ return out
596
+ q = self.d.quote
597
+ for rule, axiom, values, (sql, never) in self.with_maps(plan):
598
+ if values:
599
+ # Two values of one subject that are not one value (`value_key`).
600
+ def col(n: str) -> str:
601
+ return self.lit(None) if n in never else self.text(f"_r.{q(n)}")
602
+ key = self.value_key(col("t"), col("d"), col("l"))
603
+ group = self.value_group(col("t"), col("d"))
604
+ sql = (f"SELECT r FROM (SELECT {col('r')} AS r, {key} AS k, {group} AS g FROM ({sql}) AS _r) AS _k "
605
+ f"GROUP BY r HAVING {self.two_values()}")
606
+ else:
607
+ sql = f"SELECT {self.text('_r.' + q('r'))} FROM ({sql}) AS _r"
608
+ for (rep,) in self.db.execute(sql).fetchall():
609
+ found.setdefault(rep, (rule, axiom))
610
+ if self.rules.distinct:
611
+ listed = {x for a, b, _, _ in self.rules.distinct for x in (a, b)}
612
+ reps = dict(self.db.execute(
613
+ f"SELECT iri, rep FROM {self.comps.rows} AS _a WHERE g = {self.lit('')} AND iri IN "
614
+ f"({', '.join(self.lit(x) for x in sorted(listed))})").fetchall())
615
+ for a, b, axiom, rule in self.rules.distinct:
616
+ ra, rb = reps.get(a, a), reps.get(b, b)
617
+ if a != b and ra == rb:
618
+ found.setdefault(ra, (rule, axiom))
619
+ fresh = False
620
+ for rep, (rule, axiom) in sorted(found.items()):
621
+ if ("", rep) in self.unmerged_reps:
622
+ continue
623
+ members = [r[0] for r in self.db.execute(
624
+ f"SELECT iri FROM {self.comps.rows} AS _a WHERE g = {self.lit('')} AND rep = {self.lit(rep)} "
625
+ f"ORDER BY iri").fetchall()]
626
+ if not members: # a lone IRI: no merge to undo, an inconsistency the run's check names
627
+ continue
628
+ self.unmerged_reps.add(("", rep))
629
+ report.unmerged.append(Unmerged("inconsistent", rule, "", rep, len(members) + 1,
630
+ tuple(sorted([rep] + members)[:SAMPLE]), axiom))
631
+ read, _ = self.stage_component("unmerged", f"SELECT g, iri FROM {self.comps.rows} AS _a WHERE g = "
632
+ f"{self.lit('')} AND rep = {self.lit(rep)} UNION ALL SELECT "
633
+ f"{self.lit('')} AS g, {self.lit(rep)} AS iri")
634
+ self.poison_stages.append(read)
635
+ fresh = True
636
+ return fresh
637
+
638
+ # -- the fixpoint ----------------------------------------------------------------------------------------
639
+ def depths(self) -> List[int]:
640
+ from prograph.plan.ir import lower
641
+ ladder = list(lower.closure_ladder(self.dialect))
642
+ return ladder + [lower.DEFAULT_PATH_DEPTH, DEEPEST] if ladder else [lower.DEFAULT_PATH_DEPTH, DEEPEST]
643
+
644
+ # A round's stages: rule j's at round * RULE_STRIDE + j, the round's merged rows at round * RULE_STRIDE.
645
+ RULE_STRIDE = 100000
646
+
647
+ def one_round(self, n: int) -> Tuple[int, int]:
648
+ """Run round `n`: each rule one statement (a transitive property's closure walked by the engine's
649
+ ladder: a deeper rung only where a walk is longer), then the round's triples merged into one stage
650
+ (a triple two rules derived once, by the first rule's id), and its links into another; how many
651
+ of each."""
652
+ from prograph.plan.ir import lower
653
+ from prograph.execution import checkpoint
654
+ pending = list(range(len(self.all_rules)))
655
+ reads: Dict[int, Tuple[str, str]] = {}
656
+ batched: Tuple[List[str], List[str]] = ([], []) # (facts, links) read of each batch (Delta)
657
+ depths = self.depths()
658
+ for i, depth in enumerate(depths):
659
+ checkpoint()
660
+ sqls = self.planned(pending, depth)
661
+ again = []
662
+ single = pending
663
+ if self.db.optimistic:
664
+ single = self.stage_batches(n, i, pending, sqls, batched)
665
+ for j in single:
666
+ try:
667
+ reads[j] = self.stage(n * self.RULE_STRIDE + j + 1, f"_pg_rl_{self.key}_{n}_{j}", sqls[j])
668
+ except Exception as exc: # noqa: BLE001 - a deeper rung answers a depth guard
669
+ if not lower.depth_guard(str(exc)):
670
+ raise
671
+ if i == len(depths) - 1:
672
+ from prograph.utils.exceptions import SQLRuntimeError
673
+ raise SQLRuntimeError(
674
+ f"OWL RL materialization: a transitive property's path is longer than {depth} steps, "
675
+ f"which a round walks at most", reason="path_too_deep", sql=sqls[j]) from exc
676
+ again.append(j)
677
+ pending = again
678
+ if not pending:
679
+ break
680
+ cols = ", ".join(STATEMENT)
681
+ facts = {j: x for j, x in reads.items() if j < len(self.rules.rules)}
682
+ linked = {j: x for j, x in reads.items() if j >= len(self.rules.rules)}
683
+ count = 0
684
+ if facts or batched[0]:
685
+ union = " UNION ALL ".join(
686
+ [f"SELECT {cols}, rule FROM {read} AS _r{j}" for j, (read, _) in sorted(facts.items())] +
687
+ [f"SELECT {cols}, rule FROM {read} AS _b{k}" for k, read in enumerate(batched[0])])
688
+ read, drop = self.stage(n * self.RULE_STRIDE, f"_pg_rl_{self.key}_{n}",
689
+ f"SELECT {cols}, min(rule) AS rule FROM ({union}) AS _u GROUP BY {cols}")
690
+ count = self.db.execute(f"SELECT count(*) FROM {read} AS _c").fetchone()[0]
691
+ if count:
692
+ self.stages.append((read, drop))
693
+ for row in self.db.execute(f"SELECT DISTINCT predicate, o_kind, o_datatype, o_lang FROM {read} AS _k"
694
+ ).fetchall():
695
+ self.kinds.add(tuple(row))
696
+ links = 0
697
+ if linked or batched[1]:
698
+ union = " UNION ALL ".join(
699
+ [f"SELECT g, a, b, source FROM {read} AS _r{j}" for j, (read, _) in sorted(linked.items())] +
700
+ [f"SELECT g, a, b, source FROM {read} AS _b{k}" for k, read in enumerate(batched[1])])
701
+ read, _ = self.stage(n * self.RULE_STRIDE + self.RULE_STRIDE // 2, f"_pg_rl_{self.key}_{n}_links",
702
+ f"SELECT g, a, b, min(source) AS source FROM ({union}) AS _u GROUP BY g, a, b")
703
+ links = self.db.execute(f"SELECT count(*) FROM {read} AS _c").fetchone()[0]
704
+ if links:
705
+ self.link_stages.append(read)
706
+ if not self.db.optimistic: # the rules' own tables (on Delta every stage of the run goes at the end)
707
+ for _, rule_drop in reads.values():
708
+ self.db.execute(rule_drop)
709
+ self.cleanups.remove(rule_drop)
710
+ return count, links
711
+
712
+ # Delta: a rule's rows are one write each unless batched: at most this many rules, and this many characters
713
+ # of SQL (each rule inlines its premises, which a warehouse's statement limit and Spark's analyzer feel), go
714
+ # in one write.
715
+ BATCH = 40
716
+ BATCH_SQL_CHARS = 1_000_000
717
+
718
+ def stage_batches(self, n: int, rung: int, pending: List[int], sqls: Dict[int, str],
719
+ batched: Tuple[List[str], List[str]]) -> List[int]:
720
+ """Delta commits every statement alone, so a round's rules are staged together: the facts rules in
721
+ batches of at most `BATCH` rules and `BATCH_SQL_CHARS` characters (one write each, the rule's id in
722
+ each row), the link rules likewise. A batch a depth guard refuses is halved until the rule is found;
723
+ a batch any other error refuses is left to the rules' own writes (whose error, if they fail too,
724
+ stands). The rules so left are returned."""
725
+ from prograph.plan.ir import lower
726
+ left: List[int] = []
727
+ nfacts = len(self.rules.rules)
728
+ cols = ", ".join(STATEMENT)
729
+ for links, shape in ((False, f"{cols}, rule"), (True, "g, a, b, source")):
730
+ ids = [j for j in pending if (j >= nfacts) == links]
731
+ chunks: List[List[int]] = []
732
+ size = 0
733
+ for j in ids:
734
+ if not chunks or len(chunks[-1]) >= self.BATCH or size + len(sqls[j]) > self.BATCH_SQL_CHARS:
735
+ chunks.append([])
736
+ size = 0
737
+ chunks[-1].append(j)
738
+ size += len(sqls[j])
739
+ base = n * self.RULE_STRIDE + self.RULE_STRIDE // 4 + (rung * 2 + links) * 1000
740
+ counter = [0]
741
+
742
+ def stage_chunk(chunk: List[int]) -> None:
743
+ if len(chunk) == 1:
744
+ left.extend(chunk)
745
+ return
746
+ union = " UNION ALL ".join(f"SELECT {shape} FROM ({sqls[j]}) AS _p{j}" for j in chunk)
747
+ counter[0] += 1
748
+ try:
749
+ read, _ = self.stage(base + counter[0], f"_pg_rl_{self.key}_{n}_b{rung}{int(links)}_{counter[0]}",
750
+ union)
751
+ except Exception as exc: # noqa: BLE001 - the rules' own writes answer it, or raise as before
752
+ if lower.depth_guard(str(exc)):
753
+ half = len(chunk) // 2
754
+ stage_chunk(chunk[:half])
755
+ stage_chunk(chunk[half:])
756
+ else:
757
+ left.extend(chunk)
758
+ return
759
+ batched[links].append(read)
760
+ for chunk in chunks:
761
+ stage_chunk(chunk)
762
+ return left
763
+
764
+ def go(self) -> MaterializationReport:
765
+ from prograph.plan.ir import freshness
766
+ from prograph.plan.ir.dataset import cache_key, stored_sources
767
+ report = MaterializationReport(skipped=list(self.rules.skipped), digest=cache_key(self.engine, self.rules))
768
+ # The sources' versions, read before the rules read them: a write during the run is seen by the next check.
769
+ report.sources = freshness.read_run(self.engine, stored_sources(self.engine))
770
+ ensure = getattr(getattr(getattr(self.engine, "executor", None), "graph", None), "_ensure_temp_views", None)
771
+ if callable(ensure):
772
+ ensure() # Spark: the engine's frames are views before its first query too (a run on open)
773
+ if self.rules.rules or self.rules.links or self.maybe_stated():
774
+ self.stage_base()
775
+ self.stage_stated()
776
+ n = 0
777
+ while True: # a pass; another from its first round where a component it read is left unmerged
778
+ again, first = False, True
779
+ while True:
780
+ n += 1
781
+ if n > self.max_rounds:
782
+ from prograph.utils.exceptions import MaterializationBudgetExceeded
783
+ raise MaterializationBudgetExceeded(self.max_rounds)
784
+ report.rounds = n
785
+ if self.close_links(report, first):
786
+ again = True
787
+ break
788
+ first = False
789
+ if self.one_round(n) == (0, 0):
790
+ break
791
+ if not again:
792
+ break
793
+ self.stages, self.link_stages = [], []
794
+ report.inconsistencies = self.consistency(report)
795
+ self.forget_premise_types()
796
+ report.ran_at = _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
797
+ if self.comps is not None:
798
+ report.components, report.aliases, report.largest = self.comps.count, self.comps.aliases, \
799
+ self.comps.largest
800
+ report.unmerged.sort(key=_unmerged_order) # the order a reused report reads them in
801
+ before = self.store.alias
802
+ seq = self.write(report)
803
+ report.seq = seq
804
+ if seq is not None:
805
+ self.counts(report, seq)
806
+ self.warn_size(report, before)
807
+ return report
808
+
809
+ def maybe_stated(self) -> bool:
810
+ """Whether the graph may hold an owl:sameAs triple (some map, or the store, gives the predicate), or a
811
+ generation of the alias table is to be replaced."""
812
+ from prograph.plan.ir.dataset import maps_seen
813
+ from prograph.plan.ir.identity import SAME_AS, meta_table
814
+ if SAME_AS in maps_seen(self.engine, None).by_predicate:
815
+ return True
816
+ return self.store.db.has_table(meta_table(self.store))
817
+
818
+ def warn_size(self, report: MaterializationReport, before: Any) -> None:
819
+ """`SameAsMergeWarning` where the largest component has `WARN_MEMBERS` members or more, or grew more
820
+ than tenfold since the last generation."""
821
+ import warnings
822
+ from prograph.plan.ir.identity import CAP, GROWTH, WARN_MEMBERS, SameAsMergeWarning
823
+ if not report.largest:
824
+ return
825
+ grew = before is not None and before.largest and report.largest > GROWTH * before.largest
826
+ if report.largest >= WARN_MEMBERS or grew:
827
+ why = (f"grew from {before.largest} to {report.largest} members" if grew else
828
+ f"has {report.largest} members")
829
+ warnings.warn(f"the largest owl:sameAs component {why}: a functional or inverse-functional property "
830
+ f"(or a crosswalk) may be merging entities that are not one. engine.explain_identity(a, b) "
831
+ f"shows why two IRIs merged; a component past {CAP} members is left unmerged",
832
+ SameAsMergeWarning, stacklevel=4)
833
+
834
+ # -- the alias generation (`identity`) ------------------------------------------------------------------
835
+ def publish(self) -> bool:
836
+ """Whether this run writes a generation of the alias table: it has components or unmerged ones, or
837
+ one is written already (replaced, maybe by none)."""
838
+ from prograph.plan.ir.identity import meta_table
839
+ return bool(self.comps is not None and (self.comps.aliases or self.comps.capped)) or \
840
+ bool(self.unmerged_reps) or self.store.db.has_table(meta_table(self.store))
841
+
842
+ def alias_statements(self, report: MaterializationReport, generation: str) -> List[str]:
843
+ """The statements that write this run's generation `generation` (SQL of its number): the alias rows
844
+ (where there are some: never an empty INSERT), then one INSERT of the meta rows (the marker, the IRI
845
+ shapes, the derived links, the components left unmerged)."""
846
+ import json
847
+ from prograph.plan.ir import identity as I
848
+ null = self.lit(None)
849
+ out = []
850
+ aliases = self.comps.aliases if self.comps is not None else 0
851
+ shapes: List[str] = []
852
+ if aliases:
853
+ out.append(f"INSERT INTO {I.alias_table(self.store)} (generation, graph, iri, rep) "
854
+ f"SELECT {generation}, g, iri, rep FROM {self.comps.rows} AS _a")
855
+ ns = lambda e: I.namespace_sql(self.dialect, e) # noqa: E731
856
+ shapes = [r[0] for r in self.db.execute(
857
+ f"SELECT DISTINCT n FROM (SELECT {ns('iri')} AS n FROM {self.comps.rows} AS _a UNION "
858
+ f"SELECT {ns('rep')} AS n FROM {self.comps.rows} AS _b) AS _n LIMIT {I.SHAPES_MAX + 1}").fetchall()]
859
+ every = len(shapes) > I.SHAPES_MAX
860
+ note = json.dumps({"aliases": aliases, "largest": report.largest, "components": report.components,
861
+ "all_shapes": every}, sort_keys=True)
862
+ # The marker's `a`: the run's cache key, so a reader takes the generation only under that key.
863
+ rows = [f"SELECT {generation} AS generation, 'generation' AS kind, {null} AS graph, {self.lit(report.digest)} AS a, {null} AS b, "
864
+ f"{self.lit(note)} AS note"]
865
+ rows += [f"SELECT {generation}, 'shape', {null}, {self.lit(x)}, {null}, {null}" for x in ([] if every else shapes)]
866
+ rows += [f"SELECT {generation}, 'link', g, a, b, source FROM {x} AS _l" for x in self.link_stages]
867
+ if self.disagree is not None: # Cypher's list-valued properties (`stage_disagreeing`)
868
+ rows.append(f"SELECT {generation}, kind, {null}, c, p, {null} FROM {self.disagree} AS _d")
869
+ rows += [f"SELECT {generation}, 'unmerged', {self.lit(u.graph)}, {self.lit(u.representative)}, "
870
+ f"{self.lit(u.rule)}, {self.lit(json.dumps({'reason': u.reason, 'size': u.size, 'sample': list(u.sample), 'axiom': u.axiom}))}"
871
+ for u in report.unmerged]
872
+ out.append(f"INSERT INTO {I.meta_table(self.store)} (generation, kind, graph, a, b, note) "
873
+ + " UNION ALL ".join(rows))
874
+ return out
875
+
876
+ def canonical(self, rows: str, graph: bool, rule: bool) -> str:
877
+ """The derived statements `rows` with their subject and IRI object under their representatives (the
878
+ final components'), once each: a derived fact is stored once, on the representative."""
879
+ if self.comps is None or not self.comps.aliases:
880
+ return rows
881
+ cols = ", ".join(STATEMENT)
882
+ alias = f"SELECT iri, rep FROM {self.comps.rows} AS _a WHERE g = {self.lit('')}"
883
+ picked = (("n.graph, " if graph else "") +
884
+ "coalesce(_cs.rep, n.subject) AS subject, n.predicate, n.o_kind, "
885
+ "CASE WHEN n.o_kind = 'iri' THEN coalesce(_co.rep, n.o_text) ELSE n.o_text END AS o_text, "
886
+ "n.o_datatype, n.o_lang" + (", n.rule" if rule else ""))
887
+ canon = (f"SELECT {picked} FROM ({rows}) AS n LEFT JOIN ({alias}) AS _cs ON _cs.iri = n.subject "
888
+ f"LEFT JOIN ({alias}) AS _co ON n.o_kind = 'iri' AND _co.iri = n.o_text")
889
+ keys = ("graph, " if graph else "") + cols
890
+ return (f"SELECT {keys}, min(rule) AS rule FROM ({canon}) AS _c GROUP BY {keys}" if rule
891
+ else f"SELECT DISTINCT {keys} FROM ({canon}) AS _c")
892
+
893
+ # -- the step --------------------------------------------------------------------------------------------
894
+ def write(self, report: MaterializationReport) -> Optional[int]:
895
+ from prograph.plan.ir import freshness
896
+ from prograph.plan.ir.consistency import as_record
897
+ from prograph.store import ASSERT, INTERNAL, LITERAL, RETRACT, RL_GRAPH
898
+ store = self.store
899
+ cols = ", ".join(STATEMENT)
900
+ null = self.lit(None)
901
+ closure = self.canonical(self.derived(), True, False) if self.stages else \
902
+ (f"SELECT {self.lit(RL_GRAPH)} AS graph, " + ", ".join(f"{null} AS {c}" for c in STATEMENT) + " WHERE 1 = 0")
903
+ rule_of = self.canonical(" UNION ALL ".join(f"SELECT {cols}, rule FROM {read} AS _d" for read, _ in self.stages),
904
+ False, True) if self.stages else \
905
+ (f"SELECT {', '.join(f'{null} AS {c}' for c in STATEMENT)}, {null} AS rule WHERE 1 = 0")
906
+ held = (f"SELECT {cols} FROM ({store.effective_sql([RL_GRAPH])}) AS _h "
907
+ f"WHERE NOT {self.db.starts_with}(predicate, {self.lit(INTERNAL)})")
908
+
909
+ def same(a: str, b: str) -> str:
910
+ return " AND ".join(f"coalesce({a}.{c}, '') = coalesce({b}.{c}, '')" for c in STATEMENT)
911
+ asserts = (f"SELECT {self.lit(RL_GRAPH)} AS graph, {', '.join('n.' + c for c in STATEMENT)}, "
912
+ f"{null} AS edge_id, '{ASSERT}' AS op, {self.text(self.lit('rl:') + ' || n.rule')} AS _writer, "
913
+ f"1 AS _o FROM ({rule_of}) AS n LEFT JOIN ({held}) AS h ON {same('n', 'h')} "
914
+ f"WHERE h.subject IS NULL")
915
+ # A retraction's writer: the rule of the statement's last assert.
916
+ last = (f"SELECT {cols}, writer FROM (SELECT {cols}, writer, ROW_NUMBER() OVER (PARTITION BY {cols} "
917
+ f"ORDER BY seq DESC, pos DESC) AS _rn FROM {store._visible()} AS _v WHERE graph = "
918
+ f"{self.lit(RL_GRAPH)} AND op = '{ASSERT}') AS _w WHERE _rn = 1")
919
+ retracts = (f"SELECT {self.lit(RL_GRAPH)} AS graph, {', '.join('h.' + c for c in STATEMENT)}, "
920
+ f"{null} AS edge_id, '{RETRACT}' AS op, coalesce(w.writer, 'rl') AS _writer, 0 AS _o "
921
+ f"FROM ({held}) AS h LEFT JOIN ({closure}) AS n ON {same('h', 'n')} "
922
+ f"LEFT JOIN ({last}) AS w ON {same('h', 'w')} WHERE n.subject IS NULL")
923
+ # The run's record: the old one retracted, the new one asserted.
924
+ old = (f"SELECT graph, {cols}, edge_id, '{RETRACT}' AS op, 'rl' AS _writer, 0 AS _o FROM "
925
+ f"({store.effective_sql([RL_GRAPH])}) AS _m WHERE subject = {self.lit(RUN_SUBJECT)}")
926
+ new = " UNION ALL ".join(
927
+ f"SELECT {self.lit(RL_GRAPH)} AS graph, {self.lit(RUN_SUBJECT)} AS subject, {self.lit(p)} AS predicate, "
928
+ f"{self.lit(LITERAL)} AS o_kind, {self.lit(v)} AS o_text, {self.lit(dt)} AS o_datatype, "
929
+ f"{null} AS o_lang, {null} AS edge_id, '{ASSERT}' AS op, 'rl' AS _writer, 2 AS _o"
930
+ for p, v, dt in ((RUN_DIGEST, report.digest, xsd.STRING), (RUN_AT, report.ran_at, xsd.DATETIME),
931
+ (RUN_SOURCES, freshness.as_record(report.sources), xsd.STRING),
932
+ (RUN_CONSISTENCY, as_record(report.inconsistencies, report.not_run), xsd.STRING),
933
+ (RUN_ROUNDS, str(report.rounds), xsd.STRING)))
934
+ # One INSERT of the whole diff: on DuckDB 1.5.6 an INSERT that read the table (the retractions, here
935
+ # none) followed by a large one in the same transaction left the table's later reads without either
936
+ # (LDBC SF1, 2.9 million asserts); one statement does not.
937
+ diff = " UNION ALL ".join(f"SELECT * FROM ({q}) AS _d{i}" for i, q in enumerate((retracts, asserts, old, new)))
938
+ # The alias table's new generation (owl:sameAs): on DuckDB and PostgreSQL in the step's own transaction,
939
+ # numbered by the step's seq; on Delta after the step's marker, under its seq, its marker row last.
940
+ publish = self.publish()
941
+ if publish:
942
+ self.stage_disagreeing()
943
+ from prograph.plan.ir import identity as I
944
+ prior = None
945
+ if publish:
946
+ I.ensure_tables(store)
947
+ prior = self.db.execute(f"SELECT max(generation) FROM {I.meta_table(store)} "
948
+ f"WHERE kind = 'generation'").fetchone()[0]
949
+ runs = [("run", x) for x in self.alias_statements(report, f"(SELECT coalesce(max(seq), 0) + 1 FROM {store.table})")] \
950
+ if publish and not self.db.optimistic else []
951
+ seq, _ = store.write_select(runs + [diff], "rl", f"rl-{uuid.uuid4().hex}", writers=True, replan=True)
952
+ if publish and seq is not None:
953
+ if self.db.optimistic:
954
+ for x in self.alias_statements(report, str(int(seq))):
955
+ self.db.execute(x)
956
+ if prior is not None: # the generations before the last one: no plan reads them
957
+ for table in (I.alias_table(store), I.meta_table(store)):
958
+ try:
959
+ self.db.execute(f"DELETE FROM {table} WHERE generation < {int(prior)}")
960
+ except Exception: # noqa: BLE001 - removed by the next run
961
+ pass
962
+ store._refresh()
963
+ return seq
964
+
965
+ def counts(self, report: MaterializationReport, seq: int) -> None:
966
+ from prograph.store import ASSERT, INTERNAL, RL_GRAPH
967
+ rows = self.db.execute(
968
+ f"SELECT writer, op, count(*) FROM {self.store._visible()} AS _v WHERE seq = {int(seq)} AND graph = "
969
+ f"{self.lit(RL_GRAPH)} AND NOT {self.db.starts_with}(predicate, {self.lit(INTERNAL)}) "
970
+ f"GROUP BY writer, op").fetchall()
971
+ for writer, op, n in rows:
972
+ rule = writer[3:] if writer.startswith("rl:") else writer
973
+ (report.asserted if op == ASSERT else report.retracted)[rule] = int(n)
974
+
975
+ # -- consistency (`consistency`) ------------------------------------------------------------------------
976
+ def consistency(self, report: MaterializationReport) -> List[Inconsistency]:
977
+ """OWL 2 RL's rules that conclude false (Profiles §4.3, tables 4 to 9), once, over the closure this run
978
+ computed (the premises and its staged derivations, matched through the final components): each check's
979
+ solutions (cax-dw, prp-irp, ... dt-not-type), a stated `x owl:differentFrom x` (eq-diff1), the ontology's
980
+ owl:differentFrom or owl:AllDifferent naming one IRI twice, a subject's two values of a property it has at
981
+ most one of that are not one value (prp-fp, cls-maxc2, cls-maxqc3/4: `value_key`), and the owl:sameAs
982
+ components the run left unmerged as inconsistent. Each matched rule with a count and `SAMPLE` facts."""
983
+ from prograph.plan.ir import identity as I
984
+ from prograph.plan.ir import ql
985
+ from prograph.plan.ir.consistency import variables
986
+ from prograph.plan.ir.entry import _held
987
+ from prograph.plan.ir.lower import DATA_ERROR_MARK
988
+ from prograph.plan.ir.owl_rl_rules import Check
989
+ if self.base is None:
990
+ self.stage_base()
991
+ found: List[Inconsistency] = []
992
+ held = _held(self.engine._schema, self.dialect, True)
993
+ hints: Dict[int, str] = {}
994
+ loop = Check("eq-diff1", f"?x <{I.DIFFERENT_FROM}> ?x .", "an individual owl:differentFrom itself")
995
+
996
+ def projection(names: List[str]) -> str:
997
+ if not names:
998
+ return "(1 AS ?pgcone)"
999
+ return " ".join(f"(STR(?{n}) AS ?pgc{i}t) (STR(DATATYPE(?{n})) AS ?pgc{i}d) (LANG(?{n}) AS ?pgc{i}l) "
1000
+ f"(IF(isBlank(?{n}), \"b\", IF(isIRI(?{n}), \"i\", \"l\")) AS ?pgc{i}k)"
1001
+ for i, n in enumerate(names))
1002
+
1003
+ def plan(maps: Any, onto: Any) -> List[Tuple[str, Any, Any, List[str]]]:
1004
+ out = []
1005
+ checks = [c for c in self.rules.checks if c.range is None or c.range[2] or
1006
+ not ql.range_holds(maps, held, c.range[0], c.range[1])]
1007
+ checks += [loop] if I.DIFFERENT_FROM in maps.by_predicate else []
1008
+ for c in checks:
1009
+ if c.range is not None:
1010
+ hints[id(c)] = ql.range_hint(maps, held, c.range[0], c.range[1])
1011
+ names = variables(c.facts or c.where)
1012
+ query = f"SELECT DISTINCT {projection(names)} WHERE {{ {c.where} }}"
1013
+ out.append(("rows", c, self.planned_check(query, maps, onto), names))
1014
+ for c in self.rules.clashes:
1015
+ query = (f"SELECT DISTINCT (STR(?x) AS ?r) (STR(?v) AS ?t) (STR(DATATYPE(?v)) AS ?d) (LANG(?v) AS ?l) "
1016
+ f"WHERE {{ {c.where} FILTER(isIRI(?x) && isLiteral(?v)) }}")
1017
+ out.append(("values", c, self.planned_check(query, maps, onto), []))
1018
+ return out
1019
+ for how, c, lowered, names in self.with_maps(plan):
1020
+ try:
1021
+ got = self.rows_check(c, lowered, names) if how == "rows" else self.values_check(c, lowered)
1022
+ except Exception as exc: # noqa: BLE001 - a data error the check met (R2RML 11): not run, named
1023
+ text = str(exc)
1024
+ if DATA_ERROR_MARK not in text:
1025
+ raise
1026
+ report.not_run.append((c.rule, c.axiom, text[text.index(DATA_ERROR_MARK):].splitlines()[0]))
1027
+ continue
1028
+ if got is not None:
1029
+ found.append(replace(got, hint=hints.get(id(c), "")) if hints.get(id(c)) else got)
1030
+ for a, b, axiom, rule in self.rules.distinct:
1031
+ if a == b:
1032
+ found.append(Inconsistency(rule, axiom, ({"x": a},), 1, (f"<{a}> owl:differentFrom <{a}>",)))
1033
+ apart: Dict[Tuple[str, str], List[Unmerged]] = {}
1034
+ for u in report.unmerged:
1035
+ if u.reason == "inconsistent":
1036
+ apart.setdefault((u.rule, u.axiom), []).append(u)
1037
+ for (rule, axiom), us in apart.items():
1038
+ facts = tuple(" owl:sameAs ".join(f"<{m}>" for m in u.sample) + f" (one owl:sameAs component of {u.size} "
1039
+ f"IRIs, left unmerged)" for u in us[:SAMPLE])
1040
+ found.append(Inconsistency(rule, axiom, tuple({"x": u.representative} for u in us[:SAMPLE]), len(us),
1041
+ facts))
1042
+ return found
1043
+
1044
+ def planned_check(self, query: str, maps: Any, onto: Any) -> Tuple[str, frozenset]:
1045
+ """A check's SQL over this run's premises, matched through the run's final components."""
1046
+ return self.lowered(query, maps, onto, self.identity)
1047
+
1048
+ def rows_check(self, c: Any, lowered: Tuple[str, frozenset], names: List[str]) -> Optional[Inconsistency]:
1049
+ """A check's solutions: the first `SAMPLE` (as facts: `c.facts`, or its pattern, filled) and how many."""
1050
+ from prograph.plan.ir.consistency import fill, term
1051
+ sql, never = lowered
1052
+ q = self.d.quote
1053
+ cols = [f"pgc{i}{part}" for i in range(len(names)) for part in "tdlk"]
1054
+
1055
+ def col(n: str) -> str:
1056
+ return f"{self.lit(None)} AS {q(n)}" if n in never else f"{self.text('_r.' + q(n))} AS {q(n)}"
1057
+ picked = ", ".join(col(n) for n in cols) or "1 AS one"
1058
+ order = ", ".join(q(f"pgc{i}t") for i in range(len(names))) or "1"
1059
+ rows = self.db.execute(f"SELECT {picked} FROM ({sql}) AS _r ORDER BY {order} LIMIT {SAMPLE}").fetchall()
1060
+ if not rows:
1061
+ return None
1062
+ count = len(rows) if len(rows) < SAMPLE else \
1063
+ int(self.db.execute(f"SELECT count(*) FROM ({sql}) AS _r").fetchone()[0])
1064
+ sample, facts = [], []
1065
+ for r in rows:
1066
+ got = dict(zip(cols, r))
1067
+ terms = {n: term(got[f"pgc{i}t"], got[f"pgc{i}k"] or "l", got[f"pgc{i}d"], got[f"pgc{i}l"] or None)
1068
+ for i, n in enumerate(names)}
1069
+ sample.append({n: got[f"pgc{i}t"] for i, n in enumerate(names) if n in ("x", "y", "v", "p")})
1070
+ facts.append(fill(c.facts or c.where, terms))
1071
+ return Inconsistency(c.rule, c.axiom, tuple(sample), count, tuple(facts))
1072
+
1073
+ def values_check(self, c: Any, lowered: Tuple[str, frozenset]) -> Optional[Inconsistency]:
1074
+ """A clash: the subjects with two values of the property that are not one value (`value_key`), the first
1075
+ `SAMPLE` with their values, and how many."""
1076
+ from prograph.plan.ir.consistency import fill, term
1077
+ sql, never = lowered
1078
+ q = self.d.quote
1079
+
1080
+ def col(n: str) -> str:
1081
+ return self.lit(None) if n in never else self.text(f"_r.{q(n)}")
1082
+ keyed = (f"SELECT {col('r')} AS r, {col('t')} AS t, {col('d')} AS d, {col('l')} AS l, "
1083
+ f"{self.value_key(col('t'), col('d'), col('l'))} AS k, {self.value_group(col('t'), col('d'))} AS g "
1084
+ f"FROM ({sql}) AS _r")
1085
+ bad = f"SELECT r FROM ({keyed}) AS _k GROUP BY r HAVING {self.two_values()}"
1086
+ subjects = [r[0] for r in self.db.execute(f"SELECT r FROM ({bad}) AS _b ORDER BY r LIMIT {SAMPLE}").fetchall()]
1087
+ if not subjects:
1088
+ return None
1089
+ count = len(subjects) if len(subjects) < SAMPLE else \
1090
+ int(self.db.execute(f"SELECT count(*) FROM ({bad}) AS _b").fetchone()[0])
1091
+ values: Dict[str, List[str]] = {}
1092
+ for r, t, d, l in self.db.execute(
1093
+ f"SELECT DISTINCT r, t, d, l FROM ({keyed}) AS _k WHERE r IN ({', '.join(self.lit(x) for x in subjects)}) "
1094
+ f"ORDER BY r, t").fetchall():
1095
+ values.setdefault(r, []).append(term(t, "l", d, l or None))
1096
+ facts = tuple(fill(c.facts or c.where, {"x": f"<{r}>", "v": " , ".join(values.get(r, []))}) for r in subjects)
1097
+ return Inconsistency(c.rule, c.axiom, tuple({"x": r} for r in subjects), count, facts)
1098
+
1099
+ def cleanup(self) -> None:
1100
+ for sql in self.cleanups:
1101
+ try:
1102
+ self.db.execute(sql)
1103
+ except Exception: # noqa: BLE001 - a temporary table dropped with its session
1104
+ pass
1105
+ self.cleanups = []
1106
+ self.forget_premise_types()
1107
+
1108
+ def forget_premise_types(self) -> None:
1109
+ """The premise views' sources were typed for the rounds (`with_included`): forgotten after them."""
1110
+ types = getattr(self.engine._schema, "column_types", None)
1111
+ if isinstance(types, dict) and self.types_before is not None:
1112
+ # The store's own generated maps, as a query reads them, keep theirs: a run a query's freshness
1113
+ # check starts may be the first to type them, and the query's maps, cached by the store's version,
1114
+ # are not typed again.
1115
+ own = self.store.source_maps()
1116
+ keep = {t.table for t in own.tables()} if own is not None else set()
1117
+ for k in set(types) - self.types_before - keep:
1118
+ types.pop(k, None)
1119
+ self.types_before = None