prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,797 @@
1
+ """owl:sameAs: one entity under several IRIs (OWL 2 RL's eq-* rules; docs/reference/owl-rl.md).
2
+
3
+ Entailment changes basic graph pattern matching only (SPARQL 1.1 Entailment Regimes 2): a triple pattern
4
+ matches as if eq-sym, eq-trans, eq-rep-s, eq-rep-p and eq-rep-o were materialized over the graph SPARQL
5
+ answers. FILTER, sameTerm, VALUES, BIND and DISTINCT and GROUP BY keys keep RDF term identity.
6
+
7
+ - **The alias table** (`{store}_alias`: generation, graph, iri, rep; one row per IRI that has another name,
8
+ never a reflexive one) is rebuilt whole by each OWL RL materialization (`owl_rl`), as a new generation,
9
+ and published with the run's write step (a marker row in `{store}_alias_meta`). A plan reads the
10
+ generation current when it was made; the store's version changes with it, so every plan is made again.
11
+ - **Components** are the connected components of the sameAs links of each graph (the default graph's, and
12
+ each named graph's own), by minimum-label propagation with pointer jumping in SQL (`components`), the
13
+ representative the least IRI by code point (the dialect's binary collation, `lower.codepoint`).
14
+ - **Matching** (`pattern`): each IRI position of a triple pattern that may hold an alias is compared through
15
+ its representative (a left join on the alias rows), the matches kept once per canonical triple, then each
16
+ variable joined to every member of its component (`sameas="all"`, the default) or bound to the
17
+ representative (`sameas="representative"`). A constant IRI is looked up at planning (one read per
18
+ generation and IRI) and matches every member of its component. A predicate or class IRI with aliases
19
+ (eq-rep-p, an rdf:type object) unions its members' maps. `?a owl:sameAs ?b` is the components' cliques
20
+ (no reflexive pairs) and the graph's own sameAs triples.
21
+ - **The skip test**: a position whose term maps cannot give an IRI of the generation's IRI shapes (the
22
+ namespaces its members are in) reads its IRIs as they are, with no alias join.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import hashlib
28
+ import json
29
+ from dataclasses import dataclass, field
30
+ from typing import Any, Callable, Dict, FrozenSet, Iterable, List, Optional, Tuple
31
+
32
+ from prograph.plan.ir.dataset import active_graph
33
+ from prograph.rules import home, module_home
34
+
35
+ module_home("owl-sameas", "owl:sameAs: one entity under several IRIs, at BGP matching only",
36
+ terms=("owl:sameAs", "sameAs", "alias", "alias table", "representative", "component", "eq-rep",
37
+ "eq-sym", "eq-trans", "eq-rep-p", "sameas", "all aliases", "representative mode",
38
+ "explain_identity", "SameAsMergeWarning", "generation", "crosswalk", "one entity"))
39
+
40
+ OWL = "http://www.w3.org/2002/07/owl#"
41
+ SAME_AS = OWL + "sameAs"
42
+ DIFFERENT_FROM = OWL + "differentFrom"
43
+ RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
44
+ DEFAULT_GRAPH = "" # the graph column of the default graph's components
45
+ ALIAS_MAP = "prograph-alias"
46
+ MODES = ("all", "representative")
47
+ # A component of more members than this warns (`SameAsMergeWarning`), as does one whose largest grew tenfold.
48
+ WARN_MEMBERS = 1000
49
+ GROWTH = 10
50
+ # A component of more members than this is left unmerged and named in the report.
51
+ CAP = 100_000
52
+ # More namespaces than this among the members: every IRI position is read through the alias rows.
53
+ SHAPES_MAX = 1000
54
+ # A pattern whose expansion may give more rows than this a triple (largest^positions) is a cost hint.
55
+ EXPANSION_HINT = 1000
56
+ # A constant with more members than this is matched by a join with the alias rows, not a VALUES list.
57
+ VALUES_MAX = 64
58
+ # The most label-propagation rounds a component computation takes before it is refused.
59
+ LABEL_ROUNDS = 200
60
+
61
+
62
+ class SameAsMergeWarning(UserWarning):
63
+ """A component of owl:sameAs merged many IRIs (or grew tenfold since the last run): the classic runaway
64
+ of a bad inverse-functional property. `engine.explain_identity(a, b)` shows why two IRIs merged."""
65
+
66
+
67
+ @dataclass(frozen=True)
68
+ class AliasState:
69
+ """The current generation of a store's alias table: its number, the namespaces its members are in
70
+ (None: too many to list, every IRI may be an alias), its largest component, its aliases' count and the
71
+ cache key of the run that wrote it (`dataset.cache_key`; None for a generation written before it was kept)."""
72
+ generation: int
73
+ shapes: Optional[Tuple[str, ...]]
74
+ largest: int
75
+ aliases: int
76
+ key: Optional[str] = None
77
+
78
+
79
+ def alias_table(store: Any) -> str:
80
+ return f"{store.table}_alias"
81
+
82
+
83
+ def meta_table(store: Any) -> str:
84
+ return f"{store.table}_alias_meta"
85
+
86
+
87
+ def ensure_tables(store: Any) -> None:
88
+ """Make the alias table and its meta table where they are missing. On DuckDB and PostgreSQL the alias
89
+ rows of a generation are UNIQUE by (generation, graph, iri): one representative per IRI."""
90
+ c, x = store.db, store.db.text
91
+ unique = "" if c.optimistic else ", UNIQUE (generation, graph, iri)"
92
+ c.create(alias_table(store), f"CREATE TABLE IF NOT EXISTS {alias_table(store)} (generation BIGINT NOT NULL, "
93
+ f"graph {x} NOT NULL, iri {x} NOT NULL, rep {x} NOT NULL{unique})")
94
+ c.create(meta_table(store), f"CREATE TABLE IF NOT EXISTS {meta_table(store)} (generation BIGINT NOT NULL, "
95
+ f"kind {x} NOT NULL, graph {x}, a {x}, b {x}, note {x})")
96
+
97
+
98
+ def read_state(store: Any) -> Optional[AliasState]:
99
+ """The store's current alias generation (the latest marker), None where there is none or it holds no
100
+ alias."""
101
+ meta = meta_table(store)
102
+ try:
103
+ if not store.db.has_table(meta):
104
+ return None
105
+ rows = store.db.execute(latest_sql(store)).fetchall()
106
+ except Exception: # noqa: BLE001 - a store opened read-only before the table existed: no alias
107
+ return None
108
+ return state_of(rows)
109
+
110
+
111
+ def latest_sql(store: Any) -> str:
112
+ """SQL of the latest generation's marker and IRI shapes: (kind, a, note, generation)."""
113
+ meta = meta_table(store)
114
+ return (f"SELECT kind, a, note, generation FROM {meta} WHERE generation = (SELECT max(generation) FROM {meta} "
115
+ f"WHERE kind = 'generation') AND kind IN ('generation', 'shape')")
116
+
117
+
118
+ def state_of(rows: Any) -> Optional[AliasState]:
119
+ """The alias state of the latest generation's rows (`latest_sql`), None where it holds no alias."""
120
+ marker = [r for r in rows if r[0] == "generation"]
121
+ if not marker:
122
+ return None
123
+ note = json.loads(marker[0][2] or "{}")
124
+ if not note.get("aliases"):
125
+ return None
126
+ shapes = None if note.get("all_shapes") else tuple(sorted(r[1] for r in rows if r[0] == "shape"))
127
+ return AliasState(int(marker[0][3]), shapes, int(note.get("largest", 0)), int(note["aliases"]), marker[0][1])
128
+
129
+
130
+ # -- the identity a plan reads ---------------------------------------------------------------------------
131
+
132
+
133
+ @dataclass
134
+ class Identity:
135
+ """What a plan's BGP matching reads of owl:sameAs: `rows`, SQL of the alias rows (`g`, `iri`, `rep`);
136
+ the IRI shapes (`shapes`, None: any IRI may be an alias); the largest component; the answer mode; and
137
+ `run`, which reads a statement's rows (the constants' members are looked up at planning, one statement a query)."""
138
+ rows: str
139
+ shapes: Optional[Tuple[str, ...]]
140
+ largest: int
141
+ mode: str
142
+ run: Callable[[str], List[tuple]]
143
+ literal: Callable[[str], str]
144
+ tag: str = ""
145
+ _by_iri: Dict[str, Dict[str, FrozenSet[str]]] = field(default_factory=dict)
146
+ # The (class, predicate) pairs whose literal values differ across a component (`owl_rl.stage_disagreeing`):
147
+ # Cypher reads such a property as a list (`cypher_front`).
148
+ disagree: FrozenSet[Tuple[str, str]] = frozenset() # (class, Cypher property key)
149
+ # The classes a member of a merged component has: a Cypher pattern of no such class needs no alias join.
150
+ merged: FrozenSet[str] = frozenset()
151
+ said: set = field(default_factory=set, compare=False) # Cypher's (label, key) warnings given (`warn_disagreeing`)
152
+
153
+ def __post_init__(self) -> None:
154
+ if self.mode not in MODES:
155
+ raise ValueError(f"sameas is one of {', '.join(MODES)}, not {self.mode!r}")
156
+ self.tag = self.tag or hashlib.sha1(self.rows.encode("utf-8")).hexdigest()[:12]
157
+
158
+ # -- the skip test ----------------------------------------------------------------------------------
159
+ def may_alias(self, iri: str) -> bool:
160
+ """Whether a constant IRI may be an alias (it is in a namespace of the members)."""
161
+ return self.shapes is None or any(iri.startswith(p) for p in self.shapes)
162
+
163
+ @home("sameas-skip-test", "Where a term position needs no alias join: its maps cannot give a member's IRI",
164
+ terms=("skip test", "no alias join", "IRI shapes", "may meet an alias"))
165
+ def meets(self, slot: Any) -> bool:
166
+ """Whether a column of slot `slot` may hold an IRI that has an alias: an IRI of a term map whose
167
+ fixed prefix and a member's namespace can be one the start of the other (`TermMap.prefix`), a
168
+ relative R2RML IRI, or a column of unknown provenance."""
169
+ from prograph.schema.source_map import IRI
170
+ if slot is None:
171
+ return True
172
+ if IRI not in slot.kinds:
173
+ return False
174
+ if slot.provenance is None or self.shapes is None:
175
+ return True
176
+ for tm in slot.provenance:
177
+ if tm.term_type != IRI:
178
+ continue
179
+ if tm.relative_iri():
180
+ return True
181
+ prefix = tm.prefix()
182
+ if tm.constant is not None:
183
+ if self.may_alias(tm.constant):
184
+ return True
185
+ elif any(prefix.startswith(p) or p.startswith(prefix) for p in self.shapes):
186
+ return True
187
+ return False
188
+
189
+ # -- planning-time lookups ----------------------------------------------------------------------------
190
+ @home("sameas-planning-lookups", "A constant IRI's component read at planning: one batched statement per query",
191
+ terms=("planning-time lookup", "constant lookup", "prefetch", "round trip", "component of a constant"))
192
+ def prefetch(self, iris: Iterable[str]) -> None:
193
+ """Read the components of every IRI of `iris` (in every graph) not read yet that could be an alias, in one
194
+ statement per 400 IRIs (none where none could). The planner calls it once with the query's constants
195
+ (`sparql_front.translate`), where a lookup per constant sent one each."""
196
+ fresh = [i for i in dict.fromkeys(iris) if i not in self._by_iri]
197
+ for i in fresh:
198
+ if not self.may_alias(i):
199
+ self._by_iri[i] = {} # no statement: its namespace holds no member
200
+ wanted = [i for i in fresh if i not in self._by_iri]
201
+ for start in range(0, len(wanted), 400):
202
+ chunk = wanted[start:start + 400]
203
+ listed = ", ".join(self.literal(i) for i in chunk)
204
+ sql = (f"WITH _r AS (SELECT g, iri, rep FROM ({self.rows}) AS _a), "
205
+ f"_asked AS (SELECT g, iri, rep FROM _r WHERE iri IN ({listed}) "
206
+ f"UNION SELECT g, rep AS iri, rep FROM _r WHERE rep IN ({listed})) "
207
+ f"SELECT _asked.g AS graph, _asked.iri AS asked, _r.iri AS member, _asked.rep AS rep FROM _asked "
208
+ f"JOIN _r ON _r.rep = _asked.rep AND _r.g = _asked.g")
209
+ found: Dict[str, Dict[str, set]] = {i: {} for i in chunk}
210
+ for graph, asked, member, rep in self.run(sql): # a lookup that raises caches nothing of its chunk
211
+ found[asked].setdefault(graph, {rep}).add(member)
212
+ for i in chunk:
213
+ self._by_iri[i] = {g: frozenset(m) for g, m in found[i].items()}
214
+
215
+ def members(self, iris: Iterable[str], graph: str = DEFAULT_GRAPH) -> Dict[str, FrozenSet[str]]:
216
+ """Each IRI of `iris` that has an alias in `graph`'s components: its component's members (itself
217
+ among them). Read once per IRI (`prefetch`), in every graph."""
218
+ listed = list(dict.fromkeys(iris))
219
+ self.prefetch(listed)
220
+ return {i: self._by_iri[i][graph] for i in listed if self._by_iri[i].get(graph)}
221
+
222
+ def members_by_graph(self, iri: str) -> Dict[str, FrozenSet[str]]:
223
+ """`iri`'s component's members in each named graph where it has an alias (inside GRAPH of a variable)."""
224
+ self.prefetch([iri])
225
+ return {g: m for g, m in self._by_iri[iri].items() if g != DEFAULT_GRAPH}
226
+
227
+ # -- the alias rows as a plan ---------------------------------------------------------------------------
228
+ def scan(self, t: Any, iri: str, rep: str, role: str = "canon") -> Any:
229
+ """The alias rows of the translator's active graph as a plan of IRI terms: `iri` and its
230
+ representative `rep`, and inside GRAPH ?g the graph under the active graph's column. `role` names
231
+ the scan's use in its map's id (an expansion's `expand<n>-k<largest>`, which the cost hint
232
+ `sameas-expansion` reads)."""
233
+ from prograph.plan.ir import expr as E, ops as O
234
+ from prograph.plan.ir.sparql_front import Var
235
+ from prograph.schema.source_map import iri_column, side_table_map
236
+ graph = t.graph
237
+ if isinstance(graph, Var):
238
+ where, by_graph = f" WHERE g <> {self.literal(DEFAULT_GRAPH)}", graph.name
239
+ else:
240
+ where, by_graph = f" WHERE g = {self.literal(active_graph(t))}", None
241
+ query = f"SELECT g, iri, rep FROM ({self.rows}) AS _a{where}"
242
+ column = {c: iri_column(c) for c in ("iri", "rep", "g")}
243
+ m = side_table_map(f"{ALIAS_MAP}-{self.tag}-{role}", query, "iri")
244
+ wanted = [("iri", iri), ("rep", rep)] + ([("g", by_graph)] if by_graph else [])
245
+ names = {c: t.fresh("c") for c, _ in wanted}
246
+ from prograph.schema import xsd
247
+ scan = O.MapScan(map=m, reads=tuple(O.ScanColumn(name=names[c], column=c, held=xsd.STRING, bytewise=True)
248
+ for c, _ in wanted),
249
+ required=tuple(names.values()))
250
+ x: O.Op = scan
251
+ for c, out in wanted:
252
+ x = O.Extend(x, out, E.MakeTerm(column[c], ((c, E.Col(names[c])),)))
253
+ return O.Project(x, tuple(out for _, out in wanted))
254
+
255
+ def members_scan(self, t: Any, rep: str, member: str) -> Any:
256
+ """Each representative's members (itself and its aliases), `rep` and `member`, in the active graph:
257
+ the alias rows, and each representative once as its own member."""
258
+ from prograph.plan.ir import expr as E, ops as O
259
+ a = self.scan(t, member, rep)
260
+ own = O.Distinct(O.Project(a, tuple(c for c in a.columns if c != member)))
261
+ return O.Union((a, O.Project(O.Extend(own, member, E.Col(rep)), a.columns)))
262
+
263
+
264
+ # -- matching ---------------------------------------------------------------------------------------------
265
+
266
+
267
+ # The components of an empty default graph (a dataset of FROM NAMED graphs only): no alias row has it.
268
+ NO_GRAPH = "urn:prograph:store:no-graph"
269
+
270
+
271
+ def predicate_members(t: Any, iri: str) -> FrozenSet[str]:
272
+ """A predicate's or class's members under eq-rep-p (an rdf:type object under eq-rep-o): itself where it
273
+ has no alias. Components of the active graph's (`active_graph`); inside GRAPH of a variable see
274
+ `predicate_members_by_graph`."""
275
+ if not t.identity.may_alias(iri):
276
+ return frozenset({iri})
277
+ return t.identity.members([iri], active_graph(t)).get(iri, frozenset({iri}))
278
+
279
+
280
+ def predicate_members_by_graph(t: Any, iri: str) -> Dict[str, Optional[FrozenSet[str]]]:
281
+ """Inside GRAPH of a variable: each name `iri` answers under (itself, and the members of its component in
282
+ some named graph), with the graphs it answers in (None: every graph). A graph's entailment is its own
283
+ components' alone (Entailment Regimes 9), so a member answers in the graphs where it is `iri`'s member."""
284
+ out: Dict[str, Optional[FrozenSet[str]]] = {iri: None}
285
+ if not t.identity.may_alias(iri):
286
+ return out
287
+ wider: Dict[str, set] = {}
288
+ for graph, members in t.identity.members_by_graph(iri).items():
289
+ for m in members:
290
+ if m != iri:
291
+ wider.setdefault(m, set()).add(graph)
292
+ out.update({m: frozenset(g) for m, g in wider.items()})
293
+ return out
294
+
295
+
296
+ @home("sameas-bgp-matching", "A triple pattern matched through owl:sameAs: canonical match, then expansion",
297
+ terms=("canonical match", "expansion", "all aliases", "representative", "eq-rep-s", "eq-rep-o",
298
+ "eq-rep-p", "alias join", "constant IRI in a pattern"))
299
+ def pattern(t: Any, s: Any, p: Any, o: Any, quiet: bool) -> Any:
300
+ """The solutions of the triple pattern (s, p, o), p an IRI, in the graph its sameAs links entail
301
+ (OWL 2 RL eq-*): see the module. Where no position may hold an alias, the pattern's plan is the one
302
+ without owl:sameAs."""
303
+ from prograph.plan.ir import ops as O
304
+ from prograph.plan.ir.terms import IRITerm
305
+ if p.text == SAME_AS:
306
+ return same_as(t, s, o, quiet)
307
+ from prograph.plan.ir.sparql_front import Var
308
+ in_var_graph = isinstance(t.graph, Var)
309
+ if in_var_graph:
310
+ # Each name answers in the graphs whose own components have it as the predicate's (class's) member.
311
+ if p.text == RDF_TYPE and isinstance(o, IRITerm):
312
+ names = predicate_members_by_graph(t, o.text)
313
+ parts = [_in_graphs(t, _one(t, s, p, IRITerm(c), quiet, object_done=True), graphs)
314
+ for c, graphs in sorted(names.items())]
315
+ else:
316
+ names = predicate_members_by_graph(t, p.text)
317
+ parts = [_in_graphs(t, _one(t, s, IRITerm(q), o, quiet), graphs) for q, graphs in sorted(names.items())]
318
+ elif p.text == RDF_TYPE and isinstance(o, IRITerm):
319
+ parts = [_one(t, s, p, IRITerm(cls), quiet, object_done=True) for cls in sorted(predicate_members(t, o.text))]
320
+ else:
321
+ parts = [_one(t, s, IRITerm(q), o, quiet) for q in sorted(predicate_members(t, p.text))]
322
+ kept = [x for x in parts if not (isinstance(x, O.Values) and not x.rows)]
323
+ if len(parts) == 1:
324
+ return parts[0]
325
+ if not kept:
326
+ return parts[0]
327
+ return O.Distinct(kept[0] if len(kept) == 1 else O.Union(tuple(_aligned(kept))))
328
+
329
+
330
+ def _in_graphs(t: Any, x: Any, graphs: Optional[FrozenSet[str]]) -> Any:
331
+ """`x`'s rows whose graph (inside GRAPH of a variable) is one of `graphs`; all of them for None."""
332
+ from prograph.plan.ir import expr as E, ops as O
333
+ from prograph.plan.ir.terms import IRITerm
334
+ if graphs is None or (isinstance(x, O.Values) and not x.rows):
335
+ return x
336
+ name = E.Col(t.graph.name)
337
+ return O.Filter(x, E.Or(tuple(E.Compare("=", name, E.Const(IRITerm(g)), E.TERM) for g in sorted(graphs))))
338
+
339
+
340
+ def _aligned(parts: List[Any]) -> List[Any]:
341
+ from prograph.plan.ir import ops as O
342
+ cols = parts[0].columns
343
+ return [x if x.columns == cols else O.Project(x, cols) for x in parts]
344
+
345
+
346
+ PER_GRAPH = "per-graph" # a constant inside GRAPH ?g: matched through each graph's alias rows
347
+
348
+
349
+ def _one(t: Any, s: Any, p: Any, o: Any, quiet: bool, object_done: bool = False) -> Any:
350
+ """One predicate's pattern (see `pattern`)."""
351
+ from prograph.plan.ir import expr as E, ops as O
352
+ from prograph.plan.ir.sparql_front import Var, _empty
353
+ from prograph.plan.ir.terms import IRITerm
354
+ identity = t.identity
355
+ in_var_graph = isinstance(t.graph, Var)
356
+ # A constant IRI with aliases: its component's members, read at planning (inside GRAPH ?g each
357
+ # graph's own, matched through the alias rows).
358
+ consts: Dict[str, Any] = {}
359
+ for pos, term in (("s", s), ("o", o)):
360
+ if not isinstance(term, IRITerm) or (pos == "o" and object_done):
361
+ continue
362
+ if in_var_graph:
363
+ if identity.may_alias(term.text):
364
+ consts[pos] = PER_GRAPH
365
+ elif identity.may_alias(term.text):
366
+ got = identity.members([term.text], active_graph(t)).get(term.text)
367
+ if got:
368
+ consts[pos] = got
369
+ if not consts:
370
+ plain = t.matched(s, p, o, quiet)
371
+ slots = plain.slots
372
+ if not any(isinstance(v, Var) and v.name in slots and identity.meets(slots[v.name]) for v in (s, o)):
373
+ return plain # no position may hold an alias: the plan without owl:sameAs
374
+ terms = {"s": s, "o": o}
375
+ base = {pos: Var(t.fresh("s" + pos)) if isinstance(term, Var) or pos in consts else term
376
+ for pos, term in terms.items()}
377
+ branches = [b for b in t.branches(base["s"], p, base["o"]) if b is not None]
378
+ if not branches:
379
+ return _empty(s, p, o)
380
+ graph_col = (t.graph.name,) if in_var_graph else ()
381
+ out_cols = tuple(dict.fromkeys(v.name for v in (s, o) if isinstance(v, Var))) + graph_col
382
+ same = isinstance(s, Var) and isinstance(o, Var) and s.name == o.name
383
+ variables = [pos for pos in ("s", "o") if isinstance(terms[pos], Var)]
384
+ rep_of = {pos: t.fresh("r" + pos) for pos in ("s", "o")}
385
+ plain_parts: List[Any] = []
386
+ canon_parts: List[Any] = []
387
+ expand: set = set() # the variables some branch reads through the alias rows
388
+ for b in branches:
389
+ slots = b.slots
390
+ meets = {pos: isinstance(base[pos], Var) and base[pos].name in slots and identity.meets(slots[base[pos].name])
391
+ for pos in ("s", "o")}
392
+ x, skip = b, False
393
+ for pos, members in consts.items():
394
+ column = base[pos].name
395
+ if not meets[pos]:
396
+ if members == PER_GRAPH: # no alias here: the constant itself
397
+ x = O.Filter(x, E.Compare("=", E.Col(column), E.Const(terms[pos]), E.TERM))
398
+ x = O.Project(x, tuple(c for c in x.columns if c != column))
399
+ else:
400
+ skip = True # no IRI of this branch is a member of the constant's component
401
+ elif members != PER_GRAPH:
402
+ x = _constant(t, x, column, members)
403
+ if skip:
404
+ continue
405
+ through = [pos for pos in variables if meets[pos]] + \
406
+ [pos for pos in consts if consts[pos] == PER_GRAPH and meets[pos]]
407
+ if not through:
408
+ y = x
409
+ for pos in variables:
410
+ if terms[pos].name not in y.columns:
411
+ y = O.Extend(y, terms[pos].name, E.Col(base[pos].name))
412
+ if same:
413
+ y = O.Filter(y, E.Compare("=", E.Col(base["s"].name), E.Col(base["o"].name), E.TERM))
414
+ plain_parts.append(O.Project(y, out_cols))
415
+ continue
416
+ expand.update(pos for pos in variables if meets[pos])
417
+ for pos in variables + [q for q in consts if consts[q] == PER_GRAPH and meets[q]]:
418
+ if meets[pos]:
419
+ x = _canonical(t, x, base[pos].name, rep_of[pos])
420
+ else:
421
+ x = O.Extend(x, rep_of[pos], E.Col(base[pos].name))
422
+ if pos in consts:
423
+ x = _same_entity(t, x, rep_of[pos], terms[pos])
424
+ canon_parts.append(O.Project(x, tuple(rep_of[pos] for pos in variables) + graph_col))
425
+ out: List[Any] = list(plain_parts)
426
+ if canon_parts:
427
+ cols = canon_parts[0].columns
428
+ canon: Any = O.Distinct(canon_parts[0] if len(canon_parts) == 1 else
429
+ O.Union(tuple(c if c.columns == cols else O.Project(c, cols) for c in canon_parts)))
430
+ if same:
431
+ canon = O.Filter(canon, E.Compare("=", E.Col(rep_of["s"]), E.Col(rep_of["o"]), E.TERM))
432
+ done = set()
433
+ width = len({terms[pos].name for pos in variables if pos in expand or (same and expand)})
434
+ for pos in variables:
435
+ if terms[pos].name not in done:
436
+ done.add(terms[pos].name)
437
+ if pos in expand or (same and expand):
438
+ canon = _expanded(t, canon, rep_of[pos], terms[pos].name, width)
439
+ else:
440
+ canon = O.Extend(canon, terms[pos].name, E.Col(rep_of[pos]))
441
+ out.append(O.Project(canon, out_cols))
442
+ if not out:
443
+ return _empty(s, p, o)
444
+ if not plain_parts:
445
+ return out[0] # distinct: one row per canonical triple, each expanded to members no other has
446
+ return O.Distinct(out[0] if len(out) == 1 else O.Union(tuple(out)))
447
+
448
+
449
+ def _constant(t: Any, x: Any, column: str, members: FrozenSet[str]) -> Any:
450
+ """`x`'s rows whose `column` is one of a constant's members, `column` projected away."""
451
+ from prograph.plan.ir import expr as E, ops as O
452
+ from prograph.plan.ir.terms import IRITerm
453
+ if len(members) <= VALUES_MAX:
454
+ cond = E.Or(tuple(E.Compare("=", E.Col(column), E.Const(IRITerm(m)), E.TERM) for m in sorted(members)))
455
+ y = O.Filter(x, cond)
456
+ else:
457
+ rep = min(members, key=_codepoints)
458
+ r = t.fresh("cr")
459
+ m = t.identity.members_scan(t, r, column)
460
+ m = O.Project(O.Filter(m, E.Compare("=", E.Col(r), E.Const(IRITerm(rep)), E.TERM)), (column,))
461
+ y = O.SemiJoin(x, m)
462
+ return O.Project(y, tuple(c for c in y.columns if c != column))
463
+
464
+
465
+ def _codepoints(text: str) -> Tuple[int, ...]:
466
+ return tuple(ord(ch) for ch in text)
467
+
468
+
469
+ def _canonical(t: Any, x: Any, column: str, out: str) -> Any:
470
+ """`x` with `out` the representative of `column`'s term: its alias row's, or itself where it has none
471
+ (a join and an anti-join with the alias rows, so `out` is bound in every row)."""
472
+ from prograph.plan.ir import expr as E, ops as O
473
+ r = t.fresh("ar")
474
+ a = t.identity.scan(t, column, r)
475
+ cols = x.columns + (out,)
476
+ aliased = O.Project(O.Extend(O.Join(x, a), out, E.Col(r)), cols)
477
+ own = O.Project(O.Extend(O.AntiJoin(x, O.Project(a, tuple(c for c in a.columns if c != r))), out, E.Col(column)),
478
+ cols)
479
+ return O.Union((aliased, own))
480
+
481
+
482
+ def _same_entity(t: Any, x: Any, rep: str, constant: Any) -> Any:
483
+ """`x`'s rows whose representative `rep` is the constant's, in each row's graph (GRAPH ?g)."""
484
+ from prograph.plan.ir import expr as E, ops as O
485
+ from prograph.plan.ir.sparql_front import Var
486
+ c = Var(t.fresh("k"))
487
+ holder = O.Values((c.name,), ((constant,),))
488
+ canon = _canonical(t, holder if not isinstance(t.graph, Var) else
489
+ O.Join(holder, O.Distinct(O.Project(x, (t.graph.name,)))), c.name, t.fresh("kr"))
490
+ kr = canon.columns[-1]
491
+ joined = O.Join(x, O.Project(canon, tuple(col for col in canon.columns if col != c.name)))
492
+ return O.Project(O.Filter(joined, E.Compare("=", E.Col(rep), E.Col(kr), E.TERM)),
493
+ tuple(col for col in joined.columns if col != kr))
494
+
495
+
496
+ def _expanded(t: Any, x: Any, rep: str, name: str, width: int = 1) -> Any:
497
+ """`x` with `name` bound to each member of `rep`'s component (`sameas="all"`), or to `rep` itself
498
+ (`"representative"`). `width`: the pattern's positions expanded (up to largest^width rows a triple)."""
499
+ from prograph.plan.ir import expr as E, ops as O
500
+ own = O.Extend(x, name, E.Col(rep))
501
+ if t.identity.mode == "representative":
502
+ return own
503
+ r = t.fresh("er")
504
+ a = t.identity.scan(t, name, r, role=f"expand{width}-k{t.identity.largest}")
505
+ a = O.Project(O.Extend(a, rep, E.Col(r)), tuple(c for c in a.columns if c != r) + (rep,))
506
+ aliases = O.Join(x, a)
507
+ cols = own.columns
508
+ return O.Union((own, O.Project(aliases, cols)))
509
+
510
+
511
+ def same_as(t: Any, s: Any, o: Any, quiet: bool) -> Any:
512
+ """`s owl:sameAs o`: each ordered pair of two members of one component (the reflexive pairs aside:
513
+ eq-ref is not materialized), with the graph's own sameAs triples, once each. Under
514
+ `sameas="representative"` a variable binds each member's representative, as every pattern's does, so
515
+ an entity's pairs are its representative's one (r, r), and the stated triples are matched as any
516
+ pattern's."""
517
+ from prograph.plan.ir import expr as E, ops as O
518
+ from prograph.plan.ir.sparql_front import Var
519
+ from prograph.plan.ir.terms import IRITerm
520
+ rep_mode = t.identity.mode == "representative"
521
+ r, x, y = t.fresh("sr"), t.fresh("sx"), t.fresh("sy")
522
+ left = t.identity.members_scan(t, r, x)
523
+ right = t.identity.members_scan(t, r, y)
524
+ pairs: Any = O.Filter(O.Join(left, right), E.Not(E.Compare("=", E.Col(x), E.Col(y), E.TERM)))
525
+ graph_col = (t.graph.name,) if isinstance(t.graph, Var) else ()
526
+
527
+ def end(term: Any, member: str) -> Any:
528
+ return E.Col(r) if rep_mode and isinstance(term, Var) else E.Col(member)
529
+ terms = [(s, end(s, x)), (o, end(o, y))]
530
+ if graph_col: # the pairs' graph under a name of its own, bound to the active graph's column
531
+ g = t.fresh("sg")
532
+ pairs = O.Project(O.Extend(pairs, g, E.Col(graph_col[0])), (r, x, y, g))
533
+ terms.append((t.graph, E.Col(g)))
534
+ clique = t.bind(pairs, terms)
535
+ stated = _one(t, s, IRITerm(SAME_AS), o, quiet) if rep_mode else t.matched(s, IRITerm(SAME_AS), o, True)
536
+ parts = [clique] + ([stated] if not (isinstance(stated, O.Values) and not stated.rows) else [])
537
+ cols = parts[0].columns
538
+ return O.Distinct(parts[0] if len(parts) == 1 else O.Union(tuple(p if p.columns == cols else O.Project(p, cols)
539
+ for p in parts)))
540
+
541
+
542
+ def for_engine(engine: Any, now: Any = None) -> Optional[Identity]:
543
+ """The identity an engine's queries read, in the engine's answer mode (`sameas`): the alias generation the
544
+ dataset home gives for this planning (`dataset.store_now`, read from the store when the query is planned:
545
+ the latest generation, where it and the store's last run are of the engine's key); None where it gives none.
546
+ `now` that answer where the caller holds it."""
547
+ from prograph.plan.ir.dataset import store_now
548
+ now = now if now is not None else store_now(engine)
549
+ state = now.alias if now is not None else None
550
+ if state is None:
551
+ return None
552
+ store = engine._schema.__dict__.get("owned_store")
553
+ mode = getattr(engine, "_sameas", "all")
554
+ key = (id(store), state.generation, mode, state.key)
555
+ held = engine.__dict__.get("_identity")
556
+ if held is not None and held[0] == key:
557
+ return held[1]
558
+ from prograph.plan.sql.dialect import get_dialect_instance
559
+ rows = f"SELECT graph AS g, iri, rep FROM {alias_table(store)} WHERE generation = {int(state.generation)}"
560
+ try:
561
+ held = store.db.execute(f"SELECT kind, a, b FROM {meta_table(store)} WHERE generation = "
562
+ f"{int(state.generation)} AND kind IN ('disagree', 'merged')").fetchall()
563
+ except Exception: # noqa: BLE001 - a generation written before Cypher's rows were: every class may merge
564
+ held = None
565
+ disagree = frozenset((a, b) for k, a, b in held or () if k == "disagree")
566
+ # No merged class recorded (a generation written before they were, or records of no class): any may merge.
567
+ merged = frozenset(a for k, a, _ in held or () if k == "merged") or frozenset({"*"})
568
+ found = Identity(rows, state.shapes, state.largest, mode, run=lambda sql: store.db.execute(sql).fetchall(),
569
+ literal=get_dialect_instance(store.dialect).literal, tag=f"g{state.generation}",
570
+ disagree=disagree, merged=merged)
571
+ engine.__dict__["_identity"] = (key, found)
572
+ return found
573
+
574
+
575
+ # -- components (the fixpoint's, `owl_rl`) ------------------------------------------------------------------
576
+
577
+
578
+ def namespace_sql(dialect: str, e: str) -> str:
579
+ """SQL of the text `e`'s namespace: its text up to its last '/', '#' or ':' (the IRI shape the skip test
580
+ reads)."""
581
+ if dialect == "postgresql":
582
+ return f"coalesce(substring({e} from '^(.*[/#:])'), '')"
583
+ return f"regexp_extract({e}, '^(.*[/#:])', 1)"
584
+
585
+
586
+ def least(dialect: str, a: str, b: str) -> str:
587
+ """The lesser of two texts by code point (`lower.codepoint`'s collation)."""
588
+ return f"CASE WHEN {codepoint(dialect, a)} <= {codepoint(dialect, b)} THEN {a} ELSE {b} END"
589
+
590
+
591
+ def codepoint(dialect: str, e: str) -> str:
592
+ """`e` under the dialect's binary collation (the rule codepoint-string-order's CODEPOINT_TEXT)."""
593
+ from prograph.plan.sql.dialect import get_dialect_instance
594
+ template = get_dialect_instance(dialect).config.function_map.get("CODEPOINT_TEXT", "{0}")
595
+ return template.format(e)
596
+
597
+
598
+ @dataclass
599
+ class Components:
600
+ """One computation of the components (`components`): the alias rows' read (`rows`: g, iri, rep, as
601
+ staged), the components left unmerged for their size (`capped`: (graph, rep, size)), the largest kept
602
+ component and the count of alias rows."""
603
+ rows: str
604
+ capped: List[Tuple[str, str, int]]
605
+ largest: int
606
+ aliases: int
607
+ count: int
608
+ labels: str = ""
609
+
610
+
611
+ @home("sameas-components", "owl:sameAs components: minimum-label propagation over the links, per graph",
612
+ terms=("connected components", "label propagation", "pointer jumping", "representative", "code point",
613
+ "hard cap", "left unmerged"))
614
+ def components(run: Any, links: str, poisoned: Optional[str], cap: Optional[int] = None) -> Components:
615
+ """The connected components of the undirected links `links` (SQL of g, a, b: two IRIs' texts and the
616
+ graph they are linked in, '' the default graph), within each graph, each IRI of `poisoned` (SQL of g,
617
+ iri) left out: every node labelled with the least IRI it reaches by code point. Each round a node takes
618
+ the least label of its neighbours, then of its label's own label (pointer jumping), until no label
619
+ changes: a few statements per round, each staged (`run.stage`), on every engine alike. A component of
620
+ more than `cap` members is left unmerged (`Components.capped`). `run` is the materialization's run
621
+ (`owl_rl._Run`): its `stage`, `dialect` and `db`."""
622
+ d = run.dialect
623
+ cap = CAP if cap is None else cap
624
+ lt = lambda a, b: least(d, a, b) # noqa: E731
625
+ keep = ""
626
+ if poisoned:
627
+ keep = (f" LEFT JOIN ({poisoned}) AS _pa ON _pa.g = _l.g AND _pa.iri = _l.a "
628
+ f"LEFT JOIN ({poisoned}) AS _pb ON _pb.g = _l.g AND _pb.iri = _l.b WHERE _pa.iri IS NULL AND _pb.iri IS NULL")
629
+ kept = f"SELECT _l.g, _l.a, _l.b FROM ({links}) AS _l{keep}"
630
+ edges, _ = run.stage_component("edges", f"SELECT g, a AS x, b AS y FROM ({kept}) AS _k WHERE a <> b "
631
+ f"UNION SELECT g, b AS x, a AS y FROM ({kept}) AS _k WHERE a <> b")
632
+ first = "min(" + codepoint(d, "y") + ")" # not nested in the f-string: Python before 3.12 reads no inner quote
633
+ label, _ = run.stage_component("label0", f"SELECT g, x, {lt('x', first)} AS l FROM "
634
+ f"{edges} AS _e GROUP BY g, x")
635
+ for i in range(1, LABEL_ROUNDS + 1):
636
+ nbr = (f"SELECT _e.g, _e.x, min({codepoint(d, '_n.l')}) AS l FROM {edges} AS _e JOIN {label} AS _n "
637
+ f"ON _n.g = _e.g AND _n.x = _e.y GROUP BY _e.g, _e.x")
638
+ step, _ = run.stage_component(f"step{i}", f"SELECT _o.g, _o.x, {lt('_o.l', 'coalesce(_m.l, _o.l)')} AS l "
639
+ f"FROM {label} AS _o LEFT JOIN ({nbr}) AS _m ON _m.g = _o.g AND _m.x = _o.x")
640
+ jumped, _ = run.stage_component(f"jump{i}", f"SELECT _s.g, _s.x, {lt('_s.l', 'coalesce(_j.l, _s.l)')} AS l "
641
+ f"FROM {step} AS _s LEFT JOIN {step} AS _j ON _j.g = _s.g "
642
+ f"AND _j.x = _s.l")
643
+ changed = run.db.execute(f"SELECT count(*) FROM {jumped} AS _a JOIN {label} AS _b ON _a.g = _b.g "
644
+ f"AND _a.x = _b.x WHERE _a.l <> _b.l").fetchone()[0]
645
+ label = jumped
646
+ if not changed:
647
+ break
648
+ else:
649
+ from prograph.utils.exceptions import SQLRuntimeError
650
+ raise SQLRuntimeError(f"owl:sameAs components: labels still changed after {LABEL_ROUNDS} rounds",
651
+ reason="sameas_components")
652
+ sizes = run.db.execute(f"SELECT g, l, count(*) AS n FROM {label} AS _c GROUP BY g, l "
653
+ f"HAVING count(*) > {int(cap)}").fetchall()
654
+ capped = [(g, l, int(n)) for g, l, n in sizes]
655
+ where = ""
656
+ if capped:
657
+ listed = " OR ".join(f"(g = {run.d.literal(g)} AND l = {run.d.literal(l)})" for g, l, _ in capped)
658
+ where = f" AND NOT ({listed})"
659
+ rows, _ = run.stage_component("alias", f"SELECT g, x AS iri, l AS rep FROM {label} AS _c WHERE x <> l{where}")
660
+ stats = run.db.execute(f"SELECT count(*) AS components, coalesce(max(n), 0) AS largest FROM (SELECT g, rep, "
661
+ f"count(*) + 1 AS n "
662
+ f"FROM {rows} AS _r GROUP BY g, rep) AS _s").fetchone()
663
+ aliases = run.db.execute(f"SELECT count(*) FROM {rows} AS _r").fetchone()[0]
664
+ return Components(rows, capped, int(stats[1] or 0), int(aliases), int(stats[0] or 0), label)
665
+
666
+
667
+ # -- explaining ----------------------------------------------------------------------------------------------
668
+
669
+
670
+ @dataclass(frozen=True)
671
+ class IdentityExplanation:
672
+ """Why two IRIs are one entity, or not. `merged`: one component. `path`: a shortest chain of links
673
+ from `a` to `b`, each (one IRI, the other, its source: 'stated by <map id>' for a sameAs triple of the
674
+ graph that one map gives, 'stated; may be given by <map ids>' where several may, 'rl:<rule> <axiom>' for one
675
+ an OWL RL rule derived). `representatives`: each IRI's representative
676
+ (itself where it has no alias). `unmerged`: where a component holding either was left unmerged, why."""
677
+ a: str
678
+ b: str
679
+ merged: bool
680
+ path: Tuple[Tuple[str, str, str], ...] = ()
681
+ representatives: Tuple[str, str] = ("", "")
682
+ sizes: Tuple[int, int] = (1, 1)
683
+ unmerged: Tuple[str, ...] = ()
684
+
685
+ def __str__(self) -> str:
686
+ if not self.merged:
687
+ why = f"; {'; '.join(self.unmerged)}" if self.unmerged else ""
688
+ return (f"<{self.a}> and <{self.b}> are not merged: <{self.a}> is in a component of {self.sizes[0]} "
689
+ f"(representative <{self.representatives[0]}>), <{self.b}> in one of {self.sizes[1]} "
690
+ f"(representative <{self.representatives[1]}>){why}")
691
+ steps = "\n".join(f" <{x}> owl:sameAs <{y}>: {src}" for x, y, src in self.path)
692
+ return f"<{self.a}> and <{self.b}> are one entity (representative <{self.representatives[0]}>):\n{steps}"
693
+
694
+
695
+ def explain(engine: Any, a: str, b: str) -> IdentityExplanation:
696
+ """`engine.explain_identity(a, b)`: see `IdentityExplanation`. The default graph's components."""
697
+ from collections import deque
698
+ ident = for_engine(engine)
699
+ store = engine._schema.__dict__.get("owned_store")
700
+ unmerged = _unmerged_notes(store, (a, b))
701
+ if ident is None:
702
+ return IdentityExplanation(a, b, a == b, representatives=(a, b), unmerged=unmerged)
703
+ got = ident.members([a, b])
704
+ ma, mb = got.get(a, frozenset({a})), got.get(b, frozenset({b}))
705
+ rep_a, rep_b = min(ma, key=_codepoints), min(mb, key=_codepoints)
706
+ if ma != mb:
707
+ return IdentityExplanation(a, b, a == b, representatives=(rep_a, rep_b), sizes=(len(ma), len(mb)),
708
+ unmerged=unmerged)
709
+ edges = _component_links(engine, store, ident, ma)
710
+ graph: Dict[str, List[Tuple[str, str]]] = {}
711
+ for x, y, src in edges:
712
+ graph.setdefault(x, []).append((y, src))
713
+ graph.setdefault(y, []).append((x, src))
714
+ back: Dict[str, Tuple[str, str]] = {a: ("", "")}
715
+ todo = deque([a])
716
+ while todo:
717
+ x = todo.popleft()
718
+ if x == b:
719
+ break
720
+ for y, src in sorted(graph.get(x, ())):
721
+ if y not in back:
722
+ back[y] = (x, src)
723
+ todo.append(y)
724
+ path: List[Tuple[str, str, str]] = []
725
+ x = b
726
+ while x != a and x in back:
727
+ prev, src = back[x]
728
+ path.append((prev, x, src))
729
+ x = prev
730
+ return IdentityExplanation(a, b, True, tuple(reversed(path)), (rep_a, rep_b), (len(ma), len(mb)), unmerged)
731
+
732
+
733
+ def _unmerged_notes(store: Any, iris: Tuple[str, ...]) -> Tuple[str, ...]:
734
+ if store is None or not store.db.has_table(meta_table(store)):
735
+ return ()
736
+ meta = meta_table(store)
737
+ rows = store.db.execute(
738
+ f"SELECT a, b, note FROM {meta} WHERE kind = 'unmerged' AND generation = (SELECT max(generation) FROM {meta} "
739
+ f"WHERE kind = 'generation')").fetchall()
740
+ out = []
741
+ for rep, rule, note in rows:
742
+ held = json.loads(note or "{}")
743
+ if any(i == rep or i in held.get("sample", ()) for i in iris):
744
+ reason = "inconsistent" + (f" ({rule})" if rule else "") if held.get("reason") == "inconsistent" else \
745
+ f"past the cap of {CAP} members"
746
+ out.append(f"a component of {held.get('size')} with representative <{rep}> was left unmerged: {reason}")
747
+ return tuple(out)
748
+
749
+
750
+ def _component_links(engine: Any, store: Any, ident: Identity, members: FrozenSet[str]) -> List[Tuple[str, str, str]]:
751
+ """The links among `members`: the graph's own sameAs triples (by the maps that may give them) and those the
752
+ last run derived (its meta rows)."""
753
+ from prograph.plan.ir import lower, sparql_front
754
+ from prograph.plan.ir.dataset import maps_seen
755
+ from prograph.plan.ir.entry import _held, ontology_of
756
+ from prograph.plan.sql.dialect import get_dialect_instance
757
+ out: List[Tuple[str, str, str]] = []
758
+ onto = ontology_of(engine)
759
+ maps = maps_seen(engine, onto)
760
+ d = get_dialect_instance(store.dialect)
761
+ listed = sorted(members)
762
+ if SAME_AS in maps.by_predicate:
763
+ values = " ".join(f"<{m}>" for m in listed)
764
+ query = (f"SELECT (STR(?a) AS ?x) (STR(?b) AS ?y) WHERE {{ VALUES ?a {{ {values} }} ?a <{SAME_AS}> ?b "
765
+ f"FILTER(isIRI(?b)) }}")
766
+ t = sparql_front.translate(query, maps, _held(engine._schema, store.dialect, True), lambda m: (),
767
+ union_default=getattr(engine, "default_graph", "union") == "union", ontology=onto)
768
+ sql = lower.to_sql(t.plan, store.dialect, base_iri=maps.settings.base_iri)
769
+ givers = [g.map for g in maps.by_predicate[SAME_AS]]
770
+ for x, y in store.db.execute(f"SELECT {d.quote('x')}, {d.quote('y')} FROM ({sql}) AS _l").fetchall():
771
+ if y not in members or x == y:
772
+ continue
773
+ ids = sorted({m.id for m in givers if _gives(m, x, y)})
774
+ # One map that can give the link gave it, unless the ontology states it too; with several that may,
775
+ # the data does not say which.
776
+ layer = getattr(engine, "_ontology", None)
777
+ ontology = layer is not None and y in layer.get_same_as_group(x)
778
+ names = ids + (["the ontology"] if ontology else [])
779
+ out.append((x, y, "stated" + (f" by {ids[0]}" if len(ids) == 1 and not ontology else
780
+ f"; may be given by {', '.join(names)}" if names else "")))
781
+ meta = meta_table(store)
782
+ if store.db.has_table(meta):
783
+ rows = store.db.execute(f"SELECT a, b, note FROM {meta} WHERE kind = 'link' AND generation = (SELECT "
784
+ f"max(generation) FROM {meta} WHERE kind = 'generation')").fetchall()
785
+ out += [(x, y, src) for x, y, src in rows if x in members and y in members]
786
+ return out
787
+
788
+
789
+ def _gives(m: Any, x: str, y: str) -> bool:
790
+ """Whether map `m` may give the triple (x owl:sameAs y) (its subject and object term maps)."""
791
+ from prograph.plan.ir.sparql_front import _may_give
792
+ from prograph.plan.ir.terms import IRITerm
793
+ subject = getattr(m, "subject", None) or getattr(getattr(m, "start", None), "term", None)
794
+ obj = getattr(getattr(m, "end", None), "term", None)
795
+ if subject is None:
796
+ return True
797
+ return _may_give(subject, IRITerm(x)) and (obj is None or _may_give(obj, IRITerm(y)))