prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,3418 @@
1
+ """SPARQL's front end: a query's parse tree as the one plan (one-plan.md 5).
2
+
3
+ The ANTLR parse tree keeps each token's text, and this front end keeps it
4
+ too: a literal is the term the query wrote (`terms.LiteralTerm`, its
5
+ lexical form as written, `"01"^^xsd:integer` not `1`). The group
6
+ graph pattern is translated by SPARQL 1.1's algebra (18.2), and each
7
+ triple pattern becomes the union, over the source maps that may give its
8
+ triples, of a `MapScan` of the map's rows with `MakeTerm`s building the
9
+ terms (R2RML 11: a row with no value in a term's column gives no triple).
10
+
11
+ Anything outside what is built so far raises `NotBuilt`: reported, never
12
+ answered wrongly.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+
18
+ import dataclasses
19
+ import re
20
+ from dataclasses import dataclass, field
21
+ from typing import Any, Dict, FrozenSet, List, Optional, Sequence, Set, Tuple, Union as TUnion
22
+
23
+ from antlr4 import CommonTokenStream, InputStream
24
+ from antlr4.error.ErrorListener import ErrorListener
25
+
26
+ from prograph.parser.sparql.SparqlLexer import SparqlLexer
27
+ from antlr4.tree.Tree import TerminalNode
28
+ from prograph.parser.sparql.SparqlParser import SparqlParser
29
+ from prograph.plan.ir import expr as E
30
+ from prograph.plan.ir import ops as O
31
+ from prograph.plan.ir.facts import NotBuilt, const_slot, expr_slot
32
+ from prograph.plan.ir.slot import LOCAL, UNKNOWN, ZONED, Bound, never_one_term
33
+ from prograph.plan.ir.strings import BOOLEANS as BOOLEAN_STRING_CALLS, STRING_CALLS
34
+ from prograph.plan.ir.dates import DATE_PARTS
35
+ from prograph.plan.ir.dataset import graph_choices
36
+ from prograph.plan.ir.terms import BlankTerm, IRITerm, LiteralTerm
37
+ from prograph.schema import xsd
38
+ from prograph.rules import home
39
+ from prograph.schema.iri import LABEL_IRI_PREFIX
40
+ from prograph.schema.source_map import (
41
+ BLANK, IRI, EdgeMap, LEXICAL_R2RML, NodeMap, PredicateObject, SourceMaps, TermMap, UUID_READERS,
42
+ graphs_of,
43
+ )
44
+
45
+ RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
46
+
47
+ # Prefixes a query may use undeclared, as ProGraph has always read them.
48
+ STANDARD_PREFIXES = {
49
+ "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
50
+ "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
51
+ "xsd": xsd.XSD,
52
+ "owl": "http://www.w3.org/2002/07/owl#",
53
+ }
54
+
55
+ # A column the plan holds that is no variable of the query starts with a
56
+ # character no SPARQL variable name has (19.8, VARNAME).
57
+ HIDDEN = "#"
58
+
59
+ _ECHAR = {"t": "\t", "b": "\b", "n": "\n", "r": "\r", "f": "\f", '"': '"', "'": "'", "\\": "\\"}
60
+
61
+
62
+ class QueryError(ValueError):
63
+ """A query that is not SPARQL, or names what it does not declare."""
64
+
65
+
66
+ class _Errors(ErrorListener):
67
+ def __init__(self) -> None:
68
+ super().__init__()
69
+ self.errors: List[str] = []
70
+
71
+ def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e): # noqa: N802
72
+ self.errors.append(f"line {line}:{column} {msg}")
73
+
74
+
75
+ def codepoint_escapes(query: str) -> str:
76
+ """The query with each \\uXXXX and \\UXXXXXXXX replaced by its character
77
+ before lexing (SPARQL 1.1 19.2), in strings and IRIs too."""
78
+ def char(m):
79
+ code = int(m.group(1) or m.group(2), 16)
80
+ if 0xD800 <= code <= 0xDFFF or code > 0x10FFFF:
81
+ raise QueryError(f"{m.group(0)} is no character: a surrogate or beyond U+10FFFF")
82
+ return chr(code)
83
+ return re.sub(r"\\u([0-9A-Fa-f]{4})|\\U([0-9A-Fa-f]{8})", char, query)
84
+
85
+
86
+ def parse(query: str):
87
+ """The parse tree of a SPARQL query, in ANTLR's two stages: SLL prediction first, which
88
+ gives the same tree wherever it succeeds and is several times faster in the Python runtime;
89
+ full LL, with its error reports, only for a text SLL cannot parse (parsing
90
+ was over half of a plan's time)."""
91
+ from antlr4.atn.PredictionMode import PredictionMode
92
+ from antlr4.error.ErrorStrategy import BailErrorStrategy, DefaultErrorStrategy
93
+ from antlr4.error.Errors import ParseCancellationException
94
+ lexer = SparqlLexer(InputStream(codepoint_escapes(query)))
95
+ tokens = CommonTokenStream(lexer)
96
+ parser = SparqlParser(tokens)
97
+ errors = _Errors()
98
+ lexer.removeErrorListeners()
99
+ lexer.addErrorListener(errors)
100
+ parser.removeErrorListeners()
101
+ parser._interp.predictionMode = PredictionMode.SLL
102
+ parser._errHandler = BailErrorStrategy()
103
+ try:
104
+ tree = parser.query()
105
+ except ParseCancellationException:
106
+ tokens.seek(0)
107
+ parser.reset()
108
+ parser.addErrorListener(errors)
109
+ parser._interp.predictionMode = PredictionMode.LL
110
+ parser._errHandler = DefaultErrorStrategy()
111
+ tree = parser.query()
112
+ if errors.errors:
113
+ raise QueryError("; ".join(errors.errors))
114
+ return tree
115
+
116
+
117
+ def query_form(query: str) -> str:
118
+ """What a SPARQL text is, from its parse alone: 'select', 'ask',
119
+ 'construct', 'describe' or 'update'. For a caller that answers each
120
+ form differently before it runs one (the endpoint, the MCP tools).
121
+ Hints (`/*+ ... */`) are read off first, as `execute` reads them."""
122
+ from prograph.hints import parse_hints
123
+ tree = parse(parse_hints(query)[0])
124
+ for form in ("select", "ask", "construct", "describe"):
125
+ if getattr(tree, form + "Query")() is not None:
126
+ return form
127
+ if tree.updateCommand() is not None:
128
+ return "update"
129
+ raise QueryError("neither a query nor an update")
130
+
131
+
132
+ # -- the query ----------------------------------------------------------------------
133
+
134
+ @dataclass
135
+ class Query:
136
+ """A translated query: its form ('select', 'ask' or 'construct'), its
137
+ plan, the certificates of the rewrites applied to it
138
+ (`rewrite.Certificate`), and a CONSTRUCT's template: its triples of
139
+ terms, where a `Var` named HIDDEN... is a template blank node, fresh
140
+ per solution (16.2; `construct.instantiate`)."""
141
+ form: str
142
+ plan: O.Op
143
+ certificates: tuple = ()
144
+ template: tuple = ()
145
+ prefixes: Dict[str, str] = field(default_factory=dict)
146
+
147
+
148
+ @dataclass(frozen=True)
149
+ class Var:
150
+ """A variable of a pattern, or a blank node, which a pattern reads as
151
+ a variable it does not project (18.2.1)."""
152
+ name: str
153
+
154
+
155
+ Term = TUnion[Var, IRITerm, LiteralTerm, BlankTerm]
156
+
157
+
158
+ @dataclass(frozen=True)
159
+ class GraphSet:
160
+ """What `bind` tests a graph term against under a dataset clause: any one of these IRIs."""
161
+ iris: frozenset
162
+
163
+
164
+ @dataclass(frozen=True)
165
+ class Path:
166
+ """A property path (18.4) that is not one IRI: `inv` (^p), `seq`
167
+ (p/q), `alt` (p|q), `mod` (p*, p+, p? with `mod`), `neg` (!(...): its
168
+ parts (IRI, inverse) pairs)."""
169
+ kind: str
170
+ parts: tuple
171
+ mod: str = ""
172
+
173
+
174
+ def translate(query: str, maps: SourceMaps, held: Any, keys: Any = None,
175
+ union_default: bool = True, ontology: Any = None,
176
+ bindings: Optional[Dict[str, Term]] = None,
177
+ dataset: Optional[Tuple[Tuple[str, ...], Tuple[str, ...]]] = None,
178
+ identity: Any = None, implied_individuals: bool = False,
179
+ datatypes: str = "term") -> Query:
180
+ """`query` as the one plan over `maps`. `held(table, column)` is the
181
+ datatype a source column's values have (None where unknown) and
182
+ whether a date-time column's values have a zone. `bindings` (the
183
+ call's parameters, `prograph.parameters`) is one VALUES row joined
184
+ with the top-level pattern where its own trailing VALUES joins.
185
+ `identity` (`identity.Identity`): owl:sameAs at BGP matching, None
186
+ for RDF term identity alone."""
187
+ tree = parse(query)
188
+ t = Translator(maps, held, keys, union_default, ontology)
189
+ t.identity = identity
190
+ if implied_individuals:
191
+ # Hidden terms (`hidden_terms`): a variable every occurrence of which is in one triples block, under no
192
+ # SELECT * (which projects every variable), may be an unnamed individual, as a blank node may.
193
+ from collections import Counter
194
+ t.unnamed = True
195
+ t.occurrences = Counter(_var_names(tree))
196
+ t.by_value = datatypes == "value"
197
+ if dataset is not None:
198
+ # The protocol's dataset (default-graph-uri, named-graph-uri) is used in place of the query's own.
199
+ t.dataset = (frozenset(dataset[0]), frozenset(dataset[1]))
200
+ t.dataset_given = True
201
+ if bindings:
202
+ unknown = sorted(set(bindings) - variables_of(tree))
203
+ if unknown:
204
+ from prograph.parameters import ParameterError
205
+ raise ParameterError(unknown[0], "not a variable of the query")
206
+ names = tuple(bindings)
207
+ t.bindings = O.Values(names, (tuple(bindings[n] for n in names),))
208
+ t.params = dict(bindings)
209
+ t.prologue(tree.prologue())
210
+ if identity is not None:
211
+ identity.prefetch(t.constants_of(tree))
212
+ if tree.selectQuery() is not None:
213
+ t.top = tree.selectQuery()
214
+ return Query("select", t.select(tree.selectQuery()))
215
+ if tree.askQuery() is not None:
216
+ ask = tree.askQuery()
217
+ t.dataset_clause(ask.datasetClause())
218
+ _no_modifiers(ask.solutionModifier(), "ASK")
219
+ t.top_group = ask.whereClause().groupGraphPattern()
220
+ return Query("ask", t.trailing_values(t.group(ask.whereClause().groupGraphPattern()),
221
+ ask.valuesClause()))
222
+ if tree.constructQuery() is not None:
223
+ plan, template = t.construct(tree.constructQuery())
224
+ return Query("construct", plan, template=template, prefixes=dict(t.prefixes))
225
+ if tree.describeQuery() is not None:
226
+ plan, template = t.describe(tree.describeQuery())
227
+ return Query("construct", plan, template=template, prefixes=dict(t.prefixes))
228
+ # A write: ProGraph does not write to the user's tables (rule read-only-writes; docs/sparql/update.md).
229
+ from prograph import read_only
230
+ read_only.refuse("SPARQL Update")
231
+
232
+
233
+ def _no_modifiers(ctx, what: str) -> None:
234
+ if ctx is not None and (ctx.groupClause() or ctx.havingClause() or ctx.orderClause()
235
+ or ctx.limitOffsetClauses()):
236
+ raise NotBuilt(f"solution modifiers on {what}")
237
+
238
+
239
+ @home("decimal-text", "A DECIMAL column's text in a term: canonical, or the database's",
240
+ terms=("decimal_text", "canonical decimal", "database text", "Ontop", "1.0", "D3"))
241
+ def held_under(held: Any, decimal_text: Optional[str]) -> Any:
242
+ """How the SPARQL translator reads a column (`entry.held`'s tuple) under the schema's
243
+ `decimal_text` (sparql-choices.md D3). By default a DECIMAL column is read as its value, and
244
+ every text of it is XML Schema 1.1's canonical form (`1`, `1.5`; `lower.value_text`). Under
245
+ `database` it is read as the database's own text (`DATABASE_DECIMAL_TEXT`: `1.0`, `1.50`, as
246
+ Ontop writes it), a string whose literal is still an xsd:decimal (`Slot.text_of`): its text in a
247
+ literal, a template and an IRI column alike, its key compared as that text (a DECIMAL key and an
248
+ INTEGER or other-scale key do not meet). Cypher reads columns as they are."""
249
+ from prograph.schema.source_map import DATABASE_DECIMAL_TEXT, DECIMAL_DATABASE
250
+ if decimal_text != DECIMAL_DATABASE:
251
+ return held
252
+
253
+ def under(table: str, column: str):
254
+ dt, zoned, digits, reader, bytewise = held(table, column)
255
+ if dt == xsd.DECIMAL:
256
+ # Its declared digits kept: the text is read back by value as that DECIMAL(p, s)
257
+ # (`lower.decimal_of_text`), never as DECIMAL(38, 18), which holds 20 whole digits.
258
+ return xsd.STRING, False, digits, DATABASE_DECIMAL_TEXT, False
259
+ return dt, zoned, digits, reader, bytewise
260
+ for name in dir(held): # `kind_of` and the like
261
+ if not name.startswith("__") and not hasattr(under, name):
262
+ setattr(under, name, getattr(held, name))
263
+ return under
264
+
265
+
266
+ DIFFERENT_FROM = "http://www.w3.org/2002/07/owl#differentFrom"
267
+ NAMED_INDIVIDUAL = "http://www.w3.org/2002/07/owl#NamedIndividual"
268
+ TOP_OBJECT_PROPERTY = "http://www.w3.org/2002/07/owl#topObjectProperty"
269
+ SUB_PROPERTY_IRI = "http://www.w3.org/2000/01/rdf-schema#subPropertyOf"
270
+ def typing_of(patterns) -> FrozenSet[str]:
271
+ """The variables a typing triple of `patterns` types (Entailment Regimes 7.1.3)."""
272
+ return frozenset(t[0].name for t in patterns if isinstance(t[0], Var) and t[1] == IRITerm(RDF_TYPE)
273
+ and isinstance(t[2], IRITerm) and t[2].text in TYPING_CLASSES)
274
+
275
+
276
+ # The classes of Entailment Regimes 7.1.3's typing triples.
277
+ TYPING_CLASSES = frozenset("http://www.w3.org/2002/07/owl#" + c for c in
278
+ ("Class", "ObjectProperty", "DatatypeProperty", "NamedIndividual"))
279
+
280
+
281
+ class Translator:
282
+ """One query's translation: its prefixes and base, and fresh names."""
283
+
284
+ def __init__(self, maps: SourceMaps, held: Any, keys: Any = None, union_default: bool = True,
285
+ ontology: Any = None) -> None:
286
+ self.maps = maps
287
+ self.held = held_under(held, getattr(maps.settings, "decimal_text", None))
288
+ # The call's parameters as one VALUES row, joined with the top-level query's pattern (`bound`).
289
+ self.bindings: Optional[O.Op] = None
290
+ # Each parameter's term: where the query writes the variable, it reads the constant (Jena's
291
+ # substitution), so a scope evaluated on its own (an OPTIONAL's FILTER, MINUS, a UNION
292
+ # branch, a nested BIND) sees the value; the top-level VALUES row keeps it in the answers.
293
+ self.params: Dict[str, Term] = {}
294
+ self.top: Any = None # the top-level SELECT's context: a sub-select takes no bindings
295
+ self.top_group: Any = None # the top-level query's WHERE group, which the bindings join
296
+ # The ontology's domain and range typing (`saturate`): (property, classes, as subject).
297
+ from prograph.plan.ir.saturate import typing_properties
298
+ self.typing = typing_properties(ontology, maps.by_predicate, maps) if ontology is not None else []
299
+ self.ontology = ontology
300
+ # The ontology's own triples in the default graph (`dataset.ontology_triples`), and > 0 while a
301
+ # witness of the data's vocabulary is matched (`ontology_rows`), which reads the data alone.
302
+ from prograph.plan.ir.dataset import ontology_triples
303
+ self.onto_triples = ontology_triples(maps, ontology) if ontology is not None else None
304
+ self.witnessing = 0
305
+ self.differing = 0 # > 0 while `different_branch`'s patterns are translated: owl:differentFrom as stated
306
+ # The graph the patterns read (13, 18.5): None the default graph, else GRAPH's IRI or
307
+ # variable; and whether the default graph is the union of every graph (default_graph='union').
308
+ self.graph: Optional[Term] = None
309
+ self.patterns = 0 # triple patterns translated so far (`graph_pattern` asks whether P had one)
310
+ self.group_base: Optional[O.Op] = None # what the next group starts from (`graph_pattern`)
311
+ self.path_seed: Optional[Tuple[str, str, O.Op, str, bool]] = None # (side, variable, its terms, column, graph nodes)
312
+ # The variables a VALUES block of one row binds to an IRI in the group being translated, each with
313
+ # its IRI: the group's own triple patterns read the constant (`values_bound`).
314
+ self.narrowed: Dict[str, IRITerm] = {}
315
+ self.union_default = union_default
316
+ # The query's dataset (13.2), (FROM IRIs, FROM NAMED IRIs), None where it has no dataset clause.
317
+ self.dataset: Optional[Tuple[frozenset, frozenset]] = None
318
+ self.blank_scope: Optional[str] = None # `scope_of`
319
+ self.dataset_given = False # set by the caller (the protocol): the query's own clauses are ignored
320
+ _warn_overrides(maps, held)
321
+ # The keys a scan of a map may rely on (`node_keys.trusted_keys`).
322
+ self.keys = keys or (lambda m: ())
323
+ self.prefixes: Dict[str, str] = {}
324
+ self.base: Optional[str] = None
325
+ # NOW()'s one value for the query (17.4.5.1): the moment it is planned, UTC, to the millisecond.
326
+ import datetime as _dt
327
+ moment = _dt.datetime.now(_dt.timezone.utc)
328
+ moment = moment.replace(microsecond=moment.microsecond // 1000 * 1000)
329
+ self.now = xsd.canonical_lexical(moment, xsd.DATETIME)
330
+ self.n = 0
331
+ self.fresh_calls = 0
332
+ self.aggregates: Optional[List[Tuple[str, E.Agg]]] = None
333
+ # owl:sameAs at BGP matching (`identity.pattern`), None for RDF term identity alone; `plain` > 0
334
+ # while a pattern is matched as stated (inside a pattern owl:sameAs already handles).
335
+ self.identity: Any = None
336
+ self.plain = 0
337
+
338
+ def fresh(self, what: str = "h") -> str:
339
+ self.n += 1
340
+ return f"{HIDDEN}{what}{self.n}"
341
+
342
+ # -- terms ------------------------------------------------------------------
343
+
344
+ def prologue(self, ctx) -> None:
345
+ """PREFIX and BASE in order: a BASE applies to what follows (4.1.1)."""
346
+ for child in ctx.getChildren():
347
+ if isinstance(child, SparqlParser.BaseDeclContext):
348
+ self.base = self.resolve(child.IRI_REF().getText())
349
+ elif isinstance(child, SparqlParser.PrefixDeclContext):
350
+ self.prefixes[child.PNAME_NS().getText()[:-1]] = \
351
+ self.resolve(child.IRI_REF().getText())
352
+
353
+ def constants_of(self, tree) -> List[str]:
354
+ """Every IRI a query names (and rdf:type, which `a` writes): the constants owl:sameAs looks up at planning
355
+ (`Identity.prefetch`), read once for the whole query."""
356
+ out: List[str] = [RDF_TYPE]
357
+ stack = [tree]
358
+ while stack:
359
+ node = stack.pop()
360
+ if isinstance(node, SparqlParser.IriRefContext):
361
+ try:
362
+ out.append(self.iri(node))
363
+ except QueryError:
364
+ pass
365
+ continue
366
+ stack.extend(node.getChildren() if hasattr(node, "getChildren") else ())
367
+ return out
368
+
369
+ def dataset_clause(self, clauses) -> None:
370
+ """FROM and FROM NAMED (13.2): the graphs merged into the default graph and the named graphs,
371
+ each in place of the service's dataset (no FROM NAMED: no named graph; no FROM: an empty
372
+ default graph). A dataset the caller gave (the protocol's) is used in place of these."""
373
+ if not clauses or self.dataset_given:
374
+ return
375
+ default, named = set(), set()
376
+ for clause in clauses:
377
+ if clause.namedGraphClause() is not None:
378
+ named.add(self.iri(clause.namedGraphClause().sourceSelector().iriRef()))
379
+ else:
380
+ default.add(self.iri(clause.defaultGraphClause().sourceSelector().iriRef()))
381
+ self.dataset = (frozenset(default), frozenset(named))
382
+
383
+ def resolve(self, token: str) -> str:
384
+ """An IRI_REF's IRI, resolved against BASE where one is declared."""
385
+ iri = token[1:-1] if token.startswith("<") else token
386
+ if not self.base:
387
+ return iri
388
+ from urllib.parse import urljoin
389
+ resolved = urljoin(self.base, iri)
390
+ if iri.endswith("#") and not resolved.endswith("#"):
391
+ resolved += "#"
392
+ return resolved
393
+
394
+ def iri(self, ctx) -> str:
395
+ """An iriRef's or prefixedName's IRI."""
396
+ if isinstance(ctx, SparqlParser.IriRefContext):
397
+ if ctx.prefixedName() is None:
398
+ return self.resolve(ctx.getText())
399
+ ctx = ctx.prefixedName()
400
+ prefix, _, local = ctx.getText().partition(":")
401
+ local = re.sub(r"\\(.)", r"\1", local) # PN_LOCAL_ESC (19.8); %XX stays
402
+ if prefix in self.prefixes:
403
+ return self.prefixes[prefix] + local
404
+ if prefix in STANDARD_PREFIXES:
405
+ return STANDARD_PREFIXES[prefix] + local
406
+ raise QueryError(f"Undeclared prefix '{prefix}:' in {ctx.getText()}")
407
+
408
+ @staticmethod
409
+ def string(ctx) -> str:
410
+ text = ctx.getText()
411
+ text = text[3:-3] if text[:3] in ('"""', "'''") else text[1:-1]
412
+ return re.sub(r"""\\([tbnrf"'\\])""", lambda m: _ECHAR[m.group(1)], text)
413
+
414
+ def literal(self, ctx) -> LiteralTerm:
415
+ """An rdfLiteral, numericLiteral or booleanLiteral, as written."""
416
+ if isinstance(ctx, SparqlParser.RdfLiteralContext):
417
+ lexical = self.string(ctx.string_())
418
+ if ctx.LANGTAG() is not None:
419
+ return LiteralTerm(lexical, language=ctx.LANGTAG().getText()[1:])
420
+ if ctx.iriRef() is not None:
421
+ return LiteralTerm(lexical, self.iri(ctx.iriRef()))
422
+ return LiteralTerm(lexical)
423
+ if isinstance(ctx, SparqlParser.BooleanLiteralContext):
424
+ return LiteralTerm(ctx.getText().lower(), xsd.BOOLEAN)
425
+ return numeric(ctx.getText())
426
+
427
+ def term(self, ctx) -> Term:
428
+ """A varOrTerm, varOrIRIref or graphTerm."""
429
+ if isinstance(ctx, (SparqlParser.VarOrTermContext, SparqlParser.VarOrIRIrefContext)):
430
+ if ctx.var_() is not None:
431
+ name = ctx.var_().getText()[1:]
432
+ return self.params.get(name) or Var(name)
433
+ inner = ctx.graphTerm() if isinstance(ctx, SparqlParser.VarOrTermContext) \
434
+ else ctx.iriRef()
435
+ return self.term(inner)
436
+ if isinstance(ctx, SparqlParser.IriRefContext):
437
+ text = self.iri(ctx)
438
+ if text.startswith(LABEL_IRI_PREFIX):
439
+ raise _label_pattern(text[len(LABEL_IRI_PREFIX):], self.maps)
440
+ return IRITerm(text)
441
+ if isinstance(ctx, SparqlParser.GraphTermContext):
442
+ for get in (ctx.iriRef, ctx.rdfLiteral, ctx.numericLiteral, ctx.booleanLiteral):
443
+ found = get()
444
+ if found is not None:
445
+ return self.term(found) if get == ctx.iriRef else self.literal(found)
446
+ if ctx.blankNode() is not None:
447
+ label = ctx.blankNode().BLANK_NODE_LABEL()
448
+ return Var(f"{HIDDEN}b:{label.getText()[2:]}" if label else self.fresh("b"))
449
+ raise NotBuilt("rdf:nil in a pattern")
450
+ raise NotBuilt(f"the term {ctx.getText()}")
451
+
452
+ # -- SELECT -----------------------------------------------------------------
453
+
454
+ def select(self, ctx) -> O.Op:
455
+ """A SELECT's plan (18.2.4, 18.2.5): the pattern; its groups and
456
+ aggregates, then HAVING (18.2.4.1); the SELECT expressions; ORDER
457
+ BY; the projection; DISTINCT; the slice."""
458
+ # Under `implied_individuals`, SELECT * projects every variable of its pattern and COUNT(*) counts every
459
+ # solution of it: no variable of this SELECT's own pattern is hidden (a sub-SELECT's are its own).
460
+ held = self.select_star
461
+ self.select_star = _star_like(ctx)
462
+ try:
463
+ return self._select(ctx)
464
+ finally:
465
+ self.select_star = held
466
+
467
+ def _select(self, ctx) -> O.Op:
468
+ self.dataset_clause(ctx.datasetClause())
469
+ mods = ctx.solutionModifier()
470
+ if ctx is self.top:
471
+ self.top_group = ctx.whereClause().groupGraphPattern()
472
+ x = self.trailing_values(self.group(ctx.whereClause().groupGraphPattern()), ctx.valuesClause())
473
+ # The WHERE pattern's in-scope variables (18.2.1): no SELECT expression may bind one, grouped
474
+ # or not (`SELECT (COUNT(*) AS ?c) { ?s ?p ?c }` is a static error).
475
+ in_scope = set(x.columns)
476
+ x, keys = self.group_keys(x, mods)
477
+ # The expressions of SELECT, HAVING and ORDER BY, their aggregates
478
+ # collected (18.2.4.1: each is a column of the Aggregate).
479
+ self.aggregates: Optional[List[Tuple[str, E.Agg]]] = []
480
+ selected: List[Tuple[str, Optional[E.Expr]]] = []
481
+ for clause in ctx.selectClause():
482
+ name = clause.var_().getText()[1:]
483
+ expr = self.expression(clause.expression()) if clause.expression() is not None else None
484
+ if any(name == n for n, _ in selected):
485
+ if expr is None:
486
+ continue # the projection is a set of variables (18.2.5.2)
487
+ raise QueryError(f"?{name} is projected twice")
488
+ selected.append((name, expr))
489
+ having = [self.condition(h.constraint()) for h in mods.havingClause().havingCondition()] \
490
+ if mods.havingClause() is not None else []
491
+ order = self.order_conditions(mods)
492
+ aggregates, self.aggregates = self.aggregates, None
493
+ # 18.2.4.1: with no GROUP BY and no aggregate, the aggregate step is
494
+ # skipped, and HAVING is a filter over the solutions.
495
+ grouped = bool(keys) or bool(aggregates)
496
+ # A sub-select inside GRAPH ?g is evaluated in each graph (18.5): it keeps the active graph's
497
+ # hidden column, groups by it, and is DISTINCT within each graph.
498
+ part = self.graph.name if isinstance(self.graph, Var) and self.graph.name in x.columns else None
499
+ if part is not None:
500
+ if grouped and not keys:
501
+ raise NotBuilt("an aggregate with no GROUP BY in a sub-select inside GRAPH ?g "
502
+ "(one row per graph, an empty graph's included)")
503
+ if mods.limitOffsetClauses() is not None:
504
+ raise NotBuilt("LIMIT or OFFSET in a sub-select inside GRAPH ?g (per graph)")
505
+ if grouped:
506
+ keys.append(part)
507
+ names: List[str] = []
508
+ if ctx.STAR() is not None:
509
+ if grouped:
510
+ raise QueryError("SELECT * in a grouped query (18.2.4.1)")
511
+ names = star_order(ctx.whereClause(), [c for c in x.columns if not c.startswith(HIDDEN)])
512
+ if grouped:
513
+ # 18.2.4.1: a grouped query reads its keys and aggregates only.
514
+ visible = set(keys) | {n for n, _ in aggregates}
515
+ for name, expr in selected:
516
+ _grouped_only({name} if expr is None else expr.columns(), visible, "SELECT")
517
+ visible.add(name)
518
+ # HAVING and ORDER BY may read any variable (11.4 restricts only
519
+ # what SELECT projects). One the group does not keep is unbound
520
+ # there: 18.2.4.1's "unaggregated variable" is undefined (W3C
521
+ # errata-query-14), its editor read it as the group keys, and
522
+ # Jena and Oxigraph agree (docs/architecture/testing.md). A
523
+ # SELECT expression's variable is not visible to HAVING (18.2.4.2).
524
+ if self.by_value and self.value_columns(x, keys):
525
+ x = self.grouped_by_value(x, keys, aggregates)
526
+ else:
527
+ x = O.Aggregate(x, tuple(keys), tuple(aggregates))
528
+ _warn_outside_group(having, order, set(keys) | {n for n, _ in aggregates},
529
+ {n for n, _ in selected})
530
+ # 18.2.4: grouping, aggregates, HAVING, then the SELECT expressions:
531
+ # HAVING does not see a SELECT expression's variable.
532
+ if having:
533
+ x = O.Filter(x, having[0] if len(having) == 1 else E.And(tuple(having)))
534
+ for name, expr in selected:
535
+ if expr is not None:
536
+ if name in x.columns or name in in_scope:
537
+ raise QueryError(f"?{name} is bound already; SELECT (... AS ?{name}) "
538
+ "binds a new variable (18.2.4.4)")
539
+ x = O.Extend(x, name, expr)
540
+ names.append(name)
541
+ kept = tuple(names) + ((part,) if part is not None and part not in names else ())
542
+ if ctx.DISTINCT() is not None and self.by_value and self.value_columns(O.Project(x, kept), kept):
543
+ # One row per value, then ORDER BY (a group keeps no order): its keys read only what DISTINCT keeps.
544
+ outside = {c for e, _ in order for c in e.columns()} - set(kept)
545
+ if outside:
546
+ raise NotBuilt("ORDER BY a variable SELECT DISTINCT does not project, under datatypes='value'")
547
+ x = self.ordered(self.distinct_by_value(O.Project(x, kept)), order)
548
+ return self.sliced(x, mods)
549
+ x = self.ordered(x, order)
550
+ x = O.Project(x, kept)
551
+ if ctx.DISTINCT() is not None:
552
+ x = O.Distinct(x)
553
+ elif ctx.REDUCED() is not None:
554
+ x = O.Reduced(x)
555
+ return self.sliced(x, mods)
556
+
557
+ def group_keys(self, x: O.Op, mods) -> Tuple[O.Op, List[str]]:
558
+ """GROUP BY's keys (18.2.4.1): each a variable, or an expression bound first (`(expr AS ?v)`,
559
+ or a fresh column). The plan with them bound, and their names."""
560
+ keys: List[str] = []
561
+ if mods.groupClause() is not None:
562
+ for gc in mods.groupClause().groupCondition():
563
+ if gc.var_() is not None and gc.expression() is None:
564
+ name = gc.var_().getText()[1:]
565
+ if name not in x.columns:
566
+ x = O.Extend(x, name, E.Const(None))
567
+ else:
568
+ inner = gc.expression() if gc.expression() is not None else gc.getChild(0)
569
+ e = self.expression(inner)
570
+ if isinstance(e, E.Col) and gc.var_() is None:
571
+ name = e.name # `(?x)` is ?x itself (11.4)
572
+ if name not in x.columns:
573
+ x = O.Extend(x, name, E.Const(None))
574
+ else:
575
+ name = gc.var_().getText()[1:] if gc.var_() is not None else self.fresh("g")
576
+ if name in x.columns:
577
+ raise QueryError(f"GROUP BY (... AS ?{name}) binds ?{name}, which "
578
+ "the pattern binds already")
579
+ x = O.Extend(x, name, e)
580
+ if name not in keys:
581
+ keys.append(name)
582
+ return x, keys
583
+
584
+ def grouped(self, x: O.Op, mods) -> Tuple[O.Op, List[Tuple[E.Expr, bool]], Optional[List[str]]]:
585
+ """A CONSTRUCT's or DESCRIBE's solutions under GROUP BY and HAVING (11, 18.2.4.1, 16.2): with
586
+ either, or an aggregate in ORDER BY, one solution per group, which binds its GROUP BY keys
587
+ (`(expr AS ?v)` too) and nothing else, so a template variable the keys do not name is
588
+ unbound and leaves its triple out (16.2; Jena). HAVING and ORDER BY read aggregates. Returns
589
+ the plan, ORDER BY's keys and the group's keys (None when there is no grouping)."""
590
+ if mods is None:
591
+ return x, [], None
592
+ x, keys = self.group_keys(x, mods)
593
+ self.aggregates = []
594
+ having = [self.condition(h.constraint()) for h in mods.havingClause().havingCondition()] \
595
+ if mods.havingClause() is not None else []
596
+ order = self.order_conditions(mods)
597
+ aggregates, self.aggregates = self.aggregates, None
598
+ if not (keys or aggregates):
599
+ return (O.Filter(x, having[0] if len(having) == 1 else E.And(tuple(having))) if having else x), order, None
600
+ x = O.Aggregate(x, tuple(keys), tuple(aggregates))
601
+ _warn_outside_group(having, order, set(keys) | {n for n, _ in aggregates}, set())
602
+ if having:
603
+ x = O.Filter(x, having[0] if len(having) == 1 else E.And(tuple(having)))
604
+ return x, order, keys
605
+
606
+ def order_conditions(self, mods) -> List[Tuple[E.Expr, bool]]:
607
+ """ORDER BY's keys, each an expression and whether descending."""
608
+ order: List[Tuple[E.Expr, bool]] = []
609
+ if mods is not None and mods.orderClause() is not None:
610
+ for cond in mods.orderClause().orderCondition():
611
+ if cond.var_() is not None:
612
+ var = cond.var_().getText()[1:]
613
+ order.append((E.Const(self.params[var]) if var in self.params else E.Col(var),
614
+ cond.DESC() is not None))
615
+ else:
616
+ inner = cond.brackettedExpression() or cond.constraint()
617
+ order.append((self.expression(inner), cond.DESC() is not None))
618
+ return order
619
+
620
+ def ordered(self, x: O.Op, order: List[Tuple[E.Expr, bool]]) -> O.Op:
621
+ """`x` sorted by ORDER BY's keys (15.1)."""
622
+ sort = []
623
+ for e, descending in order:
624
+ if isinstance(e, E.Col):
625
+ column = e.name
626
+ if column not in x.columns:
627
+ continue # never bound: every row ties (15.1)
628
+ else:
629
+ column = self.fresh("k")
630
+ x = O.Extend(x, column, e)
631
+ sort.append(O.SortKey(column, descending))
632
+ return O.Order(x, tuple(sort)) if sort else x
633
+
634
+ @staticmethod
635
+ def sliced(x: O.Op, mods) -> O.Op:
636
+ """`x` under LIMIT and OFFSET (15.4, 15.5)."""
637
+ lo = mods.limitOffsetClauses() if mods is not None else None
638
+ if lo is not None:
639
+ limit = int(lo.limitClause().INTEGER().getText()) if lo.limitClause() else None
640
+ offset = int(lo.offsetClause().INTEGER().getText()) if lo.offsetClause() else 0
641
+ x = O.Slice(x, offset, limit)
642
+ return x
643
+
644
+ # -- CONSTRUCT --------------------------------------------------------------
645
+
646
+ def construct(self, ctx) -> Tuple[O.Op, tuple]:
647
+ """A CONSTRUCT's plan and template (16.2). The plan is the pattern's
648
+ solutions under ORDER BY, OFFSET and LIMIT, projected on the
649
+ template's variables; the template is instantiated above the plan
650
+ (one-plan.md 5.10). `CONSTRUCT WHERE { T }` is `CONSTRUCT { T }
651
+ WHERE { T }` (16.2.4). A template's blank node is a hidden variable
652
+ here, which no pattern binds."""
653
+ self.dataset_clause(ctx.datasetClause())
654
+ mods = ctx.solutionModifier()
655
+ template: List[Tuple[Term, Term, Term]] = []
656
+ if ctx.constructTemplate() is not None:
657
+ self.top_group = ctx.whereClause().groupGraphPattern()
658
+ x = self.group(ctx.whereClause().groupGraphPattern())
659
+ triples = ctx.constructTemplate().constructTriples()
660
+ if triples is not None:
661
+ for same in triples.triplesSameSubject():
662
+ self.same_subject(same, template)
663
+ else:
664
+ shape = ctx.triplesTemplate()
665
+ # CONSTRUCT WHERE { T }: T is the group, and has no FILTER.
666
+ x = self.bound(self.triples_block(shape) if shape is not None else O.Unit())
667
+ if shape is not None:
668
+ for same in shape.triplesSameSubject():
669
+ self.same_subject(same, template)
670
+ # The pattern's blank nodes are its variables; the template's
671
+ # are fresh per solution: two names, never one.
672
+ template = [tuple(Var(f"{HIDDEN}t{t.name}") if isinstance(t, Var)
673
+ and t.name.startswith(HIDDEN) else t for t in triple)
674
+ for triple in template]
675
+ # 18.2.4.3: a trailing VALUES is joined after the groups and HAVING.
676
+ x, order, _ = self.grouped(x, mods)
677
+ x = self.trailing_values(x, ctx.valuesClause())
678
+ x = self.ordered(x, order)
679
+ wanted = [t.name for triple in template for t in triple
680
+ if isinstance(t, Var) and not t.name.startswith(HIDDEN)]
681
+ x = O.Project(x, tuple(n for n in dict.fromkeys(wanted) if n in x.columns))
682
+ return self.sliced(x, mods), tuple(template)
683
+
684
+ def describe(self, ctx) -> Tuple[O.Op, tuple]:
685
+ """A DESCRIBE (16.4) as a CONSTRUCT (sparql-choices.md S13, as Jena and Oxigraph): each
686
+ resource it names (an IRI, a variable's values in the pattern's solutions, `*` every visible
687
+ variable) by the triples whose subject it is, and each blank node one of those triples
688
+ reaches as its object, so on (the blank-node closure: a `*` closure over the steps to a blank
689
+ object, from the named resources)."""
690
+ self.dataset_clause(ctx.datasetClause())
691
+ mods = ctx.solutionModifier()
692
+ where = ctx.whereClause()
693
+ if where is not None:
694
+ self.top_group = where.groupGraphPattern()
695
+ x = self.group(where.groupGraphPattern()) if where is not None else self.bound(O.Unit())
696
+ x, order, keys = self.grouped(x, mods)
697
+ before = set(x.columns)
698
+ x = self.trailing_values(x, ctx.valuesClause()) # 18.2.4.3: after the groups and HAVING
699
+ if keys is not None:
700
+ keys = list(keys) + [c for c in x.columns if c not in before]
701
+ x = self.sliced(self.ordered(x, order), mods)
702
+ if ctx.STAR() is not None:
703
+ named = [Var(n) for n in x.columns if not n.startswith(HIDDEN)]
704
+ else:
705
+ named = [self.term(v) for v in ctx.varOrIRIref()]
706
+ if keys is not None: # Jena: a grouped DESCRIBE names only what the groups keep
707
+ _grouped_only([t.name for t in named if isinstance(t, Var)], set(keys), "DESCRIBE")
708
+ # Names no query variable can have (`@`), and not hidden ones, which a template reads as
709
+ # blank nodes.
710
+ r, p, o = Var("@r"), Var("@p"), Var("@o")
711
+ starts: List[O.Op] = []
712
+ for thing in dict.fromkeys(named):
713
+ if isinstance(thing, Var):
714
+ if thing.name in x.columns: # a variable the pattern does not bind names nothing
715
+ starts.append(O.Project(O.Extend(O.Project(x, (thing.name,)), r.name, E.Col(thing.name)),
716
+ (r.name,)))
717
+ else:
718
+ starts.append(O.Values((r.name,), ((thing,),)))
719
+ if not starts:
720
+ return O.Project(O.Unit(), ()), ()
721
+ named_ = O.Distinct(starts[0] if len(starts) == 1 else O.Union(tuple(starts)))
722
+ nodes: O.Op = named_
723
+ if self._blank_objects():
724
+ a, b = Var(self.fresh("a")), Var(self.fresh("b"))
725
+ step = O.Project(O.Filter(self.triple(a, Var(self.fresh("p")), b),
726
+ E.Call("isBLANK", (E.Col(b.name),))), (a.name, b.name))
727
+ walk = O.Closure(step, a.name, b.name, min=0, max=None, terms=named_, terms_column=r.name)
728
+ reached = O.Join(O.Project(O.Extend(named_, a.name, E.Col(r.name)), (a.name,)), walk)
729
+ nodes = O.Distinct(O.Project(O.Extend(O.Project(reached, (b.name,)), r.name, E.Col(b.name)),
730
+ (r.name,)))
731
+ plan = O.Project(O.Join(nodes, self.triple(r, p, o)), (r.name, p.name, o.name))
732
+ return plan, ((r, p, o),)
733
+
734
+ def _blank_objects(self) -> bool:
735
+ """Whether a triple's object can be a blank node: a map's blank subject (which another map's
736
+ reference can reach) or blank object."""
737
+ for m in self.maps.nodes:
738
+ if m.subject.term_type == BLANK:
739
+ return True
740
+ if any(po.object is not None and getattr(po.object, "term_type", None) == BLANK
741
+ for po in m.predicate_objects):
742
+ return True
743
+ return False
744
+
745
+ def aggregate(self, ctx) -> E.Expr:
746
+ """An aggregate (18.5): a column of the query's Aggregate, read by
747
+ its fresh name. Only SELECT, HAVING and ORDER BY hold one."""
748
+ if getattr(self, "aggregates", None) is None:
749
+ raise QueryError(f"an aggregate outside SELECT, HAVING and ORDER BY: "
750
+ f"{ctx.getText()[:40]}")
751
+ fn = ctx.getChild(0).getText().upper()
752
+ distinct = ctx.DISTINCT() is not None
753
+ if fn == "COUNT" and ctx.STAR() is not None:
754
+ agg = E.Agg("COUNT", None, distinct)
755
+ else:
756
+ outer, self.aggregates = self.aggregates, None # no aggregate inside one
757
+ try:
758
+ arg = self.expression(ctx.expression())
759
+ finally:
760
+ self.aggregates = outer
761
+ separator = self.string(ctx.string_()) if getattr(ctx, "string_", None) and \
762
+ ctx.string_() is not None else " "
763
+ if self.by_value and fn == "COUNT" and distinct:
764
+ # COUNT(DISTINCT) counts values, as SELECT DISTINCT keeps one row per value (`value-identity-in-sql`).
765
+ arg = E.Call("valueKey", (arg,))
766
+ agg = E.Agg(fn, arg, distinct, separator)
767
+ name = self.fresh("a")
768
+ self.aggregates.append((name, agg))
769
+ return E.Col(name)
770
+
771
+ # -- graph patterns (18.2.2) ------------------------------------------------
772
+
773
+ def group(self, ctx) -> O.Op:
774
+ """A GroupGraphPattern: its FILTERs apply to the whole group
775
+ (18.2.2.6). A FILTER that is a conjunction splits into its terms,
776
+ which keep a row exactly as the conjunction does; a term that is
777
+ EXISTS or NOT EXISTS (or `!` of one) is a semi- or anti-join."""
778
+ g, filters = self.group_parts(ctx)
779
+ if ctx is self.top_group:
780
+ g = self.bound(g)
781
+ plain: List[E.Expr] = []
782
+ for f in filters:
783
+ for leaf in conjuncts_of(f.constraint()):
784
+ found = exists_leaf(leaf)
785
+ if found is None:
786
+ plain.append(self.condition(leaf))
787
+ continue
788
+ negated, body = found
789
+ right, cond = self.exists_body(body, frozenset(g.columns))
790
+ g = (O.AntiJoin if negated else O.SemiJoin)(g, right, cond)
791
+ from prograph.plan.ir.fresh import holds
792
+ lifted = []
793
+ for i, cond in enumerate(plain):
794
+ if holds(cond):
795
+ # A fresh value is made by an Extend (fresh.py): the condition's value in a hidden
796
+ # column, filtered on.
797
+ name = self.fresh("f")
798
+ g = O.Extend(g, name, cond)
799
+ lifted.append(E.Col(name))
800
+ else:
801
+ lifted.append(cond)
802
+ if lifted:
803
+ g = O.Filter(g, lifted[0] if len(lifted) == 1 else E.And(tuple(lifted)))
804
+ return g
805
+
806
+ def exists_body(self, ctx, outer: FrozenSet[str]) -> Tuple[O.Op, Optional[E.Expr]]:
807
+ """The P of EXISTS { P } (m2-functions.md 7): its patterns without
808
+ its FILTERs, and its FILTERs' conjunction, read over the merged row
809
+ of the group and P. P is triple patterns, FILTERs and nested
810
+ EXISTS; anything else is refused. A nested EXISTS may read only
811
+ variables P binds or ones the group does not have: a join of P with
812
+ its own rows cannot see the group's other values."""
813
+ if ctx.subSelect() is not None:
814
+ raise NotBuilt("a sub-select inside EXISTS")
815
+ p: O.Op = O.Unit()
816
+ filters: List[Any] = []
817
+ for child in ctx.getChildren():
818
+ if isinstance(child, SparqlParser.TriplesBlockContext):
819
+ p = join(p, self.triples_block(child))
820
+ elif isinstance(child, SparqlParser.Filter_Context):
821
+ filters.append(child)
822
+ elif isinstance(child, SparqlParser.GraphPatternNotTriplesContext):
823
+ if child.optionalGraphPattern() is not None:
824
+ raise NotBuilt("OPTIONAL inside EXISTS")
825
+ if child.groupOrUnionGraphPattern() is not None:
826
+ n = len(child.groupOrUnionGraphPattern().groupGraphPattern())
827
+ raise NotBuilt("UNION inside EXISTS" if n > 1 else "a nested group inside EXISTS")
828
+ if child.bind() is not None:
829
+ raise NotBuilt("BIND inside EXISTS")
830
+ if child.inlineData() is not None:
831
+ raise NotBuilt("VALUES inside EXISTS")
832
+ if child.minusGraphPattern() is not None:
833
+ raise NotBuilt("MINUS inside EXISTS")
834
+ if child.graphGraphPattern() is not None:
835
+ p = join(p, self.graph_pattern(child.graphGraphPattern()))
836
+ continue
837
+ raise NotBuilt("this pattern inside EXISTS")
838
+ conds: List[E.Expr] = []
839
+ for f in filters:
840
+ for leaf in conjuncts_of(f.constraint()):
841
+ found = exists_leaf(leaf)
842
+ if found is None:
843
+ conds.append(self.condition(leaf))
844
+ continue
845
+ negated, body = found
846
+ right, cond = self.exists_body(body, outer)
847
+ seen = set(right.columns) | (set(cond.columns()) if cond is not None else set())
848
+ if (seen & outer) - set(p.columns):
849
+ raise NotBuilt("a nested EXISTS that reads a variable of the outer group "
850
+ "which the enclosing EXISTS does not bind")
851
+ p = (O.AntiJoin if negated else O.SemiJoin)(p, right, cond)
852
+ return p, (conds[0] if len(conds) == 1 else E.And(tuple(conds)) if conds else None)
853
+
854
+ def group_parts(self, ctx) -> Tuple[O.Op, List[Any]]:
855
+ """A GroupGraphPattern's pattern and its own FILTERs (18.2.2.6), as
856
+ parse nodes."""
857
+ # The graph base is this group's alone (a sub-select's own WHERE is not given it: one that
858
+ # reads no graph gives the same rows in each, joined with the graphs after).
859
+ base, self.group_base = self.group_base, None
860
+ if ctx.subSelect() is not None:
861
+ return self.select(ctx.subSelect().selectQuery()), []
862
+ g: O.Op = O.Unit()
863
+ if base is not None and not isinstance(next((c for c in ctx.getChildren()
864
+ if isinstance(c, (SparqlParser.TriplesBlockContext,
865
+ SparqlParser.GraphPatternNotTriplesContext))),
866
+ None), SparqlParser.TriplesBlockContext):
867
+ # A GRAPH group that does not start with triple patterns starts from the graphs
868
+ # (18.5: P is evaluated in each graph, so its OPTIONAL's left, VALUES, BIND and a UNION
869
+ # branch with no triple pattern are, too).
870
+ g = base
871
+ filters: List[Any] = []
872
+ outer, self.narrowed = self.narrowed, self.values_bound(ctx)
873
+ try:
874
+ # One basic graph pattern is the triple patterns a FILTER alone separates (5.1): its typing triples
875
+ # type each of its blocks (`typed-variables`).
876
+ children = list(ctx.getChildren())
877
+ patterns_of: Dict[int, List[Tuple[Term, Term, Term]]] = {}
878
+ typed_of: Dict[int, FrozenSet[str]] = {}
879
+ run: List[int] = []
880
+ for i, child in enumerate(children + [None]):
881
+ if isinstance(child, SparqlParser.TriplesBlockContext):
882
+ patterns_of[i] = self.block_patterns(child)
883
+ run.append(i)
884
+ elif child is None or not isinstance(child, (SparqlParser.Filter_Context, TerminalNode)):
885
+ typed = frozenset().union(*(typing_of(patterns_of[j]) for j in run)) if run else frozenset()
886
+ for j in run:
887
+ typed_of[j] = typed
888
+ run = []
889
+ for i, child in enumerate(children):
890
+ if isinstance(child, SparqlParser.TriplesBlockContext):
891
+ g = join(g, self.triples_block(child, patterns_of[i], typed_of[i]))
892
+ elif isinstance(child, SparqlParser.Filter_Context):
893
+ filters.append(child)
894
+ elif isinstance(child, SparqlParser.GraphPatternNotTriplesContext):
895
+ mine, self.narrowed = self.narrowed, {} # a nested scope is evaluated on its own
896
+ try:
897
+ g = self.not_triples(g, child)
898
+ finally:
899
+ self.narrowed = mine
900
+ finally:
901
+ self.narrowed = outer
902
+ return g, filters
903
+
904
+ @home("values-bound-subject", "A group's one-row VALUES IRI read as a constant by the group's own triple patterns",
905
+ terms=("VALUES", "inline data", "describe", "constant subject", "VALUES-bound", "substitution",
906
+ "prune", "map pruning"))
907
+ def values_bound(self, ctx) -> Dict[str, IRITerm]:
908
+ """The variables an inline VALUES block of exactly one row binds to an IRI in the group `ctx`. The
909
+ group is the join of its elements (18.2.2.6), the VALUES block one of them, so each of the group's own
910
+ triple patterns may read that IRI in the variable's place, the variable then bound to it beside
911
+ (`triples_block`): the same rows, and the pattern's maps are chosen by the constant (`_may_give`, the
912
+ maps whose terms cannot be it read nothing), as `<iri> ?p ?o` is. Not inside an OPTIONAL, MINUS,
913
+ UNION or nested group (each evaluated on its own: an OPTIONAL's unmatched rows leave the variable
914
+ unbound, which the VALUES row then binds), nor where the variable is a parameter, nor under
915
+ owl:sameAs (a constant matches its component's members, a variable as the engine's mode binds it).
916
+ Only under an ontology: there a variable predicate's union of saturated maps is what a constant lets R10
917
+ merge into a scan per source (the turbofan walk: 1384 CTEs, refused, to 282). Without one the join is
918
+ cheaper on DuckDB: the merged scans write each member's term as text (0.5 s against 0.6 s on the
919
+ turbofan engine), so the VALUES row is left to the join, as are several rows: `meeting` drops the
920
+ branches no row can meet."""
921
+ if self.identity is not None or self.ontology is None:
922
+ return {}
923
+ out: Dict[str, IRITerm] = {}
924
+ for child in ctx.getChildren():
925
+ if not isinstance(child, SparqlParser.GraphPatternNotTriplesContext) or child.inlineData() is None:
926
+ continue
927
+ block = self.values(child.inlineData().dataBlock())
928
+ if len(block.rows) != 1:
929
+ continue
930
+ for name, term in zip(block.names, block.rows[0]):
931
+ if isinstance(term, IRITerm) and name not in self.params and out.get(name, term) == term:
932
+ out[name] = term
933
+ return out
934
+
935
+ def not_triples(self, g: O.Op, ctx) -> O.Op:
936
+ if ctx.optionalGraphPattern() is not None:
937
+ # Only the group's own filters are the LeftJoin's condition: a
938
+ # filter in a group inside it stays there (18.2.2.6, before the
939
+ # simplification of 18.2.2.8).
940
+ a, parsed = self.group_parts(ctx.optionalGraphPattern().groupGraphPattern())
941
+ filters = [self.condition(f.constraint()) for f in parsed]
942
+ if filters:
943
+ return O.LeftJoin(g, a, filters[0] if len(filters) == 1
944
+ else E.And(tuple(filters)))
945
+ return O.LeftJoin(g, a)
946
+ if ctx.groupOrUnionGraphPattern() is not None:
947
+ parts = [self.group(p) for p in ctx.groupOrUnionGraphPattern().groupGraphPattern()]
948
+ u = parts[0] if len(parts) == 1 else O.Union(tuple(parts))
949
+ return join(g, u)
950
+ if ctx.bind() is not None:
951
+ b = ctx.bind()
952
+ name = b.var_().getText()[1:]
953
+ if name in g.columns or name in self.params:
954
+ raise QueryError(f"BIND to ?{name}, which the group binds already (18.2.1)")
955
+ return O.Extend(g, name, self.expression(b.expression()))
956
+ if ctx.inlineData() is not None:
957
+ return join(g, self.values(ctx.inlineData().dataBlock()))
958
+ if ctx.minusGraphPattern() is not None:
959
+ # 18.2.2.6 / 18.5: Minus(G, the pattern).
960
+ within = (self.graph.name,) if isinstance(self.graph, Var) else ()
961
+ return O.Minus(g, self.group(ctx.minusGraphPattern().groupGraphPattern()), within)
962
+ if ctx.graphGraphPattern() is not None:
963
+ return join(g, self.graph_pattern(ctx.graphGraphPattern()))
964
+ raise NotBuilt(f"the pattern {ctx.getText()[:40]}")
965
+
966
+ def graph_pattern(self, gp) -> O.Op:
967
+ """18.2.2 / 18.5: Graph(g, P): P's triple patterns read the named graph g. Where P binds no
968
+ ?g (`GRAPH ?g {}`, only BIND or VALUES) it is joined with each named graph's name; where P
969
+ has no triple pattern, `GRAPH <g> { P }` is P if <g> is a named graph of the dataset, else
970
+ no solution (18.5's eval of Graph(IRI, P))."""
971
+ graph = self.term(gp.varOrIRIref())
972
+ # GRAPH ?g { P }: P is evaluated in each named graph, its triple patterns sharing a hidden
973
+ # graph column h, and only then joined with {?g -> that graph} (18.5): a ?g that P binds
974
+ # itself (in an OPTIONAL, say) must be compatible with the graph, not taken as it.
975
+ active = Var(self.fresh("graph")) if isinstance(graph, Var) else graph
976
+ outer, self.graph = self.graph, active
977
+ # The regime is the active graph's (Entailment Regimes 9: entailed by it, not the merge): the ontology's
978
+ # T-mapping rewriting (the maps' saturated classes and properties) and its domain and range typing are
979
+ # the default graph's; inside GRAPH the maps are read as declared.
980
+ outer_maps, outer_typing = self.maps, self.typing
981
+ self.maps, self.typing = self.maps.__dict__.get("saturated_from") or self.maps, []
982
+ before = self.patterns
983
+ try:
984
+ self.group_base = self.graph_names(active.name if isinstance(active, Var) else None)
985
+ inner = self.group(gp.groupGraphPattern())
986
+ self.group_base = None
987
+ if isinstance(graph, Var) and active.name not in inner.columns:
988
+ inner = join(inner, self.graph_names(active.name))
989
+ elif not isinstance(graph, Var) and self.patterns == before:
990
+ inner = join(inner, self.graph_names(None))
991
+ finally:
992
+ self.graph = outer
993
+ self.maps, self.typing = outer_maps, outer_typing
994
+ if not isinstance(graph, Var):
995
+ return inner
996
+ h, g = active.name, graph.name
997
+ kept = tuple(c for c in inner.columns if c not in (h, g))
998
+ if g in inner.columns:
999
+ inner = O.Filter(inner, E.Or((E.Not(E.IsBound(E.Col(g))),
1000
+ E.Compare("=", E.Col(g), E.Col(h), E.TERM))))
1001
+ inner = O.Project(inner, tuple(c for c in inner.columns if c != g))
1002
+ return O.Project(O.Extend(inner, g, E.Col(h)), kept + (g,))
1003
+
1004
+ def graph_names(self, name: Optional[str]) -> O.Op:
1005
+ """The named graphs of the dataset under the active GRAPH (those some map puts a triple
1006
+ in), one row each as `name`; for a GRAPH <g>, one row of no column if <g> is one. Under a dataset
1007
+ clause they are the FROM NAMED graphs, each a graph of the dataset even where no map gives it (13.2)."""
1008
+ if self.dataset is not None:
1009
+ listed = sorted(self.dataset[1]) if name is not None else \
1010
+ [self.graph.text] if isinstance(self.graph, IRITerm) and self.graph.text in self.dataset[1] else []
1011
+ return O.Values((name,) if name is not None else (),
1012
+ tuple((IRITerm(i),) for i in listed) if name is not None else ((),) * len(listed))
1013
+ x, y = Var(self.fresh("x")), Var(self.fresh("y"))
1014
+ self.plain += 1 # the graphs' names: the triples as stated give them all
1015
+ try:
1016
+ spo = self.any_predicate(x, Var(self.fresh("p")), y)
1017
+ finally:
1018
+ self.plain -= 1
1019
+ if name is None:
1020
+ return O.Distinct(O.Project(spo, ()))
1021
+ return O.Distinct(O.Project(spo, (name,)))
1022
+
1023
+ def bound(self, x: O.Op) -> O.Op:
1024
+ """`x` with the call's parameters joined (`translate`'s `bindings`): the top-level group's
1025
+ pattern, before its FILTERs (18.2.2.6), so that a FILTER and its EXISTS see the values, as
1026
+ with RDF4J's and Jena's initial bindings."""
1027
+ return join(x, self.bindings) if self.bindings is not None else x
1028
+
1029
+ def trailing_values(self, x: O.Op, ctx) -> O.Op:
1030
+ """A query's trailing VALUES (18.2.4.1: P := Join(P, ToMultiSet(data)),
1031
+ before grouping and the solution modifiers)."""
1032
+ if ctx is None or ctx.dataBlock() is None:
1033
+ return x
1034
+ return join(x, self.values(ctx.dataBlock()))
1035
+
1036
+ def values(self, ctx) -> O.Op:
1037
+ if ctx.inlineDataOneVar() is not None:
1038
+ one = ctx.inlineDataOneVar()
1039
+ names = (one.var_().getText()[1:],)
1040
+ rows = tuple((self.data_value(v),) for v in one.dataBlockValue())
1041
+ return O.Values(names, rows)
1042
+ full = ctx.inlineDataFull()
1043
+ names = tuple(v.getText()[1:] for v in full.var_())
1044
+ rows: List[Tuple[Any, ...]] = []
1045
+ row: List[Any] = []
1046
+ in_data = False # past the '{': a '()' before it is the empty variable list, not a row
1047
+ for child in full.getChildren():
1048
+ if child.getText() == "{":
1049
+ in_data = True
1050
+ elif isinstance(child, SparqlParser.DataBlockValueContext):
1051
+ row.append(self.data_value(child))
1052
+ elif child.getText() == ")" and names and len(row) == len(names):
1053
+ rows.append(tuple(row))
1054
+ row = []
1055
+ elif "".join(child.getText().split()) == "()" and in_data: # NIL keeps inner spaces: ( )
1056
+ rows.append(())
1057
+ if any(len(r) != len(names) for r in rows):
1058
+ raise QueryError("a VALUES row has another number of values than variables")
1059
+ return O.Values(names, tuple(rows))
1060
+
1061
+ def data_value(self, ctx) -> Any:
1062
+ if ctx.UNDEF() is not None:
1063
+ return None
1064
+ t = self.term(ctx.graphTerm())
1065
+ if isinstance(t, Var):
1066
+ raise QueryError("a blank node in VALUES")
1067
+ return t
1068
+
1069
+ # -- triples ----------------------------------------------------------------
1070
+
1071
+ by_value = False # datatypes="value": a pattern's literal matches every literal of its value (`by_value_block`)
1072
+ unnamed = False # unnamed individuals may satisfy hidden terms (`with_witnesses`)
1073
+ occurrences: Any = None
1074
+ select_star = False
1075
+ typed_vars: FrozenSet[str] = frozenset() # the variables a typing triple of the block being planned types
1076
+ direct_ok = True # false under an untyped variable predicate (`any_predicate`)
1077
+
1078
+ def block_patterns(self, ctx) -> List[Tuple[Term, Term, Term]]:
1079
+ patterns: List[Tuple[Term, Term, Term]] = []
1080
+ for same in ctx.triplesSameSubject():
1081
+ self.same_subject(same, patterns)
1082
+ return patterns
1083
+
1084
+ def triples_block(self, ctx, patterns: Optional[List[Tuple[Term, Term, Term]]] = None,
1085
+ typed: Optional[FrozenSet[str]] = None) -> O.Op:
1086
+ """A triples block's plan. A variable typed in its basic graph pattern (Entailment Regimes 7.1.3: `?x a
1087
+ owl:Class`, `owl:ObjectProperty`, `owl:DatatypeProperty`, `owl:NamedIndividual`; the blocks a FILTER
1088
+ alone separates, `typed`) is read with Direct semantics' consequences (`ontology_rows`); declarations
1089
+ are local to the basic graph pattern."""
1090
+ if patterns is None:
1091
+ patterns = self.block_patterns(ctx)
1092
+ held = self.typed_vars
1093
+ self.typed_vars = typing_of(patterns) if typed is None else typed
1094
+ try:
1095
+ return self._triples_block(patterns, ctx)
1096
+ finally:
1097
+ self.typed_vars = held
1098
+
1099
+ def _triples_block(self, patterns: List[Tuple[Term, Term, Term]], ctx) -> O.Op:
1100
+ if self.unnamed and self.graph is None and self.ql_tbox() is not None:
1101
+ # A sequence or inverse of IRIs is its triples (18.2.2.4), its middle terms hidden ones.
1102
+ patterns = [t for x in patterns for t in (self.path_triples(*x) or [x])]
1103
+ witnessed = self.with_witnesses(patterns, ctx)
1104
+ if witnessed is not None:
1105
+ return witnessed
1106
+ if self.by_value:
1107
+ # A sequence or inverse of IRIs is its triples (18.2.2.4), so its literal end matches by value too.
1108
+ patterns = [t for x in patterns for t in (self.path_triples(*x) or [x])]
1109
+ if _value_joined(patterns):
1110
+ return self.by_value_block(patterns)
1111
+ return self.block_of(patterns)
1112
+
1113
+ def path_triples(self, s: Term, p: Any, o: Term) -> Optional[List[Tuple[Term, Term, Term]]]:
1114
+ """A sequence or inverse path of IRIs as its triple patterns, joined through fresh hidden terms (18.2.2.4);
1115
+ None for any other path, or a triple."""
1116
+ if not isinstance(p, Path):
1117
+ return None
1118
+ if p.kind == "inv":
1119
+ inner = p.parts[0]
1120
+ return [(o, inner, s)] if isinstance(inner, IRITerm) else None
1121
+ if p.kind != "seq":
1122
+ return None
1123
+ out: List[Tuple[Term, Term, Term]] = []
1124
+ here = s
1125
+ for i, step in enumerate(p.parts):
1126
+ there = o if i == len(p.parts) - 1 else Var(self.fresh("q"))
1127
+ if isinstance(step, IRITerm):
1128
+ out.append((here, step, there))
1129
+ else:
1130
+ more = self.path_triples(here, step, there)
1131
+ if more is None:
1132
+ return None
1133
+ out += more
1134
+ here = there
1135
+ return out
1136
+
1137
+ def by_value_block(self, patterns: List[Tuple[Term, Term, Term]]) -> O.Op:
1138
+ """A triples block under datatypes="value" (D-entailment, OWL 2 RL's dt-eq): each literal object is a
1139
+ hidden variable whose value is the literal's (`owl_rl_rules.same_value`, OWL 2's datatype map: the integer
1140
+ types and xsd:decimal one value space, xsd:float and xsd:double each its own, any other literal by its
1141
+ term). Only matching changes: the block binds what it did."""
1142
+ from prograph.plan.ir.owl_rl_rules import least_literal
1143
+
1144
+ def expression(text: str, names: Dict[str, str], condition: bool = True) -> E.Expr:
1145
+ tree = parse("SELECT * WHERE { FILTER(" + text + ") }" if condition else
1146
+ "SELECT (" + text + " AS ?pgout) WHERE { }")
1147
+ if condition:
1148
+ group = tree.selectQuery().whereClause().groupGraphPattern()
1149
+ found = next(c for c in group.getChildren() if isinstance(c, SparqlParser.Filter_Context))
1150
+ return E.renamed(self.condition(found.constraint()), names)
1151
+ return E.renamed(self.expression(tree.selectQuery().selectClause()[0].expression()), names)
1152
+
1153
+ conds: List[E.Expr] = []
1154
+ out_patterns: List[Tuple[Term, Term, Term]] = []
1155
+ joined = _value_joined(patterns)
1156
+ copies: Dict[str, List[str]] = {v: [] for v in joined}
1157
+ for s, p, o in patterns:
1158
+ if isinstance(o, Var) and o.name in joined:
1159
+ # A variable two objects share: each a copy of its own, one value (`same_values`).
1160
+ name = self.fresh("lv")
1161
+ if copies[o.name]:
1162
+ conds.append(E.Call("sameValue", (E.Col(copies[o.name][0]), E.Col(name))))
1163
+ copies[o.name].append(name)
1164
+ o = Var(name)
1165
+ out_patterns.append((s, p, o))
1166
+ out = self.block_of(out_patterns)
1167
+ if conds:
1168
+ out = O.Filter(out, conds[0] if len(conds) == 1 else E.And(tuple(conds)))
1169
+ # One row per solution (Entailment Regimes 2: a solution's multiplicity is its blank nodes' instance
1170
+ # mappings'): the triples one value matches under several literals give it once, and a shared variable
1171
+ # shows the least of every literal of its value the solution matched (by term order).
1172
+ shown = [c for c in out.columns if not c.startswith(HIDDEN + "lv")]
1173
+ if not copies:
1174
+ return O.Distinct(O.Project(out, tuple(shown)))
1175
+ keys = []
1176
+ for v, names in copies.items():
1177
+ out = O.Extend(out, HIDDEN + "lvk" + v, E.Call("valueKey", (E.Col(names[0]),)))
1178
+ out = O.Extend(out, HIDDEN + "lvl" + v, _least_of(names, expression, least_literal))
1179
+ keys.append(HIDDEN + "lvk" + v)
1180
+ out = O.Aggregate(out, tuple(shown) + tuple(keys),
1181
+ tuple((v, E.Agg("MIN", E.Col(HIDDEN + "lvl" + v))) for v in copies))
1182
+ order = [c for c in patterns_columns(patterns) if c in out.columns]
1183
+ return O.Project(out, tuple(order + [c for c in out.columns if c not in order and not c.startswith(HIDDEN + "lv")]))
1184
+
1185
+ # -- datatypes="value": DISTINCT and GROUP BY by value ------------------------------------------------------
1186
+
1187
+ def value_columns(self, x: O.Op, columns) -> List[str]:
1188
+ """The columns of `x` among `columns` whose terms may be one value under two terms (`value-identity-in-sql`):
1189
+ literals of two of xsd:decimal's value space's datatypes, or a number, boolean or dateTime held as its text
1190
+ (a lexical form may be other than canonical, a dateTime in another timezone), or of no known datatype."""
1191
+ from prograph.plan.ir.facts import facts
1192
+ from prograph.schema.source_map import LITERAL as LITERAL_KIND
1193
+ from prograph.schema import xsd
1194
+ numbers = xsd.INTEGER_DATATYPES | {xsd.DECIMAL, xsd.DOUBLE, xsd.FLOAT}
1195
+ decimals = xsd.INTEGER_DATATYPES | {xsd.DECIMAL}
1196
+ slots = facts(x).slots
1197
+ out = []
1198
+ for c in columns:
1199
+ s = slots.get(c)
1200
+ if s is None or LITERAL_KIND not in s.kinds:
1201
+ continue
1202
+ if UNKNOWN in s.datatypes or len(s.datatypes & decimals) > 1 or \
1203
+ (s.datatypes & (numbers | {xsd.BOOLEAN, xsd.DATETIME}) and s.native is None):
1204
+ out.append(c)
1205
+ return out
1206
+
1207
+ def value_key(self, name: str) -> E.Expr:
1208
+ """One text per value of the column `name` (`lower.value_key_text`, rule `value-identity-in-sql`)."""
1209
+ return E.Call("valueKey", (E.Col(name),))
1210
+
1211
+ def grouped_by_value(self, x: O.Op, keys: List[str], aggregates) -> O.Op:
1212
+ """GROUP BY under datatypes="value": a key that may hold one value under two terms groups by its value
1213
+ (`value_key`) and shows the least of its terms (MIN: ORDER BY's order breaks a tie between equal numbers
1214
+ by datatype IRI, then lexical form)."""
1215
+ by_value = set(self.value_columns(x, keys))
1216
+ group, extra = [], []
1217
+ for k in keys:
1218
+ if k in by_value:
1219
+ key = self.fresh("vk")
1220
+ x = O.Extend(x, key, self.value_key(k))
1221
+ group.append(key)
1222
+ extra.append((k, E.Agg("MIN", E.Col(k))))
1223
+ else:
1224
+ group.append(k)
1225
+ out = O.Aggregate(x, tuple(group), tuple(extra) + tuple(aggregates))
1226
+ return O.Project(out, tuple(k for k in keys) + tuple(n for n, _ in aggregates))
1227
+
1228
+ def distinct_by_value(self, x: O.Op) -> O.Op:
1229
+ """SELECT DISTINCT under datatypes="value": rows one value column by column are one row (`grouped_by_value`
1230
+ over every column)."""
1231
+ return self.grouped_by_value(x, list(x.columns), ())
1232
+
1233
+ def block_of(self, patterns: List[Tuple[Term, Term, Term]]) -> O.Op:
1234
+ """A triples block's plan: its patterns joined in connected order."""
1235
+ written = patterns
1236
+ patterns = _connected(patterns)
1237
+ out: O.Op = O.Unit()
1238
+ for i, (s, p, o) in enumerate(patterns):
1239
+ # A path whose start (or else end) the patterns before it bind walks from those terms
1240
+ # only (`?c a :Place ; :isPartOf* ?top`), as pre-1.0.0 chose its starts first.
1241
+ self.path_seed = None
1242
+ # The variables the patterns before bind as a triple's subject or object: graph nodes,
1243
+ # so their terms may also be a `*` path's zero-length pairs (18.4 takes the graph's
1244
+ # nodes; a predicate or a path's end may hold a term that is none, M4 review).
1245
+ nodes = {v.name for s0, p0, o0 in patterns[:i] if not isinstance(p0, Path)
1246
+ for v in (s0, o0) if isinstance(v, Var)}
1247
+ if isinstance(p, Path) and not isinstance(self.graph, Var):
1248
+ for side, t in (("start", s), ("end", o)):
1249
+ if isinstance(t, Var) and t.name in out.columns:
1250
+ # Under a name of its own: a column named as a SQL alias (`s`) is read
1251
+ # for the alias inside the EXISTS that seeds the walk (Spark).
1252
+ col = self.fresh("seed")
1253
+ terms = O.Distinct(O.Project(O.Extend(out, col, E.Col(t.name)), (col,)))
1254
+ self.path_seed = (side, t.name, terms, col, t.name in nodes)
1255
+ break
1256
+ out = join(out, self.narrowed_triple(s, p, o))
1257
+ self.path_seed = None
1258
+ if patterns != written:
1259
+ # The columns in the order the patterns were written (SELECT * reads them so), whatever the
1260
+ # join order.
1261
+ first = [v.name for t in written for v in t if isinstance(v, Var)]
1262
+ rank = {c: i for i, c in enumerate(dict.fromkeys(first))}
1263
+ out = O.Project(out, tuple(sorted(out.columns, key=lambda c: rank.get(c, len(rank)))))
1264
+ return out
1265
+
1266
+ @home("implied-individuals", "Hidden terms satisfied by unnamed individuals: tree-witness rewriting under QL",
1267
+ terms=("unnamed individual", "anonymous individual", "existential reasoning", "tree witness",
1268
+ "certain answers", "hidden variable", "blank node", "implied_individuals"))
1269
+ def with_witnesses(self, patterns: List[Tuple[Term, Term, Term]], ctx: Any) -> Optional[O.Op]:
1270
+ """A triples block under `implied_individuals` (profile ql, the default graph): its named answers (the
1271
+ block as without the switch, a bag) and one row more per binding of its other terms that only unnamed
1272
+ individuals satisfy, so the answers with the switch on hold those without it. A hidden term is a blank
1273
+ node, or a variable every occurrence of which is in this block (under no SELECT *). A connected set of
1274
+ hidden terms whose patterns are class and property atoms is a tree: hanging from one term t, it holds of t
1275
+ where t is of a concept the ontology implies an unnamed such tree from (`ql.QLTBox.witnessed`), so its
1276
+ atoms are replaced by `t a B`; hanging from none (detached), it holds where some named individual is of
1277
+ such a concept. None where the block has no such tree."""
1278
+ from itertools import combinations
1279
+ ql = self.ql_tbox()
1280
+ written = Counter_of(patterns)
1281
+ here: Dict[str, int] = {}
1282
+ for n in _var_names(ctx): # the block's own occurrences in the text (`?x a :C ; :p ?y` names ?x once)
1283
+ here[n] = here.get(n, 0) + 1
1284
+ hidden = {n for n in written if n.startswith((HIDDEN + "b", HIDDEN + "q"))
1285
+ or (not self.select_star and n in here and self.occurrences.get(n, 0) == here[n]
1286
+ and n not in self.params)}
1287
+ if not hidden:
1288
+ return None
1289
+ trees = _hidden_trees(patterns, hidden)
1290
+ if not trees:
1291
+ return None
1292
+
1293
+ def subtree(node, adjacency, parent):
1294
+ """(classes, children) of `node`'s side of the tree, away from `parent`."""
1295
+ classes = frozenset(adjacency[node]["classes"])
1296
+ children = tuple((role, *subtree(child, adjacency, node))
1297
+ for role, child in adjacency[node]["edges"] if child != parent)
1298
+ return classes, children
1299
+
1300
+ def side_atoms(node, adjacency, parent, atoms_by, links):
1301
+ """The atoms of `node`'s side away from `parent`: its class atoms and each edge below it, with theirs."""
1302
+ out = list(atoms_by.get(node, ()))
1303
+ for _, child in adjacency[node]["edges"]:
1304
+ if child != parent:
1305
+ out.append(links[(node, child)])
1306
+ out += side_atoms(child, adjacency, node, atoms_by, links)
1307
+ return out
1308
+
1309
+ def holds(term, cs): # `term` of one of the concepts `cs`
1310
+ ops = []
1311
+ for c in sorted(cs, key=str):
1312
+ if c[0] == "c":
1313
+ ops.append(self.triple(term, IRITerm(RDF_TYPE), IRITerm(c[1])))
1314
+ elif c[0] == "e":
1315
+ y = Var(self.fresh("w"))
1316
+ (p, back) = c[1]
1317
+ ops.append(self.triple(y, IRITerm(p), term) if back else self.triple(term, IRITerm(p), y))
1318
+ elif c == ("top",):
1319
+ from prograph.plan.ir.ql import THING
1320
+ ops.append(self.triple(term, IRITerm(RDF_TYPE), IRITerm(THING)))
1321
+ ops = [o for o in ops if not (isinstance(o, O.Values) and not o.rows)]
1322
+ if not ops:
1323
+ return None
1324
+ cols = tuple(v.name for v in (term,) if isinstance(v, Var))
1325
+ return O.Distinct(O.Project(O.Union(tuple(O.Project(o, cols) for o in ops)) if len(ops) > 1
1326
+ else O.Project(ops[0], cols), cols))
1327
+
1328
+ # Each reading: the atoms an unnamed part replaces, and the plan saying its parent implies it. An unnamed
1329
+ # part is the side of a tree edge away from its named parent (the tree's term, or a hidden term then
1330
+ # named): a subtree hanging from the term, or from a hidden term of the tree; a detached tree may also be
1331
+ # implied whole, by a named individual it is not joined to.
1332
+ readings = []
1333
+ for atoms, attach, adjacency, atoms_by, links in trees:
1334
+ cuts = [] # (parent term, role, node, parent node or None)
1335
+ equal_to = ()
1336
+ if attach is not None:
1337
+ root_term, role, top, equal_to = attach
1338
+ cuts.append((root_term, role, top, None))
1339
+ order = [(top, None)]
1340
+ while order:
1341
+ node, parent = order.pop()
1342
+ for r, child in adjacency[node]["edges"]:
1343
+ if child != parent:
1344
+ cuts.append((Var(node), r, child, node))
1345
+ order.append((child, node))
1346
+ else:
1347
+ for node in adjacency:
1348
+ for r, child in adjacency[node]["edges"]:
1349
+ cuts.append((Var(node), r, child, node))
1350
+ for parent_term, role, node, parent in cuts:
1351
+ if isinstance(role, tuple) and role and isinstance(role[0], tuple):
1352
+ roles = role
1353
+ else:
1354
+ roles = (role,)
1355
+ classes, children = subtree(node, adjacency, parent)
1356
+ # One successor reached by every role at once: one axiom's role under each (`witnessed`).
1357
+ implied = holds(parent_term, ql.witnessed(tuple(roles) if len(roles) > 1 else roles[0], classes,
1358
+ children))
1359
+ if implied is None:
1360
+ continue
1361
+ gone = side_atoms(node, adjacency, parent, atoms_by, links) + [links[(parent, node)]]
1362
+ if parent is None:
1363
+ for other in equal_to: # the other atoms' terms: the root itself
1364
+ gone += [a for k, a in links.items() if k[0] == "extra" and k[1] == node]
1365
+ if isinstance(other, Var) and isinstance(parent_term, Var):
1366
+ implied = O.Extend(implied, other.name, E.Col(parent_term.name)) \
1367
+ if other.name != parent_term.name else implied
1368
+ elif isinstance(parent_term, Var):
1369
+ implied = O.Filter(implied, E.Compare("=", E.Col(parent_term.name), E.Const(other), E.TERM))
1370
+ elif other != parent_term:
1371
+ implied = None
1372
+ break
1373
+ if implied is None:
1374
+ continue
1375
+ readings.append((list({id(a): a for a in gone}.values()), implied))
1376
+ if attach is None:
1377
+ found = set()
1378
+ for top in adjacency:
1379
+ classes, children = subtree(top, adjacency, None)
1380
+ for role in ql.role_up:
1381
+ found |= ql.witnessed(role, classes, children)
1382
+ # And every concept with an implied successor of such a concept, at any depth: a named
1383
+ # GraduateStudent's unnamed Professor's unnamed Department holds `?d a :Department`.
1384
+ grew = True
1385
+ while grew:
1386
+ grew = False
1387
+ for concept, implied_by in ql.generates.items():
1388
+ if concept not in found and any(fill & found for _, _, fill in implied_by):
1389
+ found.add(concept)
1390
+ grew = True
1391
+ implied = holds(Var(self.fresh("w")), found)
1392
+ if implied is not None:
1393
+ readings.append((atoms, O.Distinct(O.Project(implied, ()))))
1394
+ if not readings:
1395
+ return None
1396
+ if len(readings) > WITNESS_CAP:
1397
+ # Every combination of readings is a branch: past the cap the rewrite is refused, not cut short.
1398
+ raise NotBuilt(f"a triples block with {len(readings)} ways unnamed individuals may satisfy it (at most "
1399
+ f"{WITNESS_CAP}) under implied_individuals; split the block, or project the hidden "
1400
+ f"variables")
1401
+ named = self.block_of(patterns)
1402
+ shown = tuple(c for c in named.columns if c not in hidden)
1403
+ branches = []
1404
+ for k in range(1, len(readings) + 1):
1405
+ for chosen in combinations(readings, k):
1406
+ ids = [id(a) for atoms, _ in chosen for a in atoms]
1407
+ if len(ids) != len(set(ids)):
1408
+ continue # two readings of one atom: not one model's
1409
+ gone = set(ids)
1410
+ rest = [t for t in patterns if id(t) not in gone]
1411
+ x = self.block_of(rest) if rest else O.Unit()
1412
+ for _, implied in chosen:
1413
+ x = join(x, implied)
1414
+ branches.append(O.Project(x, tuple(c for c in shown if c in x.columns)))
1415
+ implied_rows = O.Distinct(O.Project(O.Union(tuple(branches)) if len(branches) > 1 else branches[0], shown))
1416
+ only_implied = O.AntiJoin(implied_rows, O.Distinct(O.Project(named, shown)))
1417
+ return O.Union((named, only_implied))
1418
+
1419
+ def value_triple(self, s: Term, p: Any, o: LiteralTerm) -> O.Op:
1420
+ """`s p o` under datatypes="value" (`value-identity-in-sql`): every triple whose object is one value with
1421
+ the literal `o`, each solution once (a subject holding 1 and 1.00 matches `1` once). The test is in each
1422
+ branch of the pattern's union, where a map's column is of one type (a native number compares as one)."""
1423
+ name = self.fresh("lv")
1424
+ x = self.narrowed_triple(s, p, Var(name))
1425
+ cond = E.Call("sameValue", (E.Col(name), E.Const(o)))
1426
+ x = _filtered_below(x, cond, name)
1427
+ return O.Distinct(O.Project(x, tuple(c for c in x.columns if c != name)))
1428
+
1429
+ def narrowed_triple(self, s: Term, p: Any, o: Term) -> O.Op:
1430
+ """`triple`, a variable the group's one-row VALUES binds to an IRI read as that IRI and bound to it
1431
+ beside (`values_bound`)."""
1432
+ if self.by_value and isinstance(o, LiteralTerm) and not isinstance(p, Path):
1433
+ return self.value_triple(s, p, o)
1434
+ got = {} if isinstance(p, Path) else \
1435
+ {t.name: self.narrowed[t.name] for t in (s, p, o) if isinstance(t, Var) and t.name in self.narrowed}
1436
+ if not got:
1437
+ return self.triple(s, p, o)
1438
+ x = self.triple(*(self.narrowed.get(t.name, t) if isinstance(t, Var) else t for t in (s, p, o)))
1439
+ if isinstance(x, O.Values) and not x.rows:
1440
+ return _empty(s, p, o)
1441
+ for name, term in got.items():
1442
+ x = O.Extend(x, name, E.Const(term))
1443
+ return x
1444
+
1445
+ def same_subject(self, ctx, out: list) -> None:
1446
+ if ctx.varOrTerm() is not None:
1447
+ subject = self.term(ctx.varOrTerm())
1448
+ self.property_list(subject, ctx.propertyListNotEmpty(), out)
1449
+ else:
1450
+ subject = self.node(ctx.triplesNode(), out)
1451
+ if ctx.propertyList().propertyListNotEmpty() is not None:
1452
+ self.property_list(subject, ctx.propertyList().propertyListNotEmpty(), out)
1453
+
1454
+ def node(self, ctx, out: list) -> Term:
1455
+ """A triplesNode: `[ p o ]` is a fresh blank node and its triples."""
1456
+ if ctx.blankNodePropertyList() is None:
1457
+ raise NotBuilt("an RDF collection in a pattern")
1458
+ b = Var(self.fresh("b"))
1459
+ self.property_list(b, ctx.blankNodePropertyList().propertyListNotEmpty(), out)
1460
+ return b
1461
+
1462
+ def property_list(self, subject: Term, ctx, out: list) -> None:
1463
+ verbs, objects = ctx.verb(), ctx.objectList()
1464
+ for verb, object_list in zip(verbs, objects):
1465
+ predicate, inverse = self.verb(verb)
1466
+ for obj in object_list.object_():
1467
+ node = obj.graphNode()
1468
+ o = self.term(node.varOrTerm()) if node.varOrTerm() is not None \
1469
+ else self.node(node.triplesNode(), out)
1470
+ out.append((o, predicate, subject) if inverse else (subject, predicate, o))
1471
+
1472
+ def verb(self, ctx) -> Tuple[Term, bool]:
1473
+ """A verb as a predicate term, and whether it is inverted (`^p`)."""
1474
+ if ctx.A() is not None:
1475
+ return IRITerm(RDF_TYPE), False
1476
+ if ctx.varOrIRIref() is not None:
1477
+ return self.term(ctx.varOrIRIref()), False
1478
+ path = self.path(ctx.path())
1479
+ if isinstance(path, IRITerm):
1480
+ return path, False
1481
+ if isinstance(path, Path) and path.kind == "inv" and isinstance(path.parts[0], IRITerm):
1482
+ return path.parts[0], True
1483
+ return path, False
1484
+
1485
+ def path(self, ctx) -> TUnion["Path", IRITerm]:
1486
+ """A property path (18.4) as an IRI or a `Path`."""
1487
+ seqs = [self.path_sequence(q) for q in ctx.pathAlternative().pathSequence()]
1488
+ return seqs[0] if len(seqs) == 1 else Path("alt", tuple(seqs))
1489
+
1490
+ def path_sequence(self, ctx) -> TUnion["Path", IRITerm]:
1491
+ steps = []
1492
+ for step in ctx.pathEltOrInverse():
1493
+ elt = step.pathElt()
1494
+ p = self.path_primary(elt.pathPrimary())
1495
+ if elt.pathMod() is not None:
1496
+ p = Path("mod", (p,), elt.pathMod().getText())
1497
+ steps.append(Path("inv", (p,)) if step.CARET() is not None else p)
1498
+ return steps[0] if len(steps) == 1 else Path("seq", tuple(steps))
1499
+
1500
+ def path_primary(self, ctx) -> TUnion["Path", IRITerm]:
1501
+ if ctx.A() is not None:
1502
+ return IRITerm(RDF_TYPE)
1503
+ if ctx.iriRef() is not None:
1504
+ return IRITerm(self.iri(ctx.iriRef()))
1505
+ if ctx.prefixedName() is not None:
1506
+ return IRITerm(self.iri(ctx.prefixedName()))
1507
+ if ctx.path() is not None:
1508
+ return self.path(ctx.path())
1509
+ neg = ctx.pathNegatedPropertySet()
1510
+ # The grammar repeats pathOneInPropertySet, so its accessor gives a list with or without
1511
+ # parentheses (`!a`, `!^ex:p`, `!(a|^ex:p)`).
1512
+ found = neg.pathOneInPropertySet()
1513
+ members = list(found) if isinstance(found, (list, tuple)) else [found]
1514
+ out = []
1515
+ for m in members:
1516
+ iri = RDF_TYPE if m.A() is not None else self.iri(m.iriRef())
1517
+ out.append((iri, m.CARET() is not None))
1518
+ return Path("neg", tuple(out))
1519
+
1520
+ def path_op(self, s: Term, path: "Path", o: Term) -> O.Op:
1521
+ """A property path's solutions (18.4): an inverse swaps the ends, a
1522
+ sequence joins its steps through hidden variables (a bag, as 18.4's
1523
+ join and projection give), an alternative is the union of its
1524
+ branches (a bag), a negated property set the union over every other
1525
+ predicate the maps give. A closure (*, +, ?) is not built yet."""
1526
+ if isinstance(path, IRITerm):
1527
+ return self.triple(s, path, o)
1528
+ if path.kind == "inv":
1529
+ return self.path_or_triple(o, path.parts[0], s)
1530
+ if path.kind == "seq":
1531
+ x: O.Op = O.Unit()
1532
+ here = s
1533
+ for i, step in enumerate(path.parts):
1534
+ there = o if i == len(path.parts) - 1 else Var(self.fresh("m"))
1535
+ x = join(x, self.path_or_triple(here, step, there))
1536
+ here = there
1537
+ return x
1538
+ if path.kind == "alt":
1539
+ return O.Union(tuple(self.path_or_triple(s, b, o) for b in path.parts))
1540
+ if path.kind == "neg":
1541
+ forward = {iri for iri, inv in path.parts if not inv}
1542
+ backward = {iri for iri, inv in path.parts if inv}
1543
+ branches = []
1544
+ if forward or not backward:
1545
+ branches.append(self.other_predicates(s, o, forward))
1546
+ if backward:
1547
+ branches.append(self.other_predicates(o, s, backward))
1548
+ return branches[0] if len(branches) == 1 else O.Union(tuple(branches))
1549
+ if path.kind == "mod":
1550
+ return self.closure(s, path.parts[0], path.mod, o)
1551
+ raise NotBuilt(f"the property path kind {path.kind}")
1552
+
1553
+ def closure(self, s: Term, step_path, mod: str, o: Term) -> O.Op:
1554
+ """p*, p+, p? (18.4: ZeroOrMorePath, OneOrMorePath, ZeroOrOnePath,
1555
+ ALP): the set of (start, end) pairs a walk of an allowed length
1556
+ joins; a zero-length pair (t, t) for each term of the graph, or the
1557
+ bound constant (one-plan.md 5.7)."""
1558
+ lo, hi = {"*": (0, None), "+": (1, None), "?": (0, 1)}[mod]
1559
+ if not isinstance(self.graph, Var):
1560
+ folded = self.ontology_walk(s, step_path, lo, hi, o)
1561
+ if folded is not None:
1562
+ return folded
1563
+ a, b = Var(self.fresh("a")), Var(self.fresh("b"))
1564
+ # Inside GRAPH ?g the path is evaluated in each named graph (18.5 eval(D(G), Graph(var, P))):
1565
+ # one closure taken within each value of g (`Closure.partition`, one-plan 5.7), so that
1566
+ # `GRAPH ?g { ?x p+ ?y }` is one closure, not one per graph.
1567
+ part = (self.graph.name,) if isinstance(self.graph, Var) else ()
1568
+ if part and self.graph in (s, o):
1569
+ raise NotBuilt("a closure inside GRAPH ?g whose end is ?g")
1570
+ # A constant end seeds the walk (from it, not over every pair of the graph).
1571
+ seed = ("start", s) if not isinstance(s, Var) else ("end", o) if not isinstance(o, Var) else None
1572
+ seed_nodes = False
1573
+ if seed is None and self.path_seed is not None and not part:
1574
+ _, name, terms_op, col, seed_nodes = self.path_seed
1575
+ # By the variable, on whichever end it is now: an inverse (`?a ^p* ?b`) swaps the ends.
1576
+ side = "start" if s == Var(name) else "end" if o == Var(name) else None
1577
+ if side is not None:
1578
+ seed = (side, terms_op, col)
1579
+ self.path_seed = None # the outermost closure of the pattern only
1580
+ step = O.Project(self.path_or_triple(a, step_path, b), part + (a.name, b.name))
1581
+ terms = None
1582
+ t = self.fresh("t")
1583
+ if lo == 0 and seed is not None and isinstance(seed[1], O.Op) and seed_nodes:
1584
+ # The zero-length pairs the seeded end needs: its own terms, not the whole graph's
1585
+ # (every term of the graph paired with itself).
1586
+ terms = O.Project(O.Extend(seed[1], t, E.Col(seed[2])), (t,))
1587
+ elif lo == 0:
1588
+ bound = s if not isinstance(s, Var) else (o if not isinstance(o, Var) else None)
1589
+ if bound is not None and not part:
1590
+ spo = None
1591
+ else:
1592
+ x, y = Var(self.fresh("x")), Var(self.fresh("y"))
1593
+ spo = self.any_predicate(x, Var(self.fresh("p")), y)
1594
+ if spo is None:
1595
+ terms = O.Values((t,), ((bound,),))
1596
+ if isinstance(self.graph, IRITerm):
1597
+ # Inside GRAPH <g>, only where <g> is a graph of the dataset (18.5; M3 review).
1598
+ terms = join(terms, self.graph_names(None))
1599
+ elif isinstance(spo, O.Values) and not spo.rows:
1600
+ terms = O.Values(part + (t,), ())
1601
+ elif bound is not None:
1602
+ # A bound end is its own zero-length pair in every graph (ALP), each named graph here.
1603
+ graphs = O.Distinct(O.Project(spo, part))
1604
+ terms = O.Project(O.Extend(graphs, t, E.Const(bound)), part + (t,))
1605
+ else:
1606
+ terms = O.Distinct(O.Union((
1607
+ O.Project(O.Extend(spo, t, E.Col(x.name)), part + (t,)),
1608
+ O.Project(O.Extend(spo, t, E.Col(y.name)), part + (t,)))))
1609
+ walk = O.Closure(step, a.name, b.name, min=lo, max=hi, terms=terms, terms_column=t, partition=part,
1610
+ seed=seed)
1611
+ bound = self.bind(walk, [(s, E.Col(a.name)), (o, E.Col(b.name))])
1612
+ if part:
1613
+ # The graph column the closure carries stays a column of the pattern (GRAPH ?g joins on it).
1614
+ bound = O.Project(bound.input, part + bound.names)
1615
+ return bound
1616
+
1617
+ def path_or_triple(self, s: Term, p, o: Term) -> O.Op:
1618
+ return self.path_op(s, p, o) if isinstance(p, Path) else self.triple(s, p, o)
1619
+
1620
+ def other_predicates(self, s: Term, o: Term, excluded) -> O.Op:
1621
+ """The triples of every predicate the maps give but `excluded`."""
1622
+ if self.maps.open_predicates:
1623
+ raise NotBuilt("a map whose predicate is a column")
1624
+ predicates = [iri for iri in dict.fromkeys(
1625
+ ([RDF_TYPE] if any(m.classes for m in self.maps.nodes) else []) + sorted(self.maps.by_predicate)
1626
+ + self.identity_predicates() + self.ontology_predicates()) if iri not in excluded]
1627
+ parts = [self.triple(s, IRITerm(iri), o, quiet=True) for iri in predicates]
1628
+ parts = [x for x in parts if not (isinstance(x, O.Values) and not x.rows)]
1629
+ if not parts:
1630
+ return _empty(s, Var(self.fresh("p")), o)
1631
+ return parts[0] if len(parts) == 1 else O.Union(tuple(parts))
1632
+
1633
+ def triple(self, s: Term, p: Term, o: Term, quiet: bool = False) -> O.Op:
1634
+ """The solutions of one triple pattern over the maps (18.3), through owl:sameAs where the engine
1635
+ applies it (`identity.pattern`)."""
1636
+ if isinstance(p, Path):
1637
+ return self.path_op(s, p, o)
1638
+ if self.identity is not None and not self.plain and isinstance(p, IRITerm):
1639
+ from prograph.plan.ir.identity import pattern
1640
+ self.patterns += 1
1641
+ self.plain += 1
1642
+ try:
1643
+ return pattern(self, s, p, o, quiet)
1644
+ finally:
1645
+ self.plain -= 1
1646
+ return self.matched(s, p, o, quiet)
1647
+
1648
+ @home("top-object-property", "owl:topObjectProperty under profile ql: every pair of named individuals",
1649
+ terms=("topObjectProperty", "top object property", "universal role", "every pair", "Table 5"))
1650
+ def top_pairs(self, s: Term, o: Term) -> O.Op:
1651
+ """`s owl:topObjectProperty o` under profile ql (OWL 2 Syntax 5.3: it "connects all possible pairs of
1652
+ individuals"): every pair of named individuals (`individual_terms`), in the default graph, the plan marked
1653
+ for the cost guard (`cost.TOP_PROPERTY`: its size is the square of the individuals'). A variable predicate
1654
+ never binds it (`any_predicate`): a labelled known difference, as Jena and Ontop."""
1655
+ terms = self.individual_terms()
1656
+ if terms is None or isinstance(s, LiteralTerm) or isinstance(o, LiteralTerm):
1657
+ return _empty(s, IRITerm(TOP_OBJECT_PROPERTY), o)
1658
+ column, op = terms
1659
+
1660
+ def side(t: Term) -> Optional[O.Op]:
1661
+ if isinstance(t, Var):
1662
+ return O.Project(O.Extend(op, t.name, E.Col(column)), (t.name,))
1663
+ got = self._one_term(op, column, t, [])
1664
+ return None if got is None else O.Distinct(got)
1665
+ a, b = side(s), side(o)
1666
+ if a is None or b is None:
1667
+ return _empty(s, IRITerm(TOP_OBJECT_PROPERTY), o)
1668
+ if isinstance(s, Var) and isinstance(o, Var) and s.name == o.name:
1669
+ out: O.Op = a
1670
+ else:
1671
+ out = O.Join(a, b)
1672
+ if isinstance(s, Var) and isinstance(o, Var) and s.name != o.name:
1673
+ object.__setattr__(out, "_top_property", True) # the square of the individuals (`cost.TOP_PROPERTY`)
1674
+ return out
1675
+
1676
+ def identity_predicates(self) -> List[str]:
1677
+ """owl:sameAs where the engine applies it: its triples are the components' (`identity.same_as`)."""
1678
+ from prograph.plan.ir.identity import SAME_AS
1679
+ return [SAME_AS] if self.identity is not None and not self.plain else []
1680
+
1681
+ def matched(self, s: Term, p: Term, o: Term, quiet: bool = False) -> O.Op:
1682
+ """The solutions of one triple pattern over the maps as stated (18.3)."""
1683
+ if isinstance(p, Path):
1684
+ return self.path_op(s, p, o)
1685
+ self.never = [] # the maps whose objects can never be `o` (`branch`)
1686
+ self.patterns += 1
1687
+ if isinstance(p, Var):
1688
+ return self.any_predicate(s, p, o)
1689
+ if not isinstance(p, IRITerm):
1690
+ return _empty(s, p, o)
1691
+ if p.text == TOP_OBJECT_PROPERTY and self.ql_tbox() is not None and self.graph is None:
1692
+ return self.top_pairs(s, o)
1693
+ branches = self.branches(s, p, o)
1694
+ branches = [b for b in branches if b is not None]
1695
+ if not branches:
1696
+ if isinstance(o, LiteralTerm) and self.never and not quiet:
1697
+ _say_never(p, o, self.never)
1698
+ return _empty(s, p, o)
1699
+ # The graph is a set of triples (RDF 1.1 Concepts 3): a triple two
1700
+ # maps give, or one map gives from two rows, matches once.
1701
+ return O.Distinct(branches[0] if len(branches) == 1 else O.Union(tuple(branches)))
1702
+
1703
+ def branches(self, s: Term, p: IRITerm, o: Term) -> List[Optional[O.Op]]:
1704
+ """Each map's triples for the pattern (s, p, o), p an IRI: one plan per map and graph map."""
1705
+ self.never = []
1706
+ if self.maps.open_predicates:
1707
+ raise NotBuilt("a map whose predicate is a column")
1708
+ branches: List[O.Op] = []
1709
+ if p.text == RDF_TYPE and isinstance(s, Var) and s == o:
1710
+ # `?c a ?c`: the pattern `?c a ?k` whose class is ?c itself, bound once (each typing branch binds
1711
+ # its subject and its class as two columns).
1712
+ k = Var(self.fresh("class"))
1713
+ same = E.Compare("=", E.Col(s.name), E.Col(k.name), E.TERM)
1714
+ return [O.Project(O.Filter(b, same), tuple(c for c in b.columns if c != k.name))
1715
+ for b in self.branches(s, p, k) if b is not None]
1716
+ if p.text == RDF_TYPE:
1717
+ # rdfs:Resource, which saturation gives every class (the RDFS core), is the default graph's alone:
1718
+ # inside GRAPH matching is simple.
1719
+ from prograph.plan.ir.saturate import RESOURCE
1720
+ core_only = RESOURCE if self.graph is not None and self.ontology is not None else None
1721
+ declared = {d.id: d.classes for d in getattr(self.maps.__dict__.get("saturated_from"), "nodes", ())}
1722
+ for m in self.maps.nodes:
1723
+ for cls in m.classes:
1724
+ if cls == core_only and cls not in declared.get(m.id, ()):
1725
+ continue
1726
+ if isinstance(o, Var) or o == IRITerm(cls):
1727
+ for gm in self.graph_choices(m.graph_maps, m.graph):
1728
+ branches.append(self.branch(m, m.subject, None, s, o, cls, gm))
1729
+ if p.text == RDF_TYPE and self.typing:
1730
+ branches += self.typed_by_property(s, o)
1731
+ if p.text == RDF_TYPE and self.ontology is not None:
1732
+ if self.graph is None: # the ontology's subclasses are the default graph's (graph_pattern)
1733
+ branches += self.typed_by_superclass(s, o)
1734
+ branches += self.typed_resource(s, o)
1735
+ ql = self.ql_tbox()
1736
+ if ql is not None and p.text == DIFFERENT_FROM and self.graph is None and not self.differing and s != o:
1737
+ # Under 'ql' owl:differentFrom is what Direct semantics entails (`ql.difference`), never a ≠ a; a
1738
+ # pattern of one term on both sides reads the stated triples (a stated a ≠ a is an inconsistency,
1739
+ # `ql.SELF_DIFFERENT`; a consistent graph has none).
1740
+ return [self.different_branch(s, o, ql)]
1741
+ if ql is not None and p.text == RDF_TYPE:
1742
+ branches += self.typed_everything(s, o, ql)
1743
+ branches += self.typed_by_data(s, o, ql)
1744
+ if self.reads_ontology():
1745
+ branches += self.ontology_rows(s, p, o)
1746
+ for giver in self.maps.by_predicate.get(p.text, ()):
1747
+ branches += self.giver_branches(giver, s, o)
1748
+ if ql is not None and p.text != RDF_TYPE and self.graph is None:
1749
+ # Reasoning is the default graph's: inside GRAPH matching is simple.
1750
+ for q in sorted(ql.inverse_subs.get(p.text, ())):
1751
+ branches += self.inverse_branches(q, s, o)
1752
+ if p.text in ql.reflexive:
1753
+ branches += self.reflexive_branches(s, o)
1754
+ return branches
1755
+
1756
+ def different_branch(self, s: Term, o: Term, ql: Any) -> O.Op:
1757
+ """`s owl:differentFrom o` under 'ql': the pairs `ql.QLTBox.different` matches (each of its patterns under
1758
+ the same rewriting; its own owl:differentFrom patterns read the stated triples), bound to `s` and `o`."""
1759
+ from prograph.plan.ir.ql import A_, B_
1760
+ self.differing += 1
1761
+ try:
1762
+ return self.pattern_text(ql.different, ((A_, s), (B_, o)))
1763
+ finally:
1764
+ self.differing -= 1
1765
+
1766
+ def pattern_text(self, text: str, binds: Tuple[Tuple[str, Term], ...]) -> O.Op:
1767
+ """A group pattern the ontology's rewriting writes as SPARQL text (`ql`), translated here, under the same
1768
+ rewriting and graph, its variables of `binds` bound to their terms (a variable, or a constant to match),
1769
+ the others projected away (they are named `qlnd_...`, no query's own)."""
1770
+ tree = parse("SELECT * WHERE { " + text + " }")
1771
+ x = self.group(tree.selectQuery().whereClause().groupGraphPattern())
1772
+ x = O.Project(x, tuple(name for name, _ in binds))
1773
+ keep: List[str] = []
1774
+ for name, t in binds:
1775
+ if isinstance(t, Var):
1776
+ if t.name in keep:
1777
+ x = O.Filter(x, E.Compare("=", E.Col(name), E.Col(t.name), E.TERM))
1778
+ else:
1779
+ x = O.Extend(x, t.name, E.Col(name))
1780
+ keep.append(t.name)
1781
+ else:
1782
+ x = O.Filter(x, E.Compare("=", E.Col(name), E.Const(t), E.TERM))
1783
+ return O.Project(x, tuple(keep))
1784
+
1785
+ def ql_tbox(self) -> Any:
1786
+ """The ontology's QL axioms closed (`ql.QLTBox`) under profile="ql"; None otherwise."""
1787
+ return getattr(self.ontology, "ql", None) if self.ontology is not None else None
1788
+
1789
+ def inverse_branches(self, q: str, s: Term, o: Term) -> List[Optional[O.Op]]:
1790
+ """`s p o` from `o q s`, q ⊑ p⁻ (owl:inverseOf, a symmetric property, a subproperty of an inverse:
1791
+ `ql.QLTBox.inverse_subs`): q's triples with subject and object swapped, those whose object is a literal
1792
+ aside (a literal is never a subject)."""
1793
+ if isinstance(o, LiteralTerm) or isinstance(s, LiteralTerm):
1794
+ return [] # a literal is never a subject, nor the object of an object property
1795
+ found: List[Optional[O.Op]] = []
1796
+ for giver in self.maps.by_predicate.get(q, ()):
1797
+ found += self.giver_branches(giver, o, s)
1798
+ if self.reads_ontology():
1799
+ found += self.ontology_rows(o, IRITerm(q), s)
1800
+ found = [x for x in found if x is not None]
1801
+ if not isinstance(s, Var):
1802
+ return found
1803
+ term = E.Col(s.name)
1804
+ return [O.Filter(x, E.Or((E.Call("isIRI", (term,)), E.Call("isBLANK", (term,))))) for x in found]
1805
+
1806
+ def reflexive_branches(self, s: Term, o: Term) -> List[O.Op]:
1807
+ """`s p o` for a reflexive p (or one a reflexive property is under): every individual related to itself
1808
+ (`individual_terms`), in the default graph."""
1809
+ if self.graph is not None or isinstance(s, LiteralTerm) or isinstance(o, LiteralTerm):
1810
+ return []
1811
+ if not isinstance(s, Var) and not isinstance(o, Var) and s != o:
1812
+ return []
1813
+ terms = self.individual_terms()
1814
+ if terms is None:
1815
+ return []
1816
+ column, op = terms
1817
+ for t in (s, o):
1818
+ if not isinstance(t, Var):
1819
+ one = self._one_term(op, column, t, [v for v in (s, o) if isinstance(v, Var)])
1820
+ return [one] if one is not None else []
1821
+ x: O.Op = O.Extend(op, s.name, E.Col(column))
1822
+ if o.name != s.name:
1823
+ x = O.Extend(x, o.name, E.Col(column))
1824
+ return [O.Project(x, tuple(dict.fromkeys((s.name, o.name))))]
1825
+
1826
+ def _one_term(self, op: O.Op, column: str, t: Term, bound: List[Var]) -> Optional[O.Op]:
1827
+ """`op`'s rows whose `column` is the term `t`, each variable of `bound` bound to it: only the branches whose
1828
+ terms may be `t` are read (`_met`), one row at most; None where none may."""
1829
+ provenance = const_slot(t).provenance
1830
+ if provenance:
1831
+ op = _met(op, {column: provenance})
1832
+ if op is None:
1833
+ return None
1834
+ x: O.Op = O.Filter(op, E.Compare("=", E.Col(column), E.Const(t), E.TERM))
1835
+ for v in bound:
1836
+ x = O.Extend(x, v.name, E.Col(column))
1837
+ return O.Project(x, tuple(dict.fromkeys(v.name for v in bound)))
1838
+
1839
+ def typed_everything(self, s: Term, o: Term, ql: Any) -> List[O.Op]:
1840
+ """`s rdf:type C` for owl:Thing and each class every individual is (`⊤ ⊑ C`: `ql.QLTBox.everything`):
1841
+ every individual (`individual_terms`), in the default graph."""
1842
+ from prograph.plan.ir.ql import THING
1843
+ classes = sorted({THING} | set(ql.everything))
1844
+ wanted = classes if isinstance(o, Var) else [o.text] if isinstance(o, IRITerm) and o.text in classes else []
1845
+ if isinstance(o, IRITerm) and o.text == NAMED_INDIVIDUAL:
1846
+ # Every individual is a named one (OWL 2 asks no declaration of an individual): `?i a
1847
+ # owl:NamedIndividual` lists them. It is a declaration, no class: `?x a ?c` does not give it.
1848
+ wanted = [o.text]
1849
+ if not wanted or self.graph is not None or isinstance(s, LiteralTerm):
1850
+ return []
1851
+ terms = self.individual_terms()
1852
+ if terms is None:
1853
+ return []
1854
+ column, op = terms
1855
+ x = self._one_term(op, column, s, []) if not isinstance(s, Var) else \
1856
+ O.Project(O.Extend(op, s.name, E.Col(column)), (s.name,))
1857
+ if x is None:
1858
+ return []
1859
+ if not isinstance(s, Var):
1860
+ x = O.Distinct(x) # whether `s` is an individual: one row
1861
+ return [O.Extend(x, o.name, E.Const(IRITerm(c))) if isinstance(o, Var) else x for c in wanted]
1862
+
1863
+ def typed_by_data(self, s: Term, o: Term, ql: Any) -> List[O.Op]:
1864
+ """`s rdf:type C` from `∃d.DR ⊑ C` (a data property's value of some datatypes types its subject:
1865
+ `ql.QLTBox.data_typed`): the subjects of d's triples whose value's datatype is one of DR's."""
1866
+ from prograph.plan.ir.ql import in_range
1867
+ out: List[O.Op] = []
1868
+ for prop, datatypes, classes in ql.data_typed:
1869
+ wanted = sorted(classes) if isinstance(o, Var) else \
1870
+ [o.text] if isinstance(o, IRITerm) and o.text in classes else []
1871
+ if not wanted or isinstance(s, LiteralTerm):
1872
+ continue
1873
+ # The values of DR by value (`ql.in_range`: "1.0"^^xsd:decimal is an xsd:integer).
1874
+ x = self.pattern_text(f"?qlnd_s <{prop}> ?qlnd_v . FILTER({in_range('qlnd_v', datatypes)})",
1875
+ (("qlnd_s", s),))
1876
+ x = O.Project(x, self._typed_columns(s, x))
1877
+ out += [O.Extend(x, o.name, E.Const(IRITerm(c))) if isinstance(o, Var) else x for c in wanted]
1878
+ return out
1879
+
1880
+ @home("every-individual", "Every individual: what owl:Thing, a reflexive property and `⊤ ⊑ C` range over",
1881
+ terms=("every individual", "owl:Thing", "named individual", "reflexive property", "individual_terms",
1882
+ "domain of discourse"))
1883
+ def individual_terms(self) -> Optional[Tuple[str, O.Op]]:
1884
+ """Every individual of the default graph, as one column of one plan (kept per dataset): each IRI that is
1885
+ the subject of a triple the maps give, or the object of one whose predicate is not rdf:type (that object
1886
+ is a class), and each IRI the ontology asserts about or declares an owl:NamedIndividual
1887
+ (`ql.QLTBox.individuals`). Literals are values; blank nodes are no individual (OWL 2 QL has no anonymous
1888
+ individual, Profiles §3.2.1). The maps as declared are read, not their saturated copies (the same rows)."""
1889
+ key = (self.dataset, self.witnessing, self.plain)
1890
+ held = self.__dict__.setdefault("_individual_terms", {})
1891
+ if key in held:
1892
+ return held[key]
1893
+ column = self.fresh("it")
1894
+ parts: List[O.Op] = []
1895
+
1896
+ def add(x: Optional[O.Op], term: str) -> None:
1897
+ if x is None or (isinstance(x, O.Values) and not x.rows):
1898
+ return
1899
+ x = O.Filter(x, E.Call("isIRI", (E.Col(term),)))
1900
+ parts.append(O.Project(O.Extend(x, column, E.Col(term)), (column,)))
1901
+ declared = self.maps.__dict__.get("saturated_from") or self.maps
1902
+ for m in declared.nodes:
1903
+ if m.classes and m.subject.term_type == IRI:
1904
+ for gm in self.graph_choices(m.graph_maps, m.graph):
1905
+ a, b = Var(self.fresh("a")), Var(self.fresh("b"))
1906
+ add(self.branch(m, m.subject, None, a, b, m.classes[0], gm), a.name)
1907
+ for predicate, givers in sorted(declared.by_predicate.items()):
1908
+ for giver in givers:
1909
+ # Only a term map that makes IRIs gives individuals, so the column holds IRIs alone.
1910
+ node = isinstance(giver.map, NodeMap)
1911
+ subject_iri = giver.map.subject.term_type == IRI if node else True
1912
+ object_iri = giver.predicate_object.object.term_type == IRI if node else True
1913
+ classed = node and bool(giver.map.classes)
1914
+ a, b = Var(self.fresh("a")), Var(self.fresh("b"))
1915
+ for x in self.giver_branches(giver, a, b):
1916
+ if not classed and subject_iri:
1917
+ add(x, a.name)
1918
+ if predicate != RDF_TYPE and object_iri:
1919
+ add(x, b.name)
1920
+ ql = self.ql_tbox()
1921
+ if ql is not None and ql.individuals and self.reads_ontology():
1922
+ parts.append(O.Values((column,), tuple((IRITerm(i),) for i in sorted(ql.individuals))))
1923
+ held[key] = (column, O.Distinct(parts[0] if len(parts) == 1 else O.Union(tuple(parts)))) if parts else None
1924
+ return held[key]
1925
+
1926
+ def giver_branches(self, giver: Any, s: Term, o: Term) -> List[Optional[O.Op]]:
1927
+ """One `SourceMaps.by_predicate` entry's triples for the pattern (s, its predicate, o): a plan per
1928
+ graph map (None where the map can give none)."""
1929
+ m = giver.map
1930
+ out: List[Optional[O.Op]] = []
1931
+ if isinstance(m, NodeMap):
1932
+ po: PredicateObject = giver.predicate_object
1933
+ if po.predicate_map is not None:
1934
+ raise NotBuilt("a predicate map")
1935
+ for gm in self.graph_choices(po.graph_maps, po.graph):
1936
+ out.append(self.branch(m, m.subject, po.object, s, o, None, gm))
1937
+ else:
1938
+ for gm in self.graph_choices(m.graph_maps, m.graph):
1939
+ out.append(self.edge_branch(m, s, o, gm))
1940
+ return out
1941
+
1942
+ def reads_ontology(self) -> bool:
1943
+ """Whether a pattern here reads the ontology's triples: in the default graph (also one FROM builds),
1944
+ never inside GRAPH, nor in a witness (`ontology_rows`)."""
1945
+ return self.onto_triples is not None and self.graph is None and not self.witnessing
1946
+
1947
+ def ontology_predicates(self) -> List[str]:
1948
+ """The predicates of the ontology's triples a pattern here reads (`reads_ontology`)."""
1949
+ return self.onto_triples.predicates if self.reads_ontology() else []
1950
+
1951
+ def ontology_rows(self, s: Term, p: IRITerm, o: Term) -> List[O.Op]:
1952
+ """The ontology's triples matching (s, p, o) (`dataset.ontology_triples`) as constant rows: those
1953
+ stated, and those a witness of the data's vocabulary adds, each only where its witness has a triple
1954
+ in the data (a class the maps give, `?w rdf:type C`; a predicate, `?w p ?v`)."""
1955
+ held = self.onto_triples
1956
+ rows = held.stated.get(p.text, ())
1957
+ direct = held.direct.get(p.text)
1958
+ top = p.text == SUB_PROPERTY_IRI and o == IRITerm(TOP_OBJECT_PROPERTY) # every object property: a TBox answer
1959
+ if direct and not top and not (self.direct_ok and all(t.name in self.typed_vars for t in (s, o) if isinstance(t, Var))):
1960
+ # What Direct semantics adds (owl:Thing above a class, owl:Nothing below it ...) is a typed variable's
1961
+ # (or a pattern of constants'): an untyped one reads the RDFS core's graph (`typed-variables`).
1962
+ rows = tuple(r for r in rows if r not in direct)
1963
+ out: List[O.Op] = list(self.ontology_values(s, o, rows))
1964
+ # The witnessed rows that match the pattern, each with its witness's id in a column of their own, kept
1965
+ # where that witness holds (`witnesses`): one branch, whatever the number of witnesses; a witness no
1966
+ # matching row needs is not planned.
1967
+ tagged = []
1968
+ for w, pairs in sorted(held.witnessed.get(p.text, {}).items()):
1969
+ tag = LiteralTerm(f"{w[0]} {w[1]}")
1970
+ tagged += [(a, b, tag, w) for a, b in pairs if (isinstance(s, Var) or s == a)
1971
+ and (isinstance(o, Var) or o == b)]
1972
+ named, out_data = self.data_class_rows(s, p, o)
1973
+ tagged += named
1974
+ out += out_data
1975
+ ids = [w for w in dict.fromkeys(t[3] for t in tagged) if self.witness(w) is not None]
1976
+ tagged = [t for t in tagged if t[3] in ids]
1977
+ if tagged:
1978
+ column = self.witness_column()
1979
+ for rows in self.ontology_values(s, o, tagged, column):
1980
+ parts = [self.holds(w) for w in ids]
1981
+ held_ids = parts[0] if len(parts) == 1 else O.Union(tuple(parts))
1982
+ out.append(O.Project(O.SemiJoin(rows, held_ids), rows.columns[:-1]))
1983
+ return out
1984
+
1985
+ def data_class_rows(self, s: Term, p: IRITerm, o: Term) -> Tuple[List[tuple], List[O.Op]]:
1986
+ """The triples a class the data gives from a row adds that match (s, p, o) (`OntologyTriples.data_class`:
1987
+ `C rdf:type rdfs:Class` by rdfs3 over rdf:type's range, and what follows): where the pattern names the
1988
+ class, its rows tagged with its witness for `ontology_rows` (`?w rdf:type C`, an existence test); where
1989
+ it binds the class, the rows of the classes the data gives (`data_classes`). None without such a map."""
1990
+ from prograph.plan.ir.dataset import DATA_CLASS
1991
+ pairs = self.onto_triples.data_class.get(p.text, ())
1992
+ if not pairs or not self._type_givers():
1993
+ return [], []
1994
+ tagged: List[tuple] = []
1995
+ ops: List[O.Op] = []
1996
+ for a, b in pairs:
1997
+ named = [t for t, v in ((s, a), (o, b)) if v == DATA_CLASS and not isinstance(t, Var)]
1998
+ if named:
1999
+ c = named[0]
2000
+ if all(t == c for t in named) and self.may_be_data_class(c):
2001
+ pair = tuple(c if v == DATA_CLASS else v for v in (a, b))
2002
+ tagged.append(pair + (LiteralTerm(f"class {c.text}"), ("class", c.text)))
2003
+ continue
2004
+ if any(not isinstance(t, Var) and t != v for t, v in ((s, a), (o, b))):
2005
+ continue
2006
+ bound: Dict[str, Any] = {}
2007
+ equal: List[Term] = [] # a variable twice, as the class and a constant: the class is the constant
2008
+ for t, v in ((s, a), (o, b)):
2009
+ if isinstance(t, Var) and bound.setdefault(t.name, v) != v:
2010
+ equal.append(v if bound[t.name] == DATA_CLASS else bound[t.name])
2011
+ held = self.data_classes()
2012
+ if held is None:
2013
+ break
2014
+ column, x = held
2015
+ for c in equal:
2016
+ x = O.Filter(x, E.Compare("=", E.Col(column), E.Const(c), E.TERM))
2017
+ for name, v in bound.items():
2018
+ x = O.Extend(x, name, E.Col(column) if v == DATA_CLASS else E.Const(v))
2019
+ ops.append(O.Project(x, tuple(bound)))
2020
+ return tagged, ops
2021
+
2022
+ def may_be_data_class(self, c: Term) -> bool:
2023
+ """Whether `c`'s triples as a class the data gives from a row are not held already: an IRI no stated
2024
+ triple types `rdfs:Class` and no map gives as its class (whose witness has them)."""
2025
+ from prograph.plan.ir.rdfs import CLASS, RESOURCE
2026
+ if not isinstance(c, IRITerm):
2027
+ return False
2028
+ held = self.__dict__.get("_stated_classes")
2029
+ if held is None:
2030
+ held = {a for a, b in self.onto_triples.stated.get(RDF_TYPE, ()) if b == IRITerm(CLASS)}
2031
+ held |= {IRITerm(k) for m in self.maps.nodes for k in m.classes}
2032
+ held.add(IRITerm(RESOURCE))
2033
+ self.__dict__["_stated_classes"] = held
2034
+ return c not in held
2035
+
2036
+ def data_classes(self) -> Optional[Tuple[str, O.Op]]:
2037
+ """The classes the data gives from a row (the IRI and blank-node objects of the rdf:type triples a map
2038
+ takes from a column or template, in the default graph), as one column of one plan, distinct, which every
2039
+ pattern of the query that binds such a class shares; None for none."""
2040
+ key = (self.dataset, self.plain)
2041
+ held = self.__dict__.setdefault("_data_classes", {})
2042
+ if key in held:
2043
+ return held[key]
2044
+ column = self.fresh("dc")
2045
+ parts: List[O.Op] = []
2046
+ for giver in self._type_givers():
2047
+ m = giver.map
2048
+ holder = giver.predicate_object if isinstance(m, NodeMap) else m
2049
+ for gm in self.graph_choices(holder.graph_maps, holder.graph):
2050
+ a, b = Var(self.fresh("a")), Var(self.fresh("b"))
2051
+ x = (self.branch(m, m.subject, holder.object, a, b, None, gm) if isinstance(m, NodeMap)
2052
+ else self.edge_branch(m, a, b, gm))
2053
+ if x is None or (isinstance(x, O.Values) and not x.rows):
2054
+ continue
2055
+ x = O.Filter(x, E.Or((E.Call("isIRI", (E.Col(b.name),)), E.Call("isBLANK", (E.Col(b.name),)))))
2056
+ parts.append(O.Project(O.Extend(x, column, E.Col(b.name)), (column,)))
2057
+ held[key] = (column, O.Distinct(parts[0] if len(parts) == 1 else O.Union(tuple(parts)))) if parts else None
2058
+ return held[key]
2059
+
2060
+ def witness_column(self) -> str:
2061
+ """The column a witness's id is in (`holds`), one per query."""
2062
+ if "_witness_column" not in self.__dict__:
2063
+ self.__dict__["_witness_column"] = self.fresh("wid")
2064
+ return self.__dict__["_witness_column"]
2065
+
2066
+ def holds(self, w: Tuple[str, str]) -> O.Op:
2067
+ """One row, `w`'s id (as `ontology_rows` writes it) in `witness_column`, where `w`'s witness has a
2068
+ triple: an existence test (no witness is read past its first row), planned once per query and graph and
2069
+ shared by the patterns that need it."""
2070
+ key = (w, repr(self.graph), self.dataset, self.plain)
2071
+ cache = self.__dict__.setdefault("_holds", {})
2072
+ if key not in cache:
2073
+ tag = O.Values((self.witness_column(),), ((LiteralTerm(f"{w[0]} {w[1]}"),),))
2074
+ cache[key] = O.SemiJoin(tag, O.Slice(self.witness(w), limit=1))
2075
+ return cache[key]
2076
+
2077
+ def witness(self, w: Tuple[str, str]) -> Optional[O.Op]:
2078
+ """The data's triples that witness `w` (`dataset.OntologyTriples`): `?w rdf:type C` for a class,
2079
+ `?w p ?v` for a predicate; None where no map can give one. Read as an existence test (`holds`)."""
2080
+ kind, name = w
2081
+ key = (w, repr(self.graph), self.dataset, self.plain)
2082
+ cache = self.__dict__.setdefault("_witnesses", {})
2083
+ if key in cache:
2084
+ return cache[key]
2085
+ self.witnessing += 1
2086
+ never = getattr(self, "never", []) # the pattern being matched keeps its own (`branch`)
2087
+ try:
2088
+ x = Var(self.fresh("w"))
2089
+ op = (self.matched(x, IRITerm(RDF_TYPE), IRITerm(name), quiet=True) if kind == "class"
2090
+ else self.matched(x, IRITerm(name), Var(self.fresh("w")), quiet=True))
2091
+ finally:
2092
+ self.witnessing -= 1
2093
+ self.never = never
2094
+ if isinstance(op, O.Distinct):
2095
+ op = op.input # an existence test (`holds`): the triples need not be told apart
2096
+ cache[key] = None if isinstance(op, O.Values) and not op.rows else op
2097
+ return cache[key]
2098
+
2099
+ def ontology_walk(self, s: Term, step: Any, lo: int, hi: Optional[int], o: Term) -> Optional[O.Op]:
2100
+ """p*, p+ or p? from a constant end over a predicate only the ontology's triples give (no map gives
2101
+ it: `?c rdfs:subClassOf* :C`), walked here over those triples (`dataset.ontology_triples`), each end
2102
+ reached kept with the witnesses its walk needs; None where the walk is the plan's (`O.Closure`)."""
2103
+ if not self.reads_ontology() or (self.identity is not None and not self.plain):
2104
+ return None
2105
+ inverse = isinstance(step, Path) and step.kind == "inv" and isinstance(step.parts[0], IRITerm)
2106
+ p = step.parts[0] if inverse else step
2107
+ if not isinstance(p, IRITerm) or p.text in (RDF_TYPE, TOP_OBJECT_PROPERTY) or p.text in self.maps.by_predicate:
2108
+ return None # owl:topObjectProperty's pairs are `top_pairs`', not the ontology's rows
2109
+ if isinstance(s, Var) and isinstance(o, Var):
2110
+ return None
2111
+ forward = not isinstance(s, Var)
2112
+ start, other = (s, o) if forward else (o, s)
2113
+ # Direct semantics' pairs where the walk's variable end is typed (`typed-variables`).
2114
+ typed = self.direct_ok and (not isinstance(other, Var) or other.name in self.typed_vars)
2115
+ if forward == inverse: # walking against the triples' direction
2116
+ pairs = [(b, a, w) for a, b, w in self._ontology_pairs(p.text, typed)]
2117
+ else:
2118
+ pairs = self._ontology_pairs(p.text, typed)
2119
+ edges: Dict[Any, List[Tuple[Any, frozenset]]] = {}
2120
+ for a, b, w in pairs:
2121
+ edges.setdefault(a, []).append((b, w))
2122
+ # A class the data gives from a row (`data_class_rows`): its own steps, where the walk reaches it; a
2123
+ # step to any such class is the plan's walk.
2124
+ from prograph.plan.ir.dataset import DATA_CLASS
2125
+ steps = self.onto_triples.data_class.get(p.text, ()) if self._type_givers() else ()
2126
+ steps = [(b, a) for a, b in steps] if forward == inverse else list(steps)
2127
+ to_data = {a for a, b in steps if a != DATA_CLASS and b == DATA_CLASS}
2128
+
2129
+ def steps_from(node) -> List[Tuple[Any, frozenset]]:
2130
+ out = edges.get(node, [])
2131
+ if steps and self.may_be_data_class(node):
2132
+ w = frozenset({("class", node.text)})
2133
+ out = out + [(node if b == DATA_CLASS else b, w) for a, b in steps if a == DATA_CLASS]
2134
+ return out
2135
+ reached: Dict[Any, List[frozenset]] = {}
2136
+
2137
+ def keep(node, label) -> bool:
2138
+ held = reached.setdefault(node, [])
2139
+ if any(h <= label for h in held):
2140
+ return False
2141
+ held[:] = [h for h in held if not label <= h] + [label]
2142
+ return True
2143
+ frontier = [(start, frozenset())]
2144
+ if lo == 0:
2145
+ keep(start, frozenset())
2146
+ depth = 0
2147
+ while frontier and (hi is None or depth < hi):
2148
+ depth += 1
2149
+ nxt = []
2150
+ for node, label in frontier:
2151
+ for b, w in steps_from(node):
2152
+ if keep(b, label | w):
2153
+ nxt.append((b, label | w))
2154
+ frontier = nxt
2155
+ if to_data & set(reached):
2156
+ return None
2157
+ groups: Dict[frozenset, List[tuple]] = {}
2158
+ for node, labels in reached.items():
2159
+ if not isinstance(other, Var) and node != other:
2160
+ continue
2161
+ for label in labels:
2162
+ groups.setdefault(label, []).append((node,) if isinstance(other, Var) else ())
2163
+ names = (other.name,) if isinstance(other, Var) else ()
2164
+ branches: List[O.Op] = []
2165
+ for label, rows in sorted(groups.items(), key=lambda kv: sorted(kv[0])):
2166
+ x: Optional[O.Op] = O.Values(names, tuple(dict.fromkeys(rows)))
2167
+ for w in sorted(label):
2168
+ witness = self.witness(w)
2169
+ x = None if witness is None or x is None else O.SemiJoin(x, self.holds(w))
2170
+ if x is not None:
2171
+ branches.append(x)
2172
+ if not branches:
2173
+ return O.Values(names, ())
2174
+ return O.Distinct(branches[0] if len(branches) == 1 else O.Union(tuple(branches)))
2175
+
2176
+ def _ontology_pairs(self, predicate: str, typed: bool = True) -> List[Tuple[Any, Any, frozenset]]:
2177
+ """The ontology's (subject, object) pairs of `predicate`, each with the witnesses it needs; without
2178
+ `typed`, none only Direct semantics gives (`typed-variables`)."""
2179
+ held = self.onto_triples
2180
+ direct = () if typed else held.direct.get(predicate, ())
2181
+ out = [(a, b, frozenset()) for a, b in held.stated.get(predicate, ()) if (a, b) not in direct]
2182
+ for w, pairs in held.witnessed.get(predicate, {}).items():
2183
+ out += [(a, b, frozenset({w})) for a, b in pairs]
2184
+ return out
2185
+
2186
+ def ontology_values(self, s: Term, o: Term, pairs, tag: Optional[str] = None) -> List[O.Op]:
2187
+ """The pairs (subject, object) matching `s` and `o`, as rows of the pattern's variables (and, with
2188
+ `tag`, the pair's third member in that column): none, or one Values; under owl:sameAs two, the rows no
2189
+ IRI of which may be an alias (`Identity.may_alias`) apart, so their branch needs no alias join
2190
+ (`identity.pattern`)."""
2191
+ names = tuple(dict.fromkeys(t.name for t in (s, o) if isinstance(t, Var))) + ((tag,) if tag else ())
2192
+ rows: List[tuple] = []
2193
+ for pair in pairs:
2194
+ subject, obj = pair[0], pair[1]
2195
+ if (not isinstance(s, Var) and s != subject) or (not isinstance(o, Var) and o != obj):
2196
+ continue
2197
+ if isinstance(s, Var) and isinstance(o, Var) and s.name == o.name and subject != obj:
2198
+ continue
2199
+ bound = {}
2200
+ if isinstance(s, Var):
2201
+ bound[s.name] = subject
2202
+ if isinstance(o, Var):
2203
+ bound[o.name] = obj
2204
+ if tag:
2205
+ bound[tag] = pair[2]
2206
+ rows.append(tuple(bound[n] for n in names))
2207
+ rows = list(dict.fromkeys(rows))
2208
+ if not rows:
2209
+ return []
2210
+ if self.identity is None:
2211
+ return [O.Values(names, tuple(rows))]
2212
+ aliased = {r for r in rows if any(isinstance(x, IRITerm) and self.identity.may_alias(x.text) for x in r)}
2213
+ may, never = [r for r in rows if r in aliased], [r for r in rows if r not in aliased]
2214
+ return [O.Values(names, tuple(part)) for part in (never, may) if part]
2215
+
2216
+ def _type_givers(self) -> List[Any]:
2217
+ """The maps giving rdf:type whose object comes from the row (a column or template, not a class)."""
2218
+ return [g for g in self.maps.by_predicate.get(RDF_TYPE, ())
2219
+ if (isinstance(g.map, NodeMap) and g.predicate_object.object.constant is None)
2220
+ or (isinstance(g.map, EdgeMap) and g.map.end.term is not None and g.map.end.term.constant is None)]
2221
+
2222
+ def typed_resource(self, s: Term, o: Term) -> List[O.Op]:
2223
+ """`s rdf:type rdfs:Resource` (rdfs4a, rdfs4b) for the terms no class `saturate` closes types so
2224
+ already: the subjects and IRI and blank-node objects of the properties `typing_properties` names, and
2225
+ of the rdf:type triples a map gives from the row. One plan of those terms per graph the patterns read
2226
+ (`resource_terms`), which every such pattern of the query shares."""
2227
+ from prograph.plan.ir.saturate import RESOURCE
2228
+ if self.ontology.closure is None or self.graph is not None or \
2229
+ not (isinstance(o, Var) or o == IRITerm(RESOURCE)):
2230
+ return [] # the RDFS core holds in the default graph: inside GRAPH matching is simple
2231
+ terms = self.resource_terms()
2232
+ if terms is None:
2233
+ return []
2234
+ column, op = terms
2235
+ graph = (self.graph.name,) if isinstance(self.graph, Var) and self.graph.name in op.columns else ()
2236
+ if isinstance(s, Var):
2237
+ x = O.Project(O.Extend(op, s.name, E.Col(column)), (s.name,) + graph)
2238
+ else:
2239
+ # The terms that may be `s` alone: a branch whose terms cannot be it (`meeting`'s walk) goes.
2240
+ op = _met(op, {column: const_slot(s).provenance}) if const_slot(s).provenance else op
2241
+ if op is None:
2242
+ return []
2243
+ x = O.Project(O.Filter(op, E.Compare("=", E.Col(column), E.Const(s), E.TERM)), graph)
2244
+ return [O.Extend(x, o.name, E.Const(IRITerm(RESOURCE))) if isinstance(o, Var) else x]
2245
+
2246
+ def resource_terms(self) -> Optional[Tuple[str, O.Op]]:
2247
+ """The terms `typed_resource` types, as one column of one plan, kept per graph the patterns read; None
2248
+ for none."""
2249
+ from prograph.plan.ir.saturate import RESOURCE
2250
+ key = (repr(self.graph), self.dataset, self.witnessing, self.plain)
2251
+ held = self.__dict__.setdefault("_resource_terms", {})
2252
+ if key in held:
2253
+ return held[key]
2254
+ column = self.fresh("rt")
2255
+ graph = (self.graph.name,) if isinstance(self.graph, Var) else ()
2256
+ parts: List[O.Op] = []
2257
+
2258
+ def add(x: Optional[O.Op], term: str, as_object: bool) -> None:
2259
+ if x is None or (isinstance(x, O.Values) and not x.rows):
2260
+ return
2261
+ if as_object:
2262
+ x = O.Filter(x, E.Or((E.Call("isIRI", (E.Col(term),)), E.Call("isBLANK", (E.Col(term),)))))
2263
+ x = O.Extend(x, column, E.Col(term))
2264
+ parts.append(O.Project(x, (column,) + tuple(g for g in graph if g in x.columns)))
2265
+ from prograph.plan.ir.saturate import resource_typed
2266
+ for prop, classes, as_subject in self.typing:
2267
+ if RESOURCE in classes:
2268
+ # Only the maps whose terms no class of theirs types rdfs:Resource already (`resource_typed`):
2269
+ # the others' are the class branches' rows, which the pattern's Distinct keeps once.
2270
+ a, b = Var(self.fresh("a")), Var(self.fresh("b"))
2271
+ found = [x for g in self.maps.by_predicate.get(prop, ()) if not resource_typed(self.maps, g, as_subject)
2272
+ for x in self.giver_branches(g, a, b) if x is not None]
2273
+ if self.reads_ontology():
2274
+ found += self.ontology_rows(a, IRITerm(prop), b)
2275
+ if found:
2276
+ add(O.Distinct(found[0] if len(found) == 1 else O.Union(tuple(found))),
2277
+ a.name if as_subject else b.name, not as_subject)
2278
+ for giver in self._type_givers():
2279
+ m = giver.map
2280
+ holder = giver.predicate_object if isinstance(m, NodeMap) else m
2281
+ for gm in self.graph_choices(holder.graph_maps, holder.graph):
2282
+ a, b = Var(self.fresh("a")), Var(self.fresh("b"))
2283
+ x = (self.branch(m, m.subject, holder.object, a, b, None, gm) if isinstance(m, NodeMap)
2284
+ else self.edge_branch(m, a, b, gm))
2285
+ add(x, a.name, False)
2286
+ add(x, b.name, True)
2287
+ if parts:
2288
+ cols = parts[0].columns
2289
+ parts = [x if x.columns == cols else O.Project(x, cols) for x in parts]
2290
+ held[key] = (column, O.Distinct(parts[0] if len(parts) == 1 else O.Union(tuple(parts)))) if parts else None
2291
+ return held[key]
2292
+
2293
+ def typed_by_superclass(self, s: Term, o: Term) -> List[O.Op]:
2294
+ """`s rdf:type o` through the ontology's subclasses where a map's rdf:type object comes from
2295
+ the row (a column or template, not a class `saturate` closes): for each subclass C of a
2296
+ wanted class D, that map's triples typed C, typed D too (rdfs9; M3 review)."""
2297
+ out: List[O.Op] = []
2298
+ givers = self._type_givers()
2299
+ if not givers:
2300
+ return out
2301
+ pairs = [(IRITerm(sub), IRITerm(sup)) for sub, sups in sorted(self.ontology.superclasses.items())
2302
+ for sup in sorted(sups - {sub}) if isinstance(o, Var) or o == IRITerm(sup)]
2303
+ for giver in givers:
2304
+ m = giver.map
2305
+ holder = giver.predicate_object if isinstance(m, NodeMap) else m
2306
+ made = holder.object if isinstance(m, NodeMap) else m.end.term
2307
+ mine = [(a, b) for a, b in pairs if made is None or _may_give(made, a)]
2308
+ if not mine:
2309
+ continue
2310
+ for gm in self.graph_choices(holder.graph_maps, holder.graph):
2311
+ if len(mine) == 1:
2312
+ sub, sup = mine[0]
2313
+ x = (self.branch(m, m.subject, holder.object, s, sub, None, gm)
2314
+ if isinstance(m, NodeMap) else self.edge_branch(m, s, sub, gm))
2315
+ if x is None:
2316
+ continue
2317
+ x = O.Project(x, self._typed_columns(s, x))
2318
+ out.append(O.Extend(x, o.name, E.Const(sup)) if isinstance(o, Var) else x)
2319
+ continue
2320
+ # Several subclasses: the map's triples once, each typed object joined to the superclasses it
2321
+ # stands for (one Values of (subclass, superclass) pairs), not a branch per pair (the turbofan
2322
+ # ontology's 178 pairs read the RL graph's rdf:type triples 178 times).
2323
+ c = Var(self.fresh("sub"))
2324
+ x = (self.branch(m, m.subject, holder.object, s, c, None, gm)
2325
+ if isinstance(m, NodeMap) else self.edge_branch(m, s, c, gm))
2326
+ if x is None:
2327
+ continue
2328
+ if isinstance(o, Var):
2329
+ table = O.Values((c.name, o.name), tuple(mine))
2330
+ else:
2331
+ table = O.Values((c.name,), tuple(dict.fromkeys((a,) for a, _ in mine)))
2332
+ x = O.Join(x, table)
2333
+ out.append(O.Project(x, self._typed_columns(s, x) + ((o.name,) if isinstance(o, Var) else ())))
2334
+ return out
2335
+
2336
+ def _typed_columns(self, s: Term, x: O.Op) -> Tuple[str, ...]:
2337
+ """A typing branch's columns: its subject's, and GRAPH ?g's where the branch reads it."""
2338
+ return ((s.name,) if isinstance(s, Var) else ()) + \
2339
+ ((self.graph.name,) if isinstance(self.graph, Var) and self.graph.name in x.columns else ())
2340
+
2341
+ def typed_by_property(self, s: Term, o: Term) -> List[O.Op]:
2342
+ """`s rdf:type o` from an ontology's domain and range (rdfs2, rdfs3): for each
2343
+ property the maps give with a domain class C, the subjects of its own triple pattern typed C
2344
+ (its range: its IRI and blank-node objects), so a term is typed only where its triple
2345
+ exists, in the graph the pattern reads. One plan of the property's terms serves each of its
2346
+ classes; rdfs:Resource is `typed_resource`'s."""
2347
+ from prograph.plan.ir.saturate import RESOURCE
2348
+ out: List[O.Op] = []
2349
+ for prop, classes, as_subject in self.typing:
2350
+ wanted = sorted(classes - {RESOURCE}) if isinstance(o, Var) else \
2351
+ [o.text] if isinstance(o, IRITerm) and o.text in classes and o.text != RESOURCE else []
2352
+ if not wanted:
2353
+ continue
2354
+ y = Var(self.fresh("y"))
2355
+ x = self.matched(s, IRITerm(prop), y) if as_subject else self.matched(y, IRITerm(prop), s)
2356
+ if isinstance(x, O.Values) and not x.rows:
2357
+ continue
2358
+ if not as_subject and isinstance(s, Var):
2359
+ term = E.Col(s.name)
2360
+ x = O.Filter(x, E.Or((E.Call("isIRI", (term,)), E.Call("isBLANK", (term,)))))
2361
+ cols = ((s.name,) if isinstance(s, Var) else ()) + \
2362
+ ((self.graph.name,) if isinstance(self.graph, Var) and self.graph.name in x.columns else ())
2363
+ x = O.Project(x, cols)
2364
+ for cls in wanted:
2365
+ out.append(O.Extend(x, o.name, E.Const(IRITerm(cls))) if isinstance(o, Var) else x)
2366
+ return out
2367
+
2368
+ def any_predicate(self, s: Term, p: Var, o: Term) -> O.Op:
2369
+ """A variable predicate's triples: each predicate's (`_any_predicate`), Direct semantics' consequences only
2370
+ where the predicate variable is typed too (`typed-variables`)."""
2371
+ held = self.direct_ok
2372
+ self.direct_ok = held and p.name in self.typed_vars
2373
+ try:
2374
+ return self._any_predicate(s, p, o)
2375
+ finally:
2376
+ self.direct_ok = held
2377
+
2378
+ def _any_predicate(self, s: Term, p: Var, o: Term) -> O.Op:
2379
+ """A pattern whose predicate is a variable (18.3): for each predicate
2380
+ the maps give (rdf:type where a map has a class), that predicate's
2381
+ triples with `p` bound to it; their union. The predicates differ, so
2382
+ the branches share no triple."""
2383
+ if self.maps.open_predicates:
2384
+ raise NotBuilt("a map whose predicate is a column")
2385
+ if any(isinstance(t, Var) and t.name == p.name for t in (s, o)):
2386
+ raise NotBuilt("a variable that is a pattern's predicate and its subject or object")
2387
+ ql = self.ql_tbox()
2388
+ # Under 'ql' a property no map gives may have triples: from an inverse (`inverse_branches`) or reflexivity.
2389
+ derived = sorted(set(ql.reflexive) | {p for p, subs in ql.inverse_subs.items() if subs}) if ql is not None else []
2390
+ predicates = list(dict.fromkeys(
2391
+ ([RDF_TYPE] if any(m.classes for m in self.maps.nodes) or self.typing or ql is not None else [])
2392
+ + sorted(self.maps.by_predicate) + self.identity_predicates() + self.ontology_predicates() + derived))
2393
+ # owl:topObjectProperty relates every pair of individuals; a variable predicate never binds it (a labelled
2394
+ # known difference: `top_pairs`).
2395
+ predicates = [x for x in predicates if x != TOP_OBJECT_PROPERTY]
2396
+ parts: List[O.Op] = []
2397
+ named = self.predicate_names(predicates)
2398
+ per_graph = self.identity is not None and not self.plain and isinstance(self.graph, Var)
2399
+ if per_graph:
2400
+ # Inside GRAPH of a variable each graph has its own components: every predicate answers under the
2401
+ # names of its component in the graph it is read in (identity.predicate_members_by_graph).
2402
+ from prograph.plan.ir.identity import _in_graphs, predicate_members_by_graph
2403
+ self.identity.prefetch(predicates)
2404
+ for iri in predicates:
2405
+ if iri not in named:
2406
+ continue # a member of a predicate's component answered with it (eq-rep-p)
2407
+ x = self.triple(s, IRITerm(iri), o, quiet=True)
2408
+ if isinstance(x, O.Values) and not x.rows:
2409
+ continue
2410
+ if per_graph:
2411
+ for name, graphs in sorted(predicate_members_by_graph(self, iri).items()):
2412
+ parts.append(_in_graphs(self, O.Extend(x, p.name, E.Const(IRITerm(name))), graphs))
2413
+ continue
2414
+ for name in named[iri]:
2415
+ parts.append(O.Extend(x, p.name, E.Const(IRITerm(name))))
2416
+ if not parts:
2417
+ return _empty(s, p, o)
2418
+ if per_graph:
2419
+ return O.Distinct(parts[0] if len(parts) == 1 else O.Union(tuple(parts)))
2420
+ return parts[0] if len(parts) == 1 else O.Union(tuple(parts))
2421
+
2422
+ def predicate_names(self, predicates: List[str]) -> Dict[str, List[str]]:
2423
+ """Under owl:sameAs (eq-rep-p), the predicate whose pattern answers each component of `predicates`
2424
+ (its first), with the names it is answered under: every member of the component. Without, each
2425
+ predicate its own. Inside GRAPH of a variable the components differ by graph: each predicate its own
2426
+ (`predicates`' names are then read per graph)."""
2427
+ if self.identity is None or self.plain or isinstance(self.graph, Var):
2428
+ return {iri: [iri] for iri in predicates}
2429
+ self.identity.prefetch(predicates)
2430
+ from prograph.plan.ir.identity import predicate_members
2431
+ out: Dict[str, List[str]] = {}
2432
+ seen: set = set()
2433
+ for iri in predicates:
2434
+ if iri in seen:
2435
+ continue
2436
+ members = predicate_members(self, iri)
2437
+ seen |= members
2438
+ out[iri] = sorted(members)
2439
+ return out
2440
+
2441
+ def branch(self, m: NodeMap, subject: TermMap, obj: Optional[TermMap], s: Term, o: Term,
2442
+ cls: Optional[str], gm: Optional[TermMap] = None) -> Optional[O.Op]:
2443
+ """One node map's triples for the pattern: subject `subject` and
2444
+ object `obj`'s term (or the class `cls`)."""
2445
+ self.scope_of(gm)
2446
+ if not _may_give(subject, s) or (obj is not None and not _may_give(obj, o)):
2447
+ return None
2448
+ what = ill_typed_override(obj, m.source.table, self.held)
2449
+ if what:
2450
+ # R2RML 11: the processor aborts an operation that returns or inspects the term.
2451
+ from prograph.r2rml.compile import R2RMLDataError
2452
+ raise R2RMLDataError(what)
2453
+ maps = [subject] + ([obj] if obj is not None else []) + self.graph_read(gm)
2454
+ scan, names = self.scan(m, maps)
2455
+ x: O.Op = scan
2456
+ terms = [(s, self.make(subject, names))]
2457
+ if obj is not None or o != IRITerm(cls): # the pattern's class is the map's own: nothing to test
2458
+ terms.append((o, self.make(obj, names) if obj is not None else E.Const(IRITerm(cls))))
2459
+ terms += self.graph_term(gm, names)
2460
+ if obj is not None and isinstance(o, LiteralTerm):
2461
+ # A constant no term of the map can be: the map gives none of the
2462
+ # pattern's triples, and the pattern is said to match nothing.
2463
+ made = expr_slot(terms[1][1], scan.slots)
2464
+ if never_one_term(made, const_slot(o)):
2465
+ self.never.append((m, made))
2466
+ return None
2467
+ return self.bind(x, terms)
2468
+
2469
+ def graph_choices(self, graph_maps: Tuple[TermMap, ...], graph: Optional[str]) -> List[Optional[TermMap]]:
2470
+ """The graphs a triple of these graph maps (or a YAML schema's `graph`) is read in, for the graph the
2471
+ patterns read (`self.graph`) and the query's dataset: `dataset.graph_choices`."""
2472
+ return graph_choices(graphs_of(graph_maps, graph), self.graph, self.dataset, self.union_default)
2473
+
2474
+ def graph_read(self, gm: Optional[TermMap]) -> List[TermMap]:
2475
+ """A graph map whose term the row gives: its columns are read, and required (a row with
2476
+ no graph value gives no triple by that map, R2RML 11.1)."""
2477
+ return [gm] if gm is not None and gm.constant is None else []
2478
+
2479
+ def graph_term(self, gm: Optional[TermMap], names: Dict[str, str]) -> List[Tuple[Term, E.Expr]]:
2480
+ """GRAPH's term bound to the triple's graph: a variable bound, an IRI matched (a constant
2481
+ graph map that matches was chosen in `graph_choices` and needs no test)."""
2482
+ if gm is None:
2483
+ return []
2484
+ listed = None
2485
+ if self.dataset is not None and gm.constant is None:
2486
+ # A graph map read from the row: its IRI is one of the graphs the dataset lists (13.2).
2487
+ listed = (GraphSet(self.dataset[0 if self.graph is None else 1]), self.make(gm, names))
2488
+ if self.graph is None:
2489
+ return [listed] if listed else []
2490
+ if gm.constant is not None:
2491
+ return [] if isinstance(self.graph, IRITerm) else [(self.graph, E.Const(IRITerm(gm.constant)))]
2492
+ return ([listed] if listed else []) + [(self.graph, self.make(gm, names))]
2493
+
2494
+ def edge_branch(self, m: EdgeMap, s: Term, o: Term, gm: Optional[TermMap] = None) -> Optional[O.Op]:
2495
+ self.scope_of(gm)
2496
+ if m.join is None and any(e.term is None and e.exists and e.map_id for e in (m.start, m.end)):
2497
+ return self.ends_from_their_rows(m, s, o, gm)
2498
+ for end in (m.start, m.end):
2499
+ if end.term is None:
2500
+ raise NotBuilt("an edge end with no term")
2501
+ if m.join is None and any(e.parent_columns for e in (m.start, m.end)):
2502
+ raise NotBuilt("an edge end read from a parent row with no join")
2503
+ start, end = m.start.term, m.end.term
2504
+ if not _may_give(start, s) or not _may_give(end, o):
2505
+ return None
2506
+ if m.join is not None:
2507
+ if gm is not None and gm.constant is None:
2508
+ raise NotBuilt("a graph map read from a referencing object map's joint query")
2509
+ return self.joint_branch(m, s, o, gm)
2510
+ scan, names = self.scan(m, [start, end] + self.graph_read(gm))
2511
+ x: O.Op = scan
2512
+ for ref in (m.start, m.end):
2513
+ if not ref.exists:
2514
+ continue
2515
+ node = self.maps.node(ref.map_id) if ref.map_id else None
2516
+ if node is None:
2517
+ raise NotBuilt("an edge end that names no node map")
2518
+ if len(node.key.columns) != len(ref.columns):
2519
+ raise NotBuilt("an edge end of another key width")
2520
+ if not all(c in names for c in ref.columns):
2521
+ raise NotBuilt("an edge end keyed by columns its term does not read")
2522
+ x = O.SemiJoin(x, self.end_rows(ref, node, names))
2523
+ return self.bind(x, [(s, self.make(start, names)), (o, self.make(end, names))]
2524
+ + self.graph_term(gm, names))
2525
+
2526
+ def end_rows(self, ref: Any, node: Any, names: Dict[str, str]) -> O.Op:
2527
+ """The key rows of an edge end's node maps (`end_rows`)."""
2528
+ return end_rows(ref, (node,) + tuple(self.maps.node(p) for p in ref.peers), names, self.read, self.keys)
2529
+
2530
+ def ends_from_their_rows(self, m: EdgeMap, s: Term, o: Term, gm: Optional[TermMap]) -> Optional[O.Op]:
2531
+ """An edge whose end's node builds its subject from a column beyond its key (an
2532
+ `iri_template` over a whole-IRI column): the edge's rows joined to that node's
2533
+ rows on edge column = key, the end's term the node's subject from the node's row, as an
2534
+ R2RML referencing object map with a join condition gives it (R2RML 8). Every node row of
2535
+ the key gives its subject (an untrusted key's two rows, two ends); the triple's Distinct
2536
+ (`triple`) keeps the set, and the facts read the join (no edge key passes an untrusted
2537
+ node key). An end built from the edge's own row is checked as `edge_branch` checks it."""
2538
+ self.scope_of(gm)
2539
+ own = [ref.term for ref in (m.start, m.end) if ref.term is not None]
2540
+ scan, names = self.scan(m, own + self.graph_read(gm))
2541
+ # The key columns of an end built from its node's row: read and required too (R2RML 11.2).
2542
+ more = []
2543
+ for ref in (m.start, m.end):
2544
+ if ref.term is None:
2545
+ for c in ref.columns:
2546
+ if c not in names:
2547
+ names[c] = self.fresh("c")
2548
+ more.append(self.read(m, c, names[c]))
2549
+ if more:
2550
+ scan = dataclasses.replace(scan, reads=scan.reads + tuple(more),
2551
+ required=scan.required + tuple(r.name for r in more))
2552
+ x: O.Op = scan
2553
+ made: List[E.Expr] = []
2554
+ for ref in (m.start, m.end):
2555
+ node = self.maps.node(ref.map_id) if ref.map_id else None
2556
+ if ref.exists and node is None:
2557
+ raise NotBuilt("an edge end that names no node map")
2558
+ if ref.exists and len(node.key.columns) != len(ref.columns):
2559
+ raise NotBuilt("an edge end of another key width")
2560
+ if ref.term is not None:
2561
+ if ref.exists:
2562
+ for c in ref.columns:
2563
+ if c not in names:
2564
+ raise NotBuilt("an edge end keyed by columns its term does not read")
2565
+ x = O.SemiJoin(x, self.end_rows(ref, node, names))
2566
+ made.append(self.make(ref.term, names))
2567
+ continue
2568
+ # The key joins on the edge's columns, read under the edge's names; the subject's other
2569
+ # columns under names of their own.
2570
+ node_names = {k: names[c] for c, k in zip(ref.columns, node.key.columns)}
2571
+ subject = node.subject
2572
+ extra = subject.columns() + ((subject.language_column,) if subject.language_column else ())
2573
+ for c in extra:
2574
+ node_names.setdefault(c, self.fresh("c"))
2575
+ reads = tuple(self.read(node, c, n) for c, n in node_names.items())
2576
+ x = O.Join(x, O.MapScan(map=node, reads=reads, required=tuple(r.name for r in reads),
2577
+ unique_keys=self.keys(node)))
2578
+ made.append(self.make(subject, node_names))
2579
+ return self.bind(x, [(s, made[0]), (o, made[1])] + self.graph_term(gm, names))
2580
+
2581
+ def joint_branch(self, m: EdgeMap, s: Term, o: Term, gm: Optional[TermMap] = None) -> O.Op:
2582
+ """A referencing object map's triples (R2RML 8): its joint query's
2583
+ rows, the subject from the child's row and each end with
2584
+ `parent_columns` from the parent's. The join is the parent row's
2585
+ existence, so no end is checked again."""
2586
+ self.scope_of(gm)
2587
+ names: Dict[Tuple[bool, str], str] = {}
2588
+ reads: List[O.ScanColumn] = []
2589
+ made: List[E.Expr] = []
2590
+ for ref in (m.start, m.end):
2591
+ from_parent = bool(ref.parent_columns)
2592
+ table = m.join.parent.table if from_parent else m.source.table
2593
+ if ref.term.language_column is not None:
2594
+ raise NotBuilt("a language column")
2595
+ args = []
2596
+ for c in ref.term.columns():
2597
+ key = (from_parent, c)
2598
+ if key not in names:
2599
+ names[key] = self.fresh("c")
2600
+ held, zoned, digits, reader, bytewise = self.held(table, c)
2601
+ reads.append(O.ScanColumn(
2602
+ name=names[key], column=c, parent=from_parent, held=held,
2603
+ zone=(ZONED if zoned else LOCAL) if held == xsd.DATETIME else None,
2604
+ digits=tuple(digits) if digits else None, reader=reader,
2605
+ bytewise=bytewise and held == xsd.STRING and reader in (None, *UUID_READERS),
2606
+ uuid=reader in UUID_READERS,
2607
+ precision=self.precision(table, c, held)))
2608
+ args.append((c, E.Col(names[key])))
2609
+ made.append(E.MakeTerm(self.scoped(ref.term), tuple(args)))
2610
+ # The parent node map, its keys, and the child's join columns as read: what R13 reads to
2611
+ # state the join as the child's rows with the parent row's existence.
2612
+ parent = next((self.maps.node(ref.map_id) for ref in (m.start, m.end)
2613
+ if ref.parent_columns and ref.map_id), None)
2614
+ join_reads = []
2615
+ for c, _ in m.join.conditions:
2616
+ held, zoned, digits, reader, bytewise = self.held(m.source.table, c)
2617
+ join_reads.append(O.ScanColumn(
2618
+ name=c, column=c, held=held,
2619
+ zone=(ZONED if zoned else LOCAL) if held == xsd.DATETIME else None,
2620
+ digits=tuple(digits) if digits else None, reader=reader,
2621
+ bytewise=bytewise and held == xsd.STRING and reader in (None, *UUID_READERS),
2622
+ precision=self.precision(m.source.table, c, held)))
2623
+ scan = O.MapScan(map=m, reads=tuple(reads), required=tuple(r.name for r in reads),
2624
+ unique_keys=self.keys(m), parent_map=parent,
2625
+ parent_unique_keys=self.keys(parent) if parent is not None else (),
2626
+ join_reads=tuple(join_reads))
2627
+ return self.bind(scan, [(s, made[0]), (o, made[1])] + self.graph_term(gm, {}))
2628
+
2629
+ def precision(self, table: str, column: str, held: Optional[str]) -> Optional[int]:
2630
+ """A date-time column's fractional-second digits its type states (`CypherType.precision`)."""
2631
+ kind_of = getattr(self.held, "kind_of", None)
2632
+ if held != xsd.DATETIME or kind_of is None:
2633
+ return None
2634
+ return getattr(kind_of(table, column), "precision", None)
2635
+
2636
+ def scan(self, m, maps: List[TermMap]) -> Tuple[O.MapScan, Dict[str, str]]:
2637
+ """A scan of `m` reading the columns `maps` read, each under a
2638
+ hidden name, every one required (R2RML 11.2)."""
2639
+ names: Dict[str, str] = {}
2640
+ optional_: List[str] = []
2641
+ for tm in maps:
2642
+ for c in tm.columns():
2643
+ if c not in names:
2644
+ names[c] = self.fresh("c")
2645
+ for tm in maps:
2646
+ # A literal's per-row tag: read, never required (a row with none is a plain literal).
2647
+ c = tm.language_column
2648
+ if c is not None and c not in names:
2649
+ names[c] = self.fresh("c")
2650
+ optional_.append(names[c])
2651
+ reads = tuple(self.read(m, c, n) for c, n in names.items())
2652
+ return O.MapScan(map=m, reads=reads, required=tuple(n for n in names.values() if n not in optional_),
2653
+ unique_keys=self.keys(m)), names
2654
+
2655
+ def read(self, m, column: str, name: str) -> O.ScanColumn:
2656
+ """A column read: its datatype, and a date-time's zone."""
2657
+ held, zoned, digits, reader, bytewise = self.held(m.source.table, column)
2658
+ zone = None
2659
+ if held == xsd.DATETIME:
2660
+ zone = ZONED if zoned else LOCAL
2661
+ return O.ScanColumn(name=name, column=column, held=held, zone=zone,
2662
+ digits=tuple(digits) if digits else None, reader=reader,
2663
+ bytewise=bytewise and held == xsd.STRING and reader in (None, *UUID_READERS),
2664
+ uuid=reader in UUID_READERS)
2665
+
2666
+ def make(self, tm: TermMap, names: Dict[str, str]) -> E.Expr:
2667
+ tm = self.scoped(tm)
2668
+ columns = tm.columns() + ((tm.language_column,) if tm.language_column is not None else ())
2669
+ return E.MakeTerm(tm, tuple((c, E.Col(names[c])) for c in columns))
2670
+
2671
+ def scope_of(self, gm: Optional[TermMap]) -> None:
2672
+ """Where the default graph merges several FROM graphs, a blank node of one source graph is not
2673
+ one of another (RDF 1.1 Concepts 3.6, the merge): the triples a graph map reads carry their
2674
+ graph's name in their blank nodes' labels (`scoped`)."""
2675
+ merging = self.dataset is not None and self.graph is None and len(self.dataset[0]) > 1
2676
+ self.blank_scope = (gm.constant if gm.constant is not None else "") if merging and gm is not None else None
2677
+
2678
+ def scoped(self, tm: TermMap) -> TermMap:
2679
+ """`tm`, a blank node's term map, with the graph it is read in added to its label."""
2680
+ if tm.term_type != BLANK or self.blank_scope is None:
2681
+ return tm
2682
+ if not self.blank_scope:
2683
+ raise NotBuilt("blank nodes merged from several FROM graphs where a graph map reads the row's graph")
2684
+ suffix = ((f"@{self.blank_scope}", False),)
2685
+ if tm.template is not None:
2686
+ return dataclasses.replace(tm, template=tm.template + suffix)
2687
+ if tm.column is not None:
2688
+ return dataclasses.replace(tm, column=None, template=((tm.column, True),) + suffix)
2689
+ return dataclasses.replace(tm, constant=tm.constant + suffix[0][0])
2690
+
2691
+ def bind(self, x: O.Op, terms: List[Tuple[Term, E.Expr]]) -> O.Op:
2692
+ """Bind each pattern term to its built term: a variable is a column,
2693
+ a constant a filter by term identity (18.3: a pattern matches a
2694
+ triple of the same terms)."""
2695
+ out: List[str] = []
2696
+ conds: List[E.Expr] = []
2697
+ for t, e in terms:
2698
+ if isinstance(t, Var):
2699
+ if t.name in out:
2700
+ conds.append(E.Compare("=", E.Col(t.name), e, E.TERM))
2701
+ else:
2702
+ x = O.Extend(x, t.name, e)
2703
+ out.append(t.name)
2704
+ elif isinstance(t, GraphSet):
2705
+ conds.append(E.Or(tuple(E.Compare("=", e, E.Const(IRITerm(i)), E.TERM) for i in sorted(t.iris)))
2706
+ if t.iris else E.Const(LiteralTerm("false", xsd.BOOLEAN)))
2707
+ else:
2708
+ conds.append(E.Compare("=", e, E.Const(t), E.TERM))
2709
+ if conds:
2710
+ x = O.Filter(x, conds[0] if len(conds) == 1 else E.And(tuple(conds)))
2711
+ return O.Project(x, tuple(out))
2712
+
2713
+ # -- expressions (17) -------------------------------------------------------
2714
+
2715
+ def condition(self, ctx) -> E.Expr:
2716
+ """A FILTER's constraint, as its effective boolean value (17.2.2)."""
2717
+ return ebv(self.expression(ctx))
2718
+
2719
+ def expression(self, ctx) -> E.Expr:
2720
+ P = SparqlParser
2721
+ if isinstance(ctx, (P.ExpressionContext, P.ConstraintContext)):
2722
+ return self.expression(ctx.getChild(0))
2723
+ if isinstance(ctx, P.BrackettedExpressionContext):
2724
+ return self.expression(ctx.expression())
2725
+ if isinstance(ctx, P.ConditionalOrExpressionContext):
2726
+ args = [self.expression(a) for a in ctx.conditionalAndExpression()]
2727
+ return args[0] if len(args) == 1 else E.Or(tuple(ebv(a) for a in args))
2728
+ if isinstance(ctx, P.ConditionalAndExpressionContext):
2729
+ args = [self.expression(a) for a in ctx.valueLogical()]
2730
+ return args[0] if len(args) == 1 else E.And(tuple(ebv(a) for a in args))
2731
+ if isinstance(ctx, P.ValueLogicalContext):
2732
+ return self.expression(ctx.relationalExpression())
2733
+ if isinstance(ctx, P.RelationalExpressionContext):
2734
+ sides = ctx.numericExpression()
2735
+ left = self.expression(sides[0])
2736
+ if len(sides) == 2:
2737
+ return E.Compare(ctx.getChild(1).getText(), left, self.expression(sides[1]))
2738
+ if ctx.expressionList() is not None:
2739
+ items = [self.expression(x) for x in ctx.expressionList().expression()]
2740
+ # An empty list is false, an error only where the left side is one (`?v = ?v`
2741
+ # is an error exactly then).
2742
+ false = E.Const(LiteralTerm("false", xsd.BOOLEAN))
2743
+ found = E.Or(tuple(E.Compare("=", left, i) for i in items)) if items \
2744
+ else E.If(E.Compare("=", left, left), false, false)
2745
+ return E.Not(found) if ctx.NOT() is not None else found
2746
+ return left
2747
+ if isinstance(ctx, P.NumericExpressionContext):
2748
+ return self.expression(ctx.additiveExpression())
2749
+ if isinstance(ctx, P.AdditiveExpressionContext):
2750
+ # `?a -1` is `?a + -1`: a signed literal is added (19.8).
2751
+ kids = list(ctx.getChildren())
2752
+ out = self.expression(kids[0])
2753
+ i = 1
2754
+ while i < len(kids):
2755
+ k = kids[i]
2756
+ if isinstance(k, (P.NumericLiteralPositiveContext,
2757
+ P.NumericLiteralNegativeContext)):
2758
+ out = E.Call("+", (out, E.Const(numeric(k.getText()))))
2759
+ i += 1
2760
+ else:
2761
+ out = E.Call(k.getText(), (out, self.expression(kids[i + 1])))
2762
+ i += 2
2763
+ return out
2764
+ if isinstance(ctx, P.MultiplicativeExpressionContext):
2765
+ kids = list(ctx.getChildren())
2766
+ out = self.expression(kids[0])
2767
+ for i in range(1, len(kids), 2):
2768
+ out = E.Call(kids[i].getText(), (out, self.expression(kids[i + 1])))
2769
+ return out
2770
+ if isinstance(ctx, P.UnaryExpressionContext):
2771
+ inner = self.expression(ctx.primaryExpression())
2772
+ if ctx.EXCLAMATION() is not None:
2773
+ return E.Not(ebv(inner))
2774
+ if ctx.MINUS() is not None:
2775
+ return E.Call("NEG", (inner,))
2776
+ if ctx.PLUS() is not None:
2777
+ return E.Call("POS", (inner,))
2778
+ return inner
2779
+ if isinstance(ctx, P.PrimaryExpressionContext):
2780
+ return self.expression(ctx.getChild(0))
2781
+ if isinstance(ctx, P.Var_Context):
2782
+ name = ctx.getText()[1:]
2783
+ return E.Const(self.params[name]) if name in self.params else E.Col(name)
2784
+ if isinstance(ctx, (P.RdfLiteralContext, P.NumericLiteralContext,
2785
+ P.BooleanLiteralContext)):
2786
+ return E.Const(self.literal(ctx))
2787
+ if isinstance(ctx, P.IriRefOrFunctionContext):
2788
+ if ctx.argList() is not None:
2789
+ return self.function_call(ctx.iriRef(), ctx.argList())
2790
+ return E.Const(IRITerm(self.iri(ctx.iriRef())))
2791
+ if isinstance(ctx, P.FunctionCallContext):
2792
+ return self.function_call(ctx.iriRef(), ctx.argList())
2793
+ if isinstance(ctx, P.BuiltInCallContext):
2794
+ return self.builtin(ctx)
2795
+ raise NotBuilt(f"the expression {ctx.getText()[:40]}")
2796
+
2797
+ def function_call(self, iri_ctx, args_ctx) -> E.Expr:
2798
+ """A call of a function IRI: a cast (17.5, `casts`), the only one built; another an
2799
+ unknown function (S10)."""
2800
+ from prograph.plan.ir.casts import TARGETS
2801
+ name = self.iri(iri_ctx)
2802
+ exprs = args_ctx.expression() if args_ctx is not None else []
2803
+ if name in TARGETS:
2804
+ if len(exprs) != 1 or "DISTINCT" in args_ctx.getText().upper()[:9]:
2805
+ raise NotBuilt(f"a cast with {len(exprs)} arguments")
2806
+ return E.Call("CAST", (self.expression(exprs[0]), E.Const(IRITerm(name))))
2807
+ raise NotBuilt(f"the function {iri_ctx.getText()}")
2808
+
2809
+ def builtin(self, ctx) -> E.Expr:
2810
+ if ctx.aggregate() is not None:
2811
+ return self.aggregate(ctx.aggregate())
2812
+ if ctx.regexExpression() is not None:
2813
+ return E.Call("REGEX", tuple(self.expression(x) for x in ctx.regexExpression().expression()))
2814
+ if ctx.existsFunc() is not None or ctx.notExistsFunc() is not None:
2815
+ raise NotBuilt("EXISTS other than as a FILTER's own term, or terms joined by && "
2816
+ "(inside ||, a BIND, IF or a SELECT expression)")
2817
+ name = ctx.getChild(0).getText().upper()
2818
+ args = [self.expression(x) for x in ctx.expression()]
2819
+ if name == "BOUND":
2820
+ var = ctx.var_().getText()[1:]
2821
+ if var in self.params:
2822
+ return E.Const(LiteralTerm("true", xsd.BOOLEAN))
2823
+ return E.IsBound(E.Col(var))
2824
+ if name in ("UUID", "STRUUID", "RAND", "BNODE"):
2825
+ # A fresh value (fresh.py): the call's id is its place in the query.
2826
+ self.fresh_calls += 1
2827
+ return E.Call(name, (E.Const(LiteralTerm(str(self.fresh_calls))),) + tuple(args))
2828
+ if name == "NOW":
2829
+ # 17.4.5.1: one xsd:dateTime for the whole query, its moment of planning, in UTC.
2830
+ return E.Const(LiteralTerm(self.now, xsd.DATETIME))
2831
+ calls = {"SAMETERM": "sameTerm", "ISIRI": "isIRI", "ISURI": "isURI",
2832
+ "ISBLANK": "isBLANK", "ISLITERAL": "isLITERAL"}
2833
+ if name in calls:
2834
+ return E.Call(calls[name], tuple(args))
2835
+ if name == "CONCAT":
2836
+ listed = ctx.expressionList()
2837
+ items = [self.expression(x) for x in listed.expression()] if listed is not None else []
2838
+ # errata-query-4: CONCAT() is "", CONCAT(x) is x's string.
2839
+ return E.Call("CONCAT", tuple(items)) if items else E.Const(LiteralTerm(""))
2840
+ if name in STRING_CALLS or name in DATE_PARTS:
2841
+ return E.Call(name, tuple(args))
2842
+ if name in ("IRI", "URI"):
2843
+ # 17.4.2.8: resolved against the query's base (4.1.1).
2844
+ return E.Call("IRI", (args[0], E.Const(LiteralTerm(self.base or ""))))
2845
+ if name in ("ABS", "CEIL", "FLOOR", "ROUND"):
2846
+ return E.Call(name, tuple(args))
2847
+ terms = {"STR": "STR", "LANG": "LANG", "DATATYPE": "DATATYPE", "ISNUMERIC": "isNUMERIC",
2848
+ "STRDT": "STRDT", "STRLANG": "STRLANG"}
2849
+ if name in terms:
2850
+ return E.Call(terms[name], tuple(args))
2851
+ if name == "COALESCE":
2852
+ # 17.4.1.3: the first argument that evaluates with no error; none, an error.
2853
+ listed = ctx.expressionList()
2854
+ items = [self.expression(x) for x in listed.expression()] if listed is not None else []
2855
+ if not items:
2856
+ return E.Const(None) # COALESCE(): no argument has a value, an error
2857
+ return E.Coalesce(tuple(items))
2858
+ if name == "IF":
2859
+ # 17.4.1.2: the condition's effective boolean value; an error there is an error.
2860
+ return E.If(ebv(args[0]), args[1], args[2])
2861
+ raise NotBuilt(f"the function {name}")
2862
+
2863
+
2864
+ # -- helpers ------------------------------------------------------------------------
2865
+
2866
+ def end_rows(ref: Any, nodes: Sequence[Any], names: Dict[str, str], read: Any, keys: Any) -> O.Op:
2867
+ """The key rows of an edge end's node maps, its own and its `peers` (`nodes`: the label's maps giving its
2868
+ subject, compile_schema), under the edge's names for its key columns (`names`): the end exists where one
2869
+ has it. `read` makes a column read, `keys` a map's trusted keys. SPARQL's and Cypher's check (`edge-ends`)."""
2870
+ out: List[O.Op] = []
2871
+ for each in nodes:
2872
+ if len(each.key.columns) != len(ref.columns):
2873
+ raise NotBuilt("an edge end of another key width")
2874
+ reads = tuple(read(each, k, names[c]) for c, k in zip(ref.columns, each.key.columns))
2875
+ out.append(O.MapScan(map=each, reads=reads, required=tuple(r.name for r in reads),
2876
+ unique_keys=keys(each) if keys else ()))
2877
+ return out[0] if len(out) == 1 else O.Union(tuple(out))
2878
+
2879
+
2880
+ def _star_like(select) -> bool:
2881
+ """Whether a SELECT reads every variable of its pattern: SELECT * projects each, COUNT(*) (in its SELECT,
2882
+ HAVING or ORDER BY, DISTINCT or not) counts every solution."""
2883
+ if not select.selectClause():
2884
+ return True
2885
+ texts = [c.expression().getText() for c in select.selectClause() if c.expression() is not None]
2886
+ mods = select.solutionModifier() if hasattr(select, "solutionModifier") else None
2887
+ if mods is not None:
2888
+ texts.append(mods.getText()) # HAVING and ORDER BY
2889
+ text = " ".join(texts).upper().replace(" ", "")
2890
+ return "COUNT(*)" in text or "COUNT(DISTINCT*)" in text
2891
+
2892
+
2893
+ def _var_names(tree) -> List[str]:
2894
+ """Every occurrence of a variable in a parse tree that reads it, by name (`?x` and `$x`). A sub-SELECT's bare
2895
+ projection of a variable (`SELECT ?x ?p`) reads it only where the SELECT around it reads every variable
2896
+ (`_star_like`); elsewhere an outer read is counted where it is. An expression of it reads it."""
2897
+ out: List[str] = []
2898
+ stack = [(tree, False)]
2899
+ while stack:
2900
+ node, star = stack.pop()
2901
+ if isinstance(node, SparqlParser.Var_Context):
2902
+ out.append(node.getText()[1:])
2903
+ continue
2904
+ if isinstance(node, SparqlParser.SubSelectContext):
2905
+ inner = node.selectQuery()
2906
+ for clause in inner.selectClause():
2907
+ if clause.expression() is not None:
2908
+ stack.append((clause.expression(), star))
2909
+ elif star:
2910
+ out.append(clause.var_().getText()[1:])
2911
+ inside = _star_like(inner)
2912
+ stack.extend((c, inside) for c in inner.getChildren()
2913
+ if not isinstance(c, SparqlParser.SelectClauseContext))
2914
+ continue
2915
+ if isinstance(node, SparqlParser.SelectQueryContext):
2916
+ star = _star_like(node)
2917
+ stack.extend((c, star) for c in (node.getChildren() if hasattr(node, "getChildren") else ()))
2918
+ return out
2919
+
2920
+
2921
+ def variables_of(tree) -> set:
2922
+ """Every variable a parse tree names (`?x` and `$x`), for a parameter's name check."""
2923
+ out: set = set()
2924
+ stack = [tree]
2925
+ while stack:
2926
+ node = stack.pop()
2927
+ if isinstance(node, SparqlParser.Var_Context):
2928
+ out.add(node.getText()[1:])
2929
+ continue
2930
+ stack.extend(node.getChildren() if hasattr(node, "getChildren") else ())
2931
+ return out
2932
+
2933
+
2934
+ def conjuncts_of(ctx) -> List[Any]:
2935
+ """The terms of a FILTER's constraint split at `&&` (and brackets):
2936
+ parse nodes, each not itself a conjunction."""
2937
+ P = SparqlParser
2938
+ while True:
2939
+ if isinstance(ctx, P.ConditionalOrExpressionContext):
2940
+ ands = ctx.conditionalAndExpression()
2941
+ if len(ands) != 1:
2942
+ return [ctx]
2943
+ ctx = ands[0]
2944
+ elif isinstance(ctx, P.ConditionalAndExpressionContext):
2945
+ vs = ctx.valueLogical()
2946
+ if len(vs) != 1:
2947
+ return [t for v in vs for t in conjuncts_of(v)]
2948
+ ctx = vs[0]
2949
+ elif isinstance(ctx, (P.BrackettedExpressionContext,)):
2950
+ ctx = ctx.expression()
2951
+ elif isinstance(ctx, P.UnaryExpressionContext) and ctx.EXCLAMATION() is None \
2952
+ and ctx.MINUS() is None and ctx.PLUS() is None:
2953
+ ctx = ctx.primaryExpression()
2954
+ elif isinstance(ctx, (P.ExpressionContext, P.ConstraintContext, P.ValueLogicalContext,
2955
+ P.NumericExpressionContext, P.PrimaryExpressionContext)):
2956
+ ctx = ctx.getChild(0)
2957
+ elif isinstance(ctx, (P.RelationalExpressionContext, P.AdditiveExpressionContext,
2958
+ P.MultiplicativeExpressionContext)) and ctx.getChildCount() == 1:
2959
+ ctx = ctx.getChild(0)
2960
+ else:
2961
+ return [ctx]
2962
+
2963
+
2964
+ def exists_leaf(ctx) -> Optional[Tuple[bool, Any]]:
2965
+ """(negated, the pattern) where `ctx` is EXISTS { P }, NOT EXISTS { P }
2966
+ or `!` of either, else None. `!EXISTS` is NOT EXISTS: EXISTS is never
2967
+ an error."""
2968
+ P = SparqlParser
2969
+ negated = False
2970
+ if isinstance(ctx, P.UnaryExpressionContext) and ctx.EXCLAMATION() is not None:
2971
+ negated, ctx = True, ctx.primaryExpression()
2972
+ while isinstance(ctx, (P.PrimaryExpressionContext, P.ConstraintContext)):
2973
+ ctx = ctx.getChild(0)
2974
+ if isinstance(ctx, P.BuiltInCallContext):
2975
+ if ctx.existsFunc() is not None:
2976
+ return negated, ctx.existsFunc().groupGraphPattern()
2977
+ if ctx.notExistsFunc() is not None:
2978
+ return not negated, ctx.notExistsFunc().groupGraphPattern()
2979
+ return None
2980
+
2981
+
2982
+ def _grouped_only(used, visible, clause: str) -> None:
2983
+ bad = sorted(c for c in used if c not in visible and not c.startswith(HIDDEN))
2984
+ if bad:
2985
+ raise QueryError(f"?{bad[0]} is not grouped, and it is read in {clause}: a grouped "
2986
+ "query reads only its GROUP BY keys and aggregates (18.2.4.1)")
2987
+
2988
+
2989
+ _BOOLEAN_EXPRS = (E.Compare, E.And, E.Or, E.Not, E.IsBound)
2990
+ _BOOLEAN_CALLS = {"sameTerm", "sameValue", "isIRI", "isURI", "isBLANK", "isLITERAL", "EBV"} | BOOLEAN_STRING_CALLS
2991
+
2992
+
2993
+ def ebv(e: E.Expr) -> E.Expr:
2994
+ """`e`'s effective boolean value: itself where it is a boolean."""
2995
+ if isinstance(e, _BOOLEAN_EXPRS):
2996
+ return e
2997
+ if isinstance(e, E.Call) and e.name in _BOOLEAN_CALLS:
2998
+ return e
2999
+ return E.Call("EBV", (e,))
3000
+
3001
+
3002
+ def numeric(text: str) -> LiteralTerm:
3003
+ """A numeric literal: `1` an xsd:integer, `1.5` a decimal, `1e0` a
3004
+ double (19.8), its lexical form as written."""
3005
+ if "e" in text.lower():
3006
+ return LiteralTerm(text, xsd.DOUBLE)
3007
+ if "." in text:
3008
+ return LiteralTerm(text, xsd.DECIMAL)
3009
+ return LiteralTerm(text, xsd.INTEGER)
3010
+
3011
+
3012
+ def _connected(patterns: List[Tuple[Any, Any, Any]]) -> List[Tuple[Any, Any, Any]]:
3013
+ """A basic graph pattern's triples in the order to join them (rule join-order): from its most
3014
+ selective triple (a constant subject, then a constant object, then a constant class), each next one sharing
3015
+ a variable with those before, a path after what binds its ends. The lowering keeps the written order under
3016
+ a condition across the joins (`lower._across`), so this order is the one the database sees there."""
3017
+ from prograph.plan.ir.join_order import connected_order
3018
+
3019
+ def names(t) -> frozenset:
3020
+ return frozenset(v.name for v in t if isinstance(v, Var))
3021
+
3022
+ def selective(t) -> int:
3023
+ s, p, o = t
3024
+ if isinstance(p, Path):
3025
+ return -1
3026
+ if not isinstance(s, Var):
3027
+ return 3
3028
+ if not isinstance(o, Var):
3029
+ return 1 if isinstance(p, IRITerm) and p.text == RDF_TYPE else 2
3030
+ return 0
3031
+ if len(patterns) < 2:
3032
+ return patterns
3033
+ counts: Dict[str, int] = {}
3034
+ for t in patterns:
3035
+ for v in names(t):
3036
+ counts[v] = counts.get(v, 0) + 1
3037
+ order = connected_order([(i, names(t), selective(t)) for i, t in enumerate(patterns)], set(),
3038
+ [frozenset({v}) for v, n in counts.items() if n > 1])
3039
+ return [patterns[i] for i in order]
3040
+
3041
+
3042
+ WITNESS_CAP = 8 # tree-witness readings of one triples block: at most 2^8 - 1 branches
3043
+
3044
+
3045
+ def Counter_of(patterns) -> Dict[str, int]:
3046
+ """How often each variable occurs in `patterns`."""
3047
+ out: Dict[str, int] = {}
3048
+ for t in patterns:
3049
+ for v in t:
3050
+ if isinstance(v, Var):
3051
+ out[v.name] = out.get(v.name, 0) + 1
3052
+ return out
3053
+
3054
+
3055
+ def _hidden_trees(patterns, hidden) -> List[Tuple[list, Any, Dict[str, Any], Dict[str, list], Dict[Any, Any]]]:
3056
+ """The trees of hidden terms in `patterns`: (their atoms, (the term they hang from, the role to their top, the
3057
+ top) or None where detached, per hidden term its classes and its (role, neighbour) edges). A set of hidden
3058
+ terms is one only where every atom naming them is a class atom (`h a C`) or a property atom of an IRI, its
3059
+ atoms form a tree, and it hangs from one atom at most; any other shape is read by named individuals only."""
3060
+ adj: Dict[str, Dict[str, Any]] = {h: {"classes": set(), "edges": []} for h in hidden}
3061
+ atoms_of: Dict[str, list] = {h: [] for h in hidden}
3062
+ own: Dict[str, list] = {h: [] for h in hidden} # a term's class atoms
3063
+ links: Dict[Tuple[Any, str], Any] = {} # the atom of each edge (parent node or None, node)
3064
+ bad: Set[str] = set()
3065
+ hangs: Dict[str, list] = {h: [] for h in hidden}
3066
+ for t in patterns:
3067
+ s, p, o = t
3068
+ names = [v.name for v in (s, o) if isinstance(v, Var) and v.name in hidden]
3069
+ if isinstance(p, Var) and p.name in hidden:
3070
+ bad.add(p.name)
3071
+ if not names:
3072
+ continue
3073
+ if not isinstance(p, IRITerm):
3074
+ bad.update(names)
3075
+ continue
3076
+ if p.text == RDF_TYPE:
3077
+ if isinstance(s, Var) and s.name in hidden and isinstance(o, IRITerm):
3078
+ adj[s.name]["classes"].add(o.text)
3079
+ atoms_of[s.name].append(t)
3080
+ own[s.name].append(t)
3081
+ else:
3082
+ bad.update(names)
3083
+ continue
3084
+ sh = isinstance(s, Var) and s.name in hidden
3085
+ oh = isinstance(o, Var) and o.name in hidden
3086
+ if sh and oh:
3087
+ if s.name == o.name:
3088
+ bad.add(s.name)
3089
+ continue
3090
+ adj[s.name]["edges"].append(((p.text, False), o.name))
3091
+ adj[o.name]["edges"].append(((p.text, True), s.name))
3092
+ atoms_of[s.name].append(t)
3093
+ links[(s.name, o.name)] = links[(o.name, s.name)] = t
3094
+ elif sh:
3095
+ if isinstance(o, LiteralTerm):
3096
+ bad.add(s.name)
3097
+ continue
3098
+ hangs[s.name].append((o, (p.text, True), t)) # from o, by p's inverse, to s
3099
+ atoms_of[s.name].append(t)
3100
+ links.setdefault((None, s.name), t)
3101
+ else:
3102
+ if isinstance(s, LiteralTerm):
3103
+ bad.add(o.name)
3104
+ continue
3105
+ hangs[o.name].append((s, (p.text, False), t))
3106
+ atoms_of[o.name].append(t)
3107
+ links.setdefault((None, o.name), t)
3108
+ # Connected sets of hidden terms.
3109
+ out = []
3110
+ seen: Set[str] = set()
3111
+ for h in sorted(hidden):
3112
+ if h in seen:
3113
+ continue
3114
+ group, todo = set(), [h]
3115
+ while todo:
3116
+ x = todo.pop()
3117
+ if x in group:
3118
+ continue
3119
+ group.add(x)
3120
+ todo += [n for _, n in adj[x]["edges"]]
3121
+ seen |= group
3122
+ if group & bad:
3123
+ continue
3124
+ edges = sum(len(adj[x]["edges"]) for x in group) // 2
3125
+ if edges != len(group) - 1:
3126
+ continue # a cycle: no tree
3127
+ hanging = [(x, *a) for x in group for a in hangs[x]]
3128
+ if len(hanging) > 1 and len({h[0] for h in hanging}) > 1:
3129
+ continue # hangs from two of its terms: named individuals only
3130
+ atoms = list({id(a): a for x in group for a in atoms_of[x]}.values())
3131
+ by = {x: own[x] for x in group}
3132
+ if hanging:
3133
+ # Several atoms from one top (`?x p ?h . ?y q ?h`): one root, the others equal to it, the top implied by
3134
+ # roles all of them give.
3135
+ top, term, role, _ = hanging[0]
3136
+ roles = tuple(h[2] for h in hanging)
3137
+ others = tuple(h[1] for h in hanging[1:])
3138
+ for h in hanging[1:]:
3139
+ links[("extra", top, id(h[3]))] = h[3]
3140
+ out.append((atoms, (term, roles if len(roles) > 1 else role, top, others), {x: adj[x] for x in group},
3141
+ by, links))
3142
+ else:
3143
+ out.append((atoms, None, {x: adj[x] for x in group}, by, links))
3144
+ return out
3145
+
3146
+
3147
+ def _filtered_below(x: O.Op, cond: E.Expr, column: str) -> O.Op:
3148
+ """`Filter(x, cond)`, the test (of `column` alone) put inside a Distinct, a Project keeping the column and
3149
+ each branch of a Union, which it commutes with: there each map's column is of one type."""
3150
+ if isinstance(x, O.Distinct):
3151
+ return O.Distinct(_filtered_below(x.input, cond, column))
3152
+ if isinstance(x, O.Project) and column in x.input.columns:
3153
+ return O.Project(_filtered_below(x.input, cond, column), x.columns)
3154
+ if isinstance(x, O.Union):
3155
+ return O.Union(tuple(_filtered_below(b, cond, column) if column in b.columns else b for b in x.inputs))
3156
+ return O.Filter(x, cond)
3157
+
3158
+
3159
+ def patterns_columns(patterns) -> List[str]:
3160
+ """The variables of `patterns`, in the order they are written."""
3161
+ return list(dict.fromkeys(t.name for x in patterns for t in x if isinstance(t, Var)))
3162
+
3163
+
3164
+ def _value_joined(patterns) -> List[str]:
3165
+ """The variables that are the object of two or more of `patterns` and never a subject, predicate or path end:
3166
+ under datatypes="value" those join by value (`Translator.by_value_block`)."""
3167
+ objects: Dict[str, int] = {}
3168
+ elsewhere: Set[str] = set()
3169
+ for s, p, o in patterns:
3170
+ for t in (s, p):
3171
+ if isinstance(t, Var):
3172
+ elsewhere.add(t.name)
3173
+ if isinstance(o, Var):
3174
+ if isinstance(p, Path):
3175
+ elsewhere.add(o.name)
3176
+ else:
3177
+ objects[o.name] = objects.get(o.name, 0) + 1
3178
+ return sorted(v for v, n in objects.items() if n > 1 and v not in elsewhere)
3179
+
3180
+
3181
+ def _least_of(names, expression, least_literal) -> E.Expr:
3182
+ """The least literal of the columns `names` (value-equal), by `least_literal`'s term order."""
3183
+ shown: E.Expr = E.Col(names[0])
3184
+ for other in names[1:]:
3185
+ step = expression(least_literal("pga", "pgb"), {"pgb": other}, condition=False)
3186
+ shown = E.replaced(step, {"pga": shown})
3187
+ return shown
3188
+
3189
+
3190
+ def join(a: O.Op, b: O.Op) -> O.Op:
3191
+ if isinstance(a, O.Unit):
3192
+ return b
3193
+ if isinstance(b, O.Unit):
3194
+ return a
3195
+ return O.Join(meeting(a, b), meeting(b, a))
3196
+
3197
+
3198
+ def meeting(x: O.Op, other: O.Op) -> O.Op:
3199
+ """`x` without the union branches no row of `other` joins: a branch
3200
+ whose shared column, bound in every row of both, holds terms of maps
3201
+ none of which may give a term `other`'s column holds (`TermMap.
3202
+ may_equal`, by the slots' provenance; unknown provenance keeps it).
3203
+ The branches are sought through nested unions and the row-wise
3204
+ operators above them (Distinct, Filter, Extend, Project), so a
3205
+ variable predicate's union of per-predicate unions loses, under a
3206
+ VALUES-bound subject, each map whose subject cannot be one of its
3207
+ terms (the union of what each row allows; a row with UNDEF leaves the
3208
+ column unbound in some rows, so nothing goes)."""
3209
+ theirs = other.slots
3210
+ wanted = {c: theirs[c].provenance for c in x.columns if c in theirs
3211
+ and theirs[c].present is Bound.ALWAYS and theirs[c].provenance is not None}
3212
+ if not wanted:
3213
+ return x
3214
+ out = _met(x, wanted)
3215
+ if out is x:
3216
+ return x
3217
+ return O.Values(x.columns, ()) if out is None else out
3218
+
3219
+
3220
+ def _met(x: O.Op, wanted: Dict[str, Any]) -> Optional[O.Op]:
3221
+ """`meeting`'s walk: `x` with the branches that cannot meet `wanted` (column -> the provenance of the
3222
+ terms the other side holds there) gone; None where no row of `x` can meet; `x` itself where nothing goes."""
3223
+ slots = x.slots
3224
+ for c, provenance in wanted.items():
3225
+ mine = slots.get(c)
3226
+ if mine is not None and mine.present is Bound.ALWAYS and mine.provenance is not None \
3227
+ and not any(p.may_equal(q) for p in mine.provenance for q in provenance):
3228
+ return None
3229
+ if isinstance(x, O.Union):
3230
+ kids = [_met(i, wanted) for i in x.inputs]
3231
+ if all(k is i for k, i in zip(kids, x.inputs)):
3232
+ return x
3233
+ kept = [k for k in kids if k is not None]
3234
+ if not kept:
3235
+ return None
3236
+ out: O.Op = kept[0] if len(kept) == 1 and kept[0].columns == x.columns else O.Union(tuple(kept))
3237
+ for c in x.columns:
3238
+ if c not in out.columns:
3239
+ out = O.Extend(out, c, E.Const(None)) # held only by a dropped branch: unbound
3240
+ return out if out.columns == x.columns else O.Project(out, x.columns)
3241
+ if isinstance(x, (O.Distinct, O.Filter, O.Extend, O.Project)):
3242
+ inner = {c: v for c, v in wanted.items() if c in x.input.columns
3243
+ and not (isinstance(x, O.Extend) and c == x.name)}
3244
+ if isinstance(x, O.Extend) and x.name in wanted and isinstance(x.expr, E.Col) \
3245
+ and x.expr.name in x.input.columns and x.expr.name not in inner:
3246
+ inner[x.expr.name] = wanted[x.name] # a copy of a column: the same terms
3247
+ if not inner:
3248
+ return x
3249
+ got = _met(x.input, inner)
3250
+ if got is x.input:
3251
+ return x
3252
+ if got is None:
3253
+ return None
3254
+ return dataclasses.replace(x, input=got)
3255
+ return x
3256
+
3257
+
3258
+ def _may_give(tm: TermMap, t: Term) -> bool:
3259
+ """Whether a term of `tm` may be the pattern's constant `t`."""
3260
+ if isinstance(t, Var):
3261
+ return True
3262
+ if tm.term_type == "iri" and tm.lexical == LEXICAL_R2RML and \
3263
+ not re.match(r"[A-Za-z][A-Za-z0-9+.-]*:", tm.prefix()):
3264
+ return True # a relative IRI, resolved against the base (R2RML 7.3)
3265
+ provenance = const_slot(t).provenance
3266
+ return any(tm.may_equal(c) for c in provenance or ())
3267
+
3268
+
3269
+ class LabelPatternError(QueryError):
3270
+ """A query names `<urn:prograph:label:X>`, a label as a term: not SPARQL's,
3271
+ gone in ProGraph 1.0. A label is a class in the RDF view: query its class IRI."""
3272
+
3273
+
3274
+ def _label_pattern(label: str, maps: SourceMaps) -> "LabelPatternError":
3275
+ classes = sorted({c for m in maps.nodes if m.label == label for c in m.classes})
3276
+ if classes:
3277
+ use = " or ".join(f"<{c}>" for c in classes)
3278
+ return LabelPatternError(f"<{LABEL_IRI_PREFIX}{label}> is not a term in ProGraph 1.0: the label "
3279
+ f"{label} is the class {use}; write `?x a {use.split(' or ')[0]}`")
3280
+ return LabelPatternError(f"<{LABEL_IRI_PREFIX}{label}> is not a term in ProGraph 1.0, and no map has the "
3281
+ f"label {label}: query a class IRI (`?x a <class>`)")
3282
+
3283
+
3284
+ class OutsideGroupWarning(UserWarning):
3285
+ """HAVING or ORDER BY reads a variable a grouped query does not keep: it is unbound there (D2),
3286
+ almost always a mistake."""
3287
+
3288
+
3289
+ def _warn_outside_group(having, order, kept, selected) -> None:
3290
+ """Name each variable HAVING or ORDER BY reads that the group does not keep (D2: unbound
3291
+ there). HAVING sees the keys and aggregates; ORDER BY the SELECT expressions too."""
3292
+ import warnings
3293
+ for where, exprs, seen in (("HAVING", having, kept), ("ORDER BY", [e for e, _ in order], kept | selected)):
3294
+ outside = sorted({c for e in exprs for c in e.columns()} - seen)
3295
+ outside = [c for c in outside if not c.startswith(HIDDEN)]
3296
+ if outside:
3297
+ names = ", ".join("?" + c for c in outside)
3298
+ warnings.warn(f"{where} reads {names}, which the group does not keep (a GROUP BY key or an "
3299
+ f"aggregate): it is unbound there (sparql-choices.md D2). Group by it, or "
3300
+ f"aggregate it (SAMPLE, MIN, ...).", OutsideGroupWarning, stacklevel=3)
3301
+
3302
+
3303
+ class DatatypeOverrideWarning(UserWarning):
3304
+ """An object map declares a DATE or TIMESTAMP column another XSD datatype (`rr:datatype`): every
3305
+ value is an ill-typed literal, a data error (R2RML 10.3, 11), so a query that reads the map
3306
+ raises R2RMLDataError."""
3307
+
3308
+
3309
+ # R2RML 10.3's validatable datatypes (the natural mapping's table), but xsd:string, whose
3310
+ # lexical space holds every text.
3311
+ _VALIDATABLE = frozenset({xsd.INTEGER, xsd.DECIMAL, xsd.DOUBLE, xsd.BOOLEAN, xsd.DATE, xsd.DATETIME,
3312
+ xsd.XSD + "hexBinary"})
3313
+
3314
+
3315
+ @home("select-star-order", "The order of SELECT *'s columns",
3316
+ terms=("SELECT *", "column order", "first mention", "in-scope variables", "18.2.1"))
3317
+ def star_order(where: Any, names: List[str]) -> List[str]:
3318
+ """SELECT *'s variables (18.2.1's in-scope ones, `names`) in the order the WHERE clause first
3319
+ writes them, as Jena and Oxigraph give them (the standard leaves the order open):
3320
+ `?o ^p ?s` names ?o then ?s, though the plan binds ?s first. A FILTER's mention does not count
3321
+ (it binds nothing); a name never written keeps its plan order, after the written ones."""
3322
+ from antlr4.tree.Tree import TerminalNode
3323
+ seen: List[str] = []
3324
+
3325
+ def walk(node: Any) -> None:
3326
+ if isinstance(node, TerminalNode):
3327
+ text = node.getText()
3328
+ if text[:1] in "?$" and len(text) > 1 and text[1:] not in seen:
3329
+ seen.append(text[1:])
3330
+ return
3331
+ if isinstance(node, SparqlParser.Filter_Context):
3332
+ return
3333
+ for i in range(node.getChildCount()):
3334
+ walk(node.getChild(i))
3335
+
3336
+ if where is not None:
3337
+ walk(where)
3338
+ wanted = set(names)
3339
+ written = [n for n in seen if n in wanted]
3340
+ return written + [n for n in names if n not in set(written)]
3341
+
3342
+
3343
+ @home("datatype-override-ill-typed", "A native column declared a datatype its natural form is not in",
3344
+ terms=("rr:datatype", "datatype override", "ill-typed", "data error", "R2RML 10.3",
3345
+ "TIMESTAMP declared xsd:date", "DOUBLE declared xsd:decimal", "R2RMLDataError", "DatatypeOverrideWarning"))
3346
+ def ill_typed_override(tm: Optional[TermMap], table: str, held: Any) -> Optional[str]:
3347
+ """What is wrong where `tm` reads a native column whole and declares a validatable datatype
3348
+ whose lexical space never holds the column's natural lexical form (R2RML 10.2), so every value
3349
+ is an ill-typed literal, a data error (R2RML 10.3): a DATE or TIMESTAMP declared another
3350
+ datatype (`1995-06-15T13:30:00` is no xsd:date), and a DOUBLE or FLOAT declared xsd:decimal
3351
+ or an integer type (`3.0E0` is neither: M2 review B6). Known from the column's type, never
3352
+ per row (an all-NULL column too). None where nothing is."""
3353
+ if tm is None or tm.column is None or tm.template is not None or tm.datatype is None:
3354
+ return None
3355
+ native = held(table, tm.column)[0]
3356
+ short = lambda dt: dt.replace(xsd.XSD, "xsd:")
3357
+ if native in (xsd.DOUBLE, xsd.FLOAT) and (tm.datatype == xsd.DECIMAL or tm.datatype in xsd.INTEGER_DATATYPES
3358
+ or tm.datatype == xsd.INTEGER):
3359
+ return (f"column {tm.column} of {table} is a DOUBLE declared {short(tm.datatype)}: its natural lexical form "
3360
+ f"(3.0E0, R2RML 10.2) is never one, so every value is an ill-typed literal, a data error "
3361
+ f"(R2RML 10.3); map it as xsd:double (or leave rr:datatype out), or CAST it AS DECIMAL in an "
3362
+ f"rr:sqlQuery (on a pandas frame, hold Decimal values)")
3363
+ if native not in (xsd.DATE, xsd.DATETIME) or tm.datatype == native or tm.datatype not in _VALIDATABLE:
3364
+ return None
3365
+ sql = "TIMESTAMP" if native == xsd.DATETIME else "DATE"
3366
+ fix = ("map it as xsd:dateTime (or leave rr:datatype out: its natural datatype), or, for dates, "
3367
+ "CAST it AS DATE in an rr:sqlQuery (on a pandas frame, df[col] = df[col].dt.date)"
3368
+ if native == xsd.DATETIME else "leave rr:datatype out (its natural datatype is xsd:date)")
3369
+ return (f"column {tm.column} of {table} is a {sql} declared {short(tm.datatype)}: every value is an "
3370
+ f"ill-typed literal, a data error (R2RML 10.3); {fix}")
3371
+
3372
+
3373
+ def _warn_overrides(maps: SourceMaps, held: Any) -> None:
3374
+ """Name each such object map when a query is planned over the maps (once per map)."""
3375
+ import warnings
3376
+ for m in maps.nodes:
3377
+ for po in m.predicate_objects:
3378
+ what = ill_typed_override(po.object, m.source.table, held)
3379
+ if what:
3380
+ warnings.warn(what, DatatypeOverrideWarning, stacklevel=4)
3381
+
3382
+
3383
+ class PatternNeverMatchesWarning(UserWarning):
3384
+ """A triple pattern's constant no map's term can be (m2-functions.md 5)."""
3385
+
3386
+
3387
+ def _say_never(p: IRITerm, o: LiteralTerm, never) -> None:
3388
+ """Say, at planning, that a pattern's literal matches no triple because
3389
+ of its written form: what the predicate's objects are, and the FILTER
3390
+ that compares values (decided 2026-11-09)."""
3391
+ import warnings
3392
+ from decimal import Decimal
3393
+ from prograph.schema import xsd as X
3394
+ short = lambda dt: dt.replace(X.XSD, "xsd:") if dt else "a literal"
3395
+ forms = []
3396
+ for _, slot in never:
3397
+ dts = sorted(d for d in slot.datatypes if d != UNKNOWN)
3398
+ dt = dts[0] if len(dts) == 1 else None
3399
+ if dt in (X.DECIMAL, X.INTEGER) and slot.native == dt:
3400
+ try:
3401
+ value = X.parse(o.lexical, o.datatype or X.STRING)
3402
+ example = X.canonical_lexical(Decimal(value) if dt == X.DECIMAL else int(value), dt)
3403
+ forms.append(f"{short(dt)} in canonical form (this value is written {example})")
3404
+ except (ValueError, TypeError, ArithmeticError):
3405
+ forms.append(f"{short(dt)} in canonical form")
3406
+ else:
3407
+ forms.append(", ".join(short(d) for d in dts) or "literals")
3408
+ written = f'"{o.lexical}"^^{short(o.datatype)}' if o.datatype else f'"{o.lexical}"'
3409
+ warnings.warn(
3410
+ f"<{p.text}> {written} matches no triple: the objects of <{p.text}> are "
3411
+ f"{'; '.join(dict.fromkeys(forms))}, and a pattern matches a term with the same "
3412
+ f"datatype and lexical form (RDF 1.1 3.3). To compare values, write "
3413
+ f"FILTER(?v = {o.lexical}).", PatternNeverMatchesWarning, stacklevel=2)
3414
+
3415
+
3416
+ def _empty(s: Term, p: Term, o: Term) -> O.Op:
3417
+ names = tuple(dict.fromkeys(t.name for t in (s, p, o) if isinstance(t, Var)))
3418
+ return O.Values(names, ())