prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,3418 @@
|
|
|
1
|
+
"""SPARQL's front end: a query's parse tree as the one plan (one-plan.md 5).
|
|
2
|
+
|
|
3
|
+
The ANTLR parse tree keeps each token's text, and this front end keeps it
|
|
4
|
+
too: a literal is the term the query wrote (`terms.LiteralTerm`, its
|
|
5
|
+
lexical form as written, `"01"^^xsd:integer` not `1`). The group
|
|
6
|
+
graph pattern is translated by SPARQL 1.1's algebra (18.2), and each
|
|
7
|
+
triple pattern becomes the union, over the source maps that may give its
|
|
8
|
+
triples, of a `MapScan` of the map's rows with `MakeTerm`s building the
|
|
9
|
+
terms (R2RML 11: a row with no value in a term's column gives no triple).
|
|
10
|
+
|
|
11
|
+
Anything outside what is built so far raises `NotBuilt`: reported, never
|
|
12
|
+
answered wrongly.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
import dataclasses
|
|
19
|
+
import re
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from typing import Any, Dict, FrozenSet, List, Optional, Sequence, Set, Tuple, Union as TUnion
|
|
22
|
+
|
|
23
|
+
from antlr4 import CommonTokenStream, InputStream
|
|
24
|
+
from antlr4.error.ErrorListener import ErrorListener
|
|
25
|
+
|
|
26
|
+
from prograph.parser.sparql.SparqlLexer import SparqlLexer
|
|
27
|
+
from antlr4.tree.Tree import TerminalNode
|
|
28
|
+
from prograph.parser.sparql.SparqlParser import SparqlParser
|
|
29
|
+
from prograph.plan.ir import expr as E
|
|
30
|
+
from prograph.plan.ir import ops as O
|
|
31
|
+
from prograph.plan.ir.facts import NotBuilt, const_slot, expr_slot
|
|
32
|
+
from prograph.plan.ir.slot import LOCAL, UNKNOWN, ZONED, Bound, never_one_term
|
|
33
|
+
from prograph.plan.ir.strings import BOOLEANS as BOOLEAN_STRING_CALLS, STRING_CALLS
|
|
34
|
+
from prograph.plan.ir.dates import DATE_PARTS
|
|
35
|
+
from prograph.plan.ir.dataset import graph_choices
|
|
36
|
+
from prograph.plan.ir.terms import BlankTerm, IRITerm, LiteralTerm
|
|
37
|
+
from prograph.schema import xsd
|
|
38
|
+
from prograph.rules import home
|
|
39
|
+
from prograph.schema.iri import LABEL_IRI_PREFIX
|
|
40
|
+
from prograph.schema.source_map import (
|
|
41
|
+
BLANK, IRI, EdgeMap, LEXICAL_R2RML, NodeMap, PredicateObject, SourceMaps, TermMap, UUID_READERS,
|
|
42
|
+
graphs_of,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
|
|
46
|
+
|
|
47
|
+
# Prefixes a query may use undeclared, as ProGraph has always read them.
|
|
48
|
+
STANDARD_PREFIXES = {
|
|
49
|
+
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
|
50
|
+
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
|
51
|
+
"xsd": xsd.XSD,
|
|
52
|
+
"owl": "http://www.w3.org/2002/07/owl#",
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
# A column the plan holds that is no variable of the query starts with a
|
|
56
|
+
# character no SPARQL variable name has (19.8, VARNAME).
|
|
57
|
+
HIDDEN = "#"
|
|
58
|
+
|
|
59
|
+
_ECHAR = {"t": "\t", "b": "\b", "n": "\n", "r": "\r", "f": "\f", '"': '"', "'": "'", "\\": "\\"}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class QueryError(ValueError):
|
|
63
|
+
"""A query that is not SPARQL, or names what it does not declare."""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class _Errors(ErrorListener):
|
|
67
|
+
def __init__(self) -> None:
|
|
68
|
+
super().__init__()
|
|
69
|
+
self.errors: List[str] = []
|
|
70
|
+
|
|
71
|
+
def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e): # noqa: N802
|
|
72
|
+
self.errors.append(f"line {line}:{column} {msg}")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def codepoint_escapes(query: str) -> str:
|
|
76
|
+
"""The query with each \\uXXXX and \\UXXXXXXXX replaced by its character
|
|
77
|
+
before lexing (SPARQL 1.1 19.2), in strings and IRIs too."""
|
|
78
|
+
def char(m):
|
|
79
|
+
code = int(m.group(1) or m.group(2), 16)
|
|
80
|
+
if 0xD800 <= code <= 0xDFFF or code > 0x10FFFF:
|
|
81
|
+
raise QueryError(f"{m.group(0)} is no character: a surrogate or beyond U+10FFFF")
|
|
82
|
+
return chr(code)
|
|
83
|
+
return re.sub(r"\\u([0-9A-Fa-f]{4})|\\U([0-9A-Fa-f]{8})", char, query)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def parse(query: str):
|
|
87
|
+
"""The parse tree of a SPARQL query, in ANTLR's two stages: SLL prediction first, which
|
|
88
|
+
gives the same tree wherever it succeeds and is several times faster in the Python runtime;
|
|
89
|
+
full LL, with its error reports, only for a text SLL cannot parse (parsing
|
|
90
|
+
was over half of a plan's time)."""
|
|
91
|
+
from antlr4.atn.PredictionMode import PredictionMode
|
|
92
|
+
from antlr4.error.ErrorStrategy import BailErrorStrategy, DefaultErrorStrategy
|
|
93
|
+
from antlr4.error.Errors import ParseCancellationException
|
|
94
|
+
lexer = SparqlLexer(InputStream(codepoint_escapes(query)))
|
|
95
|
+
tokens = CommonTokenStream(lexer)
|
|
96
|
+
parser = SparqlParser(tokens)
|
|
97
|
+
errors = _Errors()
|
|
98
|
+
lexer.removeErrorListeners()
|
|
99
|
+
lexer.addErrorListener(errors)
|
|
100
|
+
parser.removeErrorListeners()
|
|
101
|
+
parser._interp.predictionMode = PredictionMode.SLL
|
|
102
|
+
parser._errHandler = BailErrorStrategy()
|
|
103
|
+
try:
|
|
104
|
+
tree = parser.query()
|
|
105
|
+
except ParseCancellationException:
|
|
106
|
+
tokens.seek(0)
|
|
107
|
+
parser.reset()
|
|
108
|
+
parser.addErrorListener(errors)
|
|
109
|
+
parser._interp.predictionMode = PredictionMode.LL
|
|
110
|
+
parser._errHandler = DefaultErrorStrategy()
|
|
111
|
+
tree = parser.query()
|
|
112
|
+
if errors.errors:
|
|
113
|
+
raise QueryError("; ".join(errors.errors))
|
|
114
|
+
return tree
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def query_form(query: str) -> str:
|
|
118
|
+
"""What a SPARQL text is, from its parse alone: 'select', 'ask',
|
|
119
|
+
'construct', 'describe' or 'update'. For a caller that answers each
|
|
120
|
+
form differently before it runs one (the endpoint, the MCP tools).
|
|
121
|
+
Hints (`/*+ ... */`) are read off first, as `execute` reads them."""
|
|
122
|
+
from prograph.hints import parse_hints
|
|
123
|
+
tree = parse(parse_hints(query)[0])
|
|
124
|
+
for form in ("select", "ask", "construct", "describe"):
|
|
125
|
+
if getattr(tree, form + "Query")() is not None:
|
|
126
|
+
return form
|
|
127
|
+
if tree.updateCommand() is not None:
|
|
128
|
+
return "update"
|
|
129
|
+
raise QueryError("neither a query nor an update")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
# -- the query ----------------------------------------------------------------------
|
|
133
|
+
|
|
134
|
+
@dataclass
|
|
135
|
+
class Query:
|
|
136
|
+
"""A translated query: its form ('select', 'ask' or 'construct'), its
|
|
137
|
+
plan, the certificates of the rewrites applied to it
|
|
138
|
+
(`rewrite.Certificate`), and a CONSTRUCT's template: its triples of
|
|
139
|
+
terms, where a `Var` named HIDDEN... is a template blank node, fresh
|
|
140
|
+
per solution (16.2; `construct.instantiate`)."""
|
|
141
|
+
form: str
|
|
142
|
+
plan: O.Op
|
|
143
|
+
certificates: tuple = ()
|
|
144
|
+
template: tuple = ()
|
|
145
|
+
prefixes: Dict[str, str] = field(default_factory=dict)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
@dataclass(frozen=True)
|
|
149
|
+
class Var:
|
|
150
|
+
"""A variable of a pattern, or a blank node, which a pattern reads as
|
|
151
|
+
a variable it does not project (18.2.1)."""
|
|
152
|
+
name: str
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
Term = TUnion[Var, IRITerm, LiteralTerm, BlankTerm]
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
@dataclass(frozen=True)
|
|
159
|
+
class GraphSet:
|
|
160
|
+
"""What `bind` tests a graph term against under a dataset clause: any one of these IRIs."""
|
|
161
|
+
iris: frozenset
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@dataclass(frozen=True)
|
|
165
|
+
class Path:
|
|
166
|
+
"""A property path (18.4) that is not one IRI: `inv` (^p), `seq`
|
|
167
|
+
(p/q), `alt` (p|q), `mod` (p*, p+, p? with `mod`), `neg` (!(...): its
|
|
168
|
+
parts (IRI, inverse) pairs)."""
|
|
169
|
+
kind: str
|
|
170
|
+
parts: tuple
|
|
171
|
+
mod: str = ""
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def translate(query: str, maps: SourceMaps, held: Any, keys: Any = None,
|
|
175
|
+
union_default: bool = True, ontology: Any = None,
|
|
176
|
+
bindings: Optional[Dict[str, Term]] = None,
|
|
177
|
+
dataset: Optional[Tuple[Tuple[str, ...], Tuple[str, ...]]] = None,
|
|
178
|
+
identity: Any = None, implied_individuals: bool = False,
|
|
179
|
+
datatypes: str = "term") -> Query:
|
|
180
|
+
"""`query` as the one plan over `maps`. `held(table, column)` is the
|
|
181
|
+
datatype a source column's values have (None where unknown) and
|
|
182
|
+
whether a date-time column's values have a zone. `bindings` (the
|
|
183
|
+
call's parameters, `prograph.parameters`) is one VALUES row joined
|
|
184
|
+
with the top-level pattern where its own trailing VALUES joins.
|
|
185
|
+
`identity` (`identity.Identity`): owl:sameAs at BGP matching, None
|
|
186
|
+
for RDF term identity alone."""
|
|
187
|
+
tree = parse(query)
|
|
188
|
+
t = Translator(maps, held, keys, union_default, ontology)
|
|
189
|
+
t.identity = identity
|
|
190
|
+
if implied_individuals:
|
|
191
|
+
# Hidden terms (`hidden_terms`): a variable every occurrence of which is in one triples block, under no
|
|
192
|
+
# SELECT * (which projects every variable), may be an unnamed individual, as a blank node may.
|
|
193
|
+
from collections import Counter
|
|
194
|
+
t.unnamed = True
|
|
195
|
+
t.occurrences = Counter(_var_names(tree))
|
|
196
|
+
t.by_value = datatypes == "value"
|
|
197
|
+
if dataset is not None:
|
|
198
|
+
# The protocol's dataset (default-graph-uri, named-graph-uri) is used in place of the query's own.
|
|
199
|
+
t.dataset = (frozenset(dataset[0]), frozenset(dataset[1]))
|
|
200
|
+
t.dataset_given = True
|
|
201
|
+
if bindings:
|
|
202
|
+
unknown = sorted(set(bindings) - variables_of(tree))
|
|
203
|
+
if unknown:
|
|
204
|
+
from prograph.parameters import ParameterError
|
|
205
|
+
raise ParameterError(unknown[0], "not a variable of the query")
|
|
206
|
+
names = tuple(bindings)
|
|
207
|
+
t.bindings = O.Values(names, (tuple(bindings[n] for n in names),))
|
|
208
|
+
t.params = dict(bindings)
|
|
209
|
+
t.prologue(tree.prologue())
|
|
210
|
+
if identity is not None:
|
|
211
|
+
identity.prefetch(t.constants_of(tree))
|
|
212
|
+
if tree.selectQuery() is not None:
|
|
213
|
+
t.top = tree.selectQuery()
|
|
214
|
+
return Query("select", t.select(tree.selectQuery()))
|
|
215
|
+
if tree.askQuery() is not None:
|
|
216
|
+
ask = tree.askQuery()
|
|
217
|
+
t.dataset_clause(ask.datasetClause())
|
|
218
|
+
_no_modifiers(ask.solutionModifier(), "ASK")
|
|
219
|
+
t.top_group = ask.whereClause().groupGraphPattern()
|
|
220
|
+
return Query("ask", t.trailing_values(t.group(ask.whereClause().groupGraphPattern()),
|
|
221
|
+
ask.valuesClause()))
|
|
222
|
+
if tree.constructQuery() is not None:
|
|
223
|
+
plan, template = t.construct(tree.constructQuery())
|
|
224
|
+
return Query("construct", plan, template=template, prefixes=dict(t.prefixes))
|
|
225
|
+
if tree.describeQuery() is not None:
|
|
226
|
+
plan, template = t.describe(tree.describeQuery())
|
|
227
|
+
return Query("construct", plan, template=template, prefixes=dict(t.prefixes))
|
|
228
|
+
# A write: ProGraph does not write to the user's tables (rule read-only-writes; docs/sparql/update.md).
|
|
229
|
+
from prograph import read_only
|
|
230
|
+
read_only.refuse("SPARQL Update")
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _no_modifiers(ctx, what: str) -> None:
|
|
234
|
+
if ctx is not None and (ctx.groupClause() or ctx.havingClause() or ctx.orderClause()
|
|
235
|
+
or ctx.limitOffsetClauses()):
|
|
236
|
+
raise NotBuilt(f"solution modifiers on {what}")
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
@home("decimal-text", "A DECIMAL column's text in a term: canonical, or the database's",
|
|
240
|
+
terms=("decimal_text", "canonical decimal", "database text", "Ontop", "1.0", "D3"))
|
|
241
|
+
def held_under(held: Any, decimal_text: Optional[str]) -> Any:
|
|
242
|
+
"""How the SPARQL translator reads a column (`entry.held`'s tuple) under the schema's
|
|
243
|
+
`decimal_text` (sparql-choices.md D3). By default a DECIMAL column is read as its value, and
|
|
244
|
+
every text of it is XML Schema 1.1's canonical form (`1`, `1.5`; `lower.value_text`). Under
|
|
245
|
+
`database` it is read as the database's own text (`DATABASE_DECIMAL_TEXT`: `1.0`, `1.50`, as
|
|
246
|
+
Ontop writes it), a string whose literal is still an xsd:decimal (`Slot.text_of`): its text in a
|
|
247
|
+
literal, a template and an IRI column alike, its key compared as that text (a DECIMAL key and an
|
|
248
|
+
INTEGER or other-scale key do not meet). Cypher reads columns as they are."""
|
|
249
|
+
from prograph.schema.source_map import DATABASE_DECIMAL_TEXT, DECIMAL_DATABASE
|
|
250
|
+
if decimal_text != DECIMAL_DATABASE:
|
|
251
|
+
return held
|
|
252
|
+
|
|
253
|
+
def under(table: str, column: str):
|
|
254
|
+
dt, zoned, digits, reader, bytewise = held(table, column)
|
|
255
|
+
if dt == xsd.DECIMAL:
|
|
256
|
+
# Its declared digits kept: the text is read back by value as that DECIMAL(p, s)
|
|
257
|
+
# (`lower.decimal_of_text`), never as DECIMAL(38, 18), which holds 20 whole digits.
|
|
258
|
+
return xsd.STRING, False, digits, DATABASE_DECIMAL_TEXT, False
|
|
259
|
+
return dt, zoned, digits, reader, bytewise
|
|
260
|
+
for name in dir(held): # `kind_of` and the like
|
|
261
|
+
if not name.startswith("__") and not hasattr(under, name):
|
|
262
|
+
setattr(under, name, getattr(held, name))
|
|
263
|
+
return under
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
DIFFERENT_FROM = "http://www.w3.org/2002/07/owl#differentFrom"
|
|
267
|
+
NAMED_INDIVIDUAL = "http://www.w3.org/2002/07/owl#NamedIndividual"
|
|
268
|
+
TOP_OBJECT_PROPERTY = "http://www.w3.org/2002/07/owl#topObjectProperty"
|
|
269
|
+
SUB_PROPERTY_IRI = "http://www.w3.org/2000/01/rdf-schema#subPropertyOf"
|
|
270
|
+
def typing_of(patterns) -> FrozenSet[str]:
|
|
271
|
+
"""The variables a typing triple of `patterns` types (Entailment Regimes 7.1.3)."""
|
|
272
|
+
return frozenset(t[0].name for t in patterns if isinstance(t[0], Var) and t[1] == IRITerm(RDF_TYPE)
|
|
273
|
+
and isinstance(t[2], IRITerm) and t[2].text in TYPING_CLASSES)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
# The classes of Entailment Regimes 7.1.3's typing triples.
|
|
277
|
+
TYPING_CLASSES = frozenset("http://www.w3.org/2002/07/owl#" + c for c in
|
|
278
|
+
("Class", "ObjectProperty", "DatatypeProperty", "NamedIndividual"))
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
class Translator:
|
|
282
|
+
"""One query's translation: its prefixes and base, and fresh names."""
|
|
283
|
+
|
|
284
|
+
def __init__(self, maps: SourceMaps, held: Any, keys: Any = None, union_default: bool = True,
|
|
285
|
+
ontology: Any = None) -> None:
|
|
286
|
+
self.maps = maps
|
|
287
|
+
self.held = held_under(held, getattr(maps.settings, "decimal_text", None))
|
|
288
|
+
# The call's parameters as one VALUES row, joined with the top-level query's pattern (`bound`).
|
|
289
|
+
self.bindings: Optional[O.Op] = None
|
|
290
|
+
# Each parameter's term: where the query writes the variable, it reads the constant (Jena's
|
|
291
|
+
# substitution), so a scope evaluated on its own (an OPTIONAL's FILTER, MINUS, a UNION
|
|
292
|
+
# branch, a nested BIND) sees the value; the top-level VALUES row keeps it in the answers.
|
|
293
|
+
self.params: Dict[str, Term] = {}
|
|
294
|
+
self.top: Any = None # the top-level SELECT's context: a sub-select takes no bindings
|
|
295
|
+
self.top_group: Any = None # the top-level query's WHERE group, which the bindings join
|
|
296
|
+
# The ontology's domain and range typing (`saturate`): (property, classes, as subject).
|
|
297
|
+
from prograph.plan.ir.saturate import typing_properties
|
|
298
|
+
self.typing = typing_properties(ontology, maps.by_predicate, maps) if ontology is not None else []
|
|
299
|
+
self.ontology = ontology
|
|
300
|
+
# The ontology's own triples in the default graph (`dataset.ontology_triples`), and > 0 while a
|
|
301
|
+
# witness of the data's vocabulary is matched (`ontology_rows`), which reads the data alone.
|
|
302
|
+
from prograph.plan.ir.dataset import ontology_triples
|
|
303
|
+
self.onto_triples = ontology_triples(maps, ontology) if ontology is not None else None
|
|
304
|
+
self.witnessing = 0
|
|
305
|
+
self.differing = 0 # > 0 while `different_branch`'s patterns are translated: owl:differentFrom as stated
|
|
306
|
+
# The graph the patterns read (13, 18.5): None the default graph, else GRAPH's IRI or
|
|
307
|
+
# variable; and whether the default graph is the union of every graph (default_graph='union').
|
|
308
|
+
self.graph: Optional[Term] = None
|
|
309
|
+
self.patterns = 0 # triple patterns translated so far (`graph_pattern` asks whether P had one)
|
|
310
|
+
self.group_base: Optional[O.Op] = None # what the next group starts from (`graph_pattern`)
|
|
311
|
+
self.path_seed: Optional[Tuple[str, str, O.Op, str, bool]] = None # (side, variable, its terms, column, graph nodes)
|
|
312
|
+
# The variables a VALUES block of one row binds to an IRI in the group being translated, each with
|
|
313
|
+
# its IRI: the group's own triple patterns read the constant (`values_bound`).
|
|
314
|
+
self.narrowed: Dict[str, IRITerm] = {}
|
|
315
|
+
self.union_default = union_default
|
|
316
|
+
# The query's dataset (13.2), (FROM IRIs, FROM NAMED IRIs), None where it has no dataset clause.
|
|
317
|
+
self.dataset: Optional[Tuple[frozenset, frozenset]] = None
|
|
318
|
+
self.blank_scope: Optional[str] = None # `scope_of`
|
|
319
|
+
self.dataset_given = False # set by the caller (the protocol): the query's own clauses are ignored
|
|
320
|
+
_warn_overrides(maps, held)
|
|
321
|
+
# The keys a scan of a map may rely on (`node_keys.trusted_keys`).
|
|
322
|
+
self.keys = keys or (lambda m: ())
|
|
323
|
+
self.prefixes: Dict[str, str] = {}
|
|
324
|
+
self.base: Optional[str] = None
|
|
325
|
+
# NOW()'s one value for the query (17.4.5.1): the moment it is planned, UTC, to the millisecond.
|
|
326
|
+
import datetime as _dt
|
|
327
|
+
moment = _dt.datetime.now(_dt.timezone.utc)
|
|
328
|
+
moment = moment.replace(microsecond=moment.microsecond // 1000 * 1000)
|
|
329
|
+
self.now = xsd.canonical_lexical(moment, xsd.DATETIME)
|
|
330
|
+
self.n = 0
|
|
331
|
+
self.fresh_calls = 0
|
|
332
|
+
self.aggregates: Optional[List[Tuple[str, E.Agg]]] = None
|
|
333
|
+
# owl:sameAs at BGP matching (`identity.pattern`), None for RDF term identity alone; `plain` > 0
|
|
334
|
+
# while a pattern is matched as stated (inside a pattern owl:sameAs already handles).
|
|
335
|
+
self.identity: Any = None
|
|
336
|
+
self.plain = 0
|
|
337
|
+
|
|
338
|
+
def fresh(self, what: str = "h") -> str:
|
|
339
|
+
self.n += 1
|
|
340
|
+
return f"{HIDDEN}{what}{self.n}"
|
|
341
|
+
|
|
342
|
+
# -- terms ------------------------------------------------------------------
|
|
343
|
+
|
|
344
|
+
def prologue(self, ctx) -> None:
|
|
345
|
+
"""PREFIX and BASE in order: a BASE applies to what follows (4.1.1)."""
|
|
346
|
+
for child in ctx.getChildren():
|
|
347
|
+
if isinstance(child, SparqlParser.BaseDeclContext):
|
|
348
|
+
self.base = self.resolve(child.IRI_REF().getText())
|
|
349
|
+
elif isinstance(child, SparqlParser.PrefixDeclContext):
|
|
350
|
+
self.prefixes[child.PNAME_NS().getText()[:-1]] = \
|
|
351
|
+
self.resolve(child.IRI_REF().getText())
|
|
352
|
+
|
|
353
|
+
def constants_of(self, tree) -> List[str]:
|
|
354
|
+
"""Every IRI a query names (and rdf:type, which `a` writes): the constants owl:sameAs looks up at planning
|
|
355
|
+
(`Identity.prefetch`), read once for the whole query."""
|
|
356
|
+
out: List[str] = [RDF_TYPE]
|
|
357
|
+
stack = [tree]
|
|
358
|
+
while stack:
|
|
359
|
+
node = stack.pop()
|
|
360
|
+
if isinstance(node, SparqlParser.IriRefContext):
|
|
361
|
+
try:
|
|
362
|
+
out.append(self.iri(node))
|
|
363
|
+
except QueryError:
|
|
364
|
+
pass
|
|
365
|
+
continue
|
|
366
|
+
stack.extend(node.getChildren() if hasattr(node, "getChildren") else ())
|
|
367
|
+
return out
|
|
368
|
+
|
|
369
|
+
def dataset_clause(self, clauses) -> None:
|
|
370
|
+
"""FROM and FROM NAMED (13.2): the graphs merged into the default graph and the named graphs,
|
|
371
|
+
each in place of the service's dataset (no FROM NAMED: no named graph; no FROM: an empty
|
|
372
|
+
default graph). A dataset the caller gave (the protocol's) is used in place of these."""
|
|
373
|
+
if not clauses or self.dataset_given:
|
|
374
|
+
return
|
|
375
|
+
default, named = set(), set()
|
|
376
|
+
for clause in clauses:
|
|
377
|
+
if clause.namedGraphClause() is not None:
|
|
378
|
+
named.add(self.iri(clause.namedGraphClause().sourceSelector().iriRef()))
|
|
379
|
+
else:
|
|
380
|
+
default.add(self.iri(clause.defaultGraphClause().sourceSelector().iriRef()))
|
|
381
|
+
self.dataset = (frozenset(default), frozenset(named))
|
|
382
|
+
|
|
383
|
+
def resolve(self, token: str) -> str:
|
|
384
|
+
"""An IRI_REF's IRI, resolved against BASE where one is declared."""
|
|
385
|
+
iri = token[1:-1] if token.startswith("<") else token
|
|
386
|
+
if not self.base:
|
|
387
|
+
return iri
|
|
388
|
+
from urllib.parse import urljoin
|
|
389
|
+
resolved = urljoin(self.base, iri)
|
|
390
|
+
if iri.endswith("#") and not resolved.endswith("#"):
|
|
391
|
+
resolved += "#"
|
|
392
|
+
return resolved
|
|
393
|
+
|
|
394
|
+
def iri(self, ctx) -> str:
|
|
395
|
+
"""An iriRef's or prefixedName's IRI."""
|
|
396
|
+
if isinstance(ctx, SparqlParser.IriRefContext):
|
|
397
|
+
if ctx.prefixedName() is None:
|
|
398
|
+
return self.resolve(ctx.getText())
|
|
399
|
+
ctx = ctx.prefixedName()
|
|
400
|
+
prefix, _, local = ctx.getText().partition(":")
|
|
401
|
+
local = re.sub(r"\\(.)", r"\1", local) # PN_LOCAL_ESC (19.8); %XX stays
|
|
402
|
+
if prefix in self.prefixes:
|
|
403
|
+
return self.prefixes[prefix] + local
|
|
404
|
+
if prefix in STANDARD_PREFIXES:
|
|
405
|
+
return STANDARD_PREFIXES[prefix] + local
|
|
406
|
+
raise QueryError(f"Undeclared prefix '{prefix}:' in {ctx.getText()}")
|
|
407
|
+
|
|
408
|
+
@staticmethod
|
|
409
|
+
def string(ctx) -> str:
|
|
410
|
+
text = ctx.getText()
|
|
411
|
+
text = text[3:-3] if text[:3] in ('"""', "'''") else text[1:-1]
|
|
412
|
+
return re.sub(r"""\\([tbnrf"'\\])""", lambda m: _ECHAR[m.group(1)], text)
|
|
413
|
+
|
|
414
|
+
def literal(self, ctx) -> LiteralTerm:
|
|
415
|
+
"""An rdfLiteral, numericLiteral or booleanLiteral, as written."""
|
|
416
|
+
if isinstance(ctx, SparqlParser.RdfLiteralContext):
|
|
417
|
+
lexical = self.string(ctx.string_())
|
|
418
|
+
if ctx.LANGTAG() is not None:
|
|
419
|
+
return LiteralTerm(lexical, language=ctx.LANGTAG().getText()[1:])
|
|
420
|
+
if ctx.iriRef() is not None:
|
|
421
|
+
return LiteralTerm(lexical, self.iri(ctx.iriRef()))
|
|
422
|
+
return LiteralTerm(lexical)
|
|
423
|
+
if isinstance(ctx, SparqlParser.BooleanLiteralContext):
|
|
424
|
+
return LiteralTerm(ctx.getText().lower(), xsd.BOOLEAN)
|
|
425
|
+
return numeric(ctx.getText())
|
|
426
|
+
|
|
427
|
+
def term(self, ctx) -> Term:
|
|
428
|
+
"""A varOrTerm, varOrIRIref or graphTerm."""
|
|
429
|
+
if isinstance(ctx, (SparqlParser.VarOrTermContext, SparqlParser.VarOrIRIrefContext)):
|
|
430
|
+
if ctx.var_() is not None:
|
|
431
|
+
name = ctx.var_().getText()[1:]
|
|
432
|
+
return self.params.get(name) or Var(name)
|
|
433
|
+
inner = ctx.graphTerm() if isinstance(ctx, SparqlParser.VarOrTermContext) \
|
|
434
|
+
else ctx.iriRef()
|
|
435
|
+
return self.term(inner)
|
|
436
|
+
if isinstance(ctx, SparqlParser.IriRefContext):
|
|
437
|
+
text = self.iri(ctx)
|
|
438
|
+
if text.startswith(LABEL_IRI_PREFIX):
|
|
439
|
+
raise _label_pattern(text[len(LABEL_IRI_PREFIX):], self.maps)
|
|
440
|
+
return IRITerm(text)
|
|
441
|
+
if isinstance(ctx, SparqlParser.GraphTermContext):
|
|
442
|
+
for get in (ctx.iriRef, ctx.rdfLiteral, ctx.numericLiteral, ctx.booleanLiteral):
|
|
443
|
+
found = get()
|
|
444
|
+
if found is not None:
|
|
445
|
+
return self.term(found) if get == ctx.iriRef else self.literal(found)
|
|
446
|
+
if ctx.blankNode() is not None:
|
|
447
|
+
label = ctx.blankNode().BLANK_NODE_LABEL()
|
|
448
|
+
return Var(f"{HIDDEN}b:{label.getText()[2:]}" if label else self.fresh("b"))
|
|
449
|
+
raise NotBuilt("rdf:nil in a pattern")
|
|
450
|
+
raise NotBuilt(f"the term {ctx.getText()}")
|
|
451
|
+
|
|
452
|
+
# -- SELECT -----------------------------------------------------------------
|
|
453
|
+
|
|
454
|
+
def select(self, ctx) -> O.Op:
|
|
455
|
+
"""A SELECT's plan (18.2.4, 18.2.5): the pattern; its groups and
|
|
456
|
+
aggregates, then HAVING (18.2.4.1); the SELECT expressions; ORDER
|
|
457
|
+
BY; the projection; DISTINCT; the slice."""
|
|
458
|
+
# Under `implied_individuals`, SELECT * projects every variable of its pattern and COUNT(*) counts every
|
|
459
|
+
# solution of it: no variable of this SELECT's own pattern is hidden (a sub-SELECT's are its own).
|
|
460
|
+
held = self.select_star
|
|
461
|
+
self.select_star = _star_like(ctx)
|
|
462
|
+
try:
|
|
463
|
+
return self._select(ctx)
|
|
464
|
+
finally:
|
|
465
|
+
self.select_star = held
|
|
466
|
+
|
|
467
|
+
def _select(self, ctx) -> O.Op:
|
|
468
|
+
self.dataset_clause(ctx.datasetClause())
|
|
469
|
+
mods = ctx.solutionModifier()
|
|
470
|
+
if ctx is self.top:
|
|
471
|
+
self.top_group = ctx.whereClause().groupGraphPattern()
|
|
472
|
+
x = self.trailing_values(self.group(ctx.whereClause().groupGraphPattern()), ctx.valuesClause())
|
|
473
|
+
# The WHERE pattern's in-scope variables (18.2.1): no SELECT expression may bind one, grouped
|
|
474
|
+
# or not (`SELECT (COUNT(*) AS ?c) { ?s ?p ?c }` is a static error).
|
|
475
|
+
in_scope = set(x.columns)
|
|
476
|
+
x, keys = self.group_keys(x, mods)
|
|
477
|
+
# The expressions of SELECT, HAVING and ORDER BY, their aggregates
|
|
478
|
+
# collected (18.2.4.1: each is a column of the Aggregate).
|
|
479
|
+
self.aggregates: Optional[List[Tuple[str, E.Agg]]] = []
|
|
480
|
+
selected: List[Tuple[str, Optional[E.Expr]]] = []
|
|
481
|
+
for clause in ctx.selectClause():
|
|
482
|
+
name = clause.var_().getText()[1:]
|
|
483
|
+
expr = self.expression(clause.expression()) if clause.expression() is not None else None
|
|
484
|
+
if any(name == n for n, _ in selected):
|
|
485
|
+
if expr is None:
|
|
486
|
+
continue # the projection is a set of variables (18.2.5.2)
|
|
487
|
+
raise QueryError(f"?{name} is projected twice")
|
|
488
|
+
selected.append((name, expr))
|
|
489
|
+
having = [self.condition(h.constraint()) for h in mods.havingClause().havingCondition()] \
|
|
490
|
+
if mods.havingClause() is not None else []
|
|
491
|
+
order = self.order_conditions(mods)
|
|
492
|
+
aggregates, self.aggregates = self.aggregates, None
|
|
493
|
+
# 18.2.4.1: with no GROUP BY and no aggregate, the aggregate step is
|
|
494
|
+
# skipped, and HAVING is a filter over the solutions.
|
|
495
|
+
grouped = bool(keys) or bool(aggregates)
|
|
496
|
+
# A sub-select inside GRAPH ?g is evaluated in each graph (18.5): it keeps the active graph's
|
|
497
|
+
# hidden column, groups by it, and is DISTINCT within each graph.
|
|
498
|
+
part = self.graph.name if isinstance(self.graph, Var) and self.graph.name in x.columns else None
|
|
499
|
+
if part is not None:
|
|
500
|
+
if grouped and not keys:
|
|
501
|
+
raise NotBuilt("an aggregate with no GROUP BY in a sub-select inside GRAPH ?g "
|
|
502
|
+
"(one row per graph, an empty graph's included)")
|
|
503
|
+
if mods.limitOffsetClauses() is not None:
|
|
504
|
+
raise NotBuilt("LIMIT or OFFSET in a sub-select inside GRAPH ?g (per graph)")
|
|
505
|
+
if grouped:
|
|
506
|
+
keys.append(part)
|
|
507
|
+
names: List[str] = []
|
|
508
|
+
if ctx.STAR() is not None:
|
|
509
|
+
if grouped:
|
|
510
|
+
raise QueryError("SELECT * in a grouped query (18.2.4.1)")
|
|
511
|
+
names = star_order(ctx.whereClause(), [c for c in x.columns if not c.startswith(HIDDEN)])
|
|
512
|
+
if grouped:
|
|
513
|
+
# 18.2.4.1: a grouped query reads its keys and aggregates only.
|
|
514
|
+
visible = set(keys) | {n for n, _ in aggregates}
|
|
515
|
+
for name, expr in selected:
|
|
516
|
+
_grouped_only({name} if expr is None else expr.columns(), visible, "SELECT")
|
|
517
|
+
visible.add(name)
|
|
518
|
+
# HAVING and ORDER BY may read any variable (11.4 restricts only
|
|
519
|
+
# what SELECT projects). One the group does not keep is unbound
|
|
520
|
+
# there: 18.2.4.1's "unaggregated variable" is undefined (W3C
|
|
521
|
+
# errata-query-14), its editor read it as the group keys, and
|
|
522
|
+
# Jena and Oxigraph agree (docs/architecture/testing.md). A
|
|
523
|
+
# SELECT expression's variable is not visible to HAVING (18.2.4.2).
|
|
524
|
+
if self.by_value and self.value_columns(x, keys):
|
|
525
|
+
x = self.grouped_by_value(x, keys, aggregates)
|
|
526
|
+
else:
|
|
527
|
+
x = O.Aggregate(x, tuple(keys), tuple(aggregates))
|
|
528
|
+
_warn_outside_group(having, order, set(keys) | {n for n, _ in aggregates},
|
|
529
|
+
{n for n, _ in selected})
|
|
530
|
+
# 18.2.4: grouping, aggregates, HAVING, then the SELECT expressions:
|
|
531
|
+
# HAVING does not see a SELECT expression's variable.
|
|
532
|
+
if having:
|
|
533
|
+
x = O.Filter(x, having[0] if len(having) == 1 else E.And(tuple(having)))
|
|
534
|
+
for name, expr in selected:
|
|
535
|
+
if expr is not None:
|
|
536
|
+
if name in x.columns or name in in_scope:
|
|
537
|
+
raise QueryError(f"?{name} is bound already; SELECT (... AS ?{name}) "
|
|
538
|
+
"binds a new variable (18.2.4.4)")
|
|
539
|
+
x = O.Extend(x, name, expr)
|
|
540
|
+
names.append(name)
|
|
541
|
+
kept = tuple(names) + ((part,) if part is not None and part not in names else ())
|
|
542
|
+
if ctx.DISTINCT() is not None and self.by_value and self.value_columns(O.Project(x, kept), kept):
|
|
543
|
+
# One row per value, then ORDER BY (a group keeps no order): its keys read only what DISTINCT keeps.
|
|
544
|
+
outside = {c for e, _ in order for c in e.columns()} - set(kept)
|
|
545
|
+
if outside:
|
|
546
|
+
raise NotBuilt("ORDER BY a variable SELECT DISTINCT does not project, under datatypes='value'")
|
|
547
|
+
x = self.ordered(self.distinct_by_value(O.Project(x, kept)), order)
|
|
548
|
+
return self.sliced(x, mods)
|
|
549
|
+
x = self.ordered(x, order)
|
|
550
|
+
x = O.Project(x, kept)
|
|
551
|
+
if ctx.DISTINCT() is not None:
|
|
552
|
+
x = O.Distinct(x)
|
|
553
|
+
elif ctx.REDUCED() is not None:
|
|
554
|
+
x = O.Reduced(x)
|
|
555
|
+
return self.sliced(x, mods)
|
|
556
|
+
|
|
557
|
+
def group_keys(self, x: O.Op, mods) -> Tuple[O.Op, List[str]]:
|
|
558
|
+
"""GROUP BY's keys (18.2.4.1): each a variable, or an expression bound first (`(expr AS ?v)`,
|
|
559
|
+
or a fresh column). The plan with them bound, and their names."""
|
|
560
|
+
keys: List[str] = []
|
|
561
|
+
if mods.groupClause() is not None:
|
|
562
|
+
for gc in mods.groupClause().groupCondition():
|
|
563
|
+
if gc.var_() is not None and gc.expression() is None:
|
|
564
|
+
name = gc.var_().getText()[1:]
|
|
565
|
+
if name not in x.columns:
|
|
566
|
+
x = O.Extend(x, name, E.Const(None))
|
|
567
|
+
else:
|
|
568
|
+
inner = gc.expression() if gc.expression() is not None else gc.getChild(0)
|
|
569
|
+
e = self.expression(inner)
|
|
570
|
+
if isinstance(e, E.Col) and gc.var_() is None:
|
|
571
|
+
name = e.name # `(?x)` is ?x itself (11.4)
|
|
572
|
+
if name not in x.columns:
|
|
573
|
+
x = O.Extend(x, name, E.Const(None))
|
|
574
|
+
else:
|
|
575
|
+
name = gc.var_().getText()[1:] if gc.var_() is not None else self.fresh("g")
|
|
576
|
+
if name in x.columns:
|
|
577
|
+
raise QueryError(f"GROUP BY (... AS ?{name}) binds ?{name}, which "
|
|
578
|
+
"the pattern binds already")
|
|
579
|
+
x = O.Extend(x, name, e)
|
|
580
|
+
if name not in keys:
|
|
581
|
+
keys.append(name)
|
|
582
|
+
return x, keys
|
|
583
|
+
|
|
584
|
+
def grouped(self, x: O.Op, mods) -> Tuple[O.Op, List[Tuple[E.Expr, bool]], Optional[List[str]]]:
|
|
585
|
+
"""A CONSTRUCT's or DESCRIBE's solutions under GROUP BY and HAVING (11, 18.2.4.1, 16.2): with
|
|
586
|
+
either, or an aggregate in ORDER BY, one solution per group, which binds its GROUP BY keys
|
|
587
|
+
(`(expr AS ?v)` too) and nothing else, so a template variable the keys do not name is
|
|
588
|
+
unbound and leaves its triple out (16.2; Jena). HAVING and ORDER BY read aggregates. Returns
|
|
589
|
+
the plan, ORDER BY's keys and the group's keys (None when there is no grouping)."""
|
|
590
|
+
if mods is None:
|
|
591
|
+
return x, [], None
|
|
592
|
+
x, keys = self.group_keys(x, mods)
|
|
593
|
+
self.aggregates = []
|
|
594
|
+
having = [self.condition(h.constraint()) for h in mods.havingClause().havingCondition()] \
|
|
595
|
+
if mods.havingClause() is not None else []
|
|
596
|
+
order = self.order_conditions(mods)
|
|
597
|
+
aggregates, self.aggregates = self.aggregates, None
|
|
598
|
+
if not (keys or aggregates):
|
|
599
|
+
return (O.Filter(x, having[0] if len(having) == 1 else E.And(tuple(having))) if having else x), order, None
|
|
600
|
+
x = O.Aggregate(x, tuple(keys), tuple(aggregates))
|
|
601
|
+
_warn_outside_group(having, order, set(keys) | {n for n, _ in aggregates}, set())
|
|
602
|
+
if having:
|
|
603
|
+
x = O.Filter(x, having[0] if len(having) == 1 else E.And(tuple(having)))
|
|
604
|
+
return x, order, keys
|
|
605
|
+
|
|
606
|
+
def order_conditions(self, mods) -> List[Tuple[E.Expr, bool]]:
|
|
607
|
+
"""ORDER BY's keys, each an expression and whether descending."""
|
|
608
|
+
order: List[Tuple[E.Expr, bool]] = []
|
|
609
|
+
if mods is not None and mods.orderClause() is not None:
|
|
610
|
+
for cond in mods.orderClause().orderCondition():
|
|
611
|
+
if cond.var_() is not None:
|
|
612
|
+
var = cond.var_().getText()[1:]
|
|
613
|
+
order.append((E.Const(self.params[var]) if var in self.params else E.Col(var),
|
|
614
|
+
cond.DESC() is not None))
|
|
615
|
+
else:
|
|
616
|
+
inner = cond.brackettedExpression() or cond.constraint()
|
|
617
|
+
order.append((self.expression(inner), cond.DESC() is not None))
|
|
618
|
+
return order
|
|
619
|
+
|
|
620
|
+
def ordered(self, x: O.Op, order: List[Tuple[E.Expr, bool]]) -> O.Op:
|
|
621
|
+
"""`x` sorted by ORDER BY's keys (15.1)."""
|
|
622
|
+
sort = []
|
|
623
|
+
for e, descending in order:
|
|
624
|
+
if isinstance(e, E.Col):
|
|
625
|
+
column = e.name
|
|
626
|
+
if column not in x.columns:
|
|
627
|
+
continue # never bound: every row ties (15.1)
|
|
628
|
+
else:
|
|
629
|
+
column = self.fresh("k")
|
|
630
|
+
x = O.Extend(x, column, e)
|
|
631
|
+
sort.append(O.SortKey(column, descending))
|
|
632
|
+
return O.Order(x, tuple(sort)) if sort else x
|
|
633
|
+
|
|
634
|
+
@staticmethod
|
|
635
|
+
def sliced(x: O.Op, mods) -> O.Op:
|
|
636
|
+
"""`x` under LIMIT and OFFSET (15.4, 15.5)."""
|
|
637
|
+
lo = mods.limitOffsetClauses() if mods is not None else None
|
|
638
|
+
if lo is not None:
|
|
639
|
+
limit = int(lo.limitClause().INTEGER().getText()) if lo.limitClause() else None
|
|
640
|
+
offset = int(lo.offsetClause().INTEGER().getText()) if lo.offsetClause() else 0
|
|
641
|
+
x = O.Slice(x, offset, limit)
|
|
642
|
+
return x
|
|
643
|
+
|
|
644
|
+
# -- CONSTRUCT --------------------------------------------------------------
|
|
645
|
+
|
|
646
|
+
def construct(self, ctx) -> Tuple[O.Op, tuple]:
|
|
647
|
+
"""A CONSTRUCT's plan and template (16.2). The plan is the pattern's
|
|
648
|
+
solutions under ORDER BY, OFFSET and LIMIT, projected on the
|
|
649
|
+
template's variables; the template is instantiated above the plan
|
|
650
|
+
(one-plan.md 5.10). `CONSTRUCT WHERE { T }` is `CONSTRUCT { T }
|
|
651
|
+
WHERE { T }` (16.2.4). A template's blank node is a hidden variable
|
|
652
|
+
here, which no pattern binds."""
|
|
653
|
+
self.dataset_clause(ctx.datasetClause())
|
|
654
|
+
mods = ctx.solutionModifier()
|
|
655
|
+
template: List[Tuple[Term, Term, Term]] = []
|
|
656
|
+
if ctx.constructTemplate() is not None:
|
|
657
|
+
self.top_group = ctx.whereClause().groupGraphPattern()
|
|
658
|
+
x = self.group(ctx.whereClause().groupGraphPattern())
|
|
659
|
+
triples = ctx.constructTemplate().constructTriples()
|
|
660
|
+
if triples is not None:
|
|
661
|
+
for same in triples.triplesSameSubject():
|
|
662
|
+
self.same_subject(same, template)
|
|
663
|
+
else:
|
|
664
|
+
shape = ctx.triplesTemplate()
|
|
665
|
+
# CONSTRUCT WHERE { T }: T is the group, and has no FILTER.
|
|
666
|
+
x = self.bound(self.triples_block(shape) if shape is not None else O.Unit())
|
|
667
|
+
if shape is not None:
|
|
668
|
+
for same in shape.triplesSameSubject():
|
|
669
|
+
self.same_subject(same, template)
|
|
670
|
+
# The pattern's blank nodes are its variables; the template's
|
|
671
|
+
# are fresh per solution: two names, never one.
|
|
672
|
+
template = [tuple(Var(f"{HIDDEN}t{t.name}") if isinstance(t, Var)
|
|
673
|
+
and t.name.startswith(HIDDEN) else t for t in triple)
|
|
674
|
+
for triple in template]
|
|
675
|
+
# 18.2.4.3: a trailing VALUES is joined after the groups and HAVING.
|
|
676
|
+
x, order, _ = self.grouped(x, mods)
|
|
677
|
+
x = self.trailing_values(x, ctx.valuesClause())
|
|
678
|
+
x = self.ordered(x, order)
|
|
679
|
+
wanted = [t.name for triple in template for t in triple
|
|
680
|
+
if isinstance(t, Var) and not t.name.startswith(HIDDEN)]
|
|
681
|
+
x = O.Project(x, tuple(n for n in dict.fromkeys(wanted) if n in x.columns))
|
|
682
|
+
return self.sliced(x, mods), tuple(template)
|
|
683
|
+
|
|
684
|
+
def describe(self, ctx) -> Tuple[O.Op, tuple]:
|
|
685
|
+
"""A DESCRIBE (16.4) as a CONSTRUCT (sparql-choices.md S13, as Jena and Oxigraph): each
|
|
686
|
+
resource it names (an IRI, a variable's values in the pattern's solutions, `*` every visible
|
|
687
|
+
variable) by the triples whose subject it is, and each blank node one of those triples
|
|
688
|
+
reaches as its object, so on (the blank-node closure: a `*` closure over the steps to a blank
|
|
689
|
+
object, from the named resources)."""
|
|
690
|
+
self.dataset_clause(ctx.datasetClause())
|
|
691
|
+
mods = ctx.solutionModifier()
|
|
692
|
+
where = ctx.whereClause()
|
|
693
|
+
if where is not None:
|
|
694
|
+
self.top_group = where.groupGraphPattern()
|
|
695
|
+
x = self.group(where.groupGraphPattern()) if where is not None else self.bound(O.Unit())
|
|
696
|
+
x, order, keys = self.grouped(x, mods)
|
|
697
|
+
before = set(x.columns)
|
|
698
|
+
x = self.trailing_values(x, ctx.valuesClause()) # 18.2.4.3: after the groups and HAVING
|
|
699
|
+
if keys is not None:
|
|
700
|
+
keys = list(keys) + [c for c in x.columns if c not in before]
|
|
701
|
+
x = self.sliced(self.ordered(x, order), mods)
|
|
702
|
+
if ctx.STAR() is not None:
|
|
703
|
+
named = [Var(n) for n in x.columns if not n.startswith(HIDDEN)]
|
|
704
|
+
else:
|
|
705
|
+
named = [self.term(v) for v in ctx.varOrIRIref()]
|
|
706
|
+
if keys is not None: # Jena: a grouped DESCRIBE names only what the groups keep
|
|
707
|
+
_grouped_only([t.name for t in named if isinstance(t, Var)], set(keys), "DESCRIBE")
|
|
708
|
+
# Names no query variable can have (`@`), and not hidden ones, which a template reads as
|
|
709
|
+
# blank nodes.
|
|
710
|
+
r, p, o = Var("@r"), Var("@p"), Var("@o")
|
|
711
|
+
starts: List[O.Op] = []
|
|
712
|
+
for thing in dict.fromkeys(named):
|
|
713
|
+
if isinstance(thing, Var):
|
|
714
|
+
if thing.name in x.columns: # a variable the pattern does not bind names nothing
|
|
715
|
+
starts.append(O.Project(O.Extend(O.Project(x, (thing.name,)), r.name, E.Col(thing.name)),
|
|
716
|
+
(r.name,)))
|
|
717
|
+
else:
|
|
718
|
+
starts.append(O.Values((r.name,), ((thing,),)))
|
|
719
|
+
if not starts:
|
|
720
|
+
return O.Project(O.Unit(), ()), ()
|
|
721
|
+
named_ = O.Distinct(starts[0] if len(starts) == 1 else O.Union(tuple(starts)))
|
|
722
|
+
nodes: O.Op = named_
|
|
723
|
+
if self._blank_objects():
|
|
724
|
+
a, b = Var(self.fresh("a")), Var(self.fresh("b"))
|
|
725
|
+
step = O.Project(O.Filter(self.triple(a, Var(self.fresh("p")), b),
|
|
726
|
+
E.Call("isBLANK", (E.Col(b.name),))), (a.name, b.name))
|
|
727
|
+
walk = O.Closure(step, a.name, b.name, min=0, max=None, terms=named_, terms_column=r.name)
|
|
728
|
+
reached = O.Join(O.Project(O.Extend(named_, a.name, E.Col(r.name)), (a.name,)), walk)
|
|
729
|
+
nodes = O.Distinct(O.Project(O.Extend(O.Project(reached, (b.name,)), r.name, E.Col(b.name)),
|
|
730
|
+
(r.name,)))
|
|
731
|
+
plan = O.Project(O.Join(nodes, self.triple(r, p, o)), (r.name, p.name, o.name))
|
|
732
|
+
return plan, ((r, p, o),)
|
|
733
|
+
|
|
734
|
+
def _blank_objects(self) -> bool:
|
|
735
|
+
"""Whether a triple's object can be a blank node: a map's blank subject (which another map's
|
|
736
|
+
reference can reach) or blank object."""
|
|
737
|
+
for m in self.maps.nodes:
|
|
738
|
+
if m.subject.term_type == BLANK:
|
|
739
|
+
return True
|
|
740
|
+
if any(po.object is not None and getattr(po.object, "term_type", None) == BLANK
|
|
741
|
+
for po in m.predicate_objects):
|
|
742
|
+
return True
|
|
743
|
+
return False
|
|
744
|
+
|
|
745
|
+
def aggregate(self, ctx) -> E.Expr:
|
|
746
|
+
"""An aggregate (18.5): a column of the query's Aggregate, read by
|
|
747
|
+
its fresh name. Only SELECT, HAVING and ORDER BY hold one."""
|
|
748
|
+
if getattr(self, "aggregates", None) is None:
|
|
749
|
+
raise QueryError(f"an aggregate outside SELECT, HAVING and ORDER BY: "
|
|
750
|
+
f"{ctx.getText()[:40]}")
|
|
751
|
+
fn = ctx.getChild(0).getText().upper()
|
|
752
|
+
distinct = ctx.DISTINCT() is not None
|
|
753
|
+
if fn == "COUNT" and ctx.STAR() is not None:
|
|
754
|
+
agg = E.Agg("COUNT", None, distinct)
|
|
755
|
+
else:
|
|
756
|
+
outer, self.aggregates = self.aggregates, None # no aggregate inside one
|
|
757
|
+
try:
|
|
758
|
+
arg = self.expression(ctx.expression())
|
|
759
|
+
finally:
|
|
760
|
+
self.aggregates = outer
|
|
761
|
+
separator = self.string(ctx.string_()) if getattr(ctx, "string_", None) and \
|
|
762
|
+
ctx.string_() is not None else " "
|
|
763
|
+
if self.by_value and fn == "COUNT" and distinct:
|
|
764
|
+
# COUNT(DISTINCT) counts values, as SELECT DISTINCT keeps one row per value (`value-identity-in-sql`).
|
|
765
|
+
arg = E.Call("valueKey", (arg,))
|
|
766
|
+
agg = E.Agg(fn, arg, distinct, separator)
|
|
767
|
+
name = self.fresh("a")
|
|
768
|
+
self.aggregates.append((name, agg))
|
|
769
|
+
return E.Col(name)
|
|
770
|
+
|
|
771
|
+
# -- graph patterns (18.2.2) ------------------------------------------------
|
|
772
|
+
|
|
773
|
+
def group(self, ctx) -> O.Op:
|
|
774
|
+
"""A GroupGraphPattern: its FILTERs apply to the whole group
|
|
775
|
+
(18.2.2.6). A FILTER that is a conjunction splits into its terms,
|
|
776
|
+
which keep a row exactly as the conjunction does; a term that is
|
|
777
|
+
EXISTS or NOT EXISTS (or `!` of one) is a semi- or anti-join."""
|
|
778
|
+
g, filters = self.group_parts(ctx)
|
|
779
|
+
if ctx is self.top_group:
|
|
780
|
+
g = self.bound(g)
|
|
781
|
+
plain: List[E.Expr] = []
|
|
782
|
+
for f in filters:
|
|
783
|
+
for leaf in conjuncts_of(f.constraint()):
|
|
784
|
+
found = exists_leaf(leaf)
|
|
785
|
+
if found is None:
|
|
786
|
+
plain.append(self.condition(leaf))
|
|
787
|
+
continue
|
|
788
|
+
negated, body = found
|
|
789
|
+
right, cond = self.exists_body(body, frozenset(g.columns))
|
|
790
|
+
g = (O.AntiJoin if negated else O.SemiJoin)(g, right, cond)
|
|
791
|
+
from prograph.plan.ir.fresh import holds
|
|
792
|
+
lifted = []
|
|
793
|
+
for i, cond in enumerate(plain):
|
|
794
|
+
if holds(cond):
|
|
795
|
+
# A fresh value is made by an Extend (fresh.py): the condition's value in a hidden
|
|
796
|
+
# column, filtered on.
|
|
797
|
+
name = self.fresh("f")
|
|
798
|
+
g = O.Extend(g, name, cond)
|
|
799
|
+
lifted.append(E.Col(name))
|
|
800
|
+
else:
|
|
801
|
+
lifted.append(cond)
|
|
802
|
+
if lifted:
|
|
803
|
+
g = O.Filter(g, lifted[0] if len(lifted) == 1 else E.And(tuple(lifted)))
|
|
804
|
+
return g
|
|
805
|
+
|
|
806
|
+
def exists_body(self, ctx, outer: FrozenSet[str]) -> Tuple[O.Op, Optional[E.Expr]]:
|
|
807
|
+
"""The P of EXISTS { P } (m2-functions.md 7): its patterns without
|
|
808
|
+
its FILTERs, and its FILTERs' conjunction, read over the merged row
|
|
809
|
+
of the group and P. P is triple patterns, FILTERs and nested
|
|
810
|
+
EXISTS; anything else is refused. A nested EXISTS may read only
|
|
811
|
+
variables P binds or ones the group does not have: a join of P with
|
|
812
|
+
its own rows cannot see the group's other values."""
|
|
813
|
+
if ctx.subSelect() is not None:
|
|
814
|
+
raise NotBuilt("a sub-select inside EXISTS")
|
|
815
|
+
p: O.Op = O.Unit()
|
|
816
|
+
filters: List[Any] = []
|
|
817
|
+
for child in ctx.getChildren():
|
|
818
|
+
if isinstance(child, SparqlParser.TriplesBlockContext):
|
|
819
|
+
p = join(p, self.triples_block(child))
|
|
820
|
+
elif isinstance(child, SparqlParser.Filter_Context):
|
|
821
|
+
filters.append(child)
|
|
822
|
+
elif isinstance(child, SparqlParser.GraphPatternNotTriplesContext):
|
|
823
|
+
if child.optionalGraphPattern() is not None:
|
|
824
|
+
raise NotBuilt("OPTIONAL inside EXISTS")
|
|
825
|
+
if child.groupOrUnionGraphPattern() is not None:
|
|
826
|
+
n = len(child.groupOrUnionGraphPattern().groupGraphPattern())
|
|
827
|
+
raise NotBuilt("UNION inside EXISTS" if n > 1 else "a nested group inside EXISTS")
|
|
828
|
+
if child.bind() is not None:
|
|
829
|
+
raise NotBuilt("BIND inside EXISTS")
|
|
830
|
+
if child.inlineData() is not None:
|
|
831
|
+
raise NotBuilt("VALUES inside EXISTS")
|
|
832
|
+
if child.minusGraphPattern() is not None:
|
|
833
|
+
raise NotBuilt("MINUS inside EXISTS")
|
|
834
|
+
if child.graphGraphPattern() is not None:
|
|
835
|
+
p = join(p, self.graph_pattern(child.graphGraphPattern()))
|
|
836
|
+
continue
|
|
837
|
+
raise NotBuilt("this pattern inside EXISTS")
|
|
838
|
+
conds: List[E.Expr] = []
|
|
839
|
+
for f in filters:
|
|
840
|
+
for leaf in conjuncts_of(f.constraint()):
|
|
841
|
+
found = exists_leaf(leaf)
|
|
842
|
+
if found is None:
|
|
843
|
+
conds.append(self.condition(leaf))
|
|
844
|
+
continue
|
|
845
|
+
negated, body = found
|
|
846
|
+
right, cond = self.exists_body(body, outer)
|
|
847
|
+
seen = set(right.columns) | (set(cond.columns()) if cond is not None else set())
|
|
848
|
+
if (seen & outer) - set(p.columns):
|
|
849
|
+
raise NotBuilt("a nested EXISTS that reads a variable of the outer group "
|
|
850
|
+
"which the enclosing EXISTS does not bind")
|
|
851
|
+
p = (O.AntiJoin if negated else O.SemiJoin)(p, right, cond)
|
|
852
|
+
return p, (conds[0] if len(conds) == 1 else E.And(tuple(conds)) if conds else None)
|
|
853
|
+
|
|
854
|
+
def group_parts(self, ctx) -> Tuple[O.Op, List[Any]]:
|
|
855
|
+
"""A GroupGraphPattern's pattern and its own FILTERs (18.2.2.6), as
|
|
856
|
+
parse nodes."""
|
|
857
|
+
# The graph base is this group's alone (a sub-select's own WHERE is not given it: one that
|
|
858
|
+
# reads no graph gives the same rows in each, joined with the graphs after).
|
|
859
|
+
base, self.group_base = self.group_base, None
|
|
860
|
+
if ctx.subSelect() is not None:
|
|
861
|
+
return self.select(ctx.subSelect().selectQuery()), []
|
|
862
|
+
g: O.Op = O.Unit()
|
|
863
|
+
if base is not None and not isinstance(next((c for c in ctx.getChildren()
|
|
864
|
+
if isinstance(c, (SparqlParser.TriplesBlockContext,
|
|
865
|
+
SparqlParser.GraphPatternNotTriplesContext))),
|
|
866
|
+
None), SparqlParser.TriplesBlockContext):
|
|
867
|
+
# A GRAPH group that does not start with triple patterns starts from the graphs
|
|
868
|
+
# (18.5: P is evaluated in each graph, so its OPTIONAL's left, VALUES, BIND and a UNION
|
|
869
|
+
# branch with no triple pattern are, too).
|
|
870
|
+
g = base
|
|
871
|
+
filters: List[Any] = []
|
|
872
|
+
outer, self.narrowed = self.narrowed, self.values_bound(ctx)
|
|
873
|
+
try:
|
|
874
|
+
# One basic graph pattern is the triple patterns a FILTER alone separates (5.1): its typing triples
|
|
875
|
+
# type each of its blocks (`typed-variables`).
|
|
876
|
+
children = list(ctx.getChildren())
|
|
877
|
+
patterns_of: Dict[int, List[Tuple[Term, Term, Term]]] = {}
|
|
878
|
+
typed_of: Dict[int, FrozenSet[str]] = {}
|
|
879
|
+
run: List[int] = []
|
|
880
|
+
for i, child in enumerate(children + [None]):
|
|
881
|
+
if isinstance(child, SparqlParser.TriplesBlockContext):
|
|
882
|
+
patterns_of[i] = self.block_patterns(child)
|
|
883
|
+
run.append(i)
|
|
884
|
+
elif child is None or not isinstance(child, (SparqlParser.Filter_Context, TerminalNode)):
|
|
885
|
+
typed = frozenset().union(*(typing_of(patterns_of[j]) for j in run)) if run else frozenset()
|
|
886
|
+
for j in run:
|
|
887
|
+
typed_of[j] = typed
|
|
888
|
+
run = []
|
|
889
|
+
for i, child in enumerate(children):
|
|
890
|
+
if isinstance(child, SparqlParser.TriplesBlockContext):
|
|
891
|
+
g = join(g, self.triples_block(child, patterns_of[i], typed_of[i]))
|
|
892
|
+
elif isinstance(child, SparqlParser.Filter_Context):
|
|
893
|
+
filters.append(child)
|
|
894
|
+
elif isinstance(child, SparqlParser.GraphPatternNotTriplesContext):
|
|
895
|
+
mine, self.narrowed = self.narrowed, {} # a nested scope is evaluated on its own
|
|
896
|
+
try:
|
|
897
|
+
g = self.not_triples(g, child)
|
|
898
|
+
finally:
|
|
899
|
+
self.narrowed = mine
|
|
900
|
+
finally:
|
|
901
|
+
self.narrowed = outer
|
|
902
|
+
return g, filters
|
|
903
|
+
|
|
904
|
+
@home("values-bound-subject", "A group's one-row VALUES IRI read as a constant by the group's own triple patterns",
|
|
905
|
+
terms=("VALUES", "inline data", "describe", "constant subject", "VALUES-bound", "substitution",
|
|
906
|
+
"prune", "map pruning"))
|
|
907
|
+
def values_bound(self, ctx) -> Dict[str, IRITerm]:
|
|
908
|
+
"""The variables an inline VALUES block of exactly one row binds to an IRI in the group `ctx`. The
|
|
909
|
+
group is the join of its elements (18.2.2.6), the VALUES block one of them, so each of the group's own
|
|
910
|
+
triple patterns may read that IRI in the variable's place, the variable then bound to it beside
|
|
911
|
+
(`triples_block`): the same rows, and the pattern's maps are chosen by the constant (`_may_give`, the
|
|
912
|
+
maps whose terms cannot be it read nothing), as `<iri> ?p ?o` is. Not inside an OPTIONAL, MINUS,
|
|
913
|
+
UNION or nested group (each evaluated on its own: an OPTIONAL's unmatched rows leave the variable
|
|
914
|
+
unbound, which the VALUES row then binds), nor where the variable is a parameter, nor under
|
|
915
|
+
owl:sameAs (a constant matches its component's members, a variable as the engine's mode binds it).
|
|
916
|
+
Only under an ontology: there a variable predicate's union of saturated maps is what a constant lets R10
|
|
917
|
+
merge into a scan per source (the turbofan walk: 1384 CTEs, refused, to 282). Without one the join is
|
|
918
|
+
cheaper on DuckDB: the merged scans write each member's term as text (0.5 s against 0.6 s on the
|
|
919
|
+
turbofan engine), so the VALUES row is left to the join, as are several rows: `meeting` drops the
|
|
920
|
+
branches no row can meet."""
|
|
921
|
+
if self.identity is not None or self.ontology is None:
|
|
922
|
+
return {}
|
|
923
|
+
out: Dict[str, IRITerm] = {}
|
|
924
|
+
for child in ctx.getChildren():
|
|
925
|
+
if not isinstance(child, SparqlParser.GraphPatternNotTriplesContext) or child.inlineData() is None:
|
|
926
|
+
continue
|
|
927
|
+
block = self.values(child.inlineData().dataBlock())
|
|
928
|
+
if len(block.rows) != 1:
|
|
929
|
+
continue
|
|
930
|
+
for name, term in zip(block.names, block.rows[0]):
|
|
931
|
+
if isinstance(term, IRITerm) and name not in self.params and out.get(name, term) == term:
|
|
932
|
+
out[name] = term
|
|
933
|
+
return out
|
|
934
|
+
|
|
935
|
+
def not_triples(self, g: O.Op, ctx) -> O.Op:
|
|
936
|
+
if ctx.optionalGraphPattern() is not None:
|
|
937
|
+
# Only the group's own filters are the LeftJoin's condition: a
|
|
938
|
+
# filter in a group inside it stays there (18.2.2.6, before the
|
|
939
|
+
# simplification of 18.2.2.8).
|
|
940
|
+
a, parsed = self.group_parts(ctx.optionalGraphPattern().groupGraphPattern())
|
|
941
|
+
filters = [self.condition(f.constraint()) for f in parsed]
|
|
942
|
+
if filters:
|
|
943
|
+
return O.LeftJoin(g, a, filters[0] if len(filters) == 1
|
|
944
|
+
else E.And(tuple(filters)))
|
|
945
|
+
return O.LeftJoin(g, a)
|
|
946
|
+
if ctx.groupOrUnionGraphPattern() is not None:
|
|
947
|
+
parts = [self.group(p) for p in ctx.groupOrUnionGraphPattern().groupGraphPattern()]
|
|
948
|
+
u = parts[0] if len(parts) == 1 else O.Union(tuple(parts))
|
|
949
|
+
return join(g, u)
|
|
950
|
+
if ctx.bind() is not None:
|
|
951
|
+
b = ctx.bind()
|
|
952
|
+
name = b.var_().getText()[1:]
|
|
953
|
+
if name in g.columns or name in self.params:
|
|
954
|
+
raise QueryError(f"BIND to ?{name}, which the group binds already (18.2.1)")
|
|
955
|
+
return O.Extend(g, name, self.expression(b.expression()))
|
|
956
|
+
if ctx.inlineData() is not None:
|
|
957
|
+
return join(g, self.values(ctx.inlineData().dataBlock()))
|
|
958
|
+
if ctx.minusGraphPattern() is not None:
|
|
959
|
+
# 18.2.2.6 / 18.5: Minus(G, the pattern).
|
|
960
|
+
within = (self.graph.name,) if isinstance(self.graph, Var) else ()
|
|
961
|
+
return O.Minus(g, self.group(ctx.minusGraphPattern().groupGraphPattern()), within)
|
|
962
|
+
if ctx.graphGraphPattern() is not None:
|
|
963
|
+
return join(g, self.graph_pattern(ctx.graphGraphPattern()))
|
|
964
|
+
raise NotBuilt(f"the pattern {ctx.getText()[:40]}")
|
|
965
|
+
|
|
966
|
+
def graph_pattern(self, gp) -> O.Op:
|
|
967
|
+
"""18.2.2 / 18.5: Graph(g, P): P's triple patterns read the named graph g. Where P binds no
|
|
968
|
+
?g (`GRAPH ?g {}`, only BIND or VALUES) it is joined with each named graph's name; where P
|
|
969
|
+
has no triple pattern, `GRAPH <g> { P }` is P if <g> is a named graph of the dataset, else
|
|
970
|
+
no solution (18.5's eval of Graph(IRI, P))."""
|
|
971
|
+
graph = self.term(gp.varOrIRIref())
|
|
972
|
+
# GRAPH ?g { P }: P is evaluated in each named graph, its triple patterns sharing a hidden
|
|
973
|
+
# graph column h, and only then joined with {?g -> that graph} (18.5): a ?g that P binds
|
|
974
|
+
# itself (in an OPTIONAL, say) must be compatible with the graph, not taken as it.
|
|
975
|
+
active = Var(self.fresh("graph")) if isinstance(graph, Var) else graph
|
|
976
|
+
outer, self.graph = self.graph, active
|
|
977
|
+
# The regime is the active graph's (Entailment Regimes 9: entailed by it, not the merge): the ontology's
|
|
978
|
+
# T-mapping rewriting (the maps' saturated classes and properties) and its domain and range typing are
|
|
979
|
+
# the default graph's; inside GRAPH the maps are read as declared.
|
|
980
|
+
outer_maps, outer_typing = self.maps, self.typing
|
|
981
|
+
self.maps, self.typing = self.maps.__dict__.get("saturated_from") or self.maps, []
|
|
982
|
+
before = self.patterns
|
|
983
|
+
try:
|
|
984
|
+
self.group_base = self.graph_names(active.name if isinstance(active, Var) else None)
|
|
985
|
+
inner = self.group(gp.groupGraphPattern())
|
|
986
|
+
self.group_base = None
|
|
987
|
+
if isinstance(graph, Var) and active.name not in inner.columns:
|
|
988
|
+
inner = join(inner, self.graph_names(active.name))
|
|
989
|
+
elif not isinstance(graph, Var) and self.patterns == before:
|
|
990
|
+
inner = join(inner, self.graph_names(None))
|
|
991
|
+
finally:
|
|
992
|
+
self.graph = outer
|
|
993
|
+
self.maps, self.typing = outer_maps, outer_typing
|
|
994
|
+
if not isinstance(graph, Var):
|
|
995
|
+
return inner
|
|
996
|
+
h, g = active.name, graph.name
|
|
997
|
+
kept = tuple(c for c in inner.columns if c not in (h, g))
|
|
998
|
+
if g in inner.columns:
|
|
999
|
+
inner = O.Filter(inner, E.Or((E.Not(E.IsBound(E.Col(g))),
|
|
1000
|
+
E.Compare("=", E.Col(g), E.Col(h), E.TERM))))
|
|
1001
|
+
inner = O.Project(inner, tuple(c for c in inner.columns if c != g))
|
|
1002
|
+
return O.Project(O.Extend(inner, g, E.Col(h)), kept + (g,))
|
|
1003
|
+
|
|
1004
|
+
def graph_names(self, name: Optional[str]) -> O.Op:
|
|
1005
|
+
"""The named graphs of the dataset under the active GRAPH (those some map puts a triple
|
|
1006
|
+
in), one row each as `name`; for a GRAPH <g>, one row of no column if <g> is one. Under a dataset
|
|
1007
|
+
clause they are the FROM NAMED graphs, each a graph of the dataset even where no map gives it (13.2)."""
|
|
1008
|
+
if self.dataset is not None:
|
|
1009
|
+
listed = sorted(self.dataset[1]) if name is not None else \
|
|
1010
|
+
[self.graph.text] if isinstance(self.graph, IRITerm) and self.graph.text in self.dataset[1] else []
|
|
1011
|
+
return O.Values((name,) if name is not None else (),
|
|
1012
|
+
tuple((IRITerm(i),) for i in listed) if name is not None else ((),) * len(listed))
|
|
1013
|
+
x, y = Var(self.fresh("x")), Var(self.fresh("y"))
|
|
1014
|
+
self.plain += 1 # the graphs' names: the triples as stated give them all
|
|
1015
|
+
try:
|
|
1016
|
+
spo = self.any_predicate(x, Var(self.fresh("p")), y)
|
|
1017
|
+
finally:
|
|
1018
|
+
self.plain -= 1
|
|
1019
|
+
if name is None:
|
|
1020
|
+
return O.Distinct(O.Project(spo, ()))
|
|
1021
|
+
return O.Distinct(O.Project(spo, (name,)))
|
|
1022
|
+
|
|
1023
|
+
def bound(self, x: O.Op) -> O.Op:
|
|
1024
|
+
"""`x` with the call's parameters joined (`translate`'s `bindings`): the top-level group's
|
|
1025
|
+
pattern, before its FILTERs (18.2.2.6), so that a FILTER and its EXISTS see the values, as
|
|
1026
|
+
with RDF4J's and Jena's initial bindings."""
|
|
1027
|
+
return join(x, self.bindings) if self.bindings is not None else x
|
|
1028
|
+
|
|
1029
|
+
def trailing_values(self, x: O.Op, ctx) -> O.Op:
|
|
1030
|
+
"""A query's trailing VALUES (18.2.4.1: P := Join(P, ToMultiSet(data)),
|
|
1031
|
+
before grouping and the solution modifiers)."""
|
|
1032
|
+
if ctx is None or ctx.dataBlock() is None:
|
|
1033
|
+
return x
|
|
1034
|
+
return join(x, self.values(ctx.dataBlock()))
|
|
1035
|
+
|
|
1036
|
+
def values(self, ctx) -> O.Op:
|
|
1037
|
+
if ctx.inlineDataOneVar() is not None:
|
|
1038
|
+
one = ctx.inlineDataOneVar()
|
|
1039
|
+
names = (one.var_().getText()[1:],)
|
|
1040
|
+
rows = tuple((self.data_value(v),) for v in one.dataBlockValue())
|
|
1041
|
+
return O.Values(names, rows)
|
|
1042
|
+
full = ctx.inlineDataFull()
|
|
1043
|
+
names = tuple(v.getText()[1:] for v in full.var_())
|
|
1044
|
+
rows: List[Tuple[Any, ...]] = []
|
|
1045
|
+
row: List[Any] = []
|
|
1046
|
+
in_data = False # past the '{': a '()' before it is the empty variable list, not a row
|
|
1047
|
+
for child in full.getChildren():
|
|
1048
|
+
if child.getText() == "{":
|
|
1049
|
+
in_data = True
|
|
1050
|
+
elif isinstance(child, SparqlParser.DataBlockValueContext):
|
|
1051
|
+
row.append(self.data_value(child))
|
|
1052
|
+
elif child.getText() == ")" and names and len(row) == len(names):
|
|
1053
|
+
rows.append(tuple(row))
|
|
1054
|
+
row = []
|
|
1055
|
+
elif "".join(child.getText().split()) == "()" and in_data: # NIL keeps inner spaces: ( )
|
|
1056
|
+
rows.append(())
|
|
1057
|
+
if any(len(r) != len(names) for r in rows):
|
|
1058
|
+
raise QueryError("a VALUES row has another number of values than variables")
|
|
1059
|
+
return O.Values(names, tuple(rows))
|
|
1060
|
+
|
|
1061
|
+
def data_value(self, ctx) -> Any:
|
|
1062
|
+
if ctx.UNDEF() is not None:
|
|
1063
|
+
return None
|
|
1064
|
+
t = self.term(ctx.graphTerm())
|
|
1065
|
+
if isinstance(t, Var):
|
|
1066
|
+
raise QueryError("a blank node in VALUES")
|
|
1067
|
+
return t
|
|
1068
|
+
|
|
1069
|
+
# -- triples ----------------------------------------------------------------
|
|
1070
|
+
|
|
1071
|
+
by_value = False # datatypes="value": a pattern's literal matches every literal of its value (`by_value_block`)
|
|
1072
|
+
unnamed = False # unnamed individuals may satisfy hidden terms (`with_witnesses`)
|
|
1073
|
+
occurrences: Any = None
|
|
1074
|
+
select_star = False
|
|
1075
|
+
typed_vars: FrozenSet[str] = frozenset() # the variables a typing triple of the block being planned types
|
|
1076
|
+
direct_ok = True # false under an untyped variable predicate (`any_predicate`)
|
|
1077
|
+
|
|
1078
|
+
def block_patterns(self, ctx) -> List[Tuple[Term, Term, Term]]:
|
|
1079
|
+
patterns: List[Tuple[Term, Term, Term]] = []
|
|
1080
|
+
for same in ctx.triplesSameSubject():
|
|
1081
|
+
self.same_subject(same, patterns)
|
|
1082
|
+
return patterns
|
|
1083
|
+
|
|
1084
|
+
def triples_block(self, ctx, patterns: Optional[List[Tuple[Term, Term, Term]]] = None,
|
|
1085
|
+
typed: Optional[FrozenSet[str]] = None) -> O.Op:
|
|
1086
|
+
"""A triples block's plan. A variable typed in its basic graph pattern (Entailment Regimes 7.1.3: `?x a
|
|
1087
|
+
owl:Class`, `owl:ObjectProperty`, `owl:DatatypeProperty`, `owl:NamedIndividual`; the blocks a FILTER
|
|
1088
|
+
alone separates, `typed`) is read with Direct semantics' consequences (`ontology_rows`); declarations
|
|
1089
|
+
are local to the basic graph pattern."""
|
|
1090
|
+
if patterns is None:
|
|
1091
|
+
patterns = self.block_patterns(ctx)
|
|
1092
|
+
held = self.typed_vars
|
|
1093
|
+
self.typed_vars = typing_of(patterns) if typed is None else typed
|
|
1094
|
+
try:
|
|
1095
|
+
return self._triples_block(patterns, ctx)
|
|
1096
|
+
finally:
|
|
1097
|
+
self.typed_vars = held
|
|
1098
|
+
|
|
1099
|
+
def _triples_block(self, patterns: List[Tuple[Term, Term, Term]], ctx) -> O.Op:
|
|
1100
|
+
if self.unnamed and self.graph is None and self.ql_tbox() is not None:
|
|
1101
|
+
# A sequence or inverse of IRIs is its triples (18.2.2.4), its middle terms hidden ones.
|
|
1102
|
+
patterns = [t for x in patterns for t in (self.path_triples(*x) or [x])]
|
|
1103
|
+
witnessed = self.with_witnesses(patterns, ctx)
|
|
1104
|
+
if witnessed is not None:
|
|
1105
|
+
return witnessed
|
|
1106
|
+
if self.by_value:
|
|
1107
|
+
# A sequence or inverse of IRIs is its triples (18.2.2.4), so its literal end matches by value too.
|
|
1108
|
+
patterns = [t for x in patterns for t in (self.path_triples(*x) or [x])]
|
|
1109
|
+
if _value_joined(patterns):
|
|
1110
|
+
return self.by_value_block(patterns)
|
|
1111
|
+
return self.block_of(patterns)
|
|
1112
|
+
|
|
1113
|
+
def path_triples(self, s: Term, p: Any, o: Term) -> Optional[List[Tuple[Term, Term, Term]]]:
|
|
1114
|
+
"""A sequence or inverse path of IRIs as its triple patterns, joined through fresh hidden terms (18.2.2.4);
|
|
1115
|
+
None for any other path, or a triple."""
|
|
1116
|
+
if not isinstance(p, Path):
|
|
1117
|
+
return None
|
|
1118
|
+
if p.kind == "inv":
|
|
1119
|
+
inner = p.parts[0]
|
|
1120
|
+
return [(o, inner, s)] if isinstance(inner, IRITerm) else None
|
|
1121
|
+
if p.kind != "seq":
|
|
1122
|
+
return None
|
|
1123
|
+
out: List[Tuple[Term, Term, Term]] = []
|
|
1124
|
+
here = s
|
|
1125
|
+
for i, step in enumerate(p.parts):
|
|
1126
|
+
there = o if i == len(p.parts) - 1 else Var(self.fresh("q"))
|
|
1127
|
+
if isinstance(step, IRITerm):
|
|
1128
|
+
out.append((here, step, there))
|
|
1129
|
+
else:
|
|
1130
|
+
more = self.path_triples(here, step, there)
|
|
1131
|
+
if more is None:
|
|
1132
|
+
return None
|
|
1133
|
+
out += more
|
|
1134
|
+
here = there
|
|
1135
|
+
return out
|
|
1136
|
+
|
|
1137
|
+
def by_value_block(self, patterns: List[Tuple[Term, Term, Term]]) -> O.Op:
|
|
1138
|
+
"""A triples block under datatypes="value" (D-entailment, OWL 2 RL's dt-eq): each literal object is a
|
|
1139
|
+
hidden variable whose value is the literal's (`owl_rl_rules.same_value`, OWL 2's datatype map: the integer
|
|
1140
|
+
types and xsd:decimal one value space, xsd:float and xsd:double each its own, any other literal by its
|
|
1141
|
+
term). Only matching changes: the block binds what it did."""
|
|
1142
|
+
from prograph.plan.ir.owl_rl_rules import least_literal
|
|
1143
|
+
|
|
1144
|
+
def expression(text: str, names: Dict[str, str], condition: bool = True) -> E.Expr:
|
|
1145
|
+
tree = parse("SELECT * WHERE { FILTER(" + text + ") }" if condition else
|
|
1146
|
+
"SELECT (" + text + " AS ?pgout) WHERE { }")
|
|
1147
|
+
if condition:
|
|
1148
|
+
group = tree.selectQuery().whereClause().groupGraphPattern()
|
|
1149
|
+
found = next(c for c in group.getChildren() if isinstance(c, SparqlParser.Filter_Context))
|
|
1150
|
+
return E.renamed(self.condition(found.constraint()), names)
|
|
1151
|
+
return E.renamed(self.expression(tree.selectQuery().selectClause()[0].expression()), names)
|
|
1152
|
+
|
|
1153
|
+
conds: List[E.Expr] = []
|
|
1154
|
+
out_patterns: List[Tuple[Term, Term, Term]] = []
|
|
1155
|
+
joined = _value_joined(patterns)
|
|
1156
|
+
copies: Dict[str, List[str]] = {v: [] for v in joined}
|
|
1157
|
+
for s, p, o in patterns:
|
|
1158
|
+
if isinstance(o, Var) and o.name in joined:
|
|
1159
|
+
# A variable two objects share: each a copy of its own, one value (`same_values`).
|
|
1160
|
+
name = self.fresh("lv")
|
|
1161
|
+
if copies[o.name]:
|
|
1162
|
+
conds.append(E.Call("sameValue", (E.Col(copies[o.name][0]), E.Col(name))))
|
|
1163
|
+
copies[o.name].append(name)
|
|
1164
|
+
o = Var(name)
|
|
1165
|
+
out_patterns.append((s, p, o))
|
|
1166
|
+
out = self.block_of(out_patterns)
|
|
1167
|
+
if conds:
|
|
1168
|
+
out = O.Filter(out, conds[0] if len(conds) == 1 else E.And(tuple(conds)))
|
|
1169
|
+
# One row per solution (Entailment Regimes 2: a solution's multiplicity is its blank nodes' instance
|
|
1170
|
+
# mappings'): the triples one value matches under several literals give it once, and a shared variable
|
|
1171
|
+
# shows the least of every literal of its value the solution matched (by term order).
|
|
1172
|
+
shown = [c for c in out.columns if not c.startswith(HIDDEN + "lv")]
|
|
1173
|
+
if not copies:
|
|
1174
|
+
return O.Distinct(O.Project(out, tuple(shown)))
|
|
1175
|
+
keys = []
|
|
1176
|
+
for v, names in copies.items():
|
|
1177
|
+
out = O.Extend(out, HIDDEN + "lvk" + v, E.Call("valueKey", (E.Col(names[0]),)))
|
|
1178
|
+
out = O.Extend(out, HIDDEN + "lvl" + v, _least_of(names, expression, least_literal))
|
|
1179
|
+
keys.append(HIDDEN + "lvk" + v)
|
|
1180
|
+
out = O.Aggregate(out, tuple(shown) + tuple(keys),
|
|
1181
|
+
tuple((v, E.Agg("MIN", E.Col(HIDDEN + "lvl" + v))) for v in copies))
|
|
1182
|
+
order = [c for c in patterns_columns(patterns) if c in out.columns]
|
|
1183
|
+
return O.Project(out, tuple(order + [c for c in out.columns if c not in order and not c.startswith(HIDDEN + "lv")]))
|
|
1184
|
+
|
|
1185
|
+
# -- datatypes="value": DISTINCT and GROUP BY by value ------------------------------------------------------
|
|
1186
|
+
|
|
1187
|
+
def value_columns(self, x: O.Op, columns) -> List[str]:
|
|
1188
|
+
"""The columns of `x` among `columns` whose terms may be one value under two terms (`value-identity-in-sql`):
|
|
1189
|
+
literals of two of xsd:decimal's value space's datatypes, or a number, boolean or dateTime held as its text
|
|
1190
|
+
(a lexical form may be other than canonical, a dateTime in another timezone), or of no known datatype."""
|
|
1191
|
+
from prograph.plan.ir.facts import facts
|
|
1192
|
+
from prograph.schema.source_map import LITERAL as LITERAL_KIND
|
|
1193
|
+
from prograph.schema import xsd
|
|
1194
|
+
numbers = xsd.INTEGER_DATATYPES | {xsd.DECIMAL, xsd.DOUBLE, xsd.FLOAT}
|
|
1195
|
+
decimals = xsd.INTEGER_DATATYPES | {xsd.DECIMAL}
|
|
1196
|
+
slots = facts(x).slots
|
|
1197
|
+
out = []
|
|
1198
|
+
for c in columns:
|
|
1199
|
+
s = slots.get(c)
|
|
1200
|
+
if s is None or LITERAL_KIND not in s.kinds:
|
|
1201
|
+
continue
|
|
1202
|
+
if UNKNOWN in s.datatypes or len(s.datatypes & decimals) > 1 or \
|
|
1203
|
+
(s.datatypes & (numbers | {xsd.BOOLEAN, xsd.DATETIME}) and s.native is None):
|
|
1204
|
+
out.append(c)
|
|
1205
|
+
return out
|
|
1206
|
+
|
|
1207
|
+
def value_key(self, name: str) -> E.Expr:
|
|
1208
|
+
"""One text per value of the column `name` (`lower.value_key_text`, rule `value-identity-in-sql`)."""
|
|
1209
|
+
return E.Call("valueKey", (E.Col(name),))
|
|
1210
|
+
|
|
1211
|
+
def grouped_by_value(self, x: O.Op, keys: List[str], aggregates) -> O.Op:
|
|
1212
|
+
"""GROUP BY under datatypes="value": a key that may hold one value under two terms groups by its value
|
|
1213
|
+
(`value_key`) and shows the least of its terms (MIN: ORDER BY's order breaks a tie between equal numbers
|
|
1214
|
+
by datatype IRI, then lexical form)."""
|
|
1215
|
+
by_value = set(self.value_columns(x, keys))
|
|
1216
|
+
group, extra = [], []
|
|
1217
|
+
for k in keys:
|
|
1218
|
+
if k in by_value:
|
|
1219
|
+
key = self.fresh("vk")
|
|
1220
|
+
x = O.Extend(x, key, self.value_key(k))
|
|
1221
|
+
group.append(key)
|
|
1222
|
+
extra.append((k, E.Agg("MIN", E.Col(k))))
|
|
1223
|
+
else:
|
|
1224
|
+
group.append(k)
|
|
1225
|
+
out = O.Aggregate(x, tuple(group), tuple(extra) + tuple(aggregates))
|
|
1226
|
+
return O.Project(out, tuple(k for k in keys) + tuple(n for n, _ in aggregates))
|
|
1227
|
+
|
|
1228
|
+
def distinct_by_value(self, x: O.Op) -> O.Op:
|
|
1229
|
+
"""SELECT DISTINCT under datatypes="value": rows one value column by column are one row (`grouped_by_value`
|
|
1230
|
+
over every column)."""
|
|
1231
|
+
return self.grouped_by_value(x, list(x.columns), ())
|
|
1232
|
+
|
|
1233
|
+
def block_of(self, patterns: List[Tuple[Term, Term, Term]]) -> O.Op:
|
|
1234
|
+
"""A triples block's plan: its patterns joined in connected order."""
|
|
1235
|
+
written = patterns
|
|
1236
|
+
patterns = _connected(patterns)
|
|
1237
|
+
out: O.Op = O.Unit()
|
|
1238
|
+
for i, (s, p, o) in enumerate(patterns):
|
|
1239
|
+
# A path whose start (or else end) the patterns before it bind walks from those terms
|
|
1240
|
+
# only (`?c a :Place ; :isPartOf* ?top`), as pre-1.0.0 chose its starts first.
|
|
1241
|
+
self.path_seed = None
|
|
1242
|
+
# The variables the patterns before bind as a triple's subject or object: graph nodes,
|
|
1243
|
+
# so their terms may also be a `*` path's zero-length pairs (18.4 takes the graph's
|
|
1244
|
+
# nodes; a predicate or a path's end may hold a term that is none, M4 review).
|
|
1245
|
+
nodes = {v.name for s0, p0, o0 in patterns[:i] if not isinstance(p0, Path)
|
|
1246
|
+
for v in (s0, o0) if isinstance(v, Var)}
|
|
1247
|
+
if isinstance(p, Path) and not isinstance(self.graph, Var):
|
|
1248
|
+
for side, t in (("start", s), ("end", o)):
|
|
1249
|
+
if isinstance(t, Var) and t.name in out.columns:
|
|
1250
|
+
# Under a name of its own: a column named as a SQL alias (`s`) is read
|
|
1251
|
+
# for the alias inside the EXISTS that seeds the walk (Spark).
|
|
1252
|
+
col = self.fresh("seed")
|
|
1253
|
+
terms = O.Distinct(O.Project(O.Extend(out, col, E.Col(t.name)), (col,)))
|
|
1254
|
+
self.path_seed = (side, t.name, terms, col, t.name in nodes)
|
|
1255
|
+
break
|
|
1256
|
+
out = join(out, self.narrowed_triple(s, p, o))
|
|
1257
|
+
self.path_seed = None
|
|
1258
|
+
if patterns != written:
|
|
1259
|
+
# The columns in the order the patterns were written (SELECT * reads them so), whatever the
|
|
1260
|
+
# join order.
|
|
1261
|
+
first = [v.name for t in written for v in t if isinstance(v, Var)]
|
|
1262
|
+
rank = {c: i for i, c in enumerate(dict.fromkeys(first))}
|
|
1263
|
+
out = O.Project(out, tuple(sorted(out.columns, key=lambda c: rank.get(c, len(rank)))))
|
|
1264
|
+
return out
|
|
1265
|
+
|
|
1266
|
+
@home("implied-individuals", "Hidden terms satisfied by unnamed individuals: tree-witness rewriting under QL",
|
|
1267
|
+
terms=("unnamed individual", "anonymous individual", "existential reasoning", "tree witness",
|
|
1268
|
+
"certain answers", "hidden variable", "blank node", "implied_individuals"))
|
|
1269
|
+
def with_witnesses(self, patterns: List[Tuple[Term, Term, Term]], ctx: Any) -> Optional[O.Op]:
|
|
1270
|
+
"""A triples block under `implied_individuals` (profile ql, the default graph): its named answers (the
|
|
1271
|
+
block as without the switch, a bag) and one row more per binding of its other terms that only unnamed
|
|
1272
|
+
individuals satisfy, so the answers with the switch on hold those without it. A hidden term is a blank
|
|
1273
|
+
node, or a variable every occurrence of which is in this block (under no SELECT *). A connected set of
|
|
1274
|
+
hidden terms whose patterns are class and property atoms is a tree: hanging from one term t, it holds of t
|
|
1275
|
+
where t is of a concept the ontology implies an unnamed such tree from (`ql.QLTBox.witnessed`), so its
|
|
1276
|
+
atoms are replaced by `t a B`; hanging from none (detached), it holds where some named individual is of
|
|
1277
|
+
such a concept. None where the block has no such tree."""
|
|
1278
|
+
from itertools import combinations
|
|
1279
|
+
ql = self.ql_tbox()
|
|
1280
|
+
written = Counter_of(patterns)
|
|
1281
|
+
here: Dict[str, int] = {}
|
|
1282
|
+
for n in _var_names(ctx): # the block's own occurrences in the text (`?x a :C ; :p ?y` names ?x once)
|
|
1283
|
+
here[n] = here.get(n, 0) + 1
|
|
1284
|
+
hidden = {n for n in written if n.startswith((HIDDEN + "b", HIDDEN + "q"))
|
|
1285
|
+
or (not self.select_star and n in here and self.occurrences.get(n, 0) == here[n]
|
|
1286
|
+
and n not in self.params)}
|
|
1287
|
+
if not hidden:
|
|
1288
|
+
return None
|
|
1289
|
+
trees = _hidden_trees(patterns, hidden)
|
|
1290
|
+
if not trees:
|
|
1291
|
+
return None
|
|
1292
|
+
|
|
1293
|
+
def subtree(node, adjacency, parent):
|
|
1294
|
+
"""(classes, children) of `node`'s side of the tree, away from `parent`."""
|
|
1295
|
+
classes = frozenset(adjacency[node]["classes"])
|
|
1296
|
+
children = tuple((role, *subtree(child, adjacency, node))
|
|
1297
|
+
for role, child in adjacency[node]["edges"] if child != parent)
|
|
1298
|
+
return classes, children
|
|
1299
|
+
|
|
1300
|
+
def side_atoms(node, adjacency, parent, atoms_by, links):
|
|
1301
|
+
"""The atoms of `node`'s side away from `parent`: its class atoms and each edge below it, with theirs."""
|
|
1302
|
+
out = list(atoms_by.get(node, ()))
|
|
1303
|
+
for _, child in adjacency[node]["edges"]:
|
|
1304
|
+
if child != parent:
|
|
1305
|
+
out.append(links[(node, child)])
|
|
1306
|
+
out += side_atoms(child, adjacency, node, atoms_by, links)
|
|
1307
|
+
return out
|
|
1308
|
+
|
|
1309
|
+
def holds(term, cs): # `term` of one of the concepts `cs`
|
|
1310
|
+
ops = []
|
|
1311
|
+
for c in sorted(cs, key=str):
|
|
1312
|
+
if c[0] == "c":
|
|
1313
|
+
ops.append(self.triple(term, IRITerm(RDF_TYPE), IRITerm(c[1])))
|
|
1314
|
+
elif c[0] == "e":
|
|
1315
|
+
y = Var(self.fresh("w"))
|
|
1316
|
+
(p, back) = c[1]
|
|
1317
|
+
ops.append(self.triple(y, IRITerm(p), term) if back else self.triple(term, IRITerm(p), y))
|
|
1318
|
+
elif c == ("top",):
|
|
1319
|
+
from prograph.plan.ir.ql import THING
|
|
1320
|
+
ops.append(self.triple(term, IRITerm(RDF_TYPE), IRITerm(THING)))
|
|
1321
|
+
ops = [o for o in ops if not (isinstance(o, O.Values) and not o.rows)]
|
|
1322
|
+
if not ops:
|
|
1323
|
+
return None
|
|
1324
|
+
cols = tuple(v.name for v in (term,) if isinstance(v, Var))
|
|
1325
|
+
return O.Distinct(O.Project(O.Union(tuple(O.Project(o, cols) for o in ops)) if len(ops) > 1
|
|
1326
|
+
else O.Project(ops[0], cols), cols))
|
|
1327
|
+
|
|
1328
|
+
# Each reading: the atoms an unnamed part replaces, and the plan saying its parent implies it. An unnamed
|
|
1329
|
+
# part is the side of a tree edge away from its named parent (the tree's term, or a hidden term then
|
|
1330
|
+
# named): a subtree hanging from the term, or from a hidden term of the tree; a detached tree may also be
|
|
1331
|
+
# implied whole, by a named individual it is not joined to.
|
|
1332
|
+
readings = []
|
|
1333
|
+
for atoms, attach, adjacency, atoms_by, links in trees:
|
|
1334
|
+
cuts = [] # (parent term, role, node, parent node or None)
|
|
1335
|
+
equal_to = ()
|
|
1336
|
+
if attach is not None:
|
|
1337
|
+
root_term, role, top, equal_to = attach
|
|
1338
|
+
cuts.append((root_term, role, top, None))
|
|
1339
|
+
order = [(top, None)]
|
|
1340
|
+
while order:
|
|
1341
|
+
node, parent = order.pop()
|
|
1342
|
+
for r, child in adjacency[node]["edges"]:
|
|
1343
|
+
if child != parent:
|
|
1344
|
+
cuts.append((Var(node), r, child, node))
|
|
1345
|
+
order.append((child, node))
|
|
1346
|
+
else:
|
|
1347
|
+
for node in adjacency:
|
|
1348
|
+
for r, child in adjacency[node]["edges"]:
|
|
1349
|
+
cuts.append((Var(node), r, child, node))
|
|
1350
|
+
for parent_term, role, node, parent in cuts:
|
|
1351
|
+
if isinstance(role, tuple) and role and isinstance(role[0], tuple):
|
|
1352
|
+
roles = role
|
|
1353
|
+
else:
|
|
1354
|
+
roles = (role,)
|
|
1355
|
+
classes, children = subtree(node, adjacency, parent)
|
|
1356
|
+
# One successor reached by every role at once: one axiom's role under each (`witnessed`).
|
|
1357
|
+
implied = holds(parent_term, ql.witnessed(tuple(roles) if len(roles) > 1 else roles[0], classes,
|
|
1358
|
+
children))
|
|
1359
|
+
if implied is None:
|
|
1360
|
+
continue
|
|
1361
|
+
gone = side_atoms(node, adjacency, parent, atoms_by, links) + [links[(parent, node)]]
|
|
1362
|
+
if parent is None:
|
|
1363
|
+
for other in equal_to: # the other atoms' terms: the root itself
|
|
1364
|
+
gone += [a for k, a in links.items() if k[0] == "extra" and k[1] == node]
|
|
1365
|
+
if isinstance(other, Var) and isinstance(parent_term, Var):
|
|
1366
|
+
implied = O.Extend(implied, other.name, E.Col(parent_term.name)) \
|
|
1367
|
+
if other.name != parent_term.name else implied
|
|
1368
|
+
elif isinstance(parent_term, Var):
|
|
1369
|
+
implied = O.Filter(implied, E.Compare("=", E.Col(parent_term.name), E.Const(other), E.TERM))
|
|
1370
|
+
elif other != parent_term:
|
|
1371
|
+
implied = None
|
|
1372
|
+
break
|
|
1373
|
+
if implied is None:
|
|
1374
|
+
continue
|
|
1375
|
+
readings.append((list({id(a): a for a in gone}.values()), implied))
|
|
1376
|
+
if attach is None:
|
|
1377
|
+
found = set()
|
|
1378
|
+
for top in adjacency:
|
|
1379
|
+
classes, children = subtree(top, adjacency, None)
|
|
1380
|
+
for role in ql.role_up:
|
|
1381
|
+
found |= ql.witnessed(role, classes, children)
|
|
1382
|
+
# And every concept with an implied successor of such a concept, at any depth: a named
|
|
1383
|
+
# GraduateStudent's unnamed Professor's unnamed Department holds `?d a :Department`.
|
|
1384
|
+
grew = True
|
|
1385
|
+
while grew:
|
|
1386
|
+
grew = False
|
|
1387
|
+
for concept, implied_by in ql.generates.items():
|
|
1388
|
+
if concept not in found and any(fill & found for _, _, fill in implied_by):
|
|
1389
|
+
found.add(concept)
|
|
1390
|
+
grew = True
|
|
1391
|
+
implied = holds(Var(self.fresh("w")), found)
|
|
1392
|
+
if implied is not None:
|
|
1393
|
+
readings.append((atoms, O.Distinct(O.Project(implied, ()))))
|
|
1394
|
+
if not readings:
|
|
1395
|
+
return None
|
|
1396
|
+
if len(readings) > WITNESS_CAP:
|
|
1397
|
+
# Every combination of readings is a branch: past the cap the rewrite is refused, not cut short.
|
|
1398
|
+
raise NotBuilt(f"a triples block with {len(readings)} ways unnamed individuals may satisfy it (at most "
|
|
1399
|
+
f"{WITNESS_CAP}) under implied_individuals; split the block, or project the hidden "
|
|
1400
|
+
f"variables")
|
|
1401
|
+
named = self.block_of(patterns)
|
|
1402
|
+
shown = tuple(c for c in named.columns if c not in hidden)
|
|
1403
|
+
branches = []
|
|
1404
|
+
for k in range(1, len(readings) + 1):
|
|
1405
|
+
for chosen in combinations(readings, k):
|
|
1406
|
+
ids = [id(a) for atoms, _ in chosen for a in atoms]
|
|
1407
|
+
if len(ids) != len(set(ids)):
|
|
1408
|
+
continue # two readings of one atom: not one model's
|
|
1409
|
+
gone = set(ids)
|
|
1410
|
+
rest = [t for t in patterns if id(t) not in gone]
|
|
1411
|
+
x = self.block_of(rest) if rest else O.Unit()
|
|
1412
|
+
for _, implied in chosen:
|
|
1413
|
+
x = join(x, implied)
|
|
1414
|
+
branches.append(O.Project(x, tuple(c for c in shown if c in x.columns)))
|
|
1415
|
+
implied_rows = O.Distinct(O.Project(O.Union(tuple(branches)) if len(branches) > 1 else branches[0], shown))
|
|
1416
|
+
only_implied = O.AntiJoin(implied_rows, O.Distinct(O.Project(named, shown)))
|
|
1417
|
+
return O.Union((named, only_implied))
|
|
1418
|
+
|
|
1419
|
+
def value_triple(self, s: Term, p: Any, o: LiteralTerm) -> O.Op:
|
|
1420
|
+
"""`s p o` under datatypes="value" (`value-identity-in-sql`): every triple whose object is one value with
|
|
1421
|
+
the literal `o`, each solution once (a subject holding 1 and 1.00 matches `1` once). The test is in each
|
|
1422
|
+
branch of the pattern's union, where a map's column is of one type (a native number compares as one)."""
|
|
1423
|
+
name = self.fresh("lv")
|
|
1424
|
+
x = self.narrowed_triple(s, p, Var(name))
|
|
1425
|
+
cond = E.Call("sameValue", (E.Col(name), E.Const(o)))
|
|
1426
|
+
x = _filtered_below(x, cond, name)
|
|
1427
|
+
return O.Distinct(O.Project(x, tuple(c for c in x.columns if c != name)))
|
|
1428
|
+
|
|
1429
|
+
def narrowed_triple(self, s: Term, p: Any, o: Term) -> O.Op:
|
|
1430
|
+
"""`triple`, a variable the group's one-row VALUES binds to an IRI read as that IRI and bound to it
|
|
1431
|
+
beside (`values_bound`)."""
|
|
1432
|
+
if self.by_value and isinstance(o, LiteralTerm) and not isinstance(p, Path):
|
|
1433
|
+
return self.value_triple(s, p, o)
|
|
1434
|
+
got = {} if isinstance(p, Path) else \
|
|
1435
|
+
{t.name: self.narrowed[t.name] for t in (s, p, o) if isinstance(t, Var) and t.name in self.narrowed}
|
|
1436
|
+
if not got:
|
|
1437
|
+
return self.triple(s, p, o)
|
|
1438
|
+
x = self.triple(*(self.narrowed.get(t.name, t) if isinstance(t, Var) else t for t in (s, p, o)))
|
|
1439
|
+
if isinstance(x, O.Values) and not x.rows:
|
|
1440
|
+
return _empty(s, p, o)
|
|
1441
|
+
for name, term in got.items():
|
|
1442
|
+
x = O.Extend(x, name, E.Const(term))
|
|
1443
|
+
return x
|
|
1444
|
+
|
|
1445
|
+
def same_subject(self, ctx, out: list) -> None:
|
|
1446
|
+
if ctx.varOrTerm() is not None:
|
|
1447
|
+
subject = self.term(ctx.varOrTerm())
|
|
1448
|
+
self.property_list(subject, ctx.propertyListNotEmpty(), out)
|
|
1449
|
+
else:
|
|
1450
|
+
subject = self.node(ctx.triplesNode(), out)
|
|
1451
|
+
if ctx.propertyList().propertyListNotEmpty() is not None:
|
|
1452
|
+
self.property_list(subject, ctx.propertyList().propertyListNotEmpty(), out)
|
|
1453
|
+
|
|
1454
|
+
def node(self, ctx, out: list) -> Term:
|
|
1455
|
+
"""A triplesNode: `[ p o ]` is a fresh blank node and its triples."""
|
|
1456
|
+
if ctx.blankNodePropertyList() is None:
|
|
1457
|
+
raise NotBuilt("an RDF collection in a pattern")
|
|
1458
|
+
b = Var(self.fresh("b"))
|
|
1459
|
+
self.property_list(b, ctx.blankNodePropertyList().propertyListNotEmpty(), out)
|
|
1460
|
+
return b
|
|
1461
|
+
|
|
1462
|
+
def property_list(self, subject: Term, ctx, out: list) -> None:
|
|
1463
|
+
verbs, objects = ctx.verb(), ctx.objectList()
|
|
1464
|
+
for verb, object_list in zip(verbs, objects):
|
|
1465
|
+
predicate, inverse = self.verb(verb)
|
|
1466
|
+
for obj in object_list.object_():
|
|
1467
|
+
node = obj.graphNode()
|
|
1468
|
+
o = self.term(node.varOrTerm()) if node.varOrTerm() is not None \
|
|
1469
|
+
else self.node(node.triplesNode(), out)
|
|
1470
|
+
out.append((o, predicate, subject) if inverse else (subject, predicate, o))
|
|
1471
|
+
|
|
1472
|
+
def verb(self, ctx) -> Tuple[Term, bool]:
|
|
1473
|
+
"""A verb as a predicate term, and whether it is inverted (`^p`)."""
|
|
1474
|
+
if ctx.A() is not None:
|
|
1475
|
+
return IRITerm(RDF_TYPE), False
|
|
1476
|
+
if ctx.varOrIRIref() is not None:
|
|
1477
|
+
return self.term(ctx.varOrIRIref()), False
|
|
1478
|
+
path = self.path(ctx.path())
|
|
1479
|
+
if isinstance(path, IRITerm):
|
|
1480
|
+
return path, False
|
|
1481
|
+
if isinstance(path, Path) and path.kind == "inv" and isinstance(path.parts[0], IRITerm):
|
|
1482
|
+
return path.parts[0], True
|
|
1483
|
+
return path, False
|
|
1484
|
+
|
|
1485
|
+
def path(self, ctx) -> TUnion["Path", IRITerm]:
|
|
1486
|
+
"""A property path (18.4) as an IRI or a `Path`."""
|
|
1487
|
+
seqs = [self.path_sequence(q) for q in ctx.pathAlternative().pathSequence()]
|
|
1488
|
+
return seqs[0] if len(seqs) == 1 else Path("alt", tuple(seqs))
|
|
1489
|
+
|
|
1490
|
+
def path_sequence(self, ctx) -> TUnion["Path", IRITerm]:
|
|
1491
|
+
steps = []
|
|
1492
|
+
for step in ctx.pathEltOrInverse():
|
|
1493
|
+
elt = step.pathElt()
|
|
1494
|
+
p = self.path_primary(elt.pathPrimary())
|
|
1495
|
+
if elt.pathMod() is not None:
|
|
1496
|
+
p = Path("mod", (p,), elt.pathMod().getText())
|
|
1497
|
+
steps.append(Path("inv", (p,)) if step.CARET() is not None else p)
|
|
1498
|
+
return steps[0] if len(steps) == 1 else Path("seq", tuple(steps))
|
|
1499
|
+
|
|
1500
|
+
def path_primary(self, ctx) -> TUnion["Path", IRITerm]:
|
|
1501
|
+
if ctx.A() is not None:
|
|
1502
|
+
return IRITerm(RDF_TYPE)
|
|
1503
|
+
if ctx.iriRef() is not None:
|
|
1504
|
+
return IRITerm(self.iri(ctx.iriRef()))
|
|
1505
|
+
if ctx.prefixedName() is not None:
|
|
1506
|
+
return IRITerm(self.iri(ctx.prefixedName()))
|
|
1507
|
+
if ctx.path() is not None:
|
|
1508
|
+
return self.path(ctx.path())
|
|
1509
|
+
neg = ctx.pathNegatedPropertySet()
|
|
1510
|
+
# The grammar repeats pathOneInPropertySet, so its accessor gives a list with or without
|
|
1511
|
+
# parentheses (`!a`, `!^ex:p`, `!(a|^ex:p)`).
|
|
1512
|
+
found = neg.pathOneInPropertySet()
|
|
1513
|
+
members = list(found) if isinstance(found, (list, tuple)) else [found]
|
|
1514
|
+
out = []
|
|
1515
|
+
for m in members:
|
|
1516
|
+
iri = RDF_TYPE if m.A() is not None else self.iri(m.iriRef())
|
|
1517
|
+
out.append((iri, m.CARET() is not None))
|
|
1518
|
+
return Path("neg", tuple(out))
|
|
1519
|
+
|
|
1520
|
+
def path_op(self, s: Term, path: "Path", o: Term) -> O.Op:
|
|
1521
|
+
"""A property path's solutions (18.4): an inverse swaps the ends, a
|
|
1522
|
+
sequence joins its steps through hidden variables (a bag, as 18.4's
|
|
1523
|
+
join and projection give), an alternative is the union of its
|
|
1524
|
+
branches (a bag), a negated property set the union over every other
|
|
1525
|
+
predicate the maps give. A closure (*, +, ?) is not built yet."""
|
|
1526
|
+
if isinstance(path, IRITerm):
|
|
1527
|
+
return self.triple(s, path, o)
|
|
1528
|
+
if path.kind == "inv":
|
|
1529
|
+
return self.path_or_triple(o, path.parts[0], s)
|
|
1530
|
+
if path.kind == "seq":
|
|
1531
|
+
x: O.Op = O.Unit()
|
|
1532
|
+
here = s
|
|
1533
|
+
for i, step in enumerate(path.parts):
|
|
1534
|
+
there = o if i == len(path.parts) - 1 else Var(self.fresh("m"))
|
|
1535
|
+
x = join(x, self.path_or_triple(here, step, there))
|
|
1536
|
+
here = there
|
|
1537
|
+
return x
|
|
1538
|
+
if path.kind == "alt":
|
|
1539
|
+
return O.Union(tuple(self.path_or_triple(s, b, o) for b in path.parts))
|
|
1540
|
+
if path.kind == "neg":
|
|
1541
|
+
forward = {iri for iri, inv in path.parts if not inv}
|
|
1542
|
+
backward = {iri for iri, inv in path.parts if inv}
|
|
1543
|
+
branches = []
|
|
1544
|
+
if forward or not backward:
|
|
1545
|
+
branches.append(self.other_predicates(s, o, forward))
|
|
1546
|
+
if backward:
|
|
1547
|
+
branches.append(self.other_predicates(o, s, backward))
|
|
1548
|
+
return branches[0] if len(branches) == 1 else O.Union(tuple(branches))
|
|
1549
|
+
if path.kind == "mod":
|
|
1550
|
+
return self.closure(s, path.parts[0], path.mod, o)
|
|
1551
|
+
raise NotBuilt(f"the property path kind {path.kind}")
|
|
1552
|
+
|
|
1553
|
+
def closure(self, s: Term, step_path, mod: str, o: Term) -> O.Op:
|
|
1554
|
+
"""p*, p+, p? (18.4: ZeroOrMorePath, OneOrMorePath, ZeroOrOnePath,
|
|
1555
|
+
ALP): the set of (start, end) pairs a walk of an allowed length
|
|
1556
|
+
joins; a zero-length pair (t, t) for each term of the graph, or the
|
|
1557
|
+
bound constant (one-plan.md 5.7)."""
|
|
1558
|
+
lo, hi = {"*": (0, None), "+": (1, None), "?": (0, 1)}[mod]
|
|
1559
|
+
if not isinstance(self.graph, Var):
|
|
1560
|
+
folded = self.ontology_walk(s, step_path, lo, hi, o)
|
|
1561
|
+
if folded is not None:
|
|
1562
|
+
return folded
|
|
1563
|
+
a, b = Var(self.fresh("a")), Var(self.fresh("b"))
|
|
1564
|
+
# Inside GRAPH ?g the path is evaluated in each named graph (18.5 eval(D(G), Graph(var, P))):
|
|
1565
|
+
# one closure taken within each value of g (`Closure.partition`, one-plan 5.7), so that
|
|
1566
|
+
# `GRAPH ?g { ?x p+ ?y }` is one closure, not one per graph.
|
|
1567
|
+
part = (self.graph.name,) if isinstance(self.graph, Var) else ()
|
|
1568
|
+
if part and self.graph in (s, o):
|
|
1569
|
+
raise NotBuilt("a closure inside GRAPH ?g whose end is ?g")
|
|
1570
|
+
# A constant end seeds the walk (from it, not over every pair of the graph).
|
|
1571
|
+
seed = ("start", s) if not isinstance(s, Var) else ("end", o) if not isinstance(o, Var) else None
|
|
1572
|
+
seed_nodes = False
|
|
1573
|
+
if seed is None and self.path_seed is not None and not part:
|
|
1574
|
+
_, name, terms_op, col, seed_nodes = self.path_seed
|
|
1575
|
+
# By the variable, on whichever end it is now: an inverse (`?a ^p* ?b`) swaps the ends.
|
|
1576
|
+
side = "start" if s == Var(name) else "end" if o == Var(name) else None
|
|
1577
|
+
if side is not None:
|
|
1578
|
+
seed = (side, terms_op, col)
|
|
1579
|
+
self.path_seed = None # the outermost closure of the pattern only
|
|
1580
|
+
step = O.Project(self.path_or_triple(a, step_path, b), part + (a.name, b.name))
|
|
1581
|
+
terms = None
|
|
1582
|
+
t = self.fresh("t")
|
|
1583
|
+
if lo == 0 and seed is not None and isinstance(seed[1], O.Op) and seed_nodes:
|
|
1584
|
+
# The zero-length pairs the seeded end needs: its own terms, not the whole graph's
|
|
1585
|
+
# (every term of the graph paired with itself).
|
|
1586
|
+
terms = O.Project(O.Extend(seed[1], t, E.Col(seed[2])), (t,))
|
|
1587
|
+
elif lo == 0:
|
|
1588
|
+
bound = s if not isinstance(s, Var) else (o if not isinstance(o, Var) else None)
|
|
1589
|
+
if bound is not None and not part:
|
|
1590
|
+
spo = None
|
|
1591
|
+
else:
|
|
1592
|
+
x, y = Var(self.fresh("x")), Var(self.fresh("y"))
|
|
1593
|
+
spo = self.any_predicate(x, Var(self.fresh("p")), y)
|
|
1594
|
+
if spo is None:
|
|
1595
|
+
terms = O.Values((t,), ((bound,),))
|
|
1596
|
+
if isinstance(self.graph, IRITerm):
|
|
1597
|
+
# Inside GRAPH <g>, only where <g> is a graph of the dataset (18.5; M3 review).
|
|
1598
|
+
terms = join(terms, self.graph_names(None))
|
|
1599
|
+
elif isinstance(spo, O.Values) and not spo.rows:
|
|
1600
|
+
terms = O.Values(part + (t,), ())
|
|
1601
|
+
elif bound is not None:
|
|
1602
|
+
# A bound end is its own zero-length pair in every graph (ALP), each named graph here.
|
|
1603
|
+
graphs = O.Distinct(O.Project(spo, part))
|
|
1604
|
+
terms = O.Project(O.Extend(graphs, t, E.Const(bound)), part + (t,))
|
|
1605
|
+
else:
|
|
1606
|
+
terms = O.Distinct(O.Union((
|
|
1607
|
+
O.Project(O.Extend(spo, t, E.Col(x.name)), part + (t,)),
|
|
1608
|
+
O.Project(O.Extend(spo, t, E.Col(y.name)), part + (t,)))))
|
|
1609
|
+
walk = O.Closure(step, a.name, b.name, min=lo, max=hi, terms=terms, terms_column=t, partition=part,
|
|
1610
|
+
seed=seed)
|
|
1611
|
+
bound = self.bind(walk, [(s, E.Col(a.name)), (o, E.Col(b.name))])
|
|
1612
|
+
if part:
|
|
1613
|
+
# The graph column the closure carries stays a column of the pattern (GRAPH ?g joins on it).
|
|
1614
|
+
bound = O.Project(bound.input, part + bound.names)
|
|
1615
|
+
return bound
|
|
1616
|
+
|
|
1617
|
+
def path_or_triple(self, s: Term, p, o: Term) -> O.Op:
|
|
1618
|
+
return self.path_op(s, p, o) if isinstance(p, Path) else self.triple(s, p, o)
|
|
1619
|
+
|
|
1620
|
+
def other_predicates(self, s: Term, o: Term, excluded) -> O.Op:
|
|
1621
|
+
"""The triples of every predicate the maps give but `excluded`."""
|
|
1622
|
+
if self.maps.open_predicates:
|
|
1623
|
+
raise NotBuilt("a map whose predicate is a column")
|
|
1624
|
+
predicates = [iri for iri in dict.fromkeys(
|
|
1625
|
+
([RDF_TYPE] if any(m.classes for m in self.maps.nodes) else []) + sorted(self.maps.by_predicate)
|
|
1626
|
+
+ self.identity_predicates() + self.ontology_predicates()) if iri not in excluded]
|
|
1627
|
+
parts = [self.triple(s, IRITerm(iri), o, quiet=True) for iri in predicates]
|
|
1628
|
+
parts = [x for x in parts if not (isinstance(x, O.Values) and not x.rows)]
|
|
1629
|
+
if not parts:
|
|
1630
|
+
return _empty(s, Var(self.fresh("p")), o)
|
|
1631
|
+
return parts[0] if len(parts) == 1 else O.Union(tuple(parts))
|
|
1632
|
+
|
|
1633
|
+
def triple(self, s: Term, p: Term, o: Term, quiet: bool = False) -> O.Op:
|
|
1634
|
+
"""The solutions of one triple pattern over the maps (18.3), through owl:sameAs where the engine
|
|
1635
|
+
applies it (`identity.pattern`)."""
|
|
1636
|
+
if isinstance(p, Path):
|
|
1637
|
+
return self.path_op(s, p, o)
|
|
1638
|
+
if self.identity is not None and not self.plain and isinstance(p, IRITerm):
|
|
1639
|
+
from prograph.plan.ir.identity import pattern
|
|
1640
|
+
self.patterns += 1
|
|
1641
|
+
self.plain += 1
|
|
1642
|
+
try:
|
|
1643
|
+
return pattern(self, s, p, o, quiet)
|
|
1644
|
+
finally:
|
|
1645
|
+
self.plain -= 1
|
|
1646
|
+
return self.matched(s, p, o, quiet)
|
|
1647
|
+
|
|
1648
|
+
@home("top-object-property", "owl:topObjectProperty under profile ql: every pair of named individuals",
|
|
1649
|
+
terms=("topObjectProperty", "top object property", "universal role", "every pair", "Table 5"))
|
|
1650
|
+
def top_pairs(self, s: Term, o: Term) -> O.Op:
|
|
1651
|
+
"""`s owl:topObjectProperty o` under profile ql (OWL 2 Syntax 5.3: it "connects all possible pairs of
|
|
1652
|
+
individuals"): every pair of named individuals (`individual_terms`), in the default graph, the plan marked
|
|
1653
|
+
for the cost guard (`cost.TOP_PROPERTY`: its size is the square of the individuals'). A variable predicate
|
|
1654
|
+
never binds it (`any_predicate`): a labelled known difference, as Jena and Ontop."""
|
|
1655
|
+
terms = self.individual_terms()
|
|
1656
|
+
if terms is None or isinstance(s, LiteralTerm) or isinstance(o, LiteralTerm):
|
|
1657
|
+
return _empty(s, IRITerm(TOP_OBJECT_PROPERTY), o)
|
|
1658
|
+
column, op = terms
|
|
1659
|
+
|
|
1660
|
+
def side(t: Term) -> Optional[O.Op]:
|
|
1661
|
+
if isinstance(t, Var):
|
|
1662
|
+
return O.Project(O.Extend(op, t.name, E.Col(column)), (t.name,))
|
|
1663
|
+
got = self._one_term(op, column, t, [])
|
|
1664
|
+
return None if got is None else O.Distinct(got)
|
|
1665
|
+
a, b = side(s), side(o)
|
|
1666
|
+
if a is None or b is None:
|
|
1667
|
+
return _empty(s, IRITerm(TOP_OBJECT_PROPERTY), o)
|
|
1668
|
+
if isinstance(s, Var) and isinstance(o, Var) and s.name == o.name:
|
|
1669
|
+
out: O.Op = a
|
|
1670
|
+
else:
|
|
1671
|
+
out = O.Join(a, b)
|
|
1672
|
+
if isinstance(s, Var) and isinstance(o, Var) and s.name != o.name:
|
|
1673
|
+
object.__setattr__(out, "_top_property", True) # the square of the individuals (`cost.TOP_PROPERTY`)
|
|
1674
|
+
return out
|
|
1675
|
+
|
|
1676
|
+
def identity_predicates(self) -> List[str]:
|
|
1677
|
+
"""owl:sameAs where the engine applies it: its triples are the components' (`identity.same_as`)."""
|
|
1678
|
+
from prograph.plan.ir.identity import SAME_AS
|
|
1679
|
+
return [SAME_AS] if self.identity is not None and not self.plain else []
|
|
1680
|
+
|
|
1681
|
+
def matched(self, s: Term, p: Term, o: Term, quiet: bool = False) -> O.Op:
|
|
1682
|
+
"""The solutions of one triple pattern over the maps as stated (18.3)."""
|
|
1683
|
+
if isinstance(p, Path):
|
|
1684
|
+
return self.path_op(s, p, o)
|
|
1685
|
+
self.never = [] # the maps whose objects can never be `o` (`branch`)
|
|
1686
|
+
self.patterns += 1
|
|
1687
|
+
if isinstance(p, Var):
|
|
1688
|
+
return self.any_predicate(s, p, o)
|
|
1689
|
+
if not isinstance(p, IRITerm):
|
|
1690
|
+
return _empty(s, p, o)
|
|
1691
|
+
if p.text == TOP_OBJECT_PROPERTY and self.ql_tbox() is not None and self.graph is None:
|
|
1692
|
+
return self.top_pairs(s, o)
|
|
1693
|
+
branches = self.branches(s, p, o)
|
|
1694
|
+
branches = [b for b in branches if b is not None]
|
|
1695
|
+
if not branches:
|
|
1696
|
+
if isinstance(o, LiteralTerm) and self.never and not quiet:
|
|
1697
|
+
_say_never(p, o, self.never)
|
|
1698
|
+
return _empty(s, p, o)
|
|
1699
|
+
# The graph is a set of triples (RDF 1.1 Concepts 3): a triple two
|
|
1700
|
+
# maps give, or one map gives from two rows, matches once.
|
|
1701
|
+
return O.Distinct(branches[0] if len(branches) == 1 else O.Union(tuple(branches)))
|
|
1702
|
+
|
|
1703
|
+
def branches(self, s: Term, p: IRITerm, o: Term) -> List[Optional[O.Op]]:
|
|
1704
|
+
"""Each map's triples for the pattern (s, p, o), p an IRI: one plan per map and graph map."""
|
|
1705
|
+
self.never = []
|
|
1706
|
+
if self.maps.open_predicates:
|
|
1707
|
+
raise NotBuilt("a map whose predicate is a column")
|
|
1708
|
+
branches: List[O.Op] = []
|
|
1709
|
+
if p.text == RDF_TYPE and isinstance(s, Var) and s == o:
|
|
1710
|
+
# `?c a ?c`: the pattern `?c a ?k` whose class is ?c itself, bound once (each typing branch binds
|
|
1711
|
+
# its subject and its class as two columns).
|
|
1712
|
+
k = Var(self.fresh("class"))
|
|
1713
|
+
same = E.Compare("=", E.Col(s.name), E.Col(k.name), E.TERM)
|
|
1714
|
+
return [O.Project(O.Filter(b, same), tuple(c for c in b.columns if c != k.name))
|
|
1715
|
+
for b in self.branches(s, p, k) if b is not None]
|
|
1716
|
+
if p.text == RDF_TYPE:
|
|
1717
|
+
# rdfs:Resource, which saturation gives every class (the RDFS core), is the default graph's alone:
|
|
1718
|
+
# inside GRAPH matching is simple.
|
|
1719
|
+
from prograph.plan.ir.saturate import RESOURCE
|
|
1720
|
+
core_only = RESOURCE if self.graph is not None and self.ontology is not None else None
|
|
1721
|
+
declared = {d.id: d.classes for d in getattr(self.maps.__dict__.get("saturated_from"), "nodes", ())}
|
|
1722
|
+
for m in self.maps.nodes:
|
|
1723
|
+
for cls in m.classes:
|
|
1724
|
+
if cls == core_only and cls not in declared.get(m.id, ()):
|
|
1725
|
+
continue
|
|
1726
|
+
if isinstance(o, Var) or o == IRITerm(cls):
|
|
1727
|
+
for gm in self.graph_choices(m.graph_maps, m.graph):
|
|
1728
|
+
branches.append(self.branch(m, m.subject, None, s, o, cls, gm))
|
|
1729
|
+
if p.text == RDF_TYPE and self.typing:
|
|
1730
|
+
branches += self.typed_by_property(s, o)
|
|
1731
|
+
if p.text == RDF_TYPE and self.ontology is not None:
|
|
1732
|
+
if self.graph is None: # the ontology's subclasses are the default graph's (graph_pattern)
|
|
1733
|
+
branches += self.typed_by_superclass(s, o)
|
|
1734
|
+
branches += self.typed_resource(s, o)
|
|
1735
|
+
ql = self.ql_tbox()
|
|
1736
|
+
if ql is not None and p.text == DIFFERENT_FROM and self.graph is None and not self.differing and s != o:
|
|
1737
|
+
# Under 'ql' owl:differentFrom is what Direct semantics entails (`ql.difference`), never a ≠ a; a
|
|
1738
|
+
# pattern of one term on both sides reads the stated triples (a stated a ≠ a is an inconsistency,
|
|
1739
|
+
# `ql.SELF_DIFFERENT`; a consistent graph has none).
|
|
1740
|
+
return [self.different_branch(s, o, ql)]
|
|
1741
|
+
if ql is not None and p.text == RDF_TYPE:
|
|
1742
|
+
branches += self.typed_everything(s, o, ql)
|
|
1743
|
+
branches += self.typed_by_data(s, o, ql)
|
|
1744
|
+
if self.reads_ontology():
|
|
1745
|
+
branches += self.ontology_rows(s, p, o)
|
|
1746
|
+
for giver in self.maps.by_predicate.get(p.text, ()):
|
|
1747
|
+
branches += self.giver_branches(giver, s, o)
|
|
1748
|
+
if ql is not None and p.text != RDF_TYPE and self.graph is None:
|
|
1749
|
+
# Reasoning is the default graph's: inside GRAPH matching is simple.
|
|
1750
|
+
for q in sorted(ql.inverse_subs.get(p.text, ())):
|
|
1751
|
+
branches += self.inverse_branches(q, s, o)
|
|
1752
|
+
if p.text in ql.reflexive:
|
|
1753
|
+
branches += self.reflexive_branches(s, o)
|
|
1754
|
+
return branches
|
|
1755
|
+
|
|
1756
|
+
def different_branch(self, s: Term, o: Term, ql: Any) -> O.Op:
|
|
1757
|
+
"""`s owl:differentFrom o` under 'ql': the pairs `ql.QLTBox.different` matches (each of its patterns under
|
|
1758
|
+
the same rewriting; its own owl:differentFrom patterns read the stated triples), bound to `s` and `o`."""
|
|
1759
|
+
from prograph.plan.ir.ql import A_, B_
|
|
1760
|
+
self.differing += 1
|
|
1761
|
+
try:
|
|
1762
|
+
return self.pattern_text(ql.different, ((A_, s), (B_, o)))
|
|
1763
|
+
finally:
|
|
1764
|
+
self.differing -= 1
|
|
1765
|
+
|
|
1766
|
+
def pattern_text(self, text: str, binds: Tuple[Tuple[str, Term], ...]) -> O.Op:
|
|
1767
|
+
"""A group pattern the ontology's rewriting writes as SPARQL text (`ql`), translated here, under the same
|
|
1768
|
+
rewriting and graph, its variables of `binds` bound to their terms (a variable, or a constant to match),
|
|
1769
|
+
the others projected away (they are named `qlnd_...`, no query's own)."""
|
|
1770
|
+
tree = parse("SELECT * WHERE { " + text + " }")
|
|
1771
|
+
x = self.group(tree.selectQuery().whereClause().groupGraphPattern())
|
|
1772
|
+
x = O.Project(x, tuple(name for name, _ in binds))
|
|
1773
|
+
keep: List[str] = []
|
|
1774
|
+
for name, t in binds:
|
|
1775
|
+
if isinstance(t, Var):
|
|
1776
|
+
if t.name in keep:
|
|
1777
|
+
x = O.Filter(x, E.Compare("=", E.Col(name), E.Col(t.name), E.TERM))
|
|
1778
|
+
else:
|
|
1779
|
+
x = O.Extend(x, t.name, E.Col(name))
|
|
1780
|
+
keep.append(t.name)
|
|
1781
|
+
else:
|
|
1782
|
+
x = O.Filter(x, E.Compare("=", E.Col(name), E.Const(t), E.TERM))
|
|
1783
|
+
return O.Project(x, tuple(keep))
|
|
1784
|
+
|
|
1785
|
+
def ql_tbox(self) -> Any:
|
|
1786
|
+
"""The ontology's QL axioms closed (`ql.QLTBox`) under profile="ql"; None otherwise."""
|
|
1787
|
+
return getattr(self.ontology, "ql", None) if self.ontology is not None else None
|
|
1788
|
+
|
|
1789
|
+
def inverse_branches(self, q: str, s: Term, o: Term) -> List[Optional[O.Op]]:
|
|
1790
|
+
"""`s p o` from `o q s`, q ⊑ p⁻ (owl:inverseOf, a symmetric property, a subproperty of an inverse:
|
|
1791
|
+
`ql.QLTBox.inverse_subs`): q's triples with subject and object swapped, those whose object is a literal
|
|
1792
|
+
aside (a literal is never a subject)."""
|
|
1793
|
+
if isinstance(o, LiteralTerm) or isinstance(s, LiteralTerm):
|
|
1794
|
+
return [] # a literal is never a subject, nor the object of an object property
|
|
1795
|
+
found: List[Optional[O.Op]] = []
|
|
1796
|
+
for giver in self.maps.by_predicate.get(q, ()):
|
|
1797
|
+
found += self.giver_branches(giver, o, s)
|
|
1798
|
+
if self.reads_ontology():
|
|
1799
|
+
found += self.ontology_rows(o, IRITerm(q), s)
|
|
1800
|
+
found = [x for x in found if x is not None]
|
|
1801
|
+
if not isinstance(s, Var):
|
|
1802
|
+
return found
|
|
1803
|
+
term = E.Col(s.name)
|
|
1804
|
+
return [O.Filter(x, E.Or((E.Call("isIRI", (term,)), E.Call("isBLANK", (term,))))) for x in found]
|
|
1805
|
+
|
|
1806
|
+
def reflexive_branches(self, s: Term, o: Term) -> List[O.Op]:
|
|
1807
|
+
"""`s p o` for a reflexive p (or one a reflexive property is under): every individual related to itself
|
|
1808
|
+
(`individual_terms`), in the default graph."""
|
|
1809
|
+
if self.graph is not None or isinstance(s, LiteralTerm) or isinstance(o, LiteralTerm):
|
|
1810
|
+
return []
|
|
1811
|
+
if not isinstance(s, Var) and not isinstance(o, Var) and s != o:
|
|
1812
|
+
return []
|
|
1813
|
+
terms = self.individual_terms()
|
|
1814
|
+
if terms is None:
|
|
1815
|
+
return []
|
|
1816
|
+
column, op = terms
|
|
1817
|
+
for t in (s, o):
|
|
1818
|
+
if not isinstance(t, Var):
|
|
1819
|
+
one = self._one_term(op, column, t, [v for v in (s, o) if isinstance(v, Var)])
|
|
1820
|
+
return [one] if one is not None else []
|
|
1821
|
+
x: O.Op = O.Extend(op, s.name, E.Col(column))
|
|
1822
|
+
if o.name != s.name:
|
|
1823
|
+
x = O.Extend(x, o.name, E.Col(column))
|
|
1824
|
+
return [O.Project(x, tuple(dict.fromkeys((s.name, o.name))))]
|
|
1825
|
+
|
|
1826
|
+
def _one_term(self, op: O.Op, column: str, t: Term, bound: List[Var]) -> Optional[O.Op]:
|
|
1827
|
+
"""`op`'s rows whose `column` is the term `t`, each variable of `bound` bound to it: only the branches whose
|
|
1828
|
+
terms may be `t` are read (`_met`), one row at most; None where none may."""
|
|
1829
|
+
provenance = const_slot(t).provenance
|
|
1830
|
+
if provenance:
|
|
1831
|
+
op = _met(op, {column: provenance})
|
|
1832
|
+
if op is None:
|
|
1833
|
+
return None
|
|
1834
|
+
x: O.Op = O.Filter(op, E.Compare("=", E.Col(column), E.Const(t), E.TERM))
|
|
1835
|
+
for v in bound:
|
|
1836
|
+
x = O.Extend(x, v.name, E.Col(column))
|
|
1837
|
+
return O.Project(x, tuple(dict.fromkeys(v.name for v in bound)))
|
|
1838
|
+
|
|
1839
|
+
def typed_everything(self, s: Term, o: Term, ql: Any) -> List[O.Op]:
|
|
1840
|
+
"""`s rdf:type C` for owl:Thing and each class every individual is (`⊤ ⊑ C`: `ql.QLTBox.everything`):
|
|
1841
|
+
every individual (`individual_terms`), in the default graph."""
|
|
1842
|
+
from prograph.plan.ir.ql import THING
|
|
1843
|
+
classes = sorted({THING} | set(ql.everything))
|
|
1844
|
+
wanted = classes if isinstance(o, Var) else [o.text] if isinstance(o, IRITerm) and o.text in classes else []
|
|
1845
|
+
if isinstance(o, IRITerm) and o.text == NAMED_INDIVIDUAL:
|
|
1846
|
+
# Every individual is a named one (OWL 2 asks no declaration of an individual): `?i a
|
|
1847
|
+
# owl:NamedIndividual` lists them. It is a declaration, no class: `?x a ?c` does not give it.
|
|
1848
|
+
wanted = [o.text]
|
|
1849
|
+
if not wanted or self.graph is not None or isinstance(s, LiteralTerm):
|
|
1850
|
+
return []
|
|
1851
|
+
terms = self.individual_terms()
|
|
1852
|
+
if terms is None:
|
|
1853
|
+
return []
|
|
1854
|
+
column, op = terms
|
|
1855
|
+
x = self._one_term(op, column, s, []) if not isinstance(s, Var) else \
|
|
1856
|
+
O.Project(O.Extend(op, s.name, E.Col(column)), (s.name,))
|
|
1857
|
+
if x is None:
|
|
1858
|
+
return []
|
|
1859
|
+
if not isinstance(s, Var):
|
|
1860
|
+
x = O.Distinct(x) # whether `s` is an individual: one row
|
|
1861
|
+
return [O.Extend(x, o.name, E.Const(IRITerm(c))) if isinstance(o, Var) else x for c in wanted]
|
|
1862
|
+
|
|
1863
|
+
def typed_by_data(self, s: Term, o: Term, ql: Any) -> List[O.Op]:
|
|
1864
|
+
"""`s rdf:type C` from `∃d.DR ⊑ C` (a data property's value of some datatypes types its subject:
|
|
1865
|
+
`ql.QLTBox.data_typed`): the subjects of d's triples whose value's datatype is one of DR's."""
|
|
1866
|
+
from prograph.plan.ir.ql import in_range
|
|
1867
|
+
out: List[O.Op] = []
|
|
1868
|
+
for prop, datatypes, classes in ql.data_typed:
|
|
1869
|
+
wanted = sorted(classes) if isinstance(o, Var) else \
|
|
1870
|
+
[o.text] if isinstance(o, IRITerm) and o.text in classes else []
|
|
1871
|
+
if not wanted or isinstance(s, LiteralTerm):
|
|
1872
|
+
continue
|
|
1873
|
+
# The values of DR by value (`ql.in_range`: "1.0"^^xsd:decimal is an xsd:integer).
|
|
1874
|
+
x = self.pattern_text(f"?qlnd_s <{prop}> ?qlnd_v . FILTER({in_range('qlnd_v', datatypes)})",
|
|
1875
|
+
(("qlnd_s", s),))
|
|
1876
|
+
x = O.Project(x, self._typed_columns(s, x))
|
|
1877
|
+
out += [O.Extend(x, o.name, E.Const(IRITerm(c))) if isinstance(o, Var) else x for c in wanted]
|
|
1878
|
+
return out
|
|
1879
|
+
|
|
1880
|
+
@home("every-individual", "Every individual: what owl:Thing, a reflexive property and `⊤ ⊑ C` range over",
|
|
1881
|
+
terms=("every individual", "owl:Thing", "named individual", "reflexive property", "individual_terms",
|
|
1882
|
+
"domain of discourse"))
|
|
1883
|
+
def individual_terms(self) -> Optional[Tuple[str, O.Op]]:
|
|
1884
|
+
"""Every individual of the default graph, as one column of one plan (kept per dataset): each IRI that is
|
|
1885
|
+
the subject of a triple the maps give, or the object of one whose predicate is not rdf:type (that object
|
|
1886
|
+
is a class), and each IRI the ontology asserts about or declares an owl:NamedIndividual
|
|
1887
|
+
(`ql.QLTBox.individuals`). Literals are values; blank nodes are no individual (OWL 2 QL has no anonymous
|
|
1888
|
+
individual, Profiles §3.2.1). The maps as declared are read, not their saturated copies (the same rows)."""
|
|
1889
|
+
key = (self.dataset, self.witnessing, self.plain)
|
|
1890
|
+
held = self.__dict__.setdefault("_individual_terms", {})
|
|
1891
|
+
if key in held:
|
|
1892
|
+
return held[key]
|
|
1893
|
+
column = self.fresh("it")
|
|
1894
|
+
parts: List[O.Op] = []
|
|
1895
|
+
|
|
1896
|
+
def add(x: Optional[O.Op], term: str) -> None:
|
|
1897
|
+
if x is None or (isinstance(x, O.Values) and not x.rows):
|
|
1898
|
+
return
|
|
1899
|
+
x = O.Filter(x, E.Call("isIRI", (E.Col(term),)))
|
|
1900
|
+
parts.append(O.Project(O.Extend(x, column, E.Col(term)), (column,)))
|
|
1901
|
+
declared = self.maps.__dict__.get("saturated_from") or self.maps
|
|
1902
|
+
for m in declared.nodes:
|
|
1903
|
+
if m.classes and m.subject.term_type == IRI:
|
|
1904
|
+
for gm in self.graph_choices(m.graph_maps, m.graph):
|
|
1905
|
+
a, b = Var(self.fresh("a")), Var(self.fresh("b"))
|
|
1906
|
+
add(self.branch(m, m.subject, None, a, b, m.classes[0], gm), a.name)
|
|
1907
|
+
for predicate, givers in sorted(declared.by_predicate.items()):
|
|
1908
|
+
for giver in givers:
|
|
1909
|
+
# Only a term map that makes IRIs gives individuals, so the column holds IRIs alone.
|
|
1910
|
+
node = isinstance(giver.map, NodeMap)
|
|
1911
|
+
subject_iri = giver.map.subject.term_type == IRI if node else True
|
|
1912
|
+
object_iri = giver.predicate_object.object.term_type == IRI if node else True
|
|
1913
|
+
classed = node and bool(giver.map.classes)
|
|
1914
|
+
a, b = Var(self.fresh("a")), Var(self.fresh("b"))
|
|
1915
|
+
for x in self.giver_branches(giver, a, b):
|
|
1916
|
+
if not classed and subject_iri:
|
|
1917
|
+
add(x, a.name)
|
|
1918
|
+
if predicate != RDF_TYPE and object_iri:
|
|
1919
|
+
add(x, b.name)
|
|
1920
|
+
ql = self.ql_tbox()
|
|
1921
|
+
if ql is not None and ql.individuals and self.reads_ontology():
|
|
1922
|
+
parts.append(O.Values((column,), tuple((IRITerm(i),) for i in sorted(ql.individuals))))
|
|
1923
|
+
held[key] = (column, O.Distinct(parts[0] if len(parts) == 1 else O.Union(tuple(parts)))) if parts else None
|
|
1924
|
+
return held[key]
|
|
1925
|
+
|
|
1926
|
+
def giver_branches(self, giver: Any, s: Term, o: Term) -> List[Optional[O.Op]]:
|
|
1927
|
+
"""One `SourceMaps.by_predicate` entry's triples for the pattern (s, its predicate, o): a plan per
|
|
1928
|
+
graph map (None where the map can give none)."""
|
|
1929
|
+
m = giver.map
|
|
1930
|
+
out: List[Optional[O.Op]] = []
|
|
1931
|
+
if isinstance(m, NodeMap):
|
|
1932
|
+
po: PredicateObject = giver.predicate_object
|
|
1933
|
+
if po.predicate_map is not None:
|
|
1934
|
+
raise NotBuilt("a predicate map")
|
|
1935
|
+
for gm in self.graph_choices(po.graph_maps, po.graph):
|
|
1936
|
+
out.append(self.branch(m, m.subject, po.object, s, o, None, gm))
|
|
1937
|
+
else:
|
|
1938
|
+
for gm in self.graph_choices(m.graph_maps, m.graph):
|
|
1939
|
+
out.append(self.edge_branch(m, s, o, gm))
|
|
1940
|
+
return out
|
|
1941
|
+
|
|
1942
|
+
def reads_ontology(self) -> bool:
|
|
1943
|
+
"""Whether a pattern here reads the ontology's triples: in the default graph (also one FROM builds),
|
|
1944
|
+
never inside GRAPH, nor in a witness (`ontology_rows`)."""
|
|
1945
|
+
return self.onto_triples is not None and self.graph is None and not self.witnessing
|
|
1946
|
+
|
|
1947
|
+
def ontology_predicates(self) -> List[str]:
|
|
1948
|
+
"""The predicates of the ontology's triples a pattern here reads (`reads_ontology`)."""
|
|
1949
|
+
return self.onto_triples.predicates if self.reads_ontology() else []
|
|
1950
|
+
|
|
1951
|
+
def ontology_rows(self, s: Term, p: IRITerm, o: Term) -> List[O.Op]:
|
|
1952
|
+
"""The ontology's triples matching (s, p, o) (`dataset.ontology_triples`) as constant rows: those
|
|
1953
|
+
stated, and those a witness of the data's vocabulary adds, each only where its witness has a triple
|
|
1954
|
+
in the data (a class the maps give, `?w rdf:type C`; a predicate, `?w p ?v`)."""
|
|
1955
|
+
held = self.onto_triples
|
|
1956
|
+
rows = held.stated.get(p.text, ())
|
|
1957
|
+
direct = held.direct.get(p.text)
|
|
1958
|
+
top = p.text == SUB_PROPERTY_IRI and o == IRITerm(TOP_OBJECT_PROPERTY) # every object property: a TBox answer
|
|
1959
|
+
if direct and not top and not (self.direct_ok and all(t.name in self.typed_vars for t in (s, o) if isinstance(t, Var))):
|
|
1960
|
+
# What Direct semantics adds (owl:Thing above a class, owl:Nothing below it ...) is a typed variable's
|
|
1961
|
+
# (or a pattern of constants'): an untyped one reads the RDFS core's graph (`typed-variables`).
|
|
1962
|
+
rows = tuple(r for r in rows if r not in direct)
|
|
1963
|
+
out: List[O.Op] = list(self.ontology_values(s, o, rows))
|
|
1964
|
+
# The witnessed rows that match the pattern, each with its witness's id in a column of their own, kept
|
|
1965
|
+
# where that witness holds (`witnesses`): one branch, whatever the number of witnesses; a witness no
|
|
1966
|
+
# matching row needs is not planned.
|
|
1967
|
+
tagged = []
|
|
1968
|
+
for w, pairs in sorted(held.witnessed.get(p.text, {}).items()):
|
|
1969
|
+
tag = LiteralTerm(f"{w[0]} {w[1]}")
|
|
1970
|
+
tagged += [(a, b, tag, w) for a, b in pairs if (isinstance(s, Var) or s == a)
|
|
1971
|
+
and (isinstance(o, Var) or o == b)]
|
|
1972
|
+
named, out_data = self.data_class_rows(s, p, o)
|
|
1973
|
+
tagged += named
|
|
1974
|
+
out += out_data
|
|
1975
|
+
ids = [w for w in dict.fromkeys(t[3] for t in tagged) if self.witness(w) is not None]
|
|
1976
|
+
tagged = [t for t in tagged if t[3] in ids]
|
|
1977
|
+
if tagged:
|
|
1978
|
+
column = self.witness_column()
|
|
1979
|
+
for rows in self.ontology_values(s, o, tagged, column):
|
|
1980
|
+
parts = [self.holds(w) for w in ids]
|
|
1981
|
+
held_ids = parts[0] if len(parts) == 1 else O.Union(tuple(parts))
|
|
1982
|
+
out.append(O.Project(O.SemiJoin(rows, held_ids), rows.columns[:-1]))
|
|
1983
|
+
return out
|
|
1984
|
+
|
|
1985
|
+
def data_class_rows(self, s: Term, p: IRITerm, o: Term) -> Tuple[List[tuple], List[O.Op]]:
|
|
1986
|
+
"""The triples a class the data gives from a row adds that match (s, p, o) (`OntologyTriples.data_class`:
|
|
1987
|
+
`C rdf:type rdfs:Class` by rdfs3 over rdf:type's range, and what follows): where the pattern names the
|
|
1988
|
+
class, its rows tagged with its witness for `ontology_rows` (`?w rdf:type C`, an existence test); where
|
|
1989
|
+
it binds the class, the rows of the classes the data gives (`data_classes`). None without such a map."""
|
|
1990
|
+
from prograph.plan.ir.dataset import DATA_CLASS
|
|
1991
|
+
pairs = self.onto_triples.data_class.get(p.text, ())
|
|
1992
|
+
if not pairs or not self._type_givers():
|
|
1993
|
+
return [], []
|
|
1994
|
+
tagged: List[tuple] = []
|
|
1995
|
+
ops: List[O.Op] = []
|
|
1996
|
+
for a, b in pairs:
|
|
1997
|
+
named = [t for t, v in ((s, a), (o, b)) if v == DATA_CLASS and not isinstance(t, Var)]
|
|
1998
|
+
if named:
|
|
1999
|
+
c = named[0]
|
|
2000
|
+
if all(t == c for t in named) and self.may_be_data_class(c):
|
|
2001
|
+
pair = tuple(c if v == DATA_CLASS else v for v in (a, b))
|
|
2002
|
+
tagged.append(pair + (LiteralTerm(f"class {c.text}"), ("class", c.text)))
|
|
2003
|
+
continue
|
|
2004
|
+
if any(not isinstance(t, Var) and t != v for t, v in ((s, a), (o, b))):
|
|
2005
|
+
continue
|
|
2006
|
+
bound: Dict[str, Any] = {}
|
|
2007
|
+
equal: List[Term] = [] # a variable twice, as the class and a constant: the class is the constant
|
|
2008
|
+
for t, v in ((s, a), (o, b)):
|
|
2009
|
+
if isinstance(t, Var) and bound.setdefault(t.name, v) != v:
|
|
2010
|
+
equal.append(v if bound[t.name] == DATA_CLASS else bound[t.name])
|
|
2011
|
+
held = self.data_classes()
|
|
2012
|
+
if held is None:
|
|
2013
|
+
break
|
|
2014
|
+
column, x = held
|
|
2015
|
+
for c in equal:
|
|
2016
|
+
x = O.Filter(x, E.Compare("=", E.Col(column), E.Const(c), E.TERM))
|
|
2017
|
+
for name, v in bound.items():
|
|
2018
|
+
x = O.Extend(x, name, E.Col(column) if v == DATA_CLASS else E.Const(v))
|
|
2019
|
+
ops.append(O.Project(x, tuple(bound)))
|
|
2020
|
+
return tagged, ops
|
|
2021
|
+
|
|
2022
|
+
def may_be_data_class(self, c: Term) -> bool:
|
|
2023
|
+
"""Whether `c`'s triples as a class the data gives from a row are not held already: an IRI no stated
|
|
2024
|
+
triple types `rdfs:Class` and no map gives as its class (whose witness has them)."""
|
|
2025
|
+
from prograph.plan.ir.rdfs import CLASS, RESOURCE
|
|
2026
|
+
if not isinstance(c, IRITerm):
|
|
2027
|
+
return False
|
|
2028
|
+
held = self.__dict__.get("_stated_classes")
|
|
2029
|
+
if held is None:
|
|
2030
|
+
held = {a for a, b in self.onto_triples.stated.get(RDF_TYPE, ()) if b == IRITerm(CLASS)}
|
|
2031
|
+
held |= {IRITerm(k) for m in self.maps.nodes for k in m.classes}
|
|
2032
|
+
held.add(IRITerm(RESOURCE))
|
|
2033
|
+
self.__dict__["_stated_classes"] = held
|
|
2034
|
+
return c not in held
|
|
2035
|
+
|
|
2036
|
+
def data_classes(self) -> Optional[Tuple[str, O.Op]]:
|
|
2037
|
+
"""The classes the data gives from a row (the IRI and blank-node objects of the rdf:type triples a map
|
|
2038
|
+
takes from a column or template, in the default graph), as one column of one plan, distinct, which every
|
|
2039
|
+
pattern of the query that binds such a class shares; None for none."""
|
|
2040
|
+
key = (self.dataset, self.plain)
|
|
2041
|
+
held = self.__dict__.setdefault("_data_classes", {})
|
|
2042
|
+
if key in held:
|
|
2043
|
+
return held[key]
|
|
2044
|
+
column = self.fresh("dc")
|
|
2045
|
+
parts: List[O.Op] = []
|
|
2046
|
+
for giver in self._type_givers():
|
|
2047
|
+
m = giver.map
|
|
2048
|
+
holder = giver.predicate_object if isinstance(m, NodeMap) else m
|
|
2049
|
+
for gm in self.graph_choices(holder.graph_maps, holder.graph):
|
|
2050
|
+
a, b = Var(self.fresh("a")), Var(self.fresh("b"))
|
|
2051
|
+
x = (self.branch(m, m.subject, holder.object, a, b, None, gm) if isinstance(m, NodeMap)
|
|
2052
|
+
else self.edge_branch(m, a, b, gm))
|
|
2053
|
+
if x is None or (isinstance(x, O.Values) and not x.rows):
|
|
2054
|
+
continue
|
|
2055
|
+
x = O.Filter(x, E.Or((E.Call("isIRI", (E.Col(b.name),)), E.Call("isBLANK", (E.Col(b.name),)))))
|
|
2056
|
+
parts.append(O.Project(O.Extend(x, column, E.Col(b.name)), (column,)))
|
|
2057
|
+
held[key] = (column, O.Distinct(parts[0] if len(parts) == 1 else O.Union(tuple(parts)))) if parts else None
|
|
2058
|
+
return held[key]
|
|
2059
|
+
|
|
2060
|
+
def witness_column(self) -> str:
|
|
2061
|
+
"""The column a witness's id is in (`holds`), one per query."""
|
|
2062
|
+
if "_witness_column" not in self.__dict__:
|
|
2063
|
+
self.__dict__["_witness_column"] = self.fresh("wid")
|
|
2064
|
+
return self.__dict__["_witness_column"]
|
|
2065
|
+
|
|
2066
|
+
def holds(self, w: Tuple[str, str]) -> O.Op:
|
|
2067
|
+
"""One row, `w`'s id (as `ontology_rows` writes it) in `witness_column`, where `w`'s witness has a
|
|
2068
|
+
triple: an existence test (no witness is read past its first row), planned once per query and graph and
|
|
2069
|
+
shared by the patterns that need it."""
|
|
2070
|
+
key = (w, repr(self.graph), self.dataset, self.plain)
|
|
2071
|
+
cache = self.__dict__.setdefault("_holds", {})
|
|
2072
|
+
if key not in cache:
|
|
2073
|
+
tag = O.Values((self.witness_column(),), ((LiteralTerm(f"{w[0]} {w[1]}"),),))
|
|
2074
|
+
cache[key] = O.SemiJoin(tag, O.Slice(self.witness(w), limit=1))
|
|
2075
|
+
return cache[key]
|
|
2076
|
+
|
|
2077
|
+
def witness(self, w: Tuple[str, str]) -> Optional[O.Op]:
|
|
2078
|
+
"""The data's triples that witness `w` (`dataset.OntologyTriples`): `?w rdf:type C` for a class,
|
|
2079
|
+
`?w p ?v` for a predicate; None where no map can give one. Read as an existence test (`holds`)."""
|
|
2080
|
+
kind, name = w
|
|
2081
|
+
key = (w, repr(self.graph), self.dataset, self.plain)
|
|
2082
|
+
cache = self.__dict__.setdefault("_witnesses", {})
|
|
2083
|
+
if key in cache:
|
|
2084
|
+
return cache[key]
|
|
2085
|
+
self.witnessing += 1
|
|
2086
|
+
never = getattr(self, "never", []) # the pattern being matched keeps its own (`branch`)
|
|
2087
|
+
try:
|
|
2088
|
+
x = Var(self.fresh("w"))
|
|
2089
|
+
op = (self.matched(x, IRITerm(RDF_TYPE), IRITerm(name), quiet=True) if kind == "class"
|
|
2090
|
+
else self.matched(x, IRITerm(name), Var(self.fresh("w")), quiet=True))
|
|
2091
|
+
finally:
|
|
2092
|
+
self.witnessing -= 1
|
|
2093
|
+
self.never = never
|
|
2094
|
+
if isinstance(op, O.Distinct):
|
|
2095
|
+
op = op.input # an existence test (`holds`): the triples need not be told apart
|
|
2096
|
+
cache[key] = None if isinstance(op, O.Values) and not op.rows else op
|
|
2097
|
+
return cache[key]
|
|
2098
|
+
|
|
2099
|
+
def ontology_walk(self, s: Term, step: Any, lo: int, hi: Optional[int], o: Term) -> Optional[O.Op]:
|
|
2100
|
+
"""p*, p+ or p? from a constant end over a predicate only the ontology's triples give (no map gives
|
|
2101
|
+
it: `?c rdfs:subClassOf* :C`), walked here over those triples (`dataset.ontology_triples`), each end
|
|
2102
|
+
reached kept with the witnesses its walk needs; None where the walk is the plan's (`O.Closure`)."""
|
|
2103
|
+
if not self.reads_ontology() or (self.identity is not None and not self.plain):
|
|
2104
|
+
return None
|
|
2105
|
+
inverse = isinstance(step, Path) and step.kind == "inv" and isinstance(step.parts[0], IRITerm)
|
|
2106
|
+
p = step.parts[0] if inverse else step
|
|
2107
|
+
if not isinstance(p, IRITerm) or p.text in (RDF_TYPE, TOP_OBJECT_PROPERTY) or p.text in self.maps.by_predicate:
|
|
2108
|
+
return None # owl:topObjectProperty's pairs are `top_pairs`', not the ontology's rows
|
|
2109
|
+
if isinstance(s, Var) and isinstance(o, Var):
|
|
2110
|
+
return None
|
|
2111
|
+
forward = not isinstance(s, Var)
|
|
2112
|
+
start, other = (s, o) if forward else (o, s)
|
|
2113
|
+
# Direct semantics' pairs where the walk's variable end is typed (`typed-variables`).
|
|
2114
|
+
typed = self.direct_ok and (not isinstance(other, Var) or other.name in self.typed_vars)
|
|
2115
|
+
if forward == inverse: # walking against the triples' direction
|
|
2116
|
+
pairs = [(b, a, w) for a, b, w in self._ontology_pairs(p.text, typed)]
|
|
2117
|
+
else:
|
|
2118
|
+
pairs = self._ontology_pairs(p.text, typed)
|
|
2119
|
+
edges: Dict[Any, List[Tuple[Any, frozenset]]] = {}
|
|
2120
|
+
for a, b, w in pairs:
|
|
2121
|
+
edges.setdefault(a, []).append((b, w))
|
|
2122
|
+
# A class the data gives from a row (`data_class_rows`): its own steps, where the walk reaches it; a
|
|
2123
|
+
# step to any such class is the plan's walk.
|
|
2124
|
+
from prograph.plan.ir.dataset import DATA_CLASS
|
|
2125
|
+
steps = self.onto_triples.data_class.get(p.text, ()) if self._type_givers() else ()
|
|
2126
|
+
steps = [(b, a) for a, b in steps] if forward == inverse else list(steps)
|
|
2127
|
+
to_data = {a for a, b in steps if a != DATA_CLASS and b == DATA_CLASS}
|
|
2128
|
+
|
|
2129
|
+
def steps_from(node) -> List[Tuple[Any, frozenset]]:
|
|
2130
|
+
out = edges.get(node, [])
|
|
2131
|
+
if steps and self.may_be_data_class(node):
|
|
2132
|
+
w = frozenset({("class", node.text)})
|
|
2133
|
+
out = out + [(node if b == DATA_CLASS else b, w) for a, b in steps if a == DATA_CLASS]
|
|
2134
|
+
return out
|
|
2135
|
+
reached: Dict[Any, List[frozenset]] = {}
|
|
2136
|
+
|
|
2137
|
+
def keep(node, label) -> bool:
|
|
2138
|
+
held = reached.setdefault(node, [])
|
|
2139
|
+
if any(h <= label for h in held):
|
|
2140
|
+
return False
|
|
2141
|
+
held[:] = [h for h in held if not label <= h] + [label]
|
|
2142
|
+
return True
|
|
2143
|
+
frontier = [(start, frozenset())]
|
|
2144
|
+
if lo == 0:
|
|
2145
|
+
keep(start, frozenset())
|
|
2146
|
+
depth = 0
|
|
2147
|
+
while frontier and (hi is None or depth < hi):
|
|
2148
|
+
depth += 1
|
|
2149
|
+
nxt = []
|
|
2150
|
+
for node, label in frontier:
|
|
2151
|
+
for b, w in steps_from(node):
|
|
2152
|
+
if keep(b, label | w):
|
|
2153
|
+
nxt.append((b, label | w))
|
|
2154
|
+
frontier = nxt
|
|
2155
|
+
if to_data & set(reached):
|
|
2156
|
+
return None
|
|
2157
|
+
groups: Dict[frozenset, List[tuple]] = {}
|
|
2158
|
+
for node, labels in reached.items():
|
|
2159
|
+
if not isinstance(other, Var) and node != other:
|
|
2160
|
+
continue
|
|
2161
|
+
for label in labels:
|
|
2162
|
+
groups.setdefault(label, []).append((node,) if isinstance(other, Var) else ())
|
|
2163
|
+
names = (other.name,) if isinstance(other, Var) else ()
|
|
2164
|
+
branches: List[O.Op] = []
|
|
2165
|
+
for label, rows in sorted(groups.items(), key=lambda kv: sorted(kv[0])):
|
|
2166
|
+
x: Optional[O.Op] = O.Values(names, tuple(dict.fromkeys(rows)))
|
|
2167
|
+
for w in sorted(label):
|
|
2168
|
+
witness = self.witness(w)
|
|
2169
|
+
x = None if witness is None or x is None else O.SemiJoin(x, self.holds(w))
|
|
2170
|
+
if x is not None:
|
|
2171
|
+
branches.append(x)
|
|
2172
|
+
if not branches:
|
|
2173
|
+
return O.Values(names, ())
|
|
2174
|
+
return O.Distinct(branches[0] if len(branches) == 1 else O.Union(tuple(branches)))
|
|
2175
|
+
|
|
2176
|
+
def _ontology_pairs(self, predicate: str, typed: bool = True) -> List[Tuple[Any, Any, frozenset]]:
|
|
2177
|
+
"""The ontology's (subject, object) pairs of `predicate`, each with the witnesses it needs; without
|
|
2178
|
+
`typed`, none only Direct semantics gives (`typed-variables`)."""
|
|
2179
|
+
held = self.onto_triples
|
|
2180
|
+
direct = () if typed else held.direct.get(predicate, ())
|
|
2181
|
+
out = [(a, b, frozenset()) for a, b in held.stated.get(predicate, ()) if (a, b) not in direct]
|
|
2182
|
+
for w, pairs in held.witnessed.get(predicate, {}).items():
|
|
2183
|
+
out += [(a, b, frozenset({w})) for a, b in pairs]
|
|
2184
|
+
return out
|
|
2185
|
+
|
|
2186
|
+
def ontology_values(self, s: Term, o: Term, pairs, tag: Optional[str] = None) -> List[O.Op]:
|
|
2187
|
+
"""The pairs (subject, object) matching `s` and `o`, as rows of the pattern's variables (and, with
|
|
2188
|
+
`tag`, the pair's third member in that column): none, or one Values; under owl:sameAs two, the rows no
|
|
2189
|
+
IRI of which may be an alias (`Identity.may_alias`) apart, so their branch needs no alias join
|
|
2190
|
+
(`identity.pattern`)."""
|
|
2191
|
+
names = tuple(dict.fromkeys(t.name for t in (s, o) if isinstance(t, Var))) + ((tag,) if tag else ())
|
|
2192
|
+
rows: List[tuple] = []
|
|
2193
|
+
for pair in pairs:
|
|
2194
|
+
subject, obj = pair[0], pair[1]
|
|
2195
|
+
if (not isinstance(s, Var) and s != subject) or (not isinstance(o, Var) and o != obj):
|
|
2196
|
+
continue
|
|
2197
|
+
if isinstance(s, Var) and isinstance(o, Var) and s.name == o.name and subject != obj:
|
|
2198
|
+
continue
|
|
2199
|
+
bound = {}
|
|
2200
|
+
if isinstance(s, Var):
|
|
2201
|
+
bound[s.name] = subject
|
|
2202
|
+
if isinstance(o, Var):
|
|
2203
|
+
bound[o.name] = obj
|
|
2204
|
+
if tag:
|
|
2205
|
+
bound[tag] = pair[2]
|
|
2206
|
+
rows.append(tuple(bound[n] for n in names))
|
|
2207
|
+
rows = list(dict.fromkeys(rows))
|
|
2208
|
+
if not rows:
|
|
2209
|
+
return []
|
|
2210
|
+
if self.identity is None:
|
|
2211
|
+
return [O.Values(names, tuple(rows))]
|
|
2212
|
+
aliased = {r for r in rows if any(isinstance(x, IRITerm) and self.identity.may_alias(x.text) for x in r)}
|
|
2213
|
+
may, never = [r for r in rows if r in aliased], [r for r in rows if r not in aliased]
|
|
2214
|
+
return [O.Values(names, tuple(part)) for part in (never, may) if part]
|
|
2215
|
+
|
|
2216
|
+
def _type_givers(self) -> List[Any]:
|
|
2217
|
+
"""The maps giving rdf:type whose object comes from the row (a column or template, not a class)."""
|
|
2218
|
+
return [g for g in self.maps.by_predicate.get(RDF_TYPE, ())
|
|
2219
|
+
if (isinstance(g.map, NodeMap) and g.predicate_object.object.constant is None)
|
|
2220
|
+
or (isinstance(g.map, EdgeMap) and g.map.end.term is not None and g.map.end.term.constant is None)]
|
|
2221
|
+
|
|
2222
|
+
def typed_resource(self, s: Term, o: Term) -> List[O.Op]:
|
|
2223
|
+
"""`s rdf:type rdfs:Resource` (rdfs4a, rdfs4b) for the terms no class `saturate` closes types so
|
|
2224
|
+
already: the subjects and IRI and blank-node objects of the properties `typing_properties` names, and
|
|
2225
|
+
of the rdf:type triples a map gives from the row. One plan of those terms per graph the patterns read
|
|
2226
|
+
(`resource_terms`), which every such pattern of the query shares."""
|
|
2227
|
+
from prograph.plan.ir.saturate import RESOURCE
|
|
2228
|
+
if self.ontology.closure is None or self.graph is not None or \
|
|
2229
|
+
not (isinstance(o, Var) or o == IRITerm(RESOURCE)):
|
|
2230
|
+
return [] # the RDFS core holds in the default graph: inside GRAPH matching is simple
|
|
2231
|
+
terms = self.resource_terms()
|
|
2232
|
+
if terms is None:
|
|
2233
|
+
return []
|
|
2234
|
+
column, op = terms
|
|
2235
|
+
graph = (self.graph.name,) if isinstance(self.graph, Var) and self.graph.name in op.columns else ()
|
|
2236
|
+
if isinstance(s, Var):
|
|
2237
|
+
x = O.Project(O.Extend(op, s.name, E.Col(column)), (s.name,) + graph)
|
|
2238
|
+
else:
|
|
2239
|
+
# The terms that may be `s` alone: a branch whose terms cannot be it (`meeting`'s walk) goes.
|
|
2240
|
+
op = _met(op, {column: const_slot(s).provenance}) if const_slot(s).provenance else op
|
|
2241
|
+
if op is None:
|
|
2242
|
+
return []
|
|
2243
|
+
x = O.Project(O.Filter(op, E.Compare("=", E.Col(column), E.Const(s), E.TERM)), graph)
|
|
2244
|
+
return [O.Extend(x, o.name, E.Const(IRITerm(RESOURCE))) if isinstance(o, Var) else x]
|
|
2245
|
+
|
|
2246
|
+
def resource_terms(self) -> Optional[Tuple[str, O.Op]]:
|
|
2247
|
+
"""The terms `typed_resource` types, as one column of one plan, kept per graph the patterns read; None
|
|
2248
|
+
for none."""
|
|
2249
|
+
from prograph.plan.ir.saturate import RESOURCE
|
|
2250
|
+
key = (repr(self.graph), self.dataset, self.witnessing, self.plain)
|
|
2251
|
+
held = self.__dict__.setdefault("_resource_terms", {})
|
|
2252
|
+
if key in held:
|
|
2253
|
+
return held[key]
|
|
2254
|
+
column = self.fresh("rt")
|
|
2255
|
+
graph = (self.graph.name,) if isinstance(self.graph, Var) else ()
|
|
2256
|
+
parts: List[O.Op] = []
|
|
2257
|
+
|
|
2258
|
+
def add(x: Optional[O.Op], term: str, as_object: bool) -> None:
|
|
2259
|
+
if x is None or (isinstance(x, O.Values) and not x.rows):
|
|
2260
|
+
return
|
|
2261
|
+
if as_object:
|
|
2262
|
+
x = O.Filter(x, E.Or((E.Call("isIRI", (E.Col(term),)), E.Call("isBLANK", (E.Col(term),)))))
|
|
2263
|
+
x = O.Extend(x, column, E.Col(term))
|
|
2264
|
+
parts.append(O.Project(x, (column,) + tuple(g for g in graph if g in x.columns)))
|
|
2265
|
+
from prograph.plan.ir.saturate import resource_typed
|
|
2266
|
+
for prop, classes, as_subject in self.typing:
|
|
2267
|
+
if RESOURCE in classes:
|
|
2268
|
+
# Only the maps whose terms no class of theirs types rdfs:Resource already (`resource_typed`):
|
|
2269
|
+
# the others' are the class branches' rows, which the pattern's Distinct keeps once.
|
|
2270
|
+
a, b = Var(self.fresh("a")), Var(self.fresh("b"))
|
|
2271
|
+
found = [x for g in self.maps.by_predicate.get(prop, ()) if not resource_typed(self.maps, g, as_subject)
|
|
2272
|
+
for x in self.giver_branches(g, a, b) if x is not None]
|
|
2273
|
+
if self.reads_ontology():
|
|
2274
|
+
found += self.ontology_rows(a, IRITerm(prop), b)
|
|
2275
|
+
if found:
|
|
2276
|
+
add(O.Distinct(found[0] if len(found) == 1 else O.Union(tuple(found))),
|
|
2277
|
+
a.name if as_subject else b.name, not as_subject)
|
|
2278
|
+
for giver in self._type_givers():
|
|
2279
|
+
m = giver.map
|
|
2280
|
+
holder = giver.predicate_object if isinstance(m, NodeMap) else m
|
|
2281
|
+
for gm in self.graph_choices(holder.graph_maps, holder.graph):
|
|
2282
|
+
a, b = Var(self.fresh("a")), Var(self.fresh("b"))
|
|
2283
|
+
x = (self.branch(m, m.subject, holder.object, a, b, None, gm) if isinstance(m, NodeMap)
|
|
2284
|
+
else self.edge_branch(m, a, b, gm))
|
|
2285
|
+
add(x, a.name, False)
|
|
2286
|
+
add(x, b.name, True)
|
|
2287
|
+
if parts:
|
|
2288
|
+
cols = parts[0].columns
|
|
2289
|
+
parts = [x if x.columns == cols else O.Project(x, cols) for x in parts]
|
|
2290
|
+
held[key] = (column, O.Distinct(parts[0] if len(parts) == 1 else O.Union(tuple(parts)))) if parts else None
|
|
2291
|
+
return held[key]
|
|
2292
|
+
|
|
2293
|
+
def typed_by_superclass(self, s: Term, o: Term) -> List[O.Op]:
|
|
2294
|
+
"""`s rdf:type o` through the ontology's subclasses where a map's rdf:type object comes from
|
|
2295
|
+
the row (a column or template, not a class `saturate` closes): for each subclass C of a
|
|
2296
|
+
wanted class D, that map's triples typed C, typed D too (rdfs9; M3 review)."""
|
|
2297
|
+
out: List[O.Op] = []
|
|
2298
|
+
givers = self._type_givers()
|
|
2299
|
+
if not givers:
|
|
2300
|
+
return out
|
|
2301
|
+
pairs = [(IRITerm(sub), IRITerm(sup)) for sub, sups in sorted(self.ontology.superclasses.items())
|
|
2302
|
+
for sup in sorted(sups - {sub}) if isinstance(o, Var) or o == IRITerm(sup)]
|
|
2303
|
+
for giver in givers:
|
|
2304
|
+
m = giver.map
|
|
2305
|
+
holder = giver.predicate_object if isinstance(m, NodeMap) else m
|
|
2306
|
+
made = holder.object if isinstance(m, NodeMap) else m.end.term
|
|
2307
|
+
mine = [(a, b) for a, b in pairs if made is None or _may_give(made, a)]
|
|
2308
|
+
if not mine:
|
|
2309
|
+
continue
|
|
2310
|
+
for gm in self.graph_choices(holder.graph_maps, holder.graph):
|
|
2311
|
+
if len(mine) == 1:
|
|
2312
|
+
sub, sup = mine[0]
|
|
2313
|
+
x = (self.branch(m, m.subject, holder.object, s, sub, None, gm)
|
|
2314
|
+
if isinstance(m, NodeMap) else self.edge_branch(m, s, sub, gm))
|
|
2315
|
+
if x is None:
|
|
2316
|
+
continue
|
|
2317
|
+
x = O.Project(x, self._typed_columns(s, x))
|
|
2318
|
+
out.append(O.Extend(x, o.name, E.Const(sup)) if isinstance(o, Var) else x)
|
|
2319
|
+
continue
|
|
2320
|
+
# Several subclasses: the map's triples once, each typed object joined to the superclasses it
|
|
2321
|
+
# stands for (one Values of (subclass, superclass) pairs), not a branch per pair (the turbofan
|
|
2322
|
+
# ontology's 178 pairs read the RL graph's rdf:type triples 178 times).
|
|
2323
|
+
c = Var(self.fresh("sub"))
|
|
2324
|
+
x = (self.branch(m, m.subject, holder.object, s, c, None, gm)
|
|
2325
|
+
if isinstance(m, NodeMap) else self.edge_branch(m, s, c, gm))
|
|
2326
|
+
if x is None:
|
|
2327
|
+
continue
|
|
2328
|
+
if isinstance(o, Var):
|
|
2329
|
+
table = O.Values((c.name, o.name), tuple(mine))
|
|
2330
|
+
else:
|
|
2331
|
+
table = O.Values((c.name,), tuple(dict.fromkeys((a,) for a, _ in mine)))
|
|
2332
|
+
x = O.Join(x, table)
|
|
2333
|
+
out.append(O.Project(x, self._typed_columns(s, x) + ((o.name,) if isinstance(o, Var) else ())))
|
|
2334
|
+
return out
|
|
2335
|
+
|
|
2336
|
+
def _typed_columns(self, s: Term, x: O.Op) -> Tuple[str, ...]:
|
|
2337
|
+
"""A typing branch's columns: its subject's, and GRAPH ?g's where the branch reads it."""
|
|
2338
|
+
return ((s.name,) if isinstance(s, Var) else ()) + \
|
|
2339
|
+
((self.graph.name,) if isinstance(self.graph, Var) and self.graph.name in x.columns else ())
|
|
2340
|
+
|
|
2341
|
+
def typed_by_property(self, s: Term, o: Term) -> List[O.Op]:
|
|
2342
|
+
"""`s rdf:type o` from an ontology's domain and range (rdfs2, rdfs3): for each
|
|
2343
|
+
property the maps give with a domain class C, the subjects of its own triple pattern typed C
|
|
2344
|
+
(its range: its IRI and blank-node objects), so a term is typed only where its triple
|
|
2345
|
+
exists, in the graph the pattern reads. One plan of the property's terms serves each of its
|
|
2346
|
+
classes; rdfs:Resource is `typed_resource`'s."""
|
|
2347
|
+
from prograph.plan.ir.saturate import RESOURCE
|
|
2348
|
+
out: List[O.Op] = []
|
|
2349
|
+
for prop, classes, as_subject in self.typing:
|
|
2350
|
+
wanted = sorted(classes - {RESOURCE}) if isinstance(o, Var) else \
|
|
2351
|
+
[o.text] if isinstance(o, IRITerm) and o.text in classes and o.text != RESOURCE else []
|
|
2352
|
+
if not wanted:
|
|
2353
|
+
continue
|
|
2354
|
+
y = Var(self.fresh("y"))
|
|
2355
|
+
x = self.matched(s, IRITerm(prop), y) if as_subject else self.matched(y, IRITerm(prop), s)
|
|
2356
|
+
if isinstance(x, O.Values) and not x.rows:
|
|
2357
|
+
continue
|
|
2358
|
+
if not as_subject and isinstance(s, Var):
|
|
2359
|
+
term = E.Col(s.name)
|
|
2360
|
+
x = O.Filter(x, E.Or((E.Call("isIRI", (term,)), E.Call("isBLANK", (term,)))))
|
|
2361
|
+
cols = ((s.name,) if isinstance(s, Var) else ()) + \
|
|
2362
|
+
((self.graph.name,) if isinstance(self.graph, Var) and self.graph.name in x.columns else ())
|
|
2363
|
+
x = O.Project(x, cols)
|
|
2364
|
+
for cls in wanted:
|
|
2365
|
+
out.append(O.Extend(x, o.name, E.Const(IRITerm(cls))) if isinstance(o, Var) else x)
|
|
2366
|
+
return out
|
|
2367
|
+
|
|
2368
|
+
def any_predicate(self, s: Term, p: Var, o: Term) -> O.Op:
|
|
2369
|
+
"""A variable predicate's triples: each predicate's (`_any_predicate`), Direct semantics' consequences only
|
|
2370
|
+
where the predicate variable is typed too (`typed-variables`)."""
|
|
2371
|
+
held = self.direct_ok
|
|
2372
|
+
self.direct_ok = held and p.name in self.typed_vars
|
|
2373
|
+
try:
|
|
2374
|
+
return self._any_predicate(s, p, o)
|
|
2375
|
+
finally:
|
|
2376
|
+
self.direct_ok = held
|
|
2377
|
+
|
|
2378
|
+
def _any_predicate(self, s: Term, p: Var, o: Term) -> O.Op:
|
|
2379
|
+
"""A pattern whose predicate is a variable (18.3): for each predicate
|
|
2380
|
+
the maps give (rdf:type where a map has a class), that predicate's
|
|
2381
|
+
triples with `p` bound to it; their union. The predicates differ, so
|
|
2382
|
+
the branches share no triple."""
|
|
2383
|
+
if self.maps.open_predicates:
|
|
2384
|
+
raise NotBuilt("a map whose predicate is a column")
|
|
2385
|
+
if any(isinstance(t, Var) and t.name == p.name for t in (s, o)):
|
|
2386
|
+
raise NotBuilt("a variable that is a pattern's predicate and its subject or object")
|
|
2387
|
+
ql = self.ql_tbox()
|
|
2388
|
+
# Under 'ql' a property no map gives may have triples: from an inverse (`inverse_branches`) or reflexivity.
|
|
2389
|
+
derived = sorted(set(ql.reflexive) | {p for p, subs in ql.inverse_subs.items() if subs}) if ql is not None else []
|
|
2390
|
+
predicates = list(dict.fromkeys(
|
|
2391
|
+
([RDF_TYPE] if any(m.classes for m in self.maps.nodes) or self.typing or ql is not None else [])
|
|
2392
|
+
+ sorted(self.maps.by_predicate) + self.identity_predicates() + self.ontology_predicates() + derived))
|
|
2393
|
+
# owl:topObjectProperty relates every pair of individuals; a variable predicate never binds it (a labelled
|
|
2394
|
+
# known difference: `top_pairs`).
|
|
2395
|
+
predicates = [x for x in predicates if x != TOP_OBJECT_PROPERTY]
|
|
2396
|
+
parts: List[O.Op] = []
|
|
2397
|
+
named = self.predicate_names(predicates)
|
|
2398
|
+
per_graph = self.identity is not None and not self.plain and isinstance(self.graph, Var)
|
|
2399
|
+
if per_graph:
|
|
2400
|
+
# Inside GRAPH of a variable each graph has its own components: every predicate answers under the
|
|
2401
|
+
# names of its component in the graph it is read in (identity.predicate_members_by_graph).
|
|
2402
|
+
from prograph.plan.ir.identity import _in_graphs, predicate_members_by_graph
|
|
2403
|
+
self.identity.prefetch(predicates)
|
|
2404
|
+
for iri in predicates:
|
|
2405
|
+
if iri not in named:
|
|
2406
|
+
continue # a member of a predicate's component answered with it (eq-rep-p)
|
|
2407
|
+
x = self.triple(s, IRITerm(iri), o, quiet=True)
|
|
2408
|
+
if isinstance(x, O.Values) and not x.rows:
|
|
2409
|
+
continue
|
|
2410
|
+
if per_graph:
|
|
2411
|
+
for name, graphs in sorted(predicate_members_by_graph(self, iri).items()):
|
|
2412
|
+
parts.append(_in_graphs(self, O.Extend(x, p.name, E.Const(IRITerm(name))), graphs))
|
|
2413
|
+
continue
|
|
2414
|
+
for name in named[iri]:
|
|
2415
|
+
parts.append(O.Extend(x, p.name, E.Const(IRITerm(name))))
|
|
2416
|
+
if not parts:
|
|
2417
|
+
return _empty(s, p, o)
|
|
2418
|
+
if per_graph:
|
|
2419
|
+
return O.Distinct(parts[0] if len(parts) == 1 else O.Union(tuple(parts)))
|
|
2420
|
+
return parts[0] if len(parts) == 1 else O.Union(tuple(parts))
|
|
2421
|
+
|
|
2422
|
+
def predicate_names(self, predicates: List[str]) -> Dict[str, List[str]]:
|
|
2423
|
+
"""Under owl:sameAs (eq-rep-p), the predicate whose pattern answers each component of `predicates`
|
|
2424
|
+
(its first), with the names it is answered under: every member of the component. Without, each
|
|
2425
|
+
predicate its own. Inside GRAPH of a variable the components differ by graph: each predicate its own
|
|
2426
|
+
(`predicates`' names are then read per graph)."""
|
|
2427
|
+
if self.identity is None or self.plain or isinstance(self.graph, Var):
|
|
2428
|
+
return {iri: [iri] for iri in predicates}
|
|
2429
|
+
self.identity.prefetch(predicates)
|
|
2430
|
+
from prograph.plan.ir.identity import predicate_members
|
|
2431
|
+
out: Dict[str, List[str]] = {}
|
|
2432
|
+
seen: set = set()
|
|
2433
|
+
for iri in predicates:
|
|
2434
|
+
if iri in seen:
|
|
2435
|
+
continue
|
|
2436
|
+
members = predicate_members(self, iri)
|
|
2437
|
+
seen |= members
|
|
2438
|
+
out[iri] = sorted(members)
|
|
2439
|
+
return out
|
|
2440
|
+
|
|
2441
|
+
def branch(self, m: NodeMap, subject: TermMap, obj: Optional[TermMap], s: Term, o: Term,
|
|
2442
|
+
cls: Optional[str], gm: Optional[TermMap] = None) -> Optional[O.Op]:
|
|
2443
|
+
"""One node map's triples for the pattern: subject `subject` and
|
|
2444
|
+
object `obj`'s term (or the class `cls`)."""
|
|
2445
|
+
self.scope_of(gm)
|
|
2446
|
+
if not _may_give(subject, s) or (obj is not None and not _may_give(obj, o)):
|
|
2447
|
+
return None
|
|
2448
|
+
what = ill_typed_override(obj, m.source.table, self.held)
|
|
2449
|
+
if what:
|
|
2450
|
+
# R2RML 11: the processor aborts an operation that returns or inspects the term.
|
|
2451
|
+
from prograph.r2rml.compile import R2RMLDataError
|
|
2452
|
+
raise R2RMLDataError(what)
|
|
2453
|
+
maps = [subject] + ([obj] if obj is not None else []) + self.graph_read(gm)
|
|
2454
|
+
scan, names = self.scan(m, maps)
|
|
2455
|
+
x: O.Op = scan
|
|
2456
|
+
terms = [(s, self.make(subject, names))]
|
|
2457
|
+
if obj is not None or o != IRITerm(cls): # the pattern's class is the map's own: nothing to test
|
|
2458
|
+
terms.append((o, self.make(obj, names) if obj is not None else E.Const(IRITerm(cls))))
|
|
2459
|
+
terms += self.graph_term(gm, names)
|
|
2460
|
+
if obj is not None and isinstance(o, LiteralTerm):
|
|
2461
|
+
# A constant no term of the map can be: the map gives none of the
|
|
2462
|
+
# pattern's triples, and the pattern is said to match nothing.
|
|
2463
|
+
made = expr_slot(terms[1][1], scan.slots)
|
|
2464
|
+
if never_one_term(made, const_slot(o)):
|
|
2465
|
+
self.never.append((m, made))
|
|
2466
|
+
return None
|
|
2467
|
+
return self.bind(x, terms)
|
|
2468
|
+
|
|
2469
|
+
def graph_choices(self, graph_maps: Tuple[TermMap, ...], graph: Optional[str]) -> List[Optional[TermMap]]:
|
|
2470
|
+
"""The graphs a triple of these graph maps (or a YAML schema's `graph`) is read in, for the graph the
|
|
2471
|
+
patterns read (`self.graph`) and the query's dataset: `dataset.graph_choices`."""
|
|
2472
|
+
return graph_choices(graphs_of(graph_maps, graph), self.graph, self.dataset, self.union_default)
|
|
2473
|
+
|
|
2474
|
+
def graph_read(self, gm: Optional[TermMap]) -> List[TermMap]:
|
|
2475
|
+
"""A graph map whose term the row gives: its columns are read, and required (a row with
|
|
2476
|
+
no graph value gives no triple by that map, R2RML 11.1)."""
|
|
2477
|
+
return [gm] if gm is not None and gm.constant is None else []
|
|
2478
|
+
|
|
2479
|
+
def graph_term(self, gm: Optional[TermMap], names: Dict[str, str]) -> List[Tuple[Term, E.Expr]]:
|
|
2480
|
+
"""GRAPH's term bound to the triple's graph: a variable bound, an IRI matched (a constant
|
|
2481
|
+
graph map that matches was chosen in `graph_choices` and needs no test)."""
|
|
2482
|
+
if gm is None:
|
|
2483
|
+
return []
|
|
2484
|
+
listed = None
|
|
2485
|
+
if self.dataset is not None and gm.constant is None:
|
|
2486
|
+
# A graph map read from the row: its IRI is one of the graphs the dataset lists (13.2).
|
|
2487
|
+
listed = (GraphSet(self.dataset[0 if self.graph is None else 1]), self.make(gm, names))
|
|
2488
|
+
if self.graph is None:
|
|
2489
|
+
return [listed] if listed else []
|
|
2490
|
+
if gm.constant is not None:
|
|
2491
|
+
return [] if isinstance(self.graph, IRITerm) else [(self.graph, E.Const(IRITerm(gm.constant)))]
|
|
2492
|
+
return ([listed] if listed else []) + [(self.graph, self.make(gm, names))]
|
|
2493
|
+
|
|
2494
|
+
def edge_branch(self, m: EdgeMap, s: Term, o: Term, gm: Optional[TermMap] = None) -> Optional[O.Op]:
|
|
2495
|
+
self.scope_of(gm)
|
|
2496
|
+
if m.join is None and any(e.term is None and e.exists and e.map_id for e in (m.start, m.end)):
|
|
2497
|
+
return self.ends_from_their_rows(m, s, o, gm)
|
|
2498
|
+
for end in (m.start, m.end):
|
|
2499
|
+
if end.term is None:
|
|
2500
|
+
raise NotBuilt("an edge end with no term")
|
|
2501
|
+
if m.join is None and any(e.parent_columns for e in (m.start, m.end)):
|
|
2502
|
+
raise NotBuilt("an edge end read from a parent row with no join")
|
|
2503
|
+
start, end = m.start.term, m.end.term
|
|
2504
|
+
if not _may_give(start, s) or not _may_give(end, o):
|
|
2505
|
+
return None
|
|
2506
|
+
if m.join is not None:
|
|
2507
|
+
if gm is not None and gm.constant is None:
|
|
2508
|
+
raise NotBuilt("a graph map read from a referencing object map's joint query")
|
|
2509
|
+
return self.joint_branch(m, s, o, gm)
|
|
2510
|
+
scan, names = self.scan(m, [start, end] + self.graph_read(gm))
|
|
2511
|
+
x: O.Op = scan
|
|
2512
|
+
for ref in (m.start, m.end):
|
|
2513
|
+
if not ref.exists:
|
|
2514
|
+
continue
|
|
2515
|
+
node = self.maps.node(ref.map_id) if ref.map_id else None
|
|
2516
|
+
if node is None:
|
|
2517
|
+
raise NotBuilt("an edge end that names no node map")
|
|
2518
|
+
if len(node.key.columns) != len(ref.columns):
|
|
2519
|
+
raise NotBuilt("an edge end of another key width")
|
|
2520
|
+
if not all(c in names for c in ref.columns):
|
|
2521
|
+
raise NotBuilt("an edge end keyed by columns its term does not read")
|
|
2522
|
+
x = O.SemiJoin(x, self.end_rows(ref, node, names))
|
|
2523
|
+
return self.bind(x, [(s, self.make(start, names)), (o, self.make(end, names))]
|
|
2524
|
+
+ self.graph_term(gm, names))
|
|
2525
|
+
|
|
2526
|
+
def end_rows(self, ref: Any, node: Any, names: Dict[str, str]) -> O.Op:
|
|
2527
|
+
"""The key rows of an edge end's node maps (`end_rows`)."""
|
|
2528
|
+
return end_rows(ref, (node,) + tuple(self.maps.node(p) for p in ref.peers), names, self.read, self.keys)
|
|
2529
|
+
|
|
2530
|
+
def ends_from_their_rows(self, m: EdgeMap, s: Term, o: Term, gm: Optional[TermMap]) -> Optional[O.Op]:
|
|
2531
|
+
"""An edge whose end's node builds its subject from a column beyond its key (an
|
|
2532
|
+
`iri_template` over a whole-IRI column): the edge's rows joined to that node's
|
|
2533
|
+
rows on edge column = key, the end's term the node's subject from the node's row, as an
|
|
2534
|
+
R2RML referencing object map with a join condition gives it (R2RML 8). Every node row of
|
|
2535
|
+
the key gives its subject (an untrusted key's two rows, two ends); the triple's Distinct
|
|
2536
|
+
(`triple`) keeps the set, and the facts read the join (no edge key passes an untrusted
|
|
2537
|
+
node key). An end built from the edge's own row is checked as `edge_branch` checks it."""
|
|
2538
|
+
self.scope_of(gm)
|
|
2539
|
+
own = [ref.term for ref in (m.start, m.end) if ref.term is not None]
|
|
2540
|
+
scan, names = self.scan(m, own + self.graph_read(gm))
|
|
2541
|
+
# The key columns of an end built from its node's row: read and required too (R2RML 11.2).
|
|
2542
|
+
more = []
|
|
2543
|
+
for ref in (m.start, m.end):
|
|
2544
|
+
if ref.term is None:
|
|
2545
|
+
for c in ref.columns:
|
|
2546
|
+
if c not in names:
|
|
2547
|
+
names[c] = self.fresh("c")
|
|
2548
|
+
more.append(self.read(m, c, names[c]))
|
|
2549
|
+
if more:
|
|
2550
|
+
scan = dataclasses.replace(scan, reads=scan.reads + tuple(more),
|
|
2551
|
+
required=scan.required + tuple(r.name for r in more))
|
|
2552
|
+
x: O.Op = scan
|
|
2553
|
+
made: List[E.Expr] = []
|
|
2554
|
+
for ref in (m.start, m.end):
|
|
2555
|
+
node = self.maps.node(ref.map_id) if ref.map_id else None
|
|
2556
|
+
if ref.exists and node is None:
|
|
2557
|
+
raise NotBuilt("an edge end that names no node map")
|
|
2558
|
+
if ref.exists and len(node.key.columns) != len(ref.columns):
|
|
2559
|
+
raise NotBuilt("an edge end of another key width")
|
|
2560
|
+
if ref.term is not None:
|
|
2561
|
+
if ref.exists:
|
|
2562
|
+
for c in ref.columns:
|
|
2563
|
+
if c not in names:
|
|
2564
|
+
raise NotBuilt("an edge end keyed by columns its term does not read")
|
|
2565
|
+
x = O.SemiJoin(x, self.end_rows(ref, node, names))
|
|
2566
|
+
made.append(self.make(ref.term, names))
|
|
2567
|
+
continue
|
|
2568
|
+
# The key joins on the edge's columns, read under the edge's names; the subject's other
|
|
2569
|
+
# columns under names of their own.
|
|
2570
|
+
node_names = {k: names[c] for c, k in zip(ref.columns, node.key.columns)}
|
|
2571
|
+
subject = node.subject
|
|
2572
|
+
extra = subject.columns() + ((subject.language_column,) if subject.language_column else ())
|
|
2573
|
+
for c in extra:
|
|
2574
|
+
node_names.setdefault(c, self.fresh("c"))
|
|
2575
|
+
reads = tuple(self.read(node, c, n) for c, n in node_names.items())
|
|
2576
|
+
x = O.Join(x, O.MapScan(map=node, reads=reads, required=tuple(r.name for r in reads),
|
|
2577
|
+
unique_keys=self.keys(node)))
|
|
2578
|
+
made.append(self.make(subject, node_names))
|
|
2579
|
+
return self.bind(x, [(s, made[0]), (o, made[1])] + self.graph_term(gm, names))
|
|
2580
|
+
|
|
2581
|
+
def joint_branch(self, m: EdgeMap, s: Term, o: Term, gm: Optional[TermMap] = None) -> O.Op:
|
|
2582
|
+
"""A referencing object map's triples (R2RML 8): its joint query's
|
|
2583
|
+
rows, the subject from the child's row and each end with
|
|
2584
|
+
`parent_columns` from the parent's. The join is the parent row's
|
|
2585
|
+
existence, so no end is checked again."""
|
|
2586
|
+
self.scope_of(gm)
|
|
2587
|
+
names: Dict[Tuple[bool, str], str] = {}
|
|
2588
|
+
reads: List[O.ScanColumn] = []
|
|
2589
|
+
made: List[E.Expr] = []
|
|
2590
|
+
for ref in (m.start, m.end):
|
|
2591
|
+
from_parent = bool(ref.parent_columns)
|
|
2592
|
+
table = m.join.parent.table if from_parent else m.source.table
|
|
2593
|
+
if ref.term.language_column is not None:
|
|
2594
|
+
raise NotBuilt("a language column")
|
|
2595
|
+
args = []
|
|
2596
|
+
for c in ref.term.columns():
|
|
2597
|
+
key = (from_parent, c)
|
|
2598
|
+
if key not in names:
|
|
2599
|
+
names[key] = self.fresh("c")
|
|
2600
|
+
held, zoned, digits, reader, bytewise = self.held(table, c)
|
|
2601
|
+
reads.append(O.ScanColumn(
|
|
2602
|
+
name=names[key], column=c, parent=from_parent, held=held,
|
|
2603
|
+
zone=(ZONED if zoned else LOCAL) if held == xsd.DATETIME else None,
|
|
2604
|
+
digits=tuple(digits) if digits else None, reader=reader,
|
|
2605
|
+
bytewise=bytewise and held == xsd.STRING and reader in (None, *UUID_READERS),
|
|
2606
|
+
uuid=reader in UUID_READERS,
|
|
2607
|
+
precision=self.precision(table, c, held)))
|
|
2608
|
+
args.append((c, E.Col(names[key])))
|
|
2609
|
+
made.append(E.MakeTerm(self.scoped(ref.term), tuple(args)))
|
|
2610
|
+
# The parent node map, its keys, and the child's join columns as read: what R13 reads to
|
|
2611
|
+
# state the join as the child's rows with the parent row's existence.
|
|
2612
|
+
parent = next((self.maps.node(ref.map_id) for ref in (m.start, m.end)
|
|
2613
|
+
if ref.parent_columns and ref.map_id), None)
|
|
2614
|
+
join_reads = []
|
|
2615
|
+
for c, _ in m.join.conditions:
|
|
2616
|
+
held, zoned, digits, reader, bytewise = self.held(m.source.table, c)
|
|
2617
|
+
join_reads.append(O.ScanColumn(
|
|
2618
|
+
name=c, column=c, held=held,
|
|
2619
|
+
zone=(ZONED if zoned else LOCAL) if held == xsd.DATETIME else None,
|
|
2620
|
+
digits=tuple(digits) if digits else None, reader=reader,
|
|
2621
|
+
bytewise=bytewise and held == xsd.STRING and reader in (None, *UUID_READERS),
|
|
2622
|
+
precision=self.precision(m.source.table, c, held)))
|
|
2623
|
+
scan = O.MapScan(map=m, reads=tuple(reads), required=tuple(r.name for r in reads),
|
|
2624
|
+
unique_keys=self.keys(m), parent_map=parent,
|
|
2625
|
+
parent_unique_keys=self.keys(parent) if parent is not None else (),
|
|
2626
|
+
join_reads=tuple(join_reads))
|
|
2627
|
+
return self.bind(scan, [(s, made[0]), (o, made[1])] + self.graph_term(gm, {}))
|
|
2628
|
+
|
|
2629
|
+
def precision(self, table: str, column: str, held: Optional[str]) -> Optional[int]:
|
|
2630
|
+
"""A date-time column's fractional-second digits its type states (`CypherType.precision`)."""
|
|
2631
|
+
kind_of = getattr(self.held, "kind_of", None)
|
|
2632
|
+
if held != xsd.DATETIME or kind_of is None:
|
|
2633
|
+
return None
|
|
2634
|
+
return getattr(kind_of(table, column), "precision", None)
|
|
2635
|
+
|
|
2636
|
+
def scan(self, m, maps: List[TermMap]) -> Tuple[O.MapScan, Dict[str, str]]:
|
|
2637
|
+
"""A scan of `m` reading the columns `maps` read, each under a
|
|
2638
|
+
hidden name, every one required (R2RML 11.2)."""
|
|
2639
|
+
names: Dict[str, str] = {}
|
|
2640
|
+
optional_: List[str] = []
|
|
2641
|
+
for tm in maps:
|
|
2642
|
+
for c in tm.columns():
|
|
2643
|
+
if c not in names:
|
|
2644
|
+
names[c] = self.fresh("c")
|
|
2645
|
+
for tm in maps:
|
|
2646
|
+
# A literal's per-row tag: read, never required (a row with none is a plain literal).
|
|
2647
|
+
c = tm.language_column
|
|
2648
|
+
if c is not None and c not in names:
|
|
2649
|
+
names[c] = self.fresh("c")
|
|
2650
|
+
optional_.append(names[c])
|
|
2651
|
+
reads = tuple(self.read(m, c, n) for c, n in names.items())
|
|
2652
|
+
return O.MapScan(map=m, reads=reads, required=tuple(n for n in names.values() if n not in optional_),
|
|
2653
|
+
unique_keys=self.keys(m)), names
|
|
2654
|
+
|
|
2655
|
+
def read(self, m, column: str, name: str) -> O.ScanColumn:
|
|
2656
|
+
"""A column read: its datatype, and a date-time's zone."""
|
|
2657
|
+
held, zoned, digits, reader, bytewise = self.held(m.source.table, column)
|
|
2658
|
+
zone = None
|
|
2659
|
+
if held == xsd.DATETIME:
|
|
2660
|
+
zone = ZONED if zoned else LOCAL
|
|
2661
|
+
return O.ScanColumn(name=name, column=column, held=held, zone=zone,
|
|
2662
|
+
digits=tuple(digits) if digits else None, reader=reader,
|
|
2663
|
+
bytewise=bytewise and held == xsd.STRING and reader in (None, *UUID_READERS),
|
|
2664
|
+
uuid=reader in UUID_READERS)
|
|
2665
|
+
|
|
2666
|
+
def make(self, tm: TermMap, names: Dict[str, str]) -> E.Expr:
|
|
2667
|
+
tm = self.scoped(tm)
|
|
2668
|
+
columns = tm.columns() + ((tm.language_column,) if tm.language_column is not None else ())
|
|
2669
|
+
return E.MakeTerm(tm, tuple((c, E.Col(names[c])) for c in columns))
|
|
2670
|
+
|
|
2671
|
+
def scope_of(self, gm: Optional[TermMap]) -> None:
|
|
2672
|
+
"""Where the default graph merges several FROM graphs, a blank node of one source graph is not
|
|
2673
|
+
one of another (RDF 1.1 Concepts 3.6, the merge): the triples a graph map reads carry their
|
|
2674
|
+
graph's name in their blank nodes' labels (`scoped`)."""
|
|
2675
|
+
merging = self.dataset is not None and self.graph is None and len(self.dataset[0]) > 1
|
|
2676
|
+
self.blank_scope = (gm.constant if gm.constant is not None else "") if merging and gm is not None else None
|
|
2677
|
+
|
|
2678
|
+
def scoped(self, tm: TermMap) -> TermMap:
|
|
2679
|
+
"""`tm`, a blank node's term map, with the graph it is read in added to its label."""
|
|
2680
|
+
if tm.term_type != BLANK or self.blank_scope is None:
|
|
2681
|
+
return tm
|
|
2682
|
+
if not self.blank_scope:
|
|
2683
|
+
raise NotBuilt("blank nodes merged from several FROM graphs where a graph map reads the row's graph")
|
|
2684
|
+
suffix = ((f"@{self.blank_scope}", False),)
|
|
2685
|
+
if tm.template is not None:
|
|
2686
|
+
return dataclasses.replace(tm, template=tm.template + suffix)
|
|
2687
|
+
if tm.column is not None:
|
|
2688
|
+
return dataclasses.replace(tm, column=None, template=((tm.column, True),) + suffix)
|
|
2689
|
+
return dataclasses.replace(tm, constant=tm.constant + suffix[0][0])
|
|
2690
|
+
|
|
2691
|
+
def bind(self, x: O.Op, terms: List[Tuple[Term, E.Expr]]) -> O.Op:
|
|
2692
|
+
"""Bind each pattern term to its built term: a variable is a column,
|
|
2693
|
+
a constant a filter by term identity (18.3: a pattern matches a
|
|
2694
|
+
triple of the same terms)."""
|
|
2695
|
+
out: List[str] = []
|
|
2696
|
+
conds: List[E.Expr] = []
|
|
2697
|
+
for t, e in terms:
|
|
2698
|
+
if isinstance(t, Var):
|
|
2699
|
+
if t.name in out:
|
|
2700
|
+
conds.append(E.Compare("=", E.Col(t.name), e, E.TERM))
|
|
2701
|
+
else:
|
|
2702
|
+
x = O.Extend(x, t.name, e)
|
|
2703
|
+
out.append(t.name)
|
|
2704
|
+
elif isinstance(t, GraphSet):
|
|
2705
|
+
conds.append(E.Or(tuple(E.Compare("=", e, E.Const(IRITerm(i)), E.TERM) for i in sorted(t.iris)))
|
|
2706
|
+
if t.iris else E.Const(LiteralTerm("false", xsd.BOOLEAN)))
|
|
2707
|
+
else:
|
|
2708
|
+
conds.append(E.Compare("=", e, E.Const(t), E.TERM))
|
|
2709
|
+
if conds:
|
|
2710
|
+
x = O.Filter(x, conds[0] if len(conds) == 1 else E.And(tuple(conds)))
|
|
2711
|
+
return O.Project(x, tuple(out))
|
|
2712
|
+
|
|
2713
|
+
# -- expressions (17) -------------------------------------------------------
|
|
2714
|
+
|
|
2715
|
+
def condition(self, ctx) -> E.Expr:
|
|
2716
|
+
"""A FILTER's constraint, as its effective boolean value (17.2.2)."""
|
|
2717
|
+
return ebv(self.expression(ctx))
|
|
2718
|
+
|
|
2719
|
+
def expression(self, ctx) -> E.Expr:
|
|
2720
|
+
P = SparqlParser
|
|
2721
|
+
if isinstance(ctx, (P.ExpressionContext, P.ConstraintContext)):
|
|
2722
|
+
return self.expression(ctx.getChild(0))
|
|
2723
|
+
if isinstance(ctx, P.BrackettedExpressionContext):
|
|
2724
|
+
return self.expression(ctx.expression())
|
|
2725
|
+
if isinstance(ctx, P.ConditionalOrExpressionContext):
|
|
2726
|
+
args = [self.expression(a) for a in ctx.conditionalAndExpression()]
|
|
2727
|
+
return args[0] if len(args) == 1 else E.Or(tuple(ebv(a) for a in args))
|
|
2728
|
+
if isinstance(ctx, P.ConditionalAndExpressionContext):
|
|
2729
|
+
args = [self.expression(a) for a in ctx.valueLogical()]
|
|
2730
|
+
return args[0] if len(args) == 1 else E.And(tuple(ebv(a) for a in args))
|
|
2731
|
+
if isinstance(ctx, P.ValueLogicalContext):
|
|
2732
|
+
return self.expression(ctx.relationalExpression())
|
|
2733
|
+
if isinstance(ctx, P.RelationalExpressionContext):
|
|
2734
|
+
sides = ctx.numericExpression()
|
|
2735
|
+
left = self.expression(sides[0])
|
|
2736
|
+
if len(sides) == 2:
|
|
2737
|
+
return E.Compare(ctx.getChild(1).getText(), left, self.expression(sides[1]))
|
|
2738
|
+
if ctx.expressionList() is not None:
|
|
2739
|
+
items = [self.expression(x) for x in ctx.expressionList().expression()]
|
|
2740
|
+
# An empty list is false, an error only where the left side is one (`?v = ?v`
|
|
2741
|
+
# is an error exactly then).
|
|
2742
|
+
false = E.Const(LiteralTerm("false", xsd.BOOLEAN))
|
|
2743
|
+
found = E.Or(tuple(E.Compare("=", left, i) for i in items)) if items \
|
|
2744
|
+
else E.If(E.Compare("=", left, left), false, false)
|
|
2745
|
+
return E.Not(found) if ctx.NOT() is not None else found
|
|
2746
|
+
return left
|
|
2747
|
+
if isinstance(ctx, P.NumericExpressionContext):
|
|
2748
|
+
return self.expression(ctx.additiveExpression())
|
|
2749
|
+
if isinstance(ctx, P.AdditiveExpressionContext):
|
|
2750
|
+
# `?a -1` is `?a + -1`: a signed literal is added (19.8).
|
|
2751
|
+
kids = list(ctx.getChildren())
|
|
2752
|
+
out = self.expression(kids[0])
|
|
2753
|
+
i = 1
|
|
2754
|
+
while i < len(kids):
|
|
2755
|
+
k = kids[i]
|
|
2756
|
+
if isinstance(k, (P.NumericLiteralPositiveContext,
|
|
2757
|
+
P.NumericLiteralNegativeContext)):
|
|
2758
|
+
out = E.Call("+", (out, E.Const(numeric(k.getText()))))
|
|
2759
|
+
i += 1
|
|
2760
|
+
else:
|
|
2761
|
+
out = E.Call(k.getText(), (out, self.expression(kids[i + 1])))
|
|
2762
|
+
i += 2
|
|
2763
|
+
return out
|
|
2764
|
+
if isinstance(ctx, P.MultiplicativeExpressionContext):
|
|
2765
|
+
kids = list(ctx.getChildren())
|
|
2766
|
+
out = self.expression(kids[0])
|
|
2767
|
+
for i in range(1, len(kids), 2):
|
|
2768
|
+
out = E.Call(kids[i].getText(), (out, self.expression(kids[i + 1])))
|
|
2769
|
+
return out
|
|
2770
|
+
if isinstance(ctx, P.UnaryExpressionContext):
|
|
2771
|
+
inner = self.expression(ctx.primaryExpression())
|
|
2772
|
+
if ctx.EXCLAMATION() is not None:
|
|
2773
|
+
return E.Not(ebv(inner))
|
|
2774
|
+
if ctx.MINUS() is not None:
|
|
2775
|
+
return E.Call("NEG", (inner,))
|
|
2776
|
+
if ctx.PLUS() is not None:
|
|
2777
|
+
return E.Call("POS", (inner,))
|
|
2778
|
+
return inner
|
|
2779
|
+
if isinstance(ctx, P.PrimaryExpressionContext):
|
|
2780
|
+
return self.expression(ctx.getChild(0))
|
|
2781
|
+
if isinstance(ctx, P.Var_Context):
|
|
2782
|
+
name = ctx.getText()[1:]
|
|
2783
|
+
return E.Const(self.params[name]) if name in self.params else E.Col(name)
|
|
2784
|
+
if isinstance(ctx, (P.RdfLiteralContext, P.NumericLiteralContext,
|
|
2785
|
+
P.BooleanLiteralContext)):
|
|
2786
|
+
return E.Const(self.literal(ctx))
|
|
2787
|
+
if isinstance(ctx, P.IriRefOrFunctionContext):
|
|
2788
|
+
if ctx.argList() is not None:
|
|
2789
|
+
return self.function_call(ctx.iriRef(), ctx.argList())
|
|
2790
|
+
return E.Const(IRITerm(self.iri(ctx.iriRef())))
|
|
2791
|
+
if isinstance(ctx, P.FunctionCallContext):
|
|
2792
|
+
return self.function_call(ctx.iriRef(), ctx.argList())
|
|
2793
|
+
if isinstance(ctx, P.BuiltInCallContext):
|
|
2794
|
+
return self.builtin(ctx)
|
|
2795
|
+
raise NotBuilt(f"the expression {ctx.getText()[:40]}")
|
|
2796
|
+
|
|
2797
|
+
def function_call(self, iri_ctx, args_ctx) -> E.Expr:
|
|
2798
|
+
"""A call of a function IRI: a cast (17.5, `casts`), the only one built; another an
|
|
2799
|
+
unknown function (S10)."""
|
|
2800
|
+
from prograph.plan.ir.casts import TARGETS
|
|
2801
|
+
name = self.iri(iri_ctx)
|
|
2802
|
+
exprs = args_ctx.expression() if args_ctx is not None else []
|
|
2803
|
+
if name in TARGETS:
|
|
2804
|
+
if len(exprs) != 1 or "DISTINCT" in args_ctx.getText().upper()[:9]:
|
|
2805
|
+
raise NotBuilt(f"a cast with {len(exprs)} arguments")
|
|
2806
|
+
return E.Call("CAST", (self.expression(exprs[0]), E.Const(IRITerm(name))))
|
|
2807
|
+
raise NotBuilt(f"the function {iri_ctx.getText()}")
|
|
2808
|
+
|
|
2809
|
+
def builtin(self, ctx) -> E.Expr:
|
|
2810
|
+
if ctx.aggregate() is not None:
|
|
2811
|
+
return self.aggregate(ctx.aggregate())
|
|
2812
|
+
if ctx.regexExpression() is not None:
|
|
2813
|
+
return E.Call("REGEX", tuple(self.expression(x) for x in ctx.regexExpression().expression()))
|
|
2814
|
+
if ctx.existsFunc() is not None or ctx.notExistsFunc() is not None:
|
|
2815
|
+
raise NotBuilt("EXISTS other than as a FILTER's own term, or terms joined by && "
|
|
2816
|
+
"(inside ||, a BIND, IF or a SELECT expression)")
|
|
2817
|
+
name = ctx.getChild(0).getText().upper()
|
|
2818
|
+
args = [self.expression(x) for x in ctx.expression()]
|
|
2819
|
+
if name == "BOUND":
|
|
2820
|
+
var = ctx.var_().getText()[1:]
|
|
2821
|
+
if var in self.params:
|
|
2822
|
+
return E.Const(LiteralTerm("true", xsd.BOOLEAN))
|
|
2823
|
+
return E.IsBound(E.Col(var))
|
|
2824
|
+
if name in ("UUID", "STRUUID", "RAND", "BNODE"):
|
|
2825
|
+
# A fresh value (fresh.py): the call's id is its place in the query.
|
|
2826
|
+
self.fresh_calls += 1
|
|
2827
|
+
return E.Call(name, (E.Const(LiteralTerm(str(self.fresh_calls))),) + tuple(args))
|
|
2828
|
+
if name == "NOW":
|
|
2829
|
+
# 17.4.5.1: one xsd:dateTime for the whole query, its moment of planning, in UTC.
|
|
2830
|
+
return E.Const(LiteralTerm(self.now, xsd.DATETIME))
|
|
2831
|
+
calls = {"SAMETERM": "sameTerm", "ISIRI": "isIRI", "ISURI": "isURI",
|
|
2832
|
+
"ISBLANK": "isBLANK", "ISLITERAL": "isLITERAL"}
|
|
2833
|
+
if name in calls:
|
|
2834
|
+
return E.Call(calls[name], tuple(args))
|
|
2835
|
+
if name == "CONCAT":
|
|
2836
|
+
listed = ctx.expressionList()
|
|
2837
|
+
items = [self.expression(x) for x in listed.expression()] if listed is not None else []
|
|
2838
|
+
# errata-query-4: CONCAT() is "", CONCAT(x) is x's string.
|
|
2839
|
+
return E.Call("CONCAT", tuple(items)) if items else E.Const(LiteralTerm(""))
|
|
2840
|
+
if name in STRING_CALLS or name in DATE_PARTS:
|
|
2841
|
+
return E.Call(name, tuple(args))
|
|
2842
|
+
if name in ("IRI", "URI"):
|
|
2843
|
+
# 17.4.2.8: resolved against the query's base (4.1.1).
|
|
2844
|
+
return E.Call("IRI", (args[0], E.Const(LiteralTerm(self.base or ""))))
|
|
2845
|
+
if name in ("ABS", "CEIL", "FLOOR", "ROUND"):
|
|
2846
|
+
return E.Call(name, tuple(args))
|
|
2847
|
+
terms = {"STR": "STR", "LANG": "LANG", "DATATYPE": "DATATYPE", "ISNUMERIC": "isNUMERIC",
|
|
2848
|
+
"STRDT": "STRDT", "STRLANG": "STRLANG"}
|
|
2849
|
+
if name in terms:
|
|
2850
|
+
return E.Call(terms[name], tuple(args))
|
|
2851
|
+
if name == "COALESCE":
|
|
2852
|
+
# 17.4.1.3: the first argument that evaluates with no error; none, an error.
|
|
2853
|
+
listed = ctx.expressionList()
|
|
2854
|
+
items = [self.expression(x) for x in listed.expression()] if listed is not None else []
|
|
2855
|
+
if not items:
|
|
2856
|
+
return E.Const(None) # COALESCE(): no argument has a value, an error
|
|
2857
|
+
return E.Coalesce(tuple(items))
|
|
2858
|
+
if name == "IF":
|
|
2859
|
+
# 17.4.1.2: the condition's effective boolean value; an error there is an error.
|
|
2860
|
+
return E.If(ebv(args[0]), args[1], args[2])
|
|
2861
|
+
raise NotBuilt(f"the function {name}")
|
|
2862
|
+
|
|
2863
|
+
|
|
2864
|
+
# -- helpers ------------------------------------------------------------------------
|
|
2865
|
+
|
|
2866
|
+
def end_rows(ref: Any, nodes: Sequence[Any], names: Dict[str, str], read: Any, keys: Any) -> O.Op:
|
|
2867
|
+
"""The key rows of an edge end's node maps, its own and its `peers` (`nodes`: the label's maps giving its
|
|
2868
|
+
subject, compile_schema), under the edge's names for its key columns (`names`): the end exists where one
|
|
2869
|
+
has it. `read` makes a column read, `keys` a map's trusted keys. SPARQL's and Cypher's check (`edge-ends`)."""
|
|
2870
|
+
out: List[O.Op] = []
|
|
2871
|
+
for each in nodes:
|
|
2872
|
+
if len(each.key.columns) != len(ref.columns):
|
|
2873
|
+
raise NotBuilt("an edge end of another key width")
|
|
2874
|
+
reads = tuple(read(each, k, names[c]) for c, k in zip(ref.columns, each.key.columns))
|
|
2875
|
+
out.append(O.MapScan(map=each, reads=reads, required=tuple(r.name for r in reads),
|
|
2876
|
+
unique_keys=keys(each) if keys else ()))
|
|
2877
|
+
return out[0] if len(out) == 1 else O.Union(tuple(out))
|
|
2878
|
+
|
|
2879
|
+
|
|
2880
|
+
def _star_like(select) -> bool:
|
|
2881
|
+
"""Whether a SELECT reads every variable of its pattern: SELECT * projects each, COUNT(*) (in its SELECT,
|
|
2882
|
+
HAVING or ORDER BY, DISTINCT or not) counts every solution."""
|
|
2883
|
+
if not select.selectClause():
|
|
2884
|
+
return True
|
|
2885
|
+
texts = [c.expression().getText() for c in select.selectClause() if c.expression() is not None]
|
|
2886
|
+
mods = select.solutionModifier() if hasattr(select, "solutionModifier") else None
|
|
2887
|
+
if mods is not None:
|
|
2888
|
+
texts.append(mods.getText()) # HAVING and ORDER BY
|
|
2889
|
+
text = " ".join(texts).upper().replace(" ", "")
|
|
2890
|
+
return "COUNT(*)" in text or "COUNT(DISTINCT*)" in text
|
|
2891
|
+
|
|
2892
|
+
|
|
2893
|
+
def _var_names(tree) -> List[str]:
|
|
2894
|
+
"""Every occurrence of a variable in a parse tree that reads it, by name (`?x` and `$x`). A sub-SELECT's bare
|
|
2895
|
+
projection of a variable (`SELECT ?x ?p`) reads it only where the SELECT around it reads every variable
|
|
2896
|
+
(`_star_like`); elsewhere an outer read is counted where it is. An expression of it reads it."""
|
|
2897
|
+
out: List[str] = []
|
|
2898
|
+
stack = [(tree, False)]
|
|
2899
|
+
while stack:
|
|
2900
|
+
node, star = stack.pop()
|
|
2901
|
+
if isinstance(node, SparqlParser.Var_Context):
|
|
2902
|
+
out.append(node.getText()[1:])
|
|
2903
|
+
continue
|
|
2904
|
+
if isinstance(node, SparqlParser.SubSelectContext):
|
|
2905
|
+
inner = node.selectQuery()
|
|
2906
|
+
for clause in inner.selectClause():
|
|
2907
|
+
if clause.expression() is not None:
|
|
2908
|
+
stack.append((clause.expression(), star))
|
|
2909
|
+
elif star:
|
|
2910
|
+
out.append(clause.var_().getText()[1:])
|
|
2911
|
+
inside = _star_like(inner)
|
|
2912
|
+
stack.extend((c, inside) for c in inner.getChildren()
|
|
2913
|
+
if not isinstance(c, SparqlParser.SelectClauseContext))
|
|
2914
|
+
continue
|
|
2915
|
+
if isinstance(node, SparqlParser.SelectQueryContext):
|
|
2916
|
+
star = _star_like(node)
|
|
2917
|
+
stack.extend((c, star) for c in (node.getChildren() if hasattr(node, "getChildren") else ()))
|
|
2918
|
+
return out
|
|
2919
|
+
|
|
2920
|
+
|
|
2921
|
+
def variables_of(tree) -> set:
|
|
2922
|
+
"""Every variable a parse tree names (`?x` and `$x`), for a parameter's name check."""
|
|
2923
|
+
out: set = set()
|
|
2924
|
+
stack = [tree]
|
|
2925
|
+
while stack:
|
|
2926
|
+
node = stack.pop()
|
|
2927
|
+
if isinstance(node, SparqlParser.Var_Context):
|
|
2928
|
+
out.add(node.getText()[1:])
|
|
2929
|
+
continue
|
|
2930
|
+
stack.extend(node.getChildren() if hasattr(node, "getChildren") else ())
|
|
2931
|
+
return out
|
|
2932
|
+
|
|
2933
|
+
|
|
2934
|
+
def conjuncts_of(ctx) -> List[Any]:
|
|
2935
|
+
"""The terms of a FILTER's constraint split at `&&` (and brackets):
|
|
2936
|
+
parse nodes, each not itself a conjunction."""
|
|
2937
|
+
P = SparqlParser
|
|
2938
|
+
while True:
|
|
2939
|
+
if isinstance(ctx, P.ConditionalOrExpressionContext):
|
|
2940
|
+
ands = ctx.conditionalAndExpression()
|
|
2941
|
+
if len(ands) != 1:
|
|
2942
|
+
return [ctx]
|
|
2943
|
+
ctx = ands[0]
|
|
2944
|
+
elif isinstance(ctx, P.ConditionalAndExpressionContext):
|
|
2945
|
+
vs = ctx.valueLogical()
|
|
2946
|
+
if len(vs) != 1:
|
|
2947
|
+
return [t for v in vs for t in conjuncts_of(v)]
|
|
2948
|
+
ctx = vs[0]
|
|
2949
|
+
elif isinstance(ctx, (P.BrackettedExpressionContext,)):
|
|
2950
|
+
ctx = ctx.expression()
|
|
2951
|
+
elif isinstance(ctx, P.UnaryExpressionContext) and ctx.EXCLAMATION() is None \
|
|
2952
|
+
and ctx.MINUS() is None and ctx.PLUS() is None:
|
|
2953
|
+
ctx = ctx.primaryExpression()
|
|
2954
|
+
elif isinstance(ctx, (P.ExpressionContext, P.ConstraintContext, P.ValueLogicalContext,
|
|
2955
|
+
P.NumericExpressionContext, P.PrimaryExpressionContext)):
|
|
2956
|
+
ctx = ctx.getChild(0)
|
|
2957
|
+
elif isinstance(ctx, (P.RelationalExpressionContext, P.AdditiveExpressionContext,
|
|
2958
|
+
P.MultiplicativeExpressionContext)) and ctx.getChildCount() == 1:
|
|
2959
|
+
ctx = ctx.getChild(0)
|
|
2960
|
+
else:
|
|
2961
|
+
return [ctx]
|
|
2962
|
+
|
|
2963
|
+
|
|
2964
|
+
def exists_leaf(ctx) -> Optional[Tuple[bool, Any]]:
|
|
2965
|
+
"""(negated, the pattern) where `ctx` is EXISTS { P }, NOT EXISTS { P }
|
|
2966
|
+
or `!` of either, else None. `!EXISTS` is NOT EXISTS: EXISTS is never
|
|
2967
|
+
an error."""
|
|
2968
|
+
P = SparqlParser
|
|
2969
|
+
negated = False
|
|
2970
|
+
if isinstance(ctx, P.UnaryExpressionContext) and ctx.EXCLAMATION() is not None:
|
|
2971
|
+
negated, ctx = True, ctx.primaryExpression()
|
|
2972
|
+
while isinstance(ctx, (P.PrimaryExpressionContext, P.ConstraintContext)):
|
|
2973
|
+
ctx = ctx.getChild(0)
|
|
2974
|
+
if isinstance(ctx, P.BuiltInCallContext):
|
|
2975
|
+
if ctx.existsFunc() is not None:
|
|
2976
|
+
return negated, ctx.existsFunc().groupGraphPattern()
|
|
2977
|
+
if ctx.notExistsFunc() is not None:
|
|
2978
|
+
return not negated, ctx.notExistsFunc().groupGraphPattern()
|
|
2979
|
+
return None
|
|
2980
|
+
|
|
2981
|
+
|
|
2982
|
+
def _grouped_only(used, visible, clause: str) -> None:
|
|
2983
|
+
bad = sorted(c for c in used if c not in visible and not c.startswith(HIDDEN))
|
|
2984
|
+
if bad:
|
|
2985
|
+
raise QueryError(f"?{bad[0]} is not grouped, and it is read in {clause}: a grouped "
|
|
2986
|
+
"query reads only its GROUP BY keys and aggregates (18.2.4.1)")
|
|
2987
|
+
|
|
2988
|
+
|
|
2989
|
+
_BOOLEAN_EXPRS = (E.Compare, E.And, E.Or, E.Not, E.IsBound)
|
|
2990
|
+
_BOOLEAN_CALLS = {"sameTerm", "sameValue", "isIRI", "isURI", "isBLANK", "isLITERAL", "EBV"} | BOOLEAN_STRING_CALLS
|
|
2991
|
+
|
|
2992
|
+
|
|
2993
|
+
def ebv(e: E.Expr) -> E.Expr:
|
|
2994
|
+
"""`e`'s effective boolean value: itself where it is a boolean."""
|
|
2995
|
+
if isinstance(e, _BOOLEAN_EXPRS):
|
|
2996
|
+
return e
|
|
2997
|
+
if isinstance(e, E.Call) and e.name in _BOOLEAN_CALLS:
|
|
2998
|
+
return e
|
|
2999
|
+
return E.Call("EBV", (e,))
|
|
3000
|
+
|
|
3001
|
+
|
|
3002
|
+
def numeric(text: str) -> LiteralTerm:
|
|
3003
|
+
"""A numeric literal: `1` an xsd:integer, `1.5` a decimal, `1e0` a
|
|
3004
|
+
double (19.8), its lexical form as written."""
|
|
3005
|
+
if "e" in text.lower():
|
|
3006
|
+
return LiteralTerm(text, xsd.DOUBLE)
|
|
3007
|
+
if "." in text:
|
|
3008
|
+
return LiteralTerm(text, xsd.DECIMAL)
|
|
3009
|
+
return LiteralTerm(text, xsd.INTEGER)
|
|
3010
|
+
|
|
3011
|
+
|
|
3012
|
+
def _connected(patterns: List[Tuple[Any, Any, Any]]) -> List[Tuple[Any, Any, Any]]:
|
|
3013
|
+
"""A basic graph pattern's triples in the order to join them (rule join-order): from its most
|
|
3014
|
+
selective triple (a constant subject, then a constant object, then a constant class), each next one sharing
|
|
3015
|
+
a variable with those before, a path after what binds its ends. The lowering keeps the written order under
|
|
3016
|
+
a condition across the joins (`lower._across`), so this order is the one the database sees there."""
|
|
3017
|
+
from prograph.plan.ir.join_order import connected_order
|
|
3018
|
+
|
|
3019
|
+
def names(t) -> frozenset:
|
|
3020
|
+
return frozenset(v.name for v in t if isinstance(v, Var))
|
|
3021
|
+
|
|
3022
|
+
def selective(t) -> int:
|
|
3023
|
+
s, p, o = t
|
|
3024
|
+
if isinstance(p, Path):
|
|
3025
|
+
return -1
|
|
3026
|
+
if not isinstance(s, Var):
|
|
3027
|
+
return 3
|
|
3028
|
+
if not isinstance(o, Var):
|
|
3029
|
+
return 1 if isinstance(p, IRITerm) and p.text == RDF_TYPE else 2
|
|
3030
|
+
return 0
|
|
3031
|
+
if len(patterns) < 2:
|
|
3032
|
+
return patterns
|
|
3033
|
+
counts: Dict[str, int] = {}
|
|
3034
|
+
for t in patterns:
|
|
3035
|
+
for v in names(t):
|
|
3036
|
+
counts[v] = counts.get(v, 0) + 1
|
|
3037
|
+
order = connected_order([(i, names(t), selective(t)) for i, t in enumerate(patterns)], set(),
|
|
3038
|
+
[frozenset({v}) for v, n in counts.items() if n > 1])
|
|
3039
|
+
return [patterns[i] for i in order]
|
|
3040
|
+
|
|
3041
|
+
|
|
3042
|
+
WITNESS_CAP = 8 # tree-witness readings of one triples block: at most 2^8 - 1 branches
|
|
3043
|
+
|
|
3044
|
+
|
|
3045
|
+
def Counter_of(patterns) -> Dict[str, int]:
|
|
3046
|
+
"""How often each variable occurs in `patterns`."""
|
|
3047
|
+
out: Dict[str, int] = {}
|
|
3048
|
+
for t in patterns:
|
|
3049
|
+
for v in t:
|
|
3050
|
+
if isinstance(v, Var):
|
|
3051
|
+
out[v.name] = out.get(v.name, 0) + 1
|
|
3052
|
+
return out
|
|
3053
|
+
|
|
3054
|
+
|
|
3055
|
+
def _hidden_trees(patterns, hidden) -> List[Tuple[list, Any, Dict[str, Any], Dict[str, list], Dict[Any, Any]]]:
|
|
3056
|
+
"""The trees of hidden terms in `patterns`: (their atoms, (the term they hang from, the role to their top, the
|
|
3057
|
+
top) or None where detached, per hidden term its classes and its (role, neighbour) edges). A set of hidden
|
|
3058
|
+
terms is one only where every atom naming them is a class atom (`h a C`) or a property atom of an IRI, its
|
|
3059
|
+
atoms form a tree, and it hangs from one atom at most; any other shape is read by named individuals only."""
|
|
3060
|
+
adj: Dict[str, Dict[str, Any]] = {h: {"classes": set(), "edges": []} for h in hidden}
|
|
3061
|
+
atoms_of: Dict[str, list] = {h: [] for h in hidden}
|
|
3062
|
+
own: Dict[str, list] = {h: [] for h in hidden} # a term's class atoms
|
|
3063
|
+
links: Dict[Tuple[Any, str], Any] = {} # the atom of each edge (parent node or None, node)
|
|
3064
|
+
bad: Set[str] = set()
|
|
3065
|
+
hangs: Dict[str, list] = {h: [] for h in hidden}
|
|
3066
|
+
for t in patterns:
|
|
3067
|
+
s, p, o = t
|
|
3068
|
+
names = [v.name for v in (s, o) if isinstance(v, Var) and v.name in hidden]
|
|
3069
|
+
if isinstance(p, Var) and p.name in hidden:
|
|
3070
|
+
bad.add(p.name)
|
|
3071
|
+
if not names:
|
|
3072
|
+
continue
|
|
3073
|
+
if not isinstance(p, IRITerm):
|
|
3074
|
+
bad.update(names)
|
|
3075
|
+
continue
|
|
3076
|
+
if p.text == RDF_TYPE:
|
|
3077
|
+
if isinstance(s, Var) and s.name in hidden and isinstance(o, IRITerm):
|
|
3078
|
+
adj[s.name]["classes"].add(o.text)
|
|
3079
|
+
atoms_of[s.name].append(t)
|
|
3080
|
+
own[s.name].append(t)
|
|
3081
|
+
else:
|
|
3082
|
+
bad.update(names)
|
|
3083
|
+
continue
|
|
3084
|
+
sh = isinstance(s, Var) and s.name in hidden
|
|
3085
|
+
oh = isinstance(o, Var) and o.name in hidden
|
|
3086
|
+
if sh and oh:
|
|
3087
|
+
if s.name == o.name:
|
|
3088
|
+
bad.add(s.name)
|
|
3089
|
+
continue
|
|
3090
|
+
adj[s.name]["edges"].append(((p.text, False), o.name))
|
|
3091
|
+
adj[o.name]["edges"].append(((p.text, True), s.name))
|
|
3092
|
+
atoms_of[s.name].append(t)
|
|
3093
|
+
links[(s.name, o.name)] = links[(o.name, s.name)] = t
|
|
3094
|
+
elif sh:
|
|
3095
|
+
if isinstance(o, LiteralTerm):
|
|
3096
|
+
bad.add(s.name)
|
|
3097
|
+
continue
|
|
3098
|
+
hangs[s.name].append((o, (p.text, True), t)) # from o, by p's inverse, to s
|
|
3099
|
+
atoms_of[s.name].append(t)
|
|
3100
|
+
links.setdefault((None, s.name), t)
|
|
3101
|
+
else:
|
|
3102
|
+
if isinstance(s, LiteralTerm):
|
|
3103
|
+
bad.add(o.name)
|
|
3104
|
+
continue
|
|
3105
|
+
hangs[o.name].append((s, (p.text, False), t))
|
|
3106
|
+
atoms_of[o.name].append(t)
|
|
3107
|
+
links.setdefault((None, o.name), t)
|
|
3108
|
+
# Connected sets of hidden terms.
|
|
3109
|
+
out = []
|
|
3110
|
+
seen: Set[str] = set()
|
|
3111
|
+
for h in sorted(hidden):
|
|
3112
|
+
if h in seen:
|
|
3113
|
+
continue
|
|
3114
|
+
group, todo = set(), [h]
|
|
3115
|
+
while todo:
|
|
3116
|
+
x = todo.pop()
|
|
3117
|
+
if x in group:
|
|
3118
|
+
continue
|
|
3119
|
+
group.add(x)
|
|
3120
|
+
todo += [n for _, n in adj[x]["edges"]]
|
|
3121
|
+
seen |= group
|
|
3122
|
+
if group & bad:
|
|
3123
|
+
continue
|
|
3124
|
+
edges = sum(len(adj[x]["edges"]) for x in group) // 2
|
|
3125
|
+
if edges != len(group) - 1:
|
|
3126
|
+
continue # a cycle: no tree
|
|
3127
|
+
hanging = [(x, *a) for x in group for a in hangs[x]]
|
|
3128
|
+
if len(hanging) > 1 and len({h[0] for h in hanging}) > 1:
|
|
3129
|
+
continue # hangs from two of its terms: named individuals only
|
|
3130
|
+
atoms = list({id(a): a for x in group for a in atoms_of[x]}.values())
|
|
3131
|
+
by = {x: own[x] for x in group}
|
|
3132
|
+
if hanging:
|
|
3133
|
+
# Several atoms from one top (`?x p ?h . ?y q ?h`): one root, the others equal to it, the top implied by
|
|
3134
|
+
# roles all of them give.
|
|
3135
|
+
top, term, role, _ = hanging[0]
|
|
3136
|
+
roles = tuple(h[2] for h in hanging)
|
|
3137
|
+
others = tuple(h[1] for h in hanging[1:])
|
|
3138
|
+
for h in hanging[1:]:
|
|
3139
|
+
links[("extra", top, id(h[3]))] = h[3]
|
|
3140
|
+
out.append((atoms, (term, roles if len(roles) > 1 else role, top, others), {x: adj[x] for x in group},
|
|
3141
|
+
by, links))
|
|
3142
|
+
else:
|
|
3143
|
+
out.append((atoms, None, {x: adj[x] for x in group}, by, links))
|
|
3144
|
+
return out
|
|
3145
|
+
|
|
3146
|
+
|
|
3147
|
+
def _filtered_below(x: O.Op, cond: E.Expr, column: str) -> O.Op:
|
|
3148
|
+
"""`Filter(x, cond)`, the test (of `column` alone) put inside a Distinct, a Project keeping the column and
|
|
3149
|
+
each branch of a Union, which it commutes with: there each map's column is of one type."""
|
|
3150
|
+
if isinstance(x, O.Distinct):
|
|
3151
|
+
return O.Distinct(_filtered_below(x.input, cond, column))
|
|
3152
|
+
if isinstance(x, O.Project) and column in x.input.columns:
|
|
3153
|
+
return O.Project(_filtered_below(x.input, cond, column), x.columns)
|
|
3154
|
+
if isinstance(x, O.Union):
|
|
3155
|
+
return O.Union(tuple(_filtered_below(b, cond, column) if column in b.columns else b for b in x.inputs))
|
|
3156
|
+
return O.Filter(x, cond)
|
|
3157
|
+
|
|
3158
|
+
|
|
3159
|
+
def patterns_columns(patterns) -> List[str]:
|
|
3160
|
+
"""The variables of `patterns`, in the order they are written."""
|
|
3161
|
+
return list(dict.fromkeys(t.name for x in patterns for t in x if isinstance(t, Var)))
|
|
3162
|
+
|
|
3163
|
+
|
|
3164
|
+
def _value_joined(patterns) -> List[str]:
|
|
3165
|
+
"""The variables that are the object of two or more of `patterns` and never a subject, predicate or path end:
|
|
3166
|
+
under datatypes="value" those join by value (`Translator.by_value_block`)."""
|
|
3167
|
+
objects: Dict[str, int] = {}
|
|
3168
|
+
elsewhere: Set[str] = set()
|
|
3169
|
+
for s, p, o in patterns:
|
|
3170
|
+
for t in (s, p):
|
|
3171
|
+
if isinstance(t, Var):
|
|
3172
|
+
elsewhere.add(t.name)
|
|
3173
|
+
if isinstance(o, Var):
|
|
3174
|
+
if isinstance(p, Path):
|
|
3175
|
+
elsewhere.add(o.name)
|
|
3176
|
+
else:
|
|
3177
|
+
objects[o.name] = objects.get(o.name, 0) + 1
|
|
3178
|
+
return sorted(v for v, n in objects.items() if n > 1 and v not in elsewhere)
|
|
3179
|
+
|
|
3180
|
+
|
|
3181
|
+
def _least_of(names, expression, least_literal) -> E.Expr:
|
|
3182
|
+
"""The least literal of the columns `names` (value-equal), by `least_literal`'s term order."""
|
|
3183
|
+
shown: E.Expr = E.Col(names[0])
|
|
3184
|
+
for other in names[1:]:
|
|
3185
|
+
step = expression(least_literal("pga", "pgb"), {"pgb": other}, condition=False)
|
|
3186
|
+
shown = E.replaced(step, {"pga": shown})
|
|
3187
|
+
return shown
|
|
3188
|
+
|
|
3189
|
+
|
|
3190
|
+
def join(a: O.Op, b: O.Op) -> O.Op:
|
|
3191
|
+
if isinstance(a, O.Unit):
|
|
3192
|
+
return b
|
|
3193
|
+
if isinstance(b, O.Unit):
|
|
3194
|
+
return a
|
|
3195
|
+
return O.Join(meeting(a, b), meeting(b, a))
|
|
3196
|
+
|
|
3197
|
+
|
|
3198
|
+
def meeting(x: O.Op, other: O.Op) -> O.Op:
|
|
3199
|
+
"""`x` without the union branches no row of `other` joins: a branch
|
|
3200
|
+
whose shared column, bound in every row of both, holds terms of maps
|
|
3201
|
+
none of which may give a term `other`'s column holds (`TermMap.
|
|
3202
|
+
may_equal`, by the slots' provenance; unknown provenance keeps it).
|
|
3203
|
+
The branches are sought through nested unions and the row-wise
|
|
3204
|
+
operators above them (Distinct, Filter, Extend, Project), so a
|
|
3205
|
+
variable predicate's union of per-predicate unions loses, under a
|
|
3206
|
+
VALUES-bound subject, each map whose subject cannot be one of its
|
|
3207
|
+
terms (the union of what each row allows; a row with UNDEF leaves the
|
|
3208
|
+
column unbound in some rows, so nothing goes)."""
|
|
3209
|
+
theirs = other.slots
|
|
3210
|
+
wanted = {c: theirs[c].provenance for c in x.columns if c in theirs
|
|
3211
|
+
and theirs[c].present is Bound.ALWAYS and theirs[c].provenance is not None}
|
|
3212
|
+
if not wanted:
|
|
3213
|
+
return x
|
|
3214
|
+
out = _met(x, wanted)
|
|
3215
|
+
if out is x:
|
|
3216
|
+
return x
|
|
3217
|
+
return O.Values(x.columns, ()) if out is None else out
|
|
3218
|
+
|
|
3219
|
+
|
|
3220
|
+
def _met(x: O.Op, wanted: Dict[str, Any]) -> Optional[O.Op]:
|
|
3221
|
+
"""`meeting`'s walk: `x` with the branches that cannot meet `wanted` (column -> the provenance of the
|
|
3222
|
+
terms the other side holds there) gone; None where no row of `x` can meet; `x` itself where nothing goes."""
|
|
3223
|
+
slots = x.slots
|
|
3224
|
+
for c, provenance in wanted.items():
|
|
3225
|
+
mine = slots.get(c)
|
|
3226
|
+
if mine is not None and mine.present is Bound.ALWAYS and mine.provenance is not None \
|
|
3227
|
+
and not any(p.may_equal(q) for p in mine.provenance for q in provenance):
|
|
3228
|
+
return None
|
|
3229
|
+
if isinstance(x, O.Union):
|
|
3230
|
+
kids = [_met(i, wanted) for i in x.inputs]
|
|
3231
|
+
if all(k is i for k, i in zip(kids, x.inputs)):
|
|
3232
|
+
return x
|
|
3233
|
+
kept = [k for k in kids if k is not None]
|
|
3234
|
+
if not kept:
|
|
3235
|
+
return None
|
|
3236
|
+
out: O.Op = kept[0] if len(kept) == 1 and kept[0].columns == x.columns else O.Union(tuple(kept))
|
|
3237
|
+
for c in x.columns:
|
|
3238
|
+
if c not in out.columns:
|
|
3239
|
+
out = O.Extend(out, c, E.Const(None)) # held only by a dropped branch: unbound
|
|
3240
|
+
return out if out.columns == x.columns else O.Project(out, x.columns)
|
|
3241
|
+
if isinstance(x, (O.Distinct, O.Filter, O.Extend, O.Project)):
|
|
3242
|
+
inner = {c: v for c, v in wanted.items() if c in x.input.columns
|
|
3243
|
+
and not (isinstance(x, O.Extend) and c == x.name)}
|
|
3244
|
+
if isinstance(x, O.Extend) and x.name in wanted and isinstance(x.expr, E.Col) \
|
|
3245
|
+
and x.expr.name in x.input.columns and x.expr.name not in inner:
|
|
3246
|
+
inner[x.expr.name] = wanted[x.name] # a copy of a column: the same terms
|
|
3247
|
+
if not inner:
|
|
3248
|
+
return x
|
|
3249
|
+
got = _met(x.input, inner)
|
|
3250
|
+
if got is x.input:
|
|
3251
|
+
return x
|
|
3252
|
+
if got is None:
|
|
3253
|
+
return None
|
|
3254
|
+
return dataclasses.replace(x, input=got)
|
|
3255
|
+
return x
|
|
3256
|
+
|
|
3257
|
+
|
|
3258
|
+
def _may_give(tm: TermMap, t: Term) -> bool:
|
|
3259
|
+
"""Whether a term of `tm` may be the pattern's constant `t`."""
|
|
3260
|
+
if isinstance(t, Var):
|
|
3261
|
+
return True
|
|
3262
|
+
if tm.term_type == "iri" and tm.lexical == LEXICAL_R2RML and \
|
|
3263
|
+
not re.match(r"[A-Za-z][A-Za-z0-9+.-]*:", tm.prefix()):
|
|
3264
|
+
return True # a relative IRI, resolved against the base (R2RML 7.3)
|
|
3265
|
+
provenance = const_slot(t).provenance
|
|
3266
|
+
return any(tm.may_equal(c) for c in provenance or ())
|
|
3267
|
+
|
|
3268
|
+
|
|
3269
|
+
class LabelPatternError(QueryError):
|
|
3270
|
+
"""A query names `<urn:prograph:label:X>`, a label as a term: not SPARQL's,
|
|
3271
|
+
gone in ProGraph 1.0. A label is a class in the RDF view: query its class IRI."""
|
|
3272
|
+
|
|
3273
|
+
|
|
3274
|
+
def _label_pattern(label: str, maps: SourceMaps) -> "LabelPatternError":
|
|
3275
|
+
classes = sorted({c for m in maps.nodes if m.label == label for c in m.classes})
|
|
3276
|
+
if classes:
|
|
3277
|
+
use = " or ".join(f"<{c}>" for c in classes)
|
|
3278
|
+
return LabelPatternError(f"<{LABEL_IRI_PREFIX}{label}> is not a term in ProGraph 1.0: the label "
|
|
3279
|
+
f"{label} is the class {use}; write `?x a {use.split(' or ')[0]}`")
|
|
3280
|
+
return LabelPatternError(f"<{LABEL_IRI_PREFIX}{label}> is not a term in ProGraph 1.0, and no map has the "
|
|
3281
|
+
f"label {label}: query a class IRI (`?x a <class>`)")
|
|
3282
|
+
|
|
3283
|
+
|
|
3284
|
+
class OutsideGroupWarning(UserWarning):
|
|
3285
|
+
"""HAVING or ORDER BY reads a variable a grouped query does not keep: it is unbound there (D2),
|
|
3286
|
+
almost always a mistake."""
|
|
3287
|
+
|
|
3288
|
+
|
|
3289
|
+
def _warn_outside_group(having, order, kept, selected) -> None:
|
|
3290
|
+
"""Name each variable HAVING or ORDER BY reads that the group does not keep (D2: unbound
|
|
3291
|
+
there). HAVING sees the keys and aggregates; ORDER BY the SELECT expressions too."""
|
|
3292
|
+
import warnings
|
|
3293
|
+
for where, exprs, seen in (("HAVING", having, kept), ("ORDER BY", [e for e, _ in order], kept | selected)):
|
|
3294
|
+
outside = sorted({c for e in exprs for c in e.columns()} - seen)
|
|
3295
|
+
outside = [c for c in outside if not c.startswith(HIDDEN)]
|
|
3296
|
+
if outside:
|
|
3297
|
+
names = ", ".join("?" + c for c in outside)
|
|
3298
|
+
warnings.warn(f"{where} reads {names}, which the group does not keep (a GROUP BY key or an "
|
|
3299
|
+
f"aggregate): it is unbound there (sparql-choices.md D2). Group by it, or "
|
|
3300
|
+
f"aggregate it (SAMPLE, MIN, ...).", OutsideGroupWarning, stacklevel=3)
|
|
3301
|
+
|
|
3302
|
+
|
|
3303
|
+
class DatatypeOverrideWarning(UserWarning):
|
|
3304
|
+
"""An object map declares a DATE or TIMESTAMP column another XSD datatype (`rr:datatype`): every
|
|
3305
|
+
value is an ill-typed literal, a data error (R2RML 10.3, 11), so a query that reads the map
|
|
3306
|
+
raises R2RMLDataError."""
|
|
3307
|
+
|
|
3308
|
+
|
|
3309
|
+
# R2RML 10.3's validatable datatypes (the natural mapping's table), but xsd:string, whose
|
|
3310
|
+
# lexical space holds every text.
|
|
3311
|
+
_VALIDATABLE = frozenset({xsd.INTEGER, xsd.DECIMAL, xsd.DOUBLE, xsd.BOOLEAN, xsd.DATE, xsd.DATETIME,
|
|
3312
|
+
xsd.XSD + "hexBinary"})
|
|
3313
|
+
|
|
3314
|
+
|
|
3315
|
+
@home("select-star-order", "The order of SELECT *'s columns",
|
|
3316
|
+
terms=("SELECT *", "column order", "first mention", "in-scope variables", "18.2.1"))
|
|
3317
|
+
def star_order(where: Any, names: List[str]) -> List[str]:
|
|
3318
|
+
"""SELECT *'s variables (18.2.1's in-scope ones, `names`) in the order the WHERE clause first
|
|
3319
|
+
writes them, as Jena and Oxigraph give them (the standard leaves the order open):
|
|
3320
|
+
`?o ^p ?s` names ?o then ?s, though the plan binds ?s first. A FILTER's mention does not count
|
|
3321
|
+
(it binds nothing); a name never written keeps its plan order, after the written ones."""
|
|
3322
|
+
from antlr4.tree.Tree import TerminalNode
|
|
3323
|
+
seen: List[str] = []
|
|
3324
|
+
|
|
3325
|
+
def walk(node: Any) -> None:
|
|
3326
|
+
if isinstance(node, TerminalNode):
|
|
3327
|
+
text = node.getText()
|
|
3328
|
+
if text[:1] in "?$" and len(text) > 1 and text[1:] not in seen:
|
|
3329
|
+
seen.append(text[1:])
|
|
3330
|
+
return
|
|
3331
|
+
if isinstance(node, SparqlParser.Filter_Context):
|
|
3332
|
+
return
|
|
3333
|
+
for i in range(node.getChildCount()):
|
|
3334
|
+
walk(node.getChild(i))
|
|
3335
|
+
|
|
3336
|
+
if where is not None:
|
|
3337
|
+
walk(where)
|
|
3338
|
+
wanted = set(names)
|
|
3339
|
+
written = [n for n in seen if n in wanted]
|
|
3340
|
+
return written + [n for n in names if n not in set(written)]
|
|
3341
|
+
|
|
3342
|
+
|
|
3343
|
+
@home("datatype-override-ill-typed", "A native column declared a datatype its natural form is not in",
|
|
3344
|
+
terms=("rr:datatype", "datatype override", "ill-typed", "data error", "R2RML 10.3",
|
|
3345
|
+
"TIMESTAMP declared xsd:date", "DOUBLE declared xsd:decimal", "R2RMLDataError", "DatatypeOverrideWarning"))
|
|
3346
|
+
def ill_typed_override(tm: Optional[TermMap], table: str, held: Any) -> Optional[str]:
|
|
3347
|
+
"""What is wrong where `tm` reads a native column whole and declares a validatable datatype
|
|
3348
|
+
whose lexical space never holds the column's natural lexical form (R2RML 10.2), so every value
|
|
3349
|
+
is an ill-typed literal, a data error (R2RML 10.3): a DATE or TIMESTAMP declared another
|
|
3350
|
+
datatype (`1995-06-15T13:30:00` is no xsd:date), and a DOUBLE or FLOAT declared xsd:decimal
|
|
3351
|
+
or an integer type (`3.0E0` is neither: M2 review B6). Known from the column's type, never
|
|
3352
|
+
per row (an all-NULL column too). None where nothing is."""
|
|
3353
|
+
if tm is None or tm.column is None or tm.template is not None or tm.datatype is None:
|
|
3354
|
+
return None
|
|
3355
|
+
native = held(table, tm.column)[0]
|
|
3356
|
+
short = lambda dt: dt.replace(xsd.XSD, "xsd:")
|
|
3357
|
+
if native in (xsd.DOUBLE, xsd.FLOAT) and (tm.datatype == xsd.DECIMAL or tm.datatype in xsd.INTEGER_DATATYPES
|
|
3358
|
+
or tm.datatype == xsd.INTEGER):
|
|
3359
|
+
return (f"column {tm.column} of {table} is a DOUBLE declared {short(tm.datatype)}: its natural lexical form "
|
|
3360
|
+
f"(3.0E0, R2RML 10.2) is never one, so every value is an ill-typed literal, a data error "
|
|
3361
|
+
f"(R2RML 10.3); map it as xsd:double (or leave rr:datatype out), or CAST it AS DECIMAL in an "
|
|
3362
|
+
f"rr:sqlQuery (on a pandas frame, hold Decimal values)")
|
|
3363
|
+
if native not in (xsd.DATE, xsd.DATETIME) or tm.datatype == native or tm.datatype not in _VALIDATABLE:
|
|
3364
|
+
return None
|
|
3365
|
+
sql = "TIMESTAMP" if native == xsd.DATETIME else "DATE"
|
|
3366
|
+
fix = ("map it as xsd:dateTime (or leave rr:datatype out: its natural datatype), or, for dates, "
|
|
3367
|
+
"CAST it AS DATE in an rr:sqlQuery (on a pandas frame, df[col] = df[col].dt.date)"
|
|
3368
|
+
if native == xsd.DATETIME else "leave rr:datatype out (its natural datatype is xsd:date)")
|
|
3369
|
+
return (f"column {tm.column} of {table} is a {sql} declared {short(tm.datatype)}: every value is an "
|
|
3370
|
+
f"ill-typed literal, a data error (R2RML 10.3); {fix}")
|
|
3371
|
+
|
|
3372
|
+
|
|
3373
|
+
def _warn_overrides(maps: SourceMaps, held: Any) -> None:
|
|
3374
|
+
"""Name each such object map when a query is planned over the maps (once per map)."""
|
|
3375
|
+
import warnings
|
|
3376
|
+
for m in maps.nodes:
|
|
3377
|
+
for po in m.predicate_objects:
|
|
3378
|
+
what = ill_typed_override(po.object, m.source.table, held)
|
|
3379
|
+
if what:
|
|
3380
|
+
warnings.warn(what, DatatypeOverrideWarning, stacklevel=4)
|
|
3381
|
+
|
|
3382
|
+
|
|
3383
|
+
class PatternNeverMatchesWarning(UserWarning):
|
|
3384
|
+
"""A triple pattern's constant no map's term can be (m2-functions.md 5)."""
|
|
3385
|
+
|
|
3386
|
+
|
|
3387
|
+
def _say_never(p: IRITerm, o: LiteralTerm, never) -> None:
|
|
3388
|
+
"""Say, at planning, that a pattern's literal matches no triple because
|
|
3389
|
+
of its written form: what the predicate's objects are, and the FILTER
|
|
3390
|
+
that compares values (decided 2026-11-09)."""
|
|
3391
|
+
import warnings
|
|
3392
|
+
from decimal import Decimal
|
|
3393
|
+
from prograph.schema import xsd as X
|
|
3394
|
+
short = lambda dt: dt.replace(X.XSD, "xsd:") if dt else "a literal"
|
|
3395
|
+
forms = []
|
|
3396
|
+
for _, slot in never:
|
|
3397
|
+
dts = sorted(d for d in slot.datatypes if d != UNKNOWN)
|
|
3398
|
+
dt = dts[0] if len(dts) == 1 else None
|
|
3399
|
+
if dt in (X.DECIMAL, X.INTEGER) and slot.native == dt:
|
|
3400
|
+
try:
|
|
3401
|
+
value = X.parse(o.lexical, o.datatype or X.STRING)
|
|
3402
|
+
example = X.canonical_lexical(Decimal(value) if dt == X.DECIMAL else int(value), dt)
|
|
3403
|
+
forms.append(f"{short(dt)} in canonical form (this value is written {example})")
|
|
3404
|
+
except (ValueError, TypeError, ArithmeticError):
|
|
3405
|
+
forms.append(f"{short(dt)} in canonical form")
|
|
3406
|
+
else:
|
|
3407
|
+
forms.append(", ".join(short(d) for d in dts) or "literals")
|
|
3408
|
+
written = f'"{o.lexical}"^^{short(o.datatype)}' if o.datatype else f'"{o.lexical}"'
|
|
3409
|
+
warnings.warn(
|
|
3410
|
+
f"<{p.text}> {written} matches no triple: the objects of <{p.text}> are "
|
|
3411
|
+
f"{'; '.join(dict.fromkeys(forms))}, and a pattern matches a term with the same "
|
|
3412
|
+
f"datatype and lexical form (RDF 1.1 3.3). To compare values, write "
|
|
3413
|
+
f"FILTER(?v = {o.lexical}).", PatternNeverMatchesWarning, stacklevel=2)
|
|
3414
|
+
|
|
3415
|
+
|
|
3416
|
+
def _empty(s: Term, p: Term, o: Term) -> O.Op:
|
|
3417
|
+
names = tuple(dict.fromkeys(t.name for t in (s, p, o) if isinstance(t, Var)))
|
|
3418
|
+
return O.Values(names, ())
|