prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1782 @@
|
|
|
1
|
+
"""The rewrite registry (one-plan.md piece 1f; m2-performance.md 2.2).
|
|
2
|
+
|
|
3
|
+
A rewrite turns a plan into one with the same answers and less work. Each
|
|
4
|
+
rule reads the facts (`prograph.plan.ir.facts`) and fires only where they
|
|
5
|
+
prove it keeps the answers; unknown is no. The rules run bottom-up, to a
|
|
6
|
+
fixpoint, between the front end and the lowering.
|
|
7
|
+
|
|
8
|
+
Two classes (m2-performance.md):
|
|
9
|
+
- dominance: never more work where it fires (R1 a Distinct over distinct
|
|
10
|
+
rows, R2 a contained pattern); applied wherever proven;
|
|
11
|
+
- trade-off: better on some data, worse on other (where a Distinct goes);
|
|
12
|
+
applied only where a structural rule says so.
|
|
13
|
+
|
|
14
|
+
Each rewrite that fires leaves a certificate: the rule, and what it relied
|
|
15
|
+
on (`Certificate`), so a wrong answer can be traced to the fact that allowed
|
|
16
|
+
it. `PROGRAPH_REWRITES=off` turns the registry off, for tests that compare
|
|
17
|
+
the answers with and without it; it is no engine option.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import dataclasses
|
|
23
|
+
import os
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
from typing import Any, Callable, Dict, FrozenSet, List, Optional, Tuple
|
|
26
|
+
|
|
27
|
+
from prograph.plan.ir import expr as E
|
|
28
|
+
from prograph.plan.ir import ops as O
|
|
29
|
+
from prograph.plan.ir.facts import facts
|
|
30
|
+
from prograph.plan.ir.slot import UNKNOWN, Bound, injective_over, template_shape
|
|
31
|
+
from prograph.schema.source_map import EdgeMap, NodeMap, TermMap
|
|
32
|
+
from prograph.rules import home
|
|
33
|
+
from prograph.utils.helpers import ByIdentity
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class Certificate:
|
|
38
|
+
"""A rewrite that fired: its rule and what it relied on."""
|
|
39
|
+
rule: str
|
|
40
|
+
relied_on: str
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def short(self) -> str:
|
|
44
|
+
"""The rule's number: `R4b` for R4's second part, the trusted
|
|
45
|
+
reference; `R3o` for R3's left-join form (OPTIONAL)."""
|
|
46
|
+
n = self.rule.split(" ", 1)[0]
|
|
47
|
+
if self.rule.startswith("R3 left join"):
|
|
48
|
+
return n + "o"
|
|
49
|
+
if self.rule.startswith("R3 own row"):
|
|
50
|
+
return n + "e"
|
|
51
|
+
return n + "b" if self.rule.startswith("R4 trusted reference") else n
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
Rule = Callable[[O.Op, List[Certificate]], Optional[O.Op]]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def enabled() -> bool:
|
|
58
|
+
return os.environ.get("PROGRAPH_REWRITES", "on") != "off"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@home("plan-rewrites", "The plan's rewrites and what each must prove",
|
|
62
|
+
terms=("rewrite", "Distinct removal", "self-join", "contained pattern", "certificate",
|
|
63
|
+
"union branch", "containment pruning", "duplicate branch", "R9",
|
|
64
|
+
"one scan per source", "variable predicate", "R10", "R12", "joint query child", "R14"))
|
|
65
|
+
def rewrite(plan: O.Op, engine: Optional[str] = None) -> Tuple[O.Op, Tuple[Certificate, ...]]:
|
|
66
|
+
"""`plan` with every rule applied where its facts allow, and the
|
|
67
|
+
certificates of the rewrites that fired. `engine` is the SQL dialect
|
|
68
|
+
the plan will run on: a trade-off rule reads it (R5)."""
|
|
69
|
+
if not enabled():
|
|
70
|
+
return plan, ()
|
|
71
|
+
certs: List[Certificate] = []
|
|
72
|
+
plan = _down(plan, certs, ByIdentity(), RULES)
|
|
73
|
+
# R10 after the rules that remove branches (R6's join-incompatible ones), so it merges only what
|
|
74
|
+
# is left, and before the late rules and the second pass, which read the merged scan.
|
|
75
|
+
plan = _outermost_unions(plan, certs)
|
|
76
|
+
late = LATE_RULES if engine in LIMIT_INWARD_ENGINES else \
|
|
77
|
+
tuple(r for r in LATE_RULES if r is not r7_limit_inward)
|
|
78
|
+
plan = _down(plan, certs, ByIdentity(), late)
|
|
79
|
+
plan = _down(plan, certs, ByIdentity(), RULES) # what the late rules opened up
|
|
80
|
+
if engine in RESTRICT_GROUPS_ENGINES:
|
|
81
|
+
# Top-down, once, after the bottom-up passes: a Filter is met before its join (R14 reads its
|
|
82
|
+
# conjuncts), and the plan's patterns are in their final, merged form.
|
|
83
|
+
plan = _restrict_groups(plan, certs, ByIdentity())
|
|
84
|
+
if engine in DISTINCT_BESIDE_LIMIT:
|
|
85
|
+
# A trade-off, after the rules that remove work: R2 reads the
|
|
86
|
+
# patterns' own Distincts.
|
|
87
|
+
plan = _down(plan, certs, ByIdentity(), (r5_distinct_beside_limit,))
|
|
88
|
+
_log(certs)
|
|
89
|
+
return plan, tuple(certs)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _log(certs: List[Certificate]) -> None:
|
|
93
|
+
"""For the score only: where `PROGRAPH_REWRITE_LOG` names a directory,
|
|
94
|
+
each firing is appended there with the test that caused it, so the
|
|
95
|
+
score can say how often each rule fired on cases with Jena's answers."""
|
|
96
|
+
where = os.environ.get("PROGRAPH_REWRITE_LOG")
|
|
97
|
+
if not where or not certs:
|
|
98
|
+
return
|
|
99
|
+
test = os.environ.get("PYTEST_CURRENT_TEST", "").rsplit(" (", 1)[0]
|
|
100
|
+
with open(os.path.join(where, f"fired-{os.getpid()}.tsv"), "a") as f:
|
|
101
|
+
for c in certs:
|
|
102
|
+
f.write(f"{c.short}\t{test}\n")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _down(op: O.Op, certs: List[Certificate], memo: ByIdentity,
|
|
106
|
+
rules: Tuple[Rule, ...]) -> O.Op:
|
|
107
|
+
# Keyed by identity, holding the key: rules build and drop operators,
|
|
108
|
+
# and a dropped one's id may be given to a later one (rules.md, the
|
|
109
|
+
# memo keyed by an object's identity).
|
|
110
|
+
hit = memo.get(op)
|
|
111
|
+
if hit is not None:
|
|
112
|
+
return hit
|
|
113
|
+
changes = {}
|
|
114
|
+
for f in dataclasses.fields(op):
|
|
115
|
+
v = getattr(op, f.name)
|
|
116
|
+
if isinstance(v, O.Op):
|
|
117
|
+
nv = _down(v, certs, memo, rules)
|
|
118
|
+
if nv is not v:
|
|
119
|
+
changes[f.name] = nv
|
|
120
|
+
elif isinstance(v, tuple) and v and all(isinstance(x, O.Op) for x in v):
|
|
121
|
+
nv = tuple(_down(x, certs, memo, rules) for x in v)
|
|
122
|
+
if any(a is not b for a, b in zip(nv, v)):
|
|
123
|
+
changes[f.name] = nv
|
|
124
|
+
out = dataclasses.replace(op, **changes) if changes else op
|
|
125
|
+
from prograph.plan.ir.fresh import volatile
|
|
126
|
+
if volatile(out):
|
|
127
|
+
# A fresh value made below (fresh.py): no rule moves, copies or drops it.
|
|
128
|
+
memo.put(op, out)
|
|
129
|
+
return out
|
|
130
|
+
for _ in range(64):
|
|
131
|
+
for rule in rules:
|
|
132
|
+
new = rule(out, certs)
|
|
133
|
+
if new is not None and _CHECK:
|
|
134
|
+
_check_firing(rule, out, new)
|
|
135
|
+
if new is not None:
|
|
136
|
+
out = _down(new, certs, memo, rules) if new is not out else out
|
|
137
|
+
break
|
|
138
|
+
else:
|
|
139
|
+
break
|
|
140
|
+
memo.put(op, out)
|
|
141
|
+
return out
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
_CHECK = os.environ.get("PROGRAPH_REWRITE_CHECK") == "1"
|
|
145
|
+
_NARROWER = {(Bound.MAYBE, Bound.ALWAYS), (Bound.MAYBE, Bound.NEVER)}
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _check_firing(rule: Rule, before: O.Op, after: O.Op) -> None:
|
|
149
|
+
"""For tests (the score sets `PROGRAPH_REWRITE_CHECK=1`): one firing
|
|
150
|
+
keeps the operator's columns, and no column's slot or the rows'
|
|
151
|
+
distinctness gets wider. A rule may narrow (a union branch gone leaves
|
|
152
|
+
fewer term maps); it may not widen, since what the plan above was
|
|
153
|
+
built on would no longer hold (m2-performance.md 2.3)."""
|
|
154
|
+
name = rule.__name__
|
|
155
|
+
if set(before.columns) != set(after.columns):
|
|
156
|
+
raise AssertionError(f"{name}: columns {before.columns} became {after.columns}")
|
|
157
|
+
if _no_rows(before) or _no_rows(after):
|
|
158
|
+
# Rows proven not to exist satisfy every claim: R6 removing every
|
|
159
|
+
# branch, or a rule firing inside a join the facts already prove
|
|
160
|
+
# empty (the fuzzer's cases, 2026-11-09).
|
|
161
|
+
return
|
|
162
|
+
old, new = before.slots, after.slots
|
|
163
|
+
for c in before.columns:
|
|
164
|
+
a, b = old[c], new[c]
|
|
165
|
+
bad = []
|
|
166
|
+
if not b.kinds <= a.kinds:
|
|
167
|
+
bad.append(f"kinds {sorted(a.kinds)} -> {sorted(b.kinds)}")
|
|
168
|
+
if not b.datatypes <= a.datatypes and UNKNOWN not in a.datatypes:
|
|
169
|
+
bad.append(f"datatypes {sorted(a.datatypes)} -> {sorted(b.datatypes)}")
|
|
170
|
+
for f in ("bound", "null"):
|
|
171
|
+
x, y = getattr(a, f), getattr(b, f)
|
|
172
|
+
if x is not y and (x, y) not in _NARROWER:
|
|
173
|
+
bad.append(f"{f} {x.value} -> {y.value}")
|
|
174
|
+
if a.provenance is not None and (b.provenance is None or not _same_terms(b.provenance, a.provenance)):
|
|
175
|
+
bad.append("provenance wider")
|
|
176
|
+
for f in ("language", "zone"):
|
|
177
|
+
if getattr(a, f) != getattr(b, f):
|
|
178
|
+
bad.append(f"{f} {getattr(a, f)!r} -> {getattr(b, f)!r}")
|
|
179
|
+
for f in ("native", "held_as"):
|
|
180
|
+
# None -> a form is a narrowing (one template or one datatype
|
|
181
|
+
# left after a union branch went); the lowering reads the form
|
|
182
|
+
# from the final plan. A form changed or lost is not.
|
|
183
|
+
if getattr(a, f) is not None and getattr(b, f) != getattr(a, f):
|
|
184
|
+
bad.append(f"{f} {getattr(a, f)!r} -> {getattr(b, f)!r}")
|
|
185
|
+
if bad:
|
|
186
|
+
raise AssertionError(f"{name}: column {c}: " + "; ".join(bad))
|
|
187
|
+
if facts(before).distinct and not facts(after).distinct:
|
|
188
|
+
raise AssertionError(f"{name}: distinct rows became possibly repeated")
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _no_rows(op: O.Op) -> bool:
|
|
192
|
+
"""Whether the facts prove `op` has no row: a bound of 0, or a column
|
|
193
|
+
bound in every row that no kind or no map can fill."""
|
|
194
|
+
if facts(op).rows[1] == 0:
|
|
195
|
+
return True
|
|
196
|
+
if isinstance(op, O.Values):
|
|
197
|
+
return not op.rows
|
|
198
|
+
if isinstance(op, (O.Join, O.SemiJoin)):
|
|
199
|
+
return _no_rows(op.left) or _no_rows(op.right)
|
|
200
|
+
if isinstance(op, O.LeftJoin):
|
|
201
|
+
return _no_rows(op.left)
|
|
202
|
+
if isinstance(op, O.Union):
|
|
203
|
+
return all(_no_rows(x) for x in op.inputs)
|
|
204
|
+
if isinstance(op, (O.Distinct, O.Project, O.Filter, O.Extend, O.Order, O.Slice)):
|
|
205
|
+
return _no_rows(op.input)
|
|
206
|
+
return any(sl.bound is Bound.ALWAYS and (not sl.kinds or sl.provenance == frozenset())
|
|
207
|
+
for sl in op.slots.values())
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _same_terms(new: FrozenSet[TermMap], old: FrozenSet[TermMap]) -> bool:
|
|
211
|
+
"""Each map of `new` gives only terms a map of `old` gives: the same
|
|
212
|
+
map, or a template of the same shape (a node's subject and an edge's
|
|
213
|
+
end, `template_shape`)."""
|
|
214
|
+
shapes = {template_shape(m) for m in old if m.template}
|
|
215
|
+
return all(m in old or (m.template and template_shape(m) in shapes) for m in new)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
# -- R1: a Distinct over rows already distinct ----------------------------------------
|
|
219
|
+
|
|
220
|
+
def r1_distinct_removal(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
221
|
+
"""`Distinct(X) -> X` where the facts prove X has no two equal rows: a
|
|
222
|
+
trusted key whose columns X always binds, through an injective term
|
|
223
|
+
constructor (one-plan.md 4.1); a join of distinct inputs; a union of
|
|
224
|
+
distinct branches no two of which can give one row; at most one row."""
|
|
225
|
+
if not isinstance(op, O.Distinct):
|
|
226
|
+
return None
|
|
227
|
+
f = facts(op.input)
|
|
228
|
+
if not f.distinct:
|
|
229
|
+
return None
|
|
230
|
+
why = (f"keys {sorted(sorted(k) for k in f.keys)}" if f.keys else "distinct input")
|
|
231
|
+
certs.append(Certificate("R1 distinct removal", why))
|
|
232
|
+
return op.input
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
# -- R2: a pattern another pattern of the same map contains ---------------------------
|
|
236
|
+
|
|
237
|
+
@dataclass(frozen=True)
|
|
238
|
+
class _Pattern:
|
|
239
|
+
"""A pattern's rows read off one map scan: the source, the source
|
|
240
|
+
columns a row must have, the row conditions, and each output column's
|
|
241
|
+
expression over source columns."""
|
|
242
|
+
source: object
|
|
243
|
+
map_id: str
|
|
244
|
+
required: FrozenSet[str]
|
|
245
|
+
conditions: FrozenSet[E.Expr]
|
|
246
|
+
outputs: Tuple[Tuple[str, E.Expr], ...]
|
|
247
|
+
distinct: bool
|
|
248
|
+
scan: Optional[O.MapScan] = None
|
|
249
|
+
# Node maps every row's end is proven to exist in: (node map id, the
|
|
250
|
+
# (scan source column, node key column) pairs, the node scan).
|
|
251
|
+
exists: Tuple[Tuple[str, Tuple[Tuple[str, str], ...], O.MapScan], ...] = ()
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _pattern(op: O.Op) -> Optional[_Pattern]:
|
|
255
|
+
"""`op` as one scan's pattern: Distinct? Project(Filter*(Extend*(MapScan)))."""
|
|
256
|
+
distinct = False
|
|
257
|
+
if isinstance(op, O.Distinct):
|
|
258
|
+
distinct, op = True, op.input
|
|
259
|
+
if not isinstance(op, O.Project):
|
|
260
|
+
return None
|
|
261
|
+
names = op.names
|
|
262
|
+
x = op.input
|
|
263
|
+
conditions: List[E.Expr] = []
|
|
264
|
+
exprs: Dict[str, E.Expr] = {}
|
|
265
|
+
semis: List[O.SemiJoin] = []
|
|
266
|
+
while True:
|
|
267
|
+
if isinstance(x, O.Filter):
|
|
268
|
+
conditions.extend(E.conjuncts(x.condition))
|
|
269
|
+
x = x.input
|
|
270
|
+
elif isinstance(x, O.Extend):
|
|
271
|
+
exprs.setdefault(x.name, x.expr)
|
|
272
|
+
x = x.input
|
|
273
|
+
elif isinstance(x, O.SemiJoin) and x.condition is None and isinstance(x.right, O.MapScan) \
|
|
274
|
+
and isinstance(x.right.map, NodeMap) and x.right.graph is None \
|
|
275
|
+
and x.right.joint is None:
|
|
276
|
+
semis.append(x)
|
|
277
|
+
x = x.left
|
|
278
|
+
else:
|
|
279
|
+
break
|
|
280
|
+
if not isinstance(x, O.MapScan) or x.graph is not None or x.joint is not None:
|
|
281
|
+
return None
|
|
282
|
+
to_source = {r.name: r.column for r in x.reads}
|
|
283
|
+
|
|
284
|
+
def source_expr(e: E.Expr) -> Optional[E.Expr]:
|
|
285
|
+
"""`e` with every column named by its source column."""
|
|
286
|
+
if isinstance(e, E.Col):
|
|
287
|
+
if e.name in exprs:
|
|
288
|
+
return source_expr(exprs[e.name])
|
|
289
|
+
return E.Col("src:" + to_source[e.name]) if e.name in to_source else None
|
|
290
|
+
if isinstance(e, E.Const):
|
|
291
|
+
return e
|
|
292
|
+
if isinstance(e, E.MakeTerm):
|
|
293
|
+
args = []
|
|
294
|
+
for c, a in e.args:
|
|
295
|
+
sa = source_expr(a)
|
|
296
|
+
if sa is None:
|
|
297
|
+
return None
|
|
298
|
+
args.append((c, sa))
|
|
299
|
+
return E.MakeTerm(e.map, tuple(args))
|
|
300
|
+
if isinstance(e, E.Compare):
|
|
301
|
+
l, r = source_expr(e.left), source_expr(e.right)
|
|
302
|
+
return None if l is None or r is None else E.Compare(e.op, l, r, e.semantics)
|
|
303
|
+
return None
|
|
304
|
+
|
|
305
|
+
outputs = []
|
|
306
|
+
for n in names:
|
|
307
|
+
if n not in exprs:
|
|
308
|
+
return None
|
|
309
|
+
se = source_expr(exprs[n])
|
|
310
|
+
if se is None:
|
|
311
|
+
return None
|
|
312
|
+
outputs.append((n, se))
|
|
313
|
+
conds = set()
|
|
314
|
+
for c in conditions:
|
|
315
|
+
sc = source_expr(c)
|
|
316
|
+
if sc is None:
|
|
317
|
+
return None
|
|
318
|
+
if isinstance(sc, E.Compare) and sc.op == "=" and isinstance(sc.left, E.Const) \
|
|
319
|
+
and sc.left == sc.right:
|
|
320
|
+
continue # a constant equal to itself: every row
|
|
321
|
+
conds.add(sc)
|
|
322
|
+
required = frozenset(to_source[n] for n in x.required)
|
|
323
|
+
exists = []
|
|
324
|
+
for sj in semis:
|
|
325
|
+
theirs = {r.name: r.column for r in sj.right.reads}
|
|
326
|
+
pairs = tuple((to_source[c], theirs[c]) for c in sj.shared
|
|
327
|
+
if c in to_source and c in theirs)
|
|
328
|
+
if not (pairs and len(pairs) == len(sj.shared)):
|
|
329
|
+
# A semi-join this pattern cannot state on raw columns is no part of a pattern the
|
|
330
|
+
# rewrites may merge: refused, never dropped (M2 review R10).
|
|
331
|
+
return None
|
|
332
|
+
exists.append((sj.right.map.id, pairs, sj.right))
|
|
333
|
+
return _Pattern(x.map.source, x.map.id, required, frozenset(conds), tuple(outputs), distinct, x,
|
|
334
|
+
tuple(exists))
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _contains(big: _Pattern, small: _Pattern, any_map: bool = False) -> bool:
|
|
338
|
+
"""Every row of `big`'s pattern, projected on `small`'s columns, is a row
|
|
339
|
+
of `small`'s: one map's rows (with `any_map`, one source's: two maps of
|
|
340
|
+
one logical table read the same rows), `small`'s requirements and
|
|
341
|
+
conditions among `big`'s, each of `small`'s columns built alike in `big`."""
|
|
342
|
+
if big.source != small.source or (big.map_id != small.map_id and not any_map):
|
|
343
|
+
return False
|
|
344
|
+
if not small.required <= big.required or not small.conditions <= big.conditions:
|
|
345
|
+
return False
|
|
346
|
+
mine = dict(big.outputs)
|
|
347
|
+
return all(n in mine and mine[n] == e for n, e in small.outputs)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def r2_contained_pattern(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
351
|
+
"""`Join(L, R) -> R` (or `-> L`) where one side is distinct and the
|
|
352
|
+
other contains it (`_contains`): each row of the containing side
|
|
353
|
+
matches exactly one row of the contained, on columns both bind in every
|
|
354
|
+
row (m2-performance.md 2.2, R2). Needs no key: `?p a :Post . ?p :length
|
|
355
|
+
?n` over one table is the second pattern alone."""
|
|
356
|
+
if not isinstance(op, O.Join) or op.condition is not None:
|
|
357
|
+
return None
|
|
358
|
+
for small_op, big_op in ((op.left, op.right), (op.right, op.left)):
|
|
359
|
+
small, big = _pattern(small_op), _pattern(big_op)
|
|
360
|
+
if small is None or big is None:
|
|
361
|
+
continue
|
|
362
|
+
if not (small.distinct or facts(small_op).distinct):
|
|
363
|
+
continue
|
|
364
|
+
cols = set(small_op.columns)
|
|
365
|
+
if not cols <= set(big_op.columns):
|
|
366
|
+
continue
|
|
367
|
+
if not all(small_op.slots[c].present is Bound.ALWAYS and
|
|
368
|
+
big_op.slots[c].present is Bound.ALWAYS for c in cols):
|
|
369
|
+
continue
|
|
370
|
+
if not _contains(big, small):
|
|
371
|
+
continue
|
|
372
|
+
certs.append(Certificate("R2 contained pattern",
|
|
373
|
+
f"map {small.map_id}: {sorted(cols)} within {sorted(big_op.columns)}"))
|
|
374
|
+
if big_op.columns == op.columns:
|
|
375
|
+
return big_op
|
|
376
|
+
return O.Project(big_op, op.columns)
|
|
377
|
+
return _join_tree_r2(op, certs)
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _join_tree_r2(op: O.Join, certs: List[Certificate]) -> Optional[O.Op]:
|
|
381
|
+
"""R2 across a tree of joins: a pattern contained by another anywhere in
|
|
382
|
+
one tree of inner joins with no condition goes; the tree is rebuilt
|
|
383
|
+
left-deep in its order (a compatible join is associative and
|
|
384
|
+
commutative)."""
|
|
385
|
+
leaves: List[O.Op] = []
|
|
386
|
+
|
|
387
|
+
def flat(x: O.Op) -> None:
|
|
388
|
+
if isinstance(x, O.Join) and x.condition is None:
|
|
389
|
+
flat(x.left)
|
|
390
|
+
flat(x.right)
|
|
391
|
+
else:
|
|
392
|
+
leaves.append(x)
|
|
393
|
+
flat(op)
|
|
394
|
+
if len(leaves) < 3:
|
|
395
|
+
return None
|
|
396
|
+
pats = [_pattern(x) for x in leaves]
|
|
397
|
+
for i, small in enumerate(pats):
|
|
398
|
+
if small is None or not (small.distinct or facts(leaves[i]).distinct):
|
|
399
|
+
continue
|
|
400
|
+
cols = set(leaves[i].columns)
|
|
401
|
+
for j, big in enumerate(pats):
|
|
402
|
+
if i == j or big is None or not cols <= set(leaves[j].columns):
|
|
403
|
+
continue
|
|
404
|
+
if not all(leaves[i].slots[c].present is Bound.ALWAYS and
|
|
405
|
+
leaves[j].slots[c].present is Bound.ALWAYS for c in cols):
|
|
406
|
+
continue
|
|
407
|
+
if not _contains(big, small):
|
|
408
|
+
continue
|
|
409
|
+
certs.append(Certificate("R2 contained pattern",
|
|
410
|
+
f"map {small.map_id}: {sorted(cols)} within "
|
|
411
|
+
f"{sorted(leaves[j].columns)}"))
|
|
412
|
+
rest = leaves[:i] + leaves[i + 1:]
|
|
413
|
+
out = rest[0]
|
|
414
|
+
for x in rest[1:]:
|
|
415
|
+
out = O.Join(out, x)
|
|
416
|
+
return out if out.columns == op.columns else O.Project(out, op.columns)
|
|
417
|
+
return None
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
# -- R3: two patterns of one source row joined on a trusted key ---------------------
|
|
421
|
+
|
|
422
|
+
def _one_source(a: _Pattern, b: _Pattern) -> Optional[Tuple[_Pattern, _Pattern]]:
|
|
423
|
+
"""`(narrow, wide)` where `a` and `b` read rows of one source: one map;
|
|
424
|
+
or two maps of one table (the row filter of `wide` within `narrow`'s, so
|
|
425
|
+
every row `narrow` reads is a row `wide` reads: a node's property
|
|
426
|
+
and an edge on its table, `?li :price ?p ; :part ?x`); or two maps of
|
|
427
|
+
one rr:sqlQuery text (one property per triples map), or a plain
|
|
428
|
+
select of a table (`simple_select`, a link map with its EXISTS) and a
|
|
429
|
+
map of that table, the scan then the query's, which must project the
|
|
430
|
+
columns the other reads. None otherwise."""
|
|
431
|
+
if a.map_id == b.map_id and a.source == b.source:
|
|
432
|
+
return a, b
|
|
433
|
+
for narrow, wide in ((a, b), (b, a)):
|
|
434
|
+
plain = narrow.source.query is None and wide.source.query is None
|
|
435
|
+
if narrow.source.rows_within(wide.source, folds=plain) and _projects(narrow, wide) \
|
|
436
|
+
and (plain or _folds_agree(narrow, wide)):
|
|
437
|
+
return narrow, wide
|
|
438
|
+
return None
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def _projects(narrow: _Pattern, wide: _Pattern) -> bool:
|
|
442
|
+
"""The columns `wide` reads are among those the plain select `narrow` reads projects (all of them for
|
|
443
|
+
`*`); two tables, or two maps of one query text, share their columns by name."""
|
|
444
|
+
if narrow.source.query is None or wide.source.query is not None:
|
|
445
|
+
return True
|
|
446
|
+
from prograph.schema.source_map import simple_select
|
|
447
|
+
shape = simple_select(narrow.source.query)
|
|
448
|
+
if shape is None:
|
|
449
|
+
return False
|
|
450
|
+
return shape.columns is None or {r.column for r in wide.scan.reads if not r.parent} <= set(shape.columns)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def _folds_agree(a: _Pattern, b: _Pattern) -> bool:
|
|
454
|
+
"""The merged scan folds the columns either map writes undelimited (`Source.regular_columns`: the maps of
|
|
455
|
+
one query text, or a query and its table, name only their own), so each column either pattern reads must
|
|
456
|
+
be written in the merged set as its own map wrote it: no column read exactly by one that the other writes
|
|
457
|
+
undelimited."""
|
|
458
|
+
both = a.source.regular_columns | b.source.regular_columns
|
|
459
|
+
return all((r.column in p.source.regular_columns) == (r.column in both)
|
|
460
|
+
for p in (a, b) for r in p.scan.reads if not r.parent)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _exists_restated(p: _Pattern) -> bool:
|
|
464
|
+
"""Every semi-join of `p` reads only the columns it is joined on, so
|
|
465
|
+
it can be stated again over a merged scan (`_merged`)."""
|
|
466
|
+
for _, pairs, node_scan in p.exists:
|
|
467
|
+
if sorted(r.column for r in node_scan.reads) != sorted(k for _, k in pairs) \
|
|
468
|
+
or any(r.parent for r in node_scan.reads):
|
|
469
|
+
return False
|
|
470
|
+
return True
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _key_join(a: _Pattern, b: _Pattern, a_op: O.Op, b_op: O.Op) -> Optional[str]:
|
|
474
|
+
"""The shared column on which `a` and `b` meet one row each: both read
|
|
475
|
+
one source's rows (`_one_source`), every shared column is built alike
|
|
476
|
+
on both sides, and one of them is built by an injective map from
|
|
477
|
+
columns holding a key the wider side's source is trusted unique on
|
|
478
|
+
(`MapScan.unique_keys`): equal terms there are one source row. Each
|
|
479
|
+
side's semi-joins (an edge's ends proven to exist) are kept on the
|
|
480
|
+
merged scan. None where no column proves it."""
|
|
481
|
+
if a.scan is None or b.scan is None or a.distinct or b.distinct:
|
|
482
|
+
return None
|
|
483
|
+
sides = _one_source(a, b)
|
|
484
|
+
if sides is None or not (_exists_restated(a) and _exists_restated(b)):
|
|
485
|
+
return None
|
|
486
|
+
wide = sides[1]
|
|
487
|
+
shared = set(a_op.columns) & set(b_op.columns)
|
|
488
|
+
if not shared:
|
|
489
|
+
return None
|
|
490
|
+
mine, theirs = dict(a.outputs), dict(b.outputs)
|
|
491
|
+
if any(mine.get(c) != theirs.get(c) for c in shared):
|
|
492
|
+
return None
|
|
493
|
+
keys = [frozenset(k) for k in wide.scan.unique_keys]
|
|
494
|
+
by_source = {}
|
|
495
|
+
for scan in (b.scan, a.scan):
|
|
496
|
+
held = facts(scan).slots
|
|
497
|
+
by_source.update({r.column: held.get(r.name) for r in scan.reads if not r.parent})
|
|
498
|
+
for c in sorted(shared):
|
|
499
|
+
e = mine[c]
|
|
500
|
+
if isinstance(e, E.MakeTerm) and e.args and \
|
|
501
|
+
all(isinstance(x, E.Col) and x.name.startswith("src:") for _, x in e.args) and \
|
|
502
|
+
injective_over(e.map, {t: by_source.get(x.name[4:]) for t, x in e.args}):
|
|
503
|
+
cols = frozenset(x.name[4:] for _, x in e.args)
|
|
504
|
+
if any(k <= cols for k in keys):
|
|
505
|
+
return c
|
|
506
|
+
return None
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _merged(a: _Pattern, b: _Pattern, columns: Tuple[str, ...], optional: bool = False) -> O.Op:
|
|
510
|
+
"""One scan of the map giving both patterns' columns: its rows those
|
|
511
|
+
that meet both patterns' requirements and conditions. With `optional`
|
|
512
|
+
(`b` the right side of a left join) its rows are `a`'s, and
|
|
513
|
+
each column only `b` gives is its value where `b`'s requirements and
|
|
514
|
+
conditions hold on the row, else unbound."""
|
|
515
|
+
if optional:
|
|
516
|
+
narrow, wide = a, b
|
|
517
|
+
else:
|
|
518
|
+
narrow, wide = _one_source(a, b)
|
|
519
|
+
scan = narrow.scan
|
|
520
|
+
if narrow.source.regular_columns != wide.source.regular_columns:
|
|
521
|
+
# Each column is read as its own map wrote it (`_folds_agree`: no column in both differs).
|
|
522
|
+
both = narrow.source.regular_columns | wide.source.regular_columns
|
|
523
|
+
scan = dataclasses.replace(scan, map=dataclasses.replace(
|
|
524
|
+
scan.map, source=dataclasses.replace(scan.map.source, regular_columns=both)))
|
|
525
|
+
used: List[str] = []
|
|
526
|
+
|
|
527
|
+
def walk(e: E.Expr) -> None:
|
|
528
|
+
if isinstance(e, E.Col) and e.name.startswith("src:") and e.name[4:] not in used:
|
|
529
|
+
used.append(e.name[4:])
|
|
530
|
+
for f in dataclasses.fields(e) if dataclasses.is_dataclass(e) else ():
|
|
531
|
+
v = getattr(e, f.name)
|
|
532
|
+
if isinstance(v, E.Expr):
|
|
533
|
+
walk(v)
|
|
534
|
+
elif isinstance(v, tuple):
|
|
535
|
+
for x in v:
|
|
536
|
+
if isinstance(x, E.Expr):
|
|
537
|
+
walk(x)
|
|
538
|
+
elif isinstance(x, tuple):
|
|
539
|
+
for y in x:
|
|
540
|
+
if isinstance(y, E.Expr):
|
|
541
|
+
walk(y)
|
|
542
|
+
outputs = dict(a.outputs)
|
|
543
|
+
outputs.update(dict(b.outputs))
|
|
544
|
+
conditions = list(a.conditions | b.conditions)
|
|
545
|
+
for e in list(outputs.values()) + conditions:
|
|
546
|
+
walk(e)
|
|
547
|
+
exists = list(a.exists) + ([] if optional else [e for e in b.exists if e not in a.exists])
|
|
548
|
+
for c in sorted(a.required | b.required) + [s for _, pairs, _ in exists for s, _ in pairs]:
|
|
549
|
+
if c not in used:
|
|
550
|
+
used.append(c)
|
|
551
|
+
required = a.required if optional else a.required | b.required
|
|
552
|
+
only_b = [c for c in columns if c in dict(b.outputs) and c not in dict(a.outputs)]
|
|
553
|
+
if optional:
|
|
554
|
+
conditions = list(a.conditions)
|
|
555
|
+
by_column = {r.column: r for r in a.scan.reads + b.scan.reads + scan.reads if not r.parent}
|
|
556
|
+
names = {c: f"#r3.{i}" for i, c in enumerate(used)}
|
|
557
|
+
|
|
558
|
+
def local(e: E.Expr) -> E.Expr:
|
|
559
|
+
if isinstance(e, E.Col) and e.name.startswith("src:"):
|
|
560
|
+
return E.Col(names[e.name[4:]])
|
|
561
|
+
if isinstance(e, E.MakeTerm):
|
|
562
|
+
return E.MakeTerm(e.map, tuple((c, local(x)) for c, x in e.args))
|
|
563
|
+
if isinstance(e, E.Compare):
|
|
564
|
+
return E.Compare(e.op, local(e.left), local(e.right), e.semantics)
|
|
565
|
+
return e
|
|
566
|
+
reads = tuple(dataclasses.replace(by_column[c], name=names[c]) for c in used)
|
|
567
|
+
x: O.Op = O.MapScan(map=scan.map, reads=reads,
|
|
568
|
+
required=tuple(names[c] for c in used if c in required),
|
|
569
|
+
unique_keys=tuple(dict.fromkeys(scan.unique_keys + wide.scan.unique_keys)))
|
|
570
|
+
for _, pairs, node_scan in exists:
|
|
571
|
+
# The semi-join again, over the merged scan's names (`_exists_restated`).
|
|
572
|
+
to_name = {k: names[s] for s, k in pairs}
|
|
573
|
+
old = {r.name: r.column for r in node_scan.reads}
|
|
574
|
+
x = O.SemiJoin(x, dataclasses.replace(
|
|
575
|
+
node_scan, reads=tuple(dataclasses.replace(r, name=to_name[r.column]) for r in node_scan.reads),
|
|
576
|
+
required=tuple(to_name[old[n]] for n in node_scan.required if n in old)))
|
|
577
|
+
matched: Optional[E.Expr] = None
|
|
578
|
+
if optional:
|
|
579
|
+
tests = [E.IsBound(E.Col(names[c])) for c in sorted(b.required - a.required)]
|
|
580
|
+
tests += [local(c) for c in sorted(b.conditions - a.conditions, key=repr)]
|
|
581
|
+
matched = tests[0] if len(tests) == 1 else E.And(tuple(tests)) if tests else None
|
|
582
|
+
for c in columns:
|
|
583
|
+
e = local(outputs[c])
|
|
584
|
+
if c in only_b and matched is not None:
|
|
585
|
+
e = E.If(matched, e, E.Const(None))
|
|
586
|
+
x = O.Extend(x, c, e)
|
|
587
|
+
conds = [local(c) for c in conditions]
|
|
588
|
+
if conds:
|
|
589
|
+
x = O.Filter(x, conds[0] if len(conds) == 1 else E.And(tuple(conds)))
|
|
590
|
+
return O.Project(x, columns)
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
@home("self-join-on-a-key", "Two patterns of one source row on a trusted key are one scan (R3, R3o)",
|
|
594
|
+
terms=("self-join", "self join", "R3", "R3o", "one scan", "own row", "same row", "trusted key",
|
|
595
|
+
"OPTIONAL", "left join", "optional property", "_own_row", "join elimination"))
|
|
596
|
+
def r3_self_join(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
597
|
+
"""Two patterns of one map joined on a column built from a trusted key
|
|
598
|
+
of its source (`_key_join`) are one scan of it (m2-performance.md 2.2,
|
|
599
|
+
R3): each row of either side meets exactly the one row of the other
|
|
600
|
+
read from the same source row. Without a trusted key it does not fire:
|
|
601
|
+
two rows of one key with names A, B and ages 1, 2 give four pairs, one
|
|
602
|
+
scan two."""
|
|
603
|
+
if not isinstance(op, O.Join) or op.condition is not None:
|
|
604
|
+
return None
|
|
605
|
+
leaves: List[O.Op] = []
|
|
606
|
+
|
|
607
|
+
def flat(x: O.Op) -> None:
|
|
608
|
+
if isinstance(x, O.Join) and x.condition is None:
|
|
609
|
+
flat(x.left)
|
|
610
|
+
flat(x.right)
|
|
611
|
+
else:
|
|
612
|
+
leaves.append(x)
|
|
613
|
+
flat(op)
|
|
614
|
+
pats = [_pattern(x) for x in leaves]
|
|
615
|
+
for i in range(len(leaves)):
|
|
616
|
+
for j in range(i + 1, len(leaves)):
|
|
617
|
+
a, b = pats[i], pats[j]
|
|
618
|
+
if a is None or b is None:
|
|
619
|
+
continue
|
|
620
|
+
c = _key_join(a, b, leaves[i], leaves[j])
|
|
621
|
+
if c is None:
|
|
622
|
+
continue
|
|
623
|
+
cols = tuple(dict.fromkeys(leaves[i].columns + leaves[j].columns))
|
|
624
|
+
wide = _one_source(a, b)[1]
|
|
625
|
+
maps = a.map_id if a.map_id == b.map_id else f"{a.map_id} and {b.map_id} (one table)"
|
|
626
|
+
certs.append(Certificate("R3 self-join on a trusted key",
|
|
627
|
+
f"map {maps}, column {c}, keys "
|
|
628
|
+
f"{[list(k) for k in wide.scan.unique_keys]}"))
|
|
629
|
+
merged = _merged(a, b, cols)
|
|
630
|
+
# In the first one's place: each leaf then follows at least the columns it followed, so the
|
|
631
|
+
# written order stays connected (rule join-order; at the front, a leaf tied only to a later
|
|
632
|
+
# one became a cross join, lineitem x part in TPC-H q17).
|
|
633
|
+
rest = [merged if k == i else x for k, x in enumerate(leaves) if k != j]
|
|
634
|
+
out = rest[0]
|
|
635
|
+
for x in rest[1:]:
|
|
636
|
+
out = O.Join(out, x)
|
|
637
|
+
return out if out.columns == op.columns else O.Project(out, op.columns)
|
|
638
|
+
return None
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
def r3_left_join(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
642
|
+
"""R3's left-join form (pre-1.0.0's `_own_row`): an OPTIONAL
|
|
643
|
+
pattern `b` of the map a pattern `a` of the left side reads, joined on a
|
|
644
|
+
column built from a trusted key of its source (`_key_join`), is that
|
|
645
|
+
row's columns: each row of `a` meets at most the one row of `b` read
|
|
646
|
+
from the same source row, and `b` has it exactly where its requirements
|
|
647
|
+
and conditions hold there (`_merged` with `optional`). `b` shares no
|
|
648
|
+
column with the left side outside `a`, so the rest of the left side's
|
|
649
|
+
joins are kept around it. Not with a left join's own condition."""
|
|
650
|
+
if not isinstance(op, O.LeftJoin) or op.condition is not None:
|
|
651
|
+
return None
|
|
652
|
+
b = _pattern(op.right)
|
|
653
|
+
if b is None:
|
|
654
|
+
return None
|
|
655
|
+
leaves: List[O.Op] = []
|
|
656
|
+
|
|
657
|
+
def flat(x: O.Op) -> None:
|
|
658
|
+
if isinstance(x, O.Join) and x.condition is None:
|
|
659
|
+
flat(x.left)
|
|
660
|
+
flat(x.right)
|
|
661
|
+
else:
|
|
662
|
+
leaves.append(x)
|
|
663
|
+
flat(op.left)
|
|
664
|
+
shared = set(op.left.columns) & set(op.right.columns)
|
|
665
|
+
for i, leaf in enumerate(leaves):
|
|
666
|
+
a = _pattern(leaf)
|
|
667
|
+
if a is None or not shared <= set(leaf.columns):
|
|
668
|
+
continue
|
|
669
|
+
if a.map_id != b.map_id or a.source != b.source or a.exists or b.exists:
|
|
670
|
+
continue # R3o stays within one map, no semi-join (R3's wider form is for inner joins)
|
|
671
|
+
c = _key_join(a, b, leaf, op.right)
|
|
672
|
+
if c is None:
|
|
673
|
+
continue
|
|
674
|
+
cols = tuple(dict.fromkeys(leaf.columns + op.right.columns))
|
|
675
|
+
certs.append(Certificate("R3 left join on a trusted key",
|
|
676
|
+
f"map {a.map_id}, column {c}, keys "
|
|
677
|
+
f"{[list(k) for k in a.scan.unique_keys]}"))
|
|
678
|
+
rest = [_merged(a, b, cols, optional=True) if k == i else x for k, x in enumerate(leaves)]
|
|
679
|
+
out = rest[0]
|
|
680
|
+
for x in rest[1:]:
|
|
681
|
+
out = O.Join(out, x)
|
|
682
|
+
return out if out.columns == op.columns else O.Project(out, op.columns)
|
|
683
|
+
return None
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
# -- R9: a union branch another branch already gives ---------------------------------
|
|
687
|
+
|
|
688
|
+
def r9_union_containment(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
689
|
+
"""`Distinct(Union(..., X, ..., Y, ...))` without X where every row of X is
|
|
690
|
+
a row of Y (`_contains` over one source): the same columns,
|
|
691
|
+
each built alike from the same source row, and Y requiring no more of the
|
|
692
|
+
row than X. An ontology's saturation makes such branches: a map giving
|
|
693
|
+
`ex:surname` and `ex:name` from one column gives `ex:name` twice, and two
|
|
694
|
+
maps of one table typed Manager and Employee give the Employee subjects
|
|
695
|
+
twice. Under the Distinct a row given twice is one row."""
|
|
696
|
+
if not isinstance(op, O.Distinct) or not isinstance(op.input, O.Union):
|
|
697
|
+
return None
|
|
698
|
+
branches = list(op.input.inputs)
|
|
699
|
+
pats = [_pattern(b) for b in branches]
|
|
700
|
+
for i, (x, px) in enumerate(zip(branches, pats)):
|
|
701
|
+
if px is None:
|
|
702
|
+
continue
|
|
703
|
+
for j, (y, py) in enumerate(zip(branches, pats)):
|
|
704
|
+
if i == j or py is None or set(x.columns) != set(y.columns):
|
|
705
|
+
continue
|
|
706
|
+
if px.exists or py.exists or not _contains(px, py, any_map=True):
|
|
707
|
+
continue
|
|
708
|
+
if _contains(py, px, any_map=True) and j > i:
|
|
709
|
+
continue # the same rows both ways: keep the first, drop the later one
|
|
710
|
+
certs.append(Certificate("R9 union branch within another",
|
|
711
|
+
f"map {px.map_id} within map {py.map_id}"))
|
|
712
|
+
rest = tuple(b for k, b in enumerate(branches) if k != i)
|
|
713
|
+
inner = rest[0] if len(rest) == 1 else O.Union(rest)
|
|
714
|
+
if inner.columns != op.input.columns:
|
|
715
|
+
inner = O.Project(inner, op.input.columns)
|
|
716
|
+
return O.Distinct(inner)
|
|
717
|
+
return None
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
# -- R10: one scan per source for a union of its branches ------------------------------
|
|
721
|
+
|
|
722
|
+
def _src_columns(e: E.Expr, out: List[str]) -> None:
|
|
723
|
+
"""The source columns (`src:` names) `e` reads, in order of first reading."""
|
|
724
|
+
if isinstance(e, E.Col):
|
|
725
|
+
if e.name.startswith("src:") and e.name[4:] not in out:
|
|
726
|
+
out.append(e.name[4:])
|
|
727
|
+
return
|
|
728
|
+
for child in e.children():
|
|
729
|
+
_src_columns(child, out)
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
def _branch_of(x: O.Op):
|
|
733
|
+
"""A union branch as (its constant columns, its pattern), or None: `Extend`s of constants (a
|
|
734
|
+
variable predicate's term) over one map's pattern with no existence proof and no graph map."""
|
|
735
|
+
consts: Dict[str, E.Expr] = {}
|
|
736
|
+
while isinstance(x, O.Extend) and isinstance(x.expr, E.Const):
|
|
737
|
+
consts.setdefault(x.name, x.expr)
|
|
738
|
+
x = x.input
|
|
739
|
+
narrowed = None
|
|
740
|
+
if isinstance(x, O.Project) and isinstance(x.input, O.Distinct):
|
|
741
|
+
# A narrowing of a distinct pattern (a typing branch keeps the subject of its pattern):
|
|
742
|
+
# its rows are no set, so it is not distinct; under a group's Distinct that is the same.
|
|
743
|
+
narrowed, x = x.names, x.input
|
|
744
|
+
p = _pattern(x)
|
|
745
|
+
if p is None or p.scan is None or p.scan.graph is not None or p.scan.joint is not None:
|
|
746
|
+
return None
|
|
747
|
+
if narrowed is not None:
|
|
748
|
+
if not set(narrowed) <= set(dict(p.outputs)):
|
|
749
|
+
return None
|
|
750
|
+
p = dataclasses.replace(p, outputs=tuple((n, e) for n, e in p.outputs if n in narrowed), distinct=False)
|
|
751
|
+
return consts, p
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def _as_read(e: E.Expr, p) -> E.Expr:
|
|
755
|
+
"""A pattern's expression (source columns, `src:`) over its scan's read names."""
|
|
756
|
+
names = {r.column: r.name for r in p.scan.reads if not r.parent}
|
|
757
|
+
if isinstance(e, E.Col) and e.name.startswith("src:"):
|
|
758
|
+
return E.Col(names[e.name[4:]])
|
|
759
|
+
if isinstance(e, E.MakeTerm):
|
|
760
|
+
return E.MakeTerm(e.map, tuple((c, _as_read(x, p)) for c, x in e.args))
|
|
761
|
+
if isinstance(e, E.Compare):
|
|
762
|
+
return E.Compare(e.op, _as_read(e.left, p), _as_read(e.right, p), e.semantics)
|
|
763
|
+
return e
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def _form(r) -> Tuple:
|
|
767
|
+
"""How a scan reads a column (held datatype, zone, digits, reader): one column read in two
|
|
768
|
+
forms by two members is two reads of the merged scan (Spark refused a double read as text)."""
|
|
769
|
+
return (r.parent, r.held, r.zone, r.digits, r.reader)
|
|
770
|
+
|
|
771
|
+
|
|
772
|
+
def _proof(p) -> Tuple:
|
|
773
|
+
"""A pattern's existence proofs as a key: (node map, its column pairs) each; members with one key
|
|
774
|
+
share one semi-join."""
|
|
775
|
+
return tuple(sorted((node_id, tuple(sorted(pairs))) for node_id, pairs, _ in p.exists))
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def _outermost_unions(op: O.Op, certs: List[Certificate]) -> O.Op:
|
|
779
|
+
"""R10 at each outermost union, before the bottom-up passes: it flattens the Distinct(Union)
|
|
780
|
+
groups below it (a predicate's saturated map copies), which a bottom-up pass would merge one group
|
|
781
|
+
at a time and so never across groups (turbofan ?p: 409 scans left that way)."""
|
|
782
|
+
if isinstance(op, O.Union):
|
|
783
|
+
merged = r10_one_scan_per_source(op, certs)
|
|
784
|
+
if merged is not None:
|
|
785
|
+
if _CHECK:
|
|
786
|
+
_check_firing(r10_one_scan_per_source, op, merged)
|
|
787
|
+
return merged
|
|
788
|
+
changes = {}
|
|
789
|
+
for f in dataclasses.fields(op):
|
|
790
|
+
v = getattr(op, f.name)
|
|
791
|
+
if isinstance(v, O.Op):
|
|
792
|
+
nv = _outermost_unions(v, certs)
|
|
793
|
+
if nv is not v:
|
|
794
|
+
changes[f.name] = nv
|
|
795
|
+
elif isinstance(v, tuple) and v and all(isinstance(x, O.Op) for x in v):
|
|
796
|
+
nv = tuple(_outermost_unions(x, certs) for x in v)
|
|
797
|
+
if any(a is not b for a, b in zip(nv, v)):
|
|
798
|
+
changes[f.name] = nv
|
|
799
|
+
return dataclasses.replace(op, **changes) if changes else op
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def _groups_of(op: O.Union):
|
|
803
|
+
"""The union's inputs as groups of members, each member (its constant columns, its pattern):
|
|
804
|
+
an input is a group of one, or, under a Distinct, the union it holds (a predicate's map copies,
|
|
805
|
+
saturated). None where an input is neither; each group's Distinct flag beside it."""
|
|
806
|
+
groups = []
|
|
807
|
+
for x in op.inputs:
|
|
808
|
+
consts: Dict[str, E.Expr] = {}
|
|
809
|
+
y = x
|
|
810
|
+
while isinstance(y, O.Extend) and isinstance(y.expr, E.Const):
|
|
811
|
+
consts.setdefault(y.name, y.expr)
|
|
812
|
+
y = y.input
|
|
813
|
+
if isinstance(y, O.Distinct) and isinstance(y.input, O.Union):
|
|
814
|
+
members, raw = [], []
|
|
815
|
+
|
|
816
|
+
def expand(m: O.Op, outer: Dict[str, E.Expr]) -> None:
|
|
817
|
+
if _no_rows(m):
|
|
818
|
+
# A member the facts prove empty (a required column never bound) gives no row:
|
|
819
|
+
# left out, as the passes below would drop it, never merged into a scan where
|
|
820
|
+
# its term would still be built.
|
|
821
|
+
return
|
|
822
|
+
b = _branch_of(m)
|
|
823
|
+
if b is not None:
|
|
824
|
+
members.append(({**outer, **b[0]}, b[1], m, dict(outer)))
|
|
825
|
+
return
|
|
826
|
+
inner_consts = dict(outer)
|
|
827
|
+
z = m
|
|
828
|
+
while isinstance(z, O.Extend) and isinstance(z.expr, E.Const):
|
|
829
|
+
inner_consts.setdefault(z.name, z.expr)
|
|
830
|
+
z = z.input
|
|
831
|
+
if isinstance(z, O.Project) and isinstance(z.input, O.Distinct) and isinstance(z.input.input, O.Union):
|
|
832
|
+
z = z.input
|
|
833
|
+
if isinstance(z, O.Distinct) and isinstance(z.input, O.Union):
|
|
834
|
+
# A group within the group (a superclass's typing branches): the outer Distinct
|
|
835
|
+
# covers its own, so its members join the group.
|
|
836
|
+
for n in z.input.inputs:
|
|
837
|
+
expand(n, inner_consts)
|
|
838
|
+
return
|
|
839
|
+
raw.append((m, outer)) # kept as it is, its rows deduplicated with the group's
|
|
840
|
+
for m in y.input.inputs:
|
|
841
|
+
expand(m, dict(consts))
|
|
842
|
+
groups.append((members, True, x, consts, raw, False))
|
|
843
|
+
continue
|
|
844
|
+
m = None if _no_rows(y) else _branch_of(y)
|
|
845
|
+
if m is None:
|
|
846
|
+
groups.append((None, isinstance(x, O.Distinct) or facts(x).distinct, x, {}, [], False))
|
|
847
|
+
continue
|
|
848
|
+
# A set without its own Distinct (its rows keyed, as R1 reads the facts): then no Distinct is
|
|
849
|
+
# needed above it once merged.
|
|
850
|
+
below = y.input if isinstance(y, O.Distinct) else y
|
|
851
|
+
keyed = facts(below).distinct
|
|
852
|
+
# A set: its own Distinct, or rows the facts prove distinct (R1 removed that Distinct).
|
|
853
|
+
groups.append(([({**consts, **m[0]}, m[1], y, dict(consts))], m[1].distinct or keyed, x, {}, [], keyed))
|
|
854
|
+
return groups
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
def r10_one_scan_per_source(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
858
|
+
"""A union's branches over one source (pre-1.0.0's
|
|
859
|
+
`_each_row_once`) read it once: one scan of the source, joined to a Values row per branch (its
|
|
860
|
+
index), each output the branch's term for its row (`If` on the index, so each branch's term
|
|
861
|
+
keeps its own kind, datatype and SQL type, `lower.if_cell`), the rows a branch gives kept by its
|
|
862
|
+
own requirements and conditions. The same rows: each source row meets each branch's Values row,
|
|
863
|
+
and the filter keeps the pairs that branch gave. Every input distinct (a predicate's saturated
|
|
864
|
+
map copies under their Distinct, or a pattern's own): one Distinct over (input, outputs) above
|
|
865
|
+
the merged scans keeps each input's set. `?s ?p ?o` over the turbofan ontology's saturated maps
|
|
866
|
+
read one scan per map copy (1270 CTEs, past DuckDB's binder); now one per source."""
|
|
867
|
+
if not isinstance(op, O.Union):
|
|
868
|
+
return None
|
|
869
|
+
groups = _groups_of(op)
|
|
870
|
+
if groups is None or not all(g[1] for g in groups):
|
|
871
|
+
return None # an input that is not a set: one Distinct above would change its rows
|
|
872
|
+
members = [(gi, consts, p, (mop, outer)) for gi, (ms, _, _, _, _, _) in enumerate(groups) if ms is not None
|
|
873
|
+
for consts, p, mop, outer in ms]
|
|
874
|
+
by_source: Dict[Any, List[int]] = {}
|
|
875
|
+
for k, (_, _, p, _) in enumerate(members):
|
|
876
|
+
by_source.setdefault((p.source, _proof(p)), []).append(k)
|
|
877
|
+
# A member whose own terms or conditions may raise (`lower.never_raises_over`, NEVER_RAISES, the
|
|
878
|
+
# check the one-plan rewrites use): kept as its own branch, since in a merged scan another member's row could reach
|
|
879
|
+
# them (Spark does not promise a CASE guards an error; a date's text raises outside years 1-9999).
|
|
880
|
+
from prograph.plan.ir.lower import never_raises_over
|
|
881
|
+
apart: List[int] = []
|
|
882
|
+
for key, ks in list(by_source.items()):
|
|
883
|
+
if len(ks) < 2:
|
|
884
|
+
continue
|
|
885
|
+
keep = []
|
|
886
|
+
# The merged scan requires only what every member requires (`_one_scan`): each member's terms
|
|
887
|
+
# and conditions are checked over that scan, its other columns possibly unbound (fresh review
|
|
888
|
+
# 2026-11-11, an IRI column one member required was built, and raised, on a row
|
|
889
|
+
# where it did not hold a valid IRI).
|
|
890
|
+
every = set.intersection(*[set(members[j][2].required) for j in ks])
|
|
891
|
+
for k in ks:
|
|
892
|
+
_, _, p, _ = members[k]
|
|
893
|
+
p_merged = dataclasses.replace(p, scan=dataclasses.replace(
|
|
894
|
+
p.scan, required=tuple(r.name for r in p.scan.reads if not r.parent and r.column in every)))
|
|
895
|
+
outs = dict(p.outputs)
|
|
896
|
+
own = [(e, op.slots.get(c)) for c, e in outs.items()
|
|
897
|
+
if any(dict(members[j][2].outputs).get(c) != e for j in ks if j != k)]
|
|
898
|
+
others = set.intersection(*[set(members[j][2].conditions) for j in ks if j != k])
|
|
899
|
+
conds = [c for c in p.conditions if c not in others]
|
|
900
|
+
if never_raises_over(p_merged.scan, [(_as_read(e, p), s) for e, s in own], [_as_read(c, p) for c in conds]):
|
|
901
|
+
keep.append(k)
|
|
902
|
+
else:
|
|
903
|
+
apart.append(k)
|
|
904
|
+
by_source[key] = keep
|
|
905
|
+
# Merged only where a condition narrows each member's rows (a constant subject or object):
|
|
906
|
+
# there one scan of the few rows beats a scan per branch (LDBC <person/14> ?p ?o, 1.02 ->
|
|
907
|
+
# 0.8 s on the warehouse); over a whole table it multiplies every row by the Values list and reads
|
|
908
|
+
# every column, where the branches each read one (COUNT(*) { ?s ?p ?o } 2.0 s as branches, 7 s
|
|
909
|
+
# merged), and the lowering keeps the branches in one CTE (`Lowering._Union`).
|
|
910
|
+
# A source left unmerged keeps each of its members as its own branch, as one kept apart.
|
|
911
|
+
narrowed = {key: ks for key, ks in by_source.items() if ks and all(members[k][2].conditions for k in ks)}
|
|
912
|
+
apart += [k for key, ks in by_source.items() if key not in narrowed for k in ks]
|
|
913
|
+
by_source = narrowed
|
|
914
|
+
if all(len(v) < 2 for v in by_source.values()):
|
|
915
|
+
return None # nothing merges (members kept apart alone are no reason to rebuild the union)
|
|
916
|
+
cols = op.columns
|
|
917
|
+
g = "#r10.g"
|
|
918
|
+
parts: List[O.Op] = []
|
|
919
|
+
# The columns the union binds in every row: each merged part says so (an IsBound that every row
|
|
920
|
+
# meets, since each member's own requirements are in its guard), so no slot gets wider.
|
|
921
|
+
always = [c for c in cols if op.slots[c].present is Bound.ALWAYS]
|
|
922
|
+
# One Values for every group (rows 0 .. the largest group's size - 1), one CTE however many
|
|
923
|
+
# groups read it (the lowering keeps one CTE per operator object); a group's guards keep its own.
|
|
924
|
+
from prograph.plan.ir.terms import LiteralTerm as _Lit
|
|
925
|
+
from prograph.schema import xsd as _x
|
|
926
|
+
size = max(len(ks) for ks in by_source.values())
|
|
927
|
+
shared = O.Values(("#r10.b",), tuple((_Lit(str(i), _x.INTEGER),) for i in range(size)))
|
|
928
|
+
for ks in by_source.values():
|
|
929
|
+
part = _one_scan([members[k][:3] for k in ks], cols, g, shared)
|
|
930
|
+
if always:
|
|
931
|
+
keep = [E.IsBound(E.Col(c)) for c in always]
|
|
932
|
+
part = O.Filter(part, keep[0] if len(keep) == 1 else E.And(tuple(keep)))
|
|
933
|
+
parts.append(part)
|
|
934
|
+
from prograph.plan.ir.terms import LiteralTerm
|
|
935
|
+
from prograph.schema import xsd as _xsd
|
|
936
|
+
for k in apart: # a member kept apart: its own branch, with its group's id and constants
|
|
937
|
+
gi, _, _, (mop, outer) = members[k]
|
|
938
|
+
for name_, e in outer.items():
|
|
939
|
+
if name_ not in mop.columns:
|
|
940
|
+
mop = O.Extend(mop, name_, e)
|
|
941
|
+
part = O.Project(O.Extend(mop, g, E.Const(LiteralTerm(str(gi), _xsd.INTEGER))), (g,) + tuple(cols))
|
|
942
|
+
parts.append(part)
|
|
943
|
+
for gi, (ms, _, x, consts, raw, _) in enumerate(groups):
|
|
944
|
+
tag = E.Const(LiteralTerm(str(gi), _xsd.INTEGER))
|
|
945
|
+
if ms is None: # an input no scan merges: as it is, with its id
|
|
946
|
+
parts.append(O.Project(O.Extend(x, g, tag), (g,) + tuple(cols)))
|
|
947
|
+
for m, outer in raw: # a group's member no scan merges: with the constants around it
|
|
948
|
+
for name, e in outer.items():
|
|
949
|
+
if name not in m.columns:
|
|
950
|
+
m = O.Extend(m, name, e)
|
|
951
|
+
parts.append(O.Project(O.Extend(m, g, tag), (g,) + tuple(cols)))
|
|
952
|
+
certs.append(Certificate("R10 one scan per source",
|
|
953
|
+
f"{len(members)} branches over {len(by_source)} sources"))
|
|
954
|
+
body = parts[0] if len(parts) == 1 else O.Union(tuple(parts))
|
|
955
|
+
if facts(op).distinct:
|
|
956
|
+
# The union's rows were a set (its branches apart, a predicate each): one Distinct over its
|
|
957
|
+
# columns removes no row and keeps that fact for the plan above.
|
|
958
|
+
return O.Distinct(O.Project(body, tuple(cols)))
|
|
959
|
+
if all(gr[5] for gr in groups):
|
|
960
|
+
# Every input a single pattern whose rows are a set by their key: the union of their rows as
|
|
961
|
+
# it was, no Distinct (R1's facts must still hold after the merge).
|
|
962
|
+
return O.Project(body, tuple(cols))
|
|
963
|
+
return O.Project(O.Distinct(O.Project(body, (g,) + tuple(cols))), tuple(cols))
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def _one_scan(members, cols: Tuple[str, ...], g: str, values: Optional[O.Op] = None) -> O.Op:
|
|
967
|
+
"""One scan of the members' source giving each member's rows, with its group's id in `g`."""
|
|
968
|
+
forms = [{r.column: _form(r) for r in p.scan.reads if not r.parent} for _, _, p in members]
|
|
969
|
+
used: List[Tuple[str, Tuple]] = []
|
|
970
|
+
for k, (_, consts, p) in enumerate(members):
|
|
971
|
+
cols_k: List[str] = []
|
|
972
|
+
for _, e in p.outputs:
|
|
973
|
+
_src_columns(e, cols_k)
|
|
974
|
+
for c in sorted(p.conditions, key=repr):
|
|
975
|
+
_src_columns(c, cols_k)
|
|
976
|
+
for c in sorted(p.required):
|
|
977
|
+
if c not in cols_k:
|
|
978
|
+
cols_k.append(c)
|
|
979
|
+
for _, pairs, _ in p.exists:
|
|
980
|
+
for src, _ in pairs:
|
|
981
|
+
if src not in cols_k:
|
|
982
|
+
cols_k.append(src)
|
|
983
|
+
for c in cols_k:
|
|
984
|
+
key = (c, forms[k].get(c))
|
|
985
|
+
if key not in used:
|
|
986
|
+
used.append(key)
|
|
987
|
+
by_key = {}
|
|
988
|
+
for k, (_, _, p) in enumerate(members):
|
|
989
|
+
for r in p.scan.reads:
|
|
990
|
+
if not r.parent:
|
|
991
|
+
by_key.setdefault((r.column, _form(r)), r)
|
|
992
|
+
names = {key: f"#r10.{i}" for i, key in enumerate(used)}
|
|
993
|
+
|
|
994
|
+
def local(e: E.Expr, k: int = 0) -> E.Expr:
|
|
995
|
+
if isinstance(e, E.Col) and e.name.startswith("src:"):
|
|
996
|
+
c = e.name[4:]
|
|
997
|
+
return E.Col(names[(c, forms[k].get(c))])
|
|
998
|
+
if isinstance(e, E.MakeTerm):
|
|
999
|
+
return E.MakeTerm(e.map, tuple((c, local(x, k)) for c, x in e.args))
|
|
1000
|
+
if isinstance(e, E.Compare):
|
|
1001
|
+
return E.Compare(e.op, local(e.left, k), local(e.right, k), e.semantics)
|
|
1002
|
+
return e
|
|
1003
|
+
|
|
1004
|
+
def name(c: str, k: int) -> str:
|
|
1005
|
+
return names[(c, forms[k].get(c))]
|
|
1006
|
+
from prograph.plan.ir.terms import LiteralTerm
|
|
1007
|
+
from prograph.schema import xsd as _xsd
|
|
1008
|
+
|
|
1009
|
+
def number(i: int) -> E.Const:
|
|
1010
|
+
return E.Const(LiteralTerm(str(i), _xsd.INTEGER))
|
|
1011
|
+
first = members[0][2].scan
|
|
1012
|
+
# Required: the columns every member requires (in one form), so their slots are bound in every
|
|
1013
|
+
# row, as in each member's own scan.
|
|
1014
|
+
every_req = set.intersection(*[set(p.required) for _, _, p in members])
|
|
1015
|
+
x: O.Op = O.MapScan(map=first.map, reads=tuple(dataclasses.replace(by_key[key], name=names[key]) for key in used),
|
|
1016
|
+
required=tuple(names[key] for key in used if key[0] in every_req
|
|
1017
|
+
and all(f.get(key[0]) == key[1] for f in forms)),
|
|
1018
|
+
unique_keys=first.unique_keys)
|
|
1019
|
+
# The members' one existence proof (they share it: `_proof`): a semi-join into the node map on
|
|
1020
|
+
# the source's columns, as each member had it.
|
|
1021
|
+
for node_id, pairs, node_scan in members[0][2].exists:
|
|
1022
|
+
theirs = {r.column: r for r in node_scan.reads}
|
|
1023
|
+
right_reads = tuple(dataclasses.replace(theirs[key], name=name(src, 0)) for src, key in pairs)
|
|
1024
|
+
right = O.MapScan(map=node_scan.map, reads=right_reads, required=tuple(r.name for r in right_reads),
|
|
1025
|
+
unique_keys=node_scan.unique_keys)
|
|
1026
|
+
x = O.SemiJoin(x, right)
|
|
1027
|
+
if len(members) == 1:
|
|
1028
|
+
gi, consts, p = members[0]
|
|
1029
|
+
pre = [E.IsBound(E.Col(name(c, 0))) for c in sorted(p.required)] + [local(c, 0) for c in sorted(p.conditions, key=repr)]
|
|
1030
|
+
if pre:
|
|
1031
|
+
x = O.Filter(x, pre[0] if len(pre) == 1 else E.And(tuple(pre)))
|
|
1032
|
+
x = O.Extend(x, g, number(gi))
|
|
1033
|
+
outs = dict(p.outputs)
|
|
1034
|
+
for c in cols:
|
|
1035
|
+
x = O.Extend(x, c, consts[c] if c in consts else local(outs[c], 0) if c in outs else E.Const(None))
|
|
1036
|
+
return O.Project(x, (g,) + tuple(cols))
|
|
1037
|
+
# What every member requires and every condition they share filter the scan itself (a constant
|
|
1038
|
+
# subject is pushed into it, as each branch had it).
|
|
1039
|
+
one_form = {c for c in set.intersection(*[set(f) for f in forms]) if len({f[c] for f in forms}) == 1}
|
|
1040
|
+
|
|
1041
|
+
def reads_one_form(e: E.Expr) -> bool:
|
|
1042
|
+
cs: List[str] = []
|
|
1043
|
+
_src_columns(e, cs)
|
|
1044
|
+
return all(c in one_form for c in cs)
|
|
1045
|
+
shared_req = {c for c in set.intersection(*[set(p.required) for _, _, p in members]) if c in one_form}
|
|
1046
|
+
shared_cond = {c for c in set.intersection(*[set(p.conditions) for _, _, p in members]) if reads_one_form(c)}
|
|
1047
|
+
pre = [E.IsBound(E.Col(name(c, 0))) for c in sorted(shared_req)] + [local(c, 0) for c in sorted(shared_cond, key=repr)]
|
|
1048
|
+
if pre:
|
|
1049
|
+
x = O.Filter(x, pre[0] if len(pre) == 1 else E.And(tuple(pre)))
|
|
1050
|
+
b = "#r10.b"
|
|
1051
|
+
x = O.Join(x, values if values is not None else
|
|
1052
|
+
O.Values((b,), tuple((LiteralTerm(str(i), _xsd.INTEGER),) for i in range(len(members)))))
|
|
1053
|
+
|
|
1054
|
+
def is_member(i: int) -> E.Expr:
|
|
1055
|
+
return E.Compare("=", E.Col(b), number(i), E.TERM)
|
|
1056
|
+
guards = []
|
|
1057
|
+
for i, (_, _, p) in enumerate(members):
|
|
1058
|
+
own = [E.IsBound(E.Col(name(c, i))) for c in sorted(set(p.required) - shared_req)]
|
|
1059
|
+
own += [local(c, i) for c in sorted(p.conditions - shared_cond, key=repr)]
|
|
1060
|
+
guards.append(E.And((is_member(i), *own)) if own else is_member(i))
|
|
1061
|
+
x = O.Filter(x, E.Or(tuple(guards)))
|
|
1062
|
+
|
|
1063
|
+
def chain(pick) -> E.Expr:
|
|
1064
|
+
# The last member's value is the chain's end: the filter above keeps only rows of some
|
|
1065
|
+
# member, so no row reaches an unbound end, and a column bound in every member stays so.
|
|
1066
|
+
out: E.Expr = pick(len(members) - 1)
|
|
1067
|
+
for i in reversed(range(len(members) - 1)):
|
|
1068
|
+
out = E.If(is_member(i), pick(i), out)
|
|
1069
|
+
return out
|
|
1070
|
+
x = O.Extend(x, g, chain(lambda i: number(members[i][0])))
|
|
1071
|
+
for c in cols:
|
|
1072
|
+
def pick(i: int, c=c) -> E.Expr:
|
|
1073
|
+
_, consts, p = members[i]
|
|
1074
|
+
outs = dict(p.outputs)
|
|
1075
|
+
return consts[c] if c in consts else local(outs[c], i) if c in outs else E.Const(None)
|
|
1076
|
+
x = O.Extend(x, c, chain(pick))
|
|
1077
|
+
return O.Project(x, (g,) + tuple(cols))
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
# -- R4: a type pattern an existence proof implies ----------------------------------
|
|
1081
|
+
|
|
1082
|
+
def _subject_only(t: _Pattern) -> Optional[Tuple[str, E.MakeTerm]]:
|
|
1083
|
+
"""`t` as a node map's subjects and nothing else: one output, its
|
|
1084
|
+
subject term built from the map's key columns, no condition, nothing
|
|
1085
|
+
required beyond those columns (a type pattern, `?x a :C`)."""
|
|
1086
|
+
if t.scan is None or not isinstance(t.scan.map, NodeMap) or len(t.outputs) != 1 or t.conditions:
|
|
1087
|
+
return None
|
|
1088
|
+
name, e = t.outputs[0]
|
|
1089
|
+
m = t.scan.map
|
|
1090
|
+
if not isinstance(e, E.MakeTerm) or e.map != m.subject:
|
|
1091
|
+
return None
|
|
1092
|
+
cols = [x.name[4:] for _, x in e.args if isinstance(x, E.Col) and x.name.startswith("src:")]
|
|
1093
|
+
if len(cols) != len(e.args) or set(cols) != set(m.key.columns) or not t.required <= set(cols):
|
|
1094
|
+
return None
|
|
1095
|
+
return name, e
|
|
1096
|
+
|
|
1097
|
+
|
|
1098
|
+
def _implied(t: _Pattern, t_op: O.Op, x: _Pattern, x_op: O.Op) -> Optional[str]:
|
|
1099
|
+
"""Every row of `x` meets exactly one row of `t` (`t` the subjects of a
|
|
1100
|
+
node map N, distinct): `x` builds the shared column from columns it
|
|
1101
|
+
has proven to exist among N's keys (a semi-join into N's scan, on
|
|
1102
|
+
columns of the same types), by a template of the same shape as N's
|
|
1103
|
+
subject, the same values in the same places. Then the join adds
|
|
1104
|
+
nothing."""
|
|
1105
|
+
sub = _subject_only(t)
|
|
1106
|
+
if sub is None or x.scan is None:
|
|
1107
|
+
return None
|
|
1108
|
+
name, t_expr = sub
|
|
1109
|
+
if not (t.distinct or facts(t_op).distinct) or name not in x_op.columns:
|
|
1110
|
+
return None
|
|
1111
|
+
if x_op.slots[name].present is not Bound.ALWAYS:
|
|
1112
|
+
return None
|
|
1113
|
+
x_expr = dict(x.outputs).get(name)
|
|
1114
|
+
if not isinstance(x_expr, E.MakeTerm) or x_expr.map.template is None:
|
|
1115
|
+
return None
|
|
1116
|
+
if template_shape(x_expr.map) != template_shape(t_expr.map):
|
|
1117
|
+
return None
|
|
1118
|
+
x_cols = [a.name[4:] for _, a in x_expr.args if isinstance(a, E.Col) and a.name.startswith("src:")]
|
|
1119
|
+
t_cols = [a.name[4:] for _, a in t_expr.args]
|
|
1120
|
+
if len(x_cols) != len(x_expr.args):
|
|
1121
|
+
return None
|
|
1122
|
+
# the same value in each place of the template
|
|
1123
|
+
order_x = _value_order(x_expr.map)
|
|
1124
|
+
order_t = _value_order(t_expr.map)
|
|
1125
|
+
by_place_x = dict(zip(order_x, x_cols)) if len(order_x) == len(x_cols) else None
|
|
1126
|
+
by_place_t = dict(zip(order_t, t_cols)) if len(order_t) == len(t_cols) else None
|
|
1127
|
+
if by_place_x is None or by_place_t is None:
|
|
1128
|
+
return None
|
|
1129
|
+
held_x = {r.column: r.held for r in x.scan.reads}
|
|
1130
|
+
held_t = {r.column: r.held for r in t.scan.reads}
|
|
1131
|
+
node = t.scan.map
|
|
1132
|
+
for node_id, pairs, node_scan in x.exists:
|
|
1133
|
+
if node_id != node.id:
|
|
1134
|
+
continue
|
|
1135
|
+
to_key = dict(pairs)
|
|
1136
|
+
held_k = {r.column: r.held for r in node_scan.reads}
|
|
1137
|
+
if all(to_key.get(by_place_x[i]) == by_place_t[i] and held_x.get(by_place_x[i]) is not None
|
|
1138
|
+
and held_x.get(by_place_x[i]) == held_k.get(by_place_t[i]) == held_t.get(by_place_t[i])
|
|
1139
|
+
for i in by_place_t):
|
|
1140
|
+
return name
|
|
1141
|
+
return None
|
|
1142
|
+
|
|
1143
|
+
|
|
1144
|
+
def _value_order(m: TermMap) -> List[int]:
|
|
1145
|
+
"""The places of a template's values, in order of first reading."""
|
|
1146
|
+
out: List[int] = []
|
|
1147
|
+
seen: List[str] = []
|
|
1148
|
+
for run, is_column in m.template:
|
|
1149
|
+
if is_column and run not in seen:
|
|
1150
|
+
seen.append(run)
|
|
1151
|
+
out.append(len(seen) - 1)
|
|
1152
|
+
return out
|
|
1153
|
+
|
|
1154
|
+
|
|
1155
|
+
def r4_implied_type(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
1156
|
+
"""`Join(T, X) -> X` where T is a node map's subjects (a type pattern)
|
|
1157
|
+
and X's rows are proven to exist among them (`_implied`): `?c :replyOf
|
|
1158
|
+
?p . ?p a :Post` is the first pattern alone where the edge's end is
|
|
1159
|
+
semi-joined into the post table (m2-performance.md 2.2, R4)."""
|
|
1160
|
+
if not isinstance(op, O.Join) or op.condition is not None:
|
|
1161
|
+
return None
|
|
1162
|
+
leaves: List[O.Op] = []
|
|
1163
|
+
|
|
1164
|
+
def flat(x: O.Op) -> None:
|
|
1165
|
+
if isinstance(x, O.Join) and x.condition is None:
|
|
1166
|
+
flat(x.left)
|
|
1167
|
+
flat(x.right)
|
|
1168
|
+
else:
|
|
1169
|
+
leaves.append(x)
|
|
1170
|
+
flat(op)
|
|
1171
|
+
pats = [_pattern(x) for x in leaves]
|
|
1172
|
+
for i, t in enumerate(pats):
|
|
1173
|
+
if t is None:
|
|
1174
|
+
continue
|
|
1175
|
+
for j, x in enumerate(pats):
|
|
1176
|
+
if i == j or x is None:
|
|
1177
|
+
continue
|
|
1178
|
+
name = _implied(t, leaves[i], x, leaves[j])
|
|
1179
|
+
if name is None:
|
|
1180
|
+
continue
|
|
1181
|
+
certs.append(Certificate("R4 implied type pattern",
|
|
1182
|
+
f"map {t.map_id}: {name} exists among its keys in {x.map_id}"))
|
|
1183
|
+
rest = [y for k, y in enumerate(leaves) if k != i]
|
|
1184
|
+
out = rest[0]
|
|
1185
|
+
for y in rest[1:]:
|
|
1186
|
+
out = O.Join(out, y)
|
|
1187
|
+
return out if out.columns == op.columns else O.Project(out, op.columns)
|
|
1188
|
+
return None
|
|
1189
|
+
|
|
1190
|
+
|
|
1191
|
+
def r4_trusted_reference(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
1192
|
+
"""`SemiJoin(edge scan, node scan) -> edge scan` where the edge map
|
|
1193
|
+
asserts its ends exist (`referential_integrity`, the mapping's
|
|
1194
|
+
statement, trusted as a key is: one-plan.md 2.4) and the node map reads
|
|
1195
|
+
all its table's rows (no row filter, no query). Run after the other
|
|
1196
|
+
rules, which read the semi-join as the proof R4 needs."""
|
|
1197
|
+
if not isinstance(op, O.SemiJoin) or op.condition is not None:
|
|
1198
|
+
return None
|
|
1199
|
+
left = op.left
|
|
1200
|
+
while isinstance(left, O.SemiJoin):
|
|
1201
|
+
left = left.left
|
|
1202
|
+
if not isinstance(left, O.MapScan) or not isinstance(left.map, EdgeMap):
|
|
1203
|
+
return None
|
|
1204
|
+
# A split relationship (its end label's maps give several subjects) checks each end in its own maps:
|
|
1205
|
+
# every key has a node of the label, not of each branch's maps, so no check is dropped.
|
|
1206
|
+
if not left.map.referential_integrity or getattr(left.map, "split", False) or not isinstance(op.right, O.MapScan):
|
|
1207
|
+
return None
|
|
1208
|
+
source = op.right.map.source
|
|
1209
|
+
if source.filter or source.query is not None:
|
|
1210
|
+
# The assertion is that each end is a row of the node's table; a
|
|
1211
|
+
# node map holding only some of its rows (a row filter, a view)
|
|
1212
|
+
# still needs the check.
|
|
1213
|
+
return None
|
|
1214
|
+
certs.append(Certificate("R4 trusted reference",
|
|
1215
|
+
f"edge map {left.map.id} asserts its ends exist in {op.right.map.id}"))
|
|
1216
|
+
return op.left
|
|
1217
|
+
|
|
1218
|
+
|
|
1219
|
+
@home("own-row-exists", "A row's existence in its own table's node map on that map's key is the row itself (R3e)",
|
|
1220
|
+
terms=("own row", "start exists", "semi-join", "R3e", "existence check", "same table", "edge on its node's table"))
|
|
1221
|
+
def r3_own_row_exists(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
1222
|
+
"""`SemiJoin(scan of T, node map N's scan of T) -> scan of T`:
|
|
1223
|
+
an edge's start proven to exist among the node's keys, the edge read
|
|
1224
|
+
from the node's own table. Fires only where the left scan's source is
|
|
1225
|
+
N's (one table, no query), its row filter holds N's (every row it reads
|
|
1226
|
+
is a row of N), it is joined on exactly a trusted key of N, each key
|
|
1227
|
+
column paired with itself and required on the left: the edge row is
|
|
1228
|
+
the node row. The end's semi-join (another table) stays, unless R4b.
|
|
1229
|
+
Run after the other rules, which read the semi-join as the proof R4
|
|
1230
|
+
needs."""
|
|
1231
|
+
if not isinstance(op, O.SemiJoin) or op.condition is not None:
|
|
1232
|
+
return None
|
|
1233
|
+
right = op.right
|
|
1234
|
+
if not isinstance(right, O.MapScan) or not isinstance(right.map, NodeMap) or right.graph is not None \
|
|
1235
|
+
or right.joint is not None:
|
|
1236
|
+
return None
|
|
1237
|
+
left = op.left
|
|
1238
|
+
while isinstance(left, O.SemiJoin):
|
|
1239
|
+
left = left.left
|
|
1240
|
+
if not isinstance(left, O.MapScan) or left.graph is not None or left.joint is not None:
|
|
1241
|
+
return None
|
|
1242
|
+
if not left.map.source.rows_within(right.map.source, folds=False):
|
|
1243
|
+
return None # another table, a query, or a node map holding only some of the rows the left reads
|
|
1244
|
+
# The columns folded may differ (a merged scan names the columns of both its maps undelimited):
|
|
1245
|
+
# the node's own columns must be written as it wrote them.
|
|
1246
|
+
if any((r.column in left.map.source.regular_columns) != (r.column in right.map.source.regular_columns)
|
|
1247
|
+
for r in right.reads if not r.parent):
|
|
1248
|
+
return None
|
|
1249
|
+
mine = {r.name: r.column for r in left.reads if not r.parent}
|
|
1250
|
+
theirs = {r.name: r.column for r in right.reads if not r.parent}
|
|
1251
|
+
shared = op.shared
|
|
1252
|
+
if not shared or any(n not in mine or n not in theirs or mine[n] != theirs[n] for n in shared):
|
|
1253
|
+
return None # each key column joined with itself
|
|
1254
|
+
if not set(right.required) <= set(shared) or not set(shared) <= set(left.required):
|
|
1255
|
+
return None
|
|
1256
|
+
cols = frozenset(mine[n] for n in shared)
|
|
1257
|
+
if not any(frozenset(k) == cols for k in right.unique_keys):
|
|
1258
|
+
return None
|
|
1259
|
+
certs.append(Certificate("R3 own row exists",
|
|
1260
|
+
f"map {left.map.id} reads rows of {right.map.id}'s table on its key {sorted(cols)}"))
|
|
1261
|
+
return op.left
|
|
1262
|
+
|
|
1263
|
+
|
|
1264
|
+
# -- R6: a union branch no row of the join can meet --------------------------------
|
|
1265
|
+
|
|
1266
|
+
def r6_union_branches(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
1267
|
+
"""In a tree of joins, a union branch whose terms in a shared column,
|
|
1268
|
+
bound in every row, come from maps none of which may give a term the
|
|
1269
|
+
other pattern's column holds (`TermMap.may_equal`) joins no row and
|
|
1270
|
+
goes: `?c :replyOf ?p . ?p a :Post` keeps only the reply-to-a-post
|
|
1271
|
+
branch. The front end does this for two neighbours (`sparql_front.
|
|
1272
|
+
meeting`); this does it across the whole tree."""
|
|
1273
|
+
if not isinstance(op, O.Join) or op.condition is not None:
|
|
1274
|
+
return None
|
|
1275
|
+
from prograph.plan.ir.sparql_front import meeting
|
|
1276
|
+
leaves: List[O.Op] = []
|
|
1277
|
+
|
|
1278
|
+
def flat(x: O.Op) -> None:
|
|
1279
|
+
if isinstance(x, O.Join) and x.condition is None:
|
|
1280
|
+
flat(x.left)
|
|
1281
|
+
flat(x.right)
|
|
1282
|
+
else:
|
|
1283
|
+
leaves.append(x)
|
|
1284
|
+
flat(op)
|
|
1285
|
+
if len(leaves) < 3:
|
|
1286
|
+
return None
|
|
1287
|
+
for i, x in enumerate(leaves):
|
|
1288
|
+
inner = x.input if isinstance(x, O.Distinct) else x
|
|
1289
|
+
if not isinstance(inner, O.Union):
|
|
1290
|
+
continue
|
|
1291
|
+
for j, other in enumerate(leaves):
|
|
1292
|
+
if i == j:
|
|
1293
|
+
continue
|
|
1294
|
+
pruned = meeting(x, other)
|
|
1295
|
+
if pruned is x:
|
|
1296
|
+
continue
|
|
1297
|
+
certs.append(Certificate("R6 union branches",
|
|
1298
|
+
f"{len(inner.inputs)} branches against {sorted(other.columns)}"))
|
|
1299
|
+
rest = list(leaves)
|
|
1300
|
+
rest[i] = pruned
|
|
1301
|
+
out = rest[0]
|
|
1302
|
+
for y in rest[1:]:
|
|
1303
|
+
out = O.Join(out, y)
|
|
1304
|
+
return out if out.columns == op.columns else O.Project(out, op.columns)
|
|
1305
|
+
return None
|
|
1306
|
+
|
|
1307
|
+
|
|
1308
|
+
# -- R5: one Distinct beside the LIMIT (a trade-off: engine-aware) --------------------
|
|
1309
|
+
|
|
1310
|
+
# Engines measured to stop early on `SELECT DISTINCT ... LIMIT n` and to lose
|
|
1311
|
+
# on a Distinct per pattern (m2-performance.md 6: on 41 LDBC queries with no
|
|
1312
|
+
# keys, PostgreSQL 2.4 s with one Distinct beside the LIMIT against 25.8 s with
|
|
1313
|
+
# one per pattern; DuckDB 2.6 s against 1.4 s, so not DuckDB). Spark and
|
|
1314
|
+
# Databricks join when measured.
|
|
1315
|
+
DISTINCT_BESIDE_LIMIT = frozenset({"postgresql", "postgres"})
|
|
1316
|
+
|
|
1317
|
+
|
|
1318
|
+
def r5_distinct_beside_limit(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
1319
|
+
"""`Slice(limit)` over Project, Filter and Extend over a tree of inner
|
|
1320
|
+
joins whose leaves are Distinct: the leaves' Distincts become one over
|
|
1321
|
+
the join, beside the LIMIT, so the database can stop at the first rows.
|
|
1322
|
+
Sound without keys: a join of inputs without equal rows has none, so
|
|
1323
|
+
Distinct(A) join Distinct(B) and Distinct(A join B) are the same rows.
|
|
1324
|
+
The one Distinct sits directly over the join, beneath any Project, so a
|
|
1325
|
+
projection dropping columns above it changes nothing. Not through an
|
|
1326
|
+
aggregate, ORDER BY, OPTIONAL or UNION (m2-performance.md 2.2, R5)."""
|
|
1327
|
+
if not isinstance(op, O.Slice) or op.limit is None:
|
|
1328
|
+
return None
|
|
1329
|
+
chain: List[O.Op] = []
|
|
1330
|
+
x = op.input
|
|
1331
|
+
while isinstance(x, (O.Project, O.Filter, O.Extend)):
|
|
1332
|
+
chain.append(x)
|
|
1333
|
+
x = x.input
|
|
1334
|
+
if not isinstance(x, O.Join) or x.condition is not None:
|
|
1335
|
+
return None
|
|
1336
|
+
leaves: List[O.Op] = []
|
|
1337
|
+
firm = [True]
|
|
1338
|
+
|
|
1339
|
+
def flat(y: O.Op) -> None:
|
|
1340
|
+
if isinstance(y, O.Join) and y.condition is None:
|
|
1341
|
+
if not all(y.left.slots[c].present is Bound.ALWAYS and
|
|
1342
|
+
y.right.slots[c].present is Bound.ALWAYS for c in y.shared):
|
|
1343
|
+
firm[0] = False # a shared column unbound: pairs may merge
|
|
1344
|
+
flat(y.left)
|
|
1345
|
+
flat(y.right)
|
|
1346
|
+
else:
|
|
1347
|
+
leaves.append(y)
|
|
1348
|
+
flat(x)
|
|
1349
|
+
if not firm[0]:
|
|
1350
|
+
return None
|
|
1351
|
+
if sum(isinstance(y, O.Distinct) for y in leaves) < 2:
|
|
1352
|
+
return None
|
|
1353
|
+
if not all(isinstance(y, O.Distinct) or facts(y).distinct for y in leaves):
|
|
1354
|
+
return None
|
|
1355
|
+
bare = [y.input if isinstance(y, O.Distinct) else y for y in leaves]
|
|
1356
|
+
joined = bare[0]
|
|
1357
|
+
for y in bare[1:]:
|
|
1358
|
+
joined = O.Join(joined, y)
|
|
1359
|
+
if joined.columns != x.columns:
|
|
1360
|
+
joined = O.Project(joined, x.columns)
|
|
1361
|
+
out: O.Op = O.Distinct(joined)
|
|
1362
|
+
for c in reversed(chain):
|
|
1363
|
+
out = dataclasses.replace(c, input=out)
|
|
1364
|
+
certs.append(Certificate("R5 one Distinct beside the LIMIT",
|
|
1365
|
+
f"{len(leaves)} patterns, LIMIT {op.limit}"))
|
|
1366
|
+
return dataclasses.replace(op, input=out)
|
|
1367
|
+
|
|
1368
|
+
|
|
1369
|
+
# -- R7: a LIMIT into the rows that decide it ---------------------------------
|
|
1370
|
+
|
|
1371
|
+
# The engines R7 applies on: a trade-off (m2-performance.md, the two classes), applied only where measured
|
|
1372
|
+
# to help, on the 11 LDBC SF1 queries it fires on (medians of 5, 2026-11-12, totals R7 on / off):
|
|
1373
|
+
# DuckDB 296 / 351 ms (S77 0.77x, S80 0.73x, S82 0.81x; a few small queries 1-3 ms slower); PostgreSQL
|
|
1374
|
+
# 58.3 / 17.0 s (S77 3.05x, S80 97x slower with it; S82 0.13x, S86 0.31x); local Spark 8 of 11 slower
|
|
1375
|
+
# (S77 1.29x); a Databricks warehouse (FEVM) all five measured slower (S77 1.31x, S81 1.45x). Not measured
|
|
1376
|
+
# elsewhere, so not applied. `None` (no engine named) keeps the rule, as the plan's own tests read it.
|
|
1377
|
+
LIMIT_INWARD_ENGINES = frozenset({"duckdb", None})
|
|
1378
|
+
|
|
1379
|
+
def r7_limit_inward(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
1380
|
+
"""`Slice(offset k, limit n)` with no ORDER BY, over Project and Extend,
|
|
1381
|
+
over a LeftJoin: the left side limited to k + n rows first. Each left
|
|
1382
|
+
row gives at least one row of the left join, so the first k + n rows
|
|
1383
|
+
come from at most k + n left rows, and SPARQL's LIMIT without ORDER BY
|
|
1384
|
+
may answer any n solutions (15.4). Over a Union, each branch limited to
|
|
1385
|
+
k + n rows the same way. Not through Filter, Distinct, an inner join
|
|
1386
|
+
or an aggregate, which may drop or merge rows (m2-performance.md 3)."""
|
|
1387
|
+
if not isinstance(op, O.Slice) or op.limit is None:
|
|
1388
|
+
return None
|
|
1389
|
+
cap = op.offset + op.limit
|
|
1390
|
+
chain: List[O.Op] = []
|
|
1391
|
+
x = op.input
|
|
1392
|
+
while isinstance(x, (O.Project, O.Extend)):
|
|
1393
|
+
chain.append(x)
|
|
1394
|
+
x = x.input
|
|
1395
|
+
if isinstance(x, O.LeftJoin):
|
|
1396
|
+
if isinstance(x.left, O.Slice) and x.left.offset == 0 and x.left.limit is not None \
|
|
1397
|
+
and x.left.limit <= cap:
|
|
1398
|
+
return None
|
|
1399
|
+
new: O.Op = dataclasses.replace(x, left=O.Slice(x.left, 0, cap))
|
|
1400
|
+
elif isinstance(x, O.Union):
|
|
1401
|
+
if all(isinstance(b, O.Slice) and b.offset == 0 and b.limit is not None and b.limit <= cap
|
|
1402
|
+
for b in x.inputs):
|
|
1403
|
+
return None
|
|
1404
|
+
new = dataclasses.replace(x, inputs=tuple(
|
|
1405
|
+
b if isinstance(b, O.Slice) and b.offset == 0 and b.limit is not None and b.limit <= cap
|
|
1406
|
+
else O.Slice(b, 0, cap) for b in x.inputs))
|
|
1407
|
+
else:
|
|
1408
|
+
return None
|
|
1409
|
+
for c in reversed(chain):
|
|
1410
|
+
new = dataclasses.replace(c, input=new)
|
|
1411
|
+
certs.append(Certificate("R7 limit inward", f"LIMIT {cap} into the {type(x).__name__}"))
|
|
1412
|
+
return dataclasses.replace(op, input=new)
|
|
1413
|
+
|
|
1414
|
+
|
|
1415
|
+
# -- R8: a projected Distinct into a join's inputs ------------------------------------
|
|
1416
|
+
|
|
1417
|
+
def r8_distinct_into_join(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
1418
|
+
"""`Distinct(Project(X, names))` over a tree of inner joins whose shared
|
|
1419
|
+
columns are bound on both sides: each input keeps only the columns the
|
|
1420
|
+
answer or the joins read, once each. Distinct(Project_X(L join R)) =
|
|
1421
|
+
Distinct(Project_X(Distinct(Project_{X,J}(L)) join Distinct(Project_{X,J}(R))))
|
|
1422
|
+
for J the join columns: a join of sets, then a projection to a set,
|
|
1423
|
+
reaches the same set (S50: `SELECT DISTINCT ?tagName` over post authors,
|
|
1424
|
+
friends and interests built 967 million rows to keep 9,489)."""
|
|
1425
|
+
if not isinstance(op, O.Distinct) or not isinstance(op.input, O.Project):
|
|
1426
|
+
return None
|
|
1427
|
+
proj = op.input
|
|
1428
|
+
x = proj.input
|
|
1429
|
+
wrappers: List[O.Op] = []
|
|
1430
|
+
while isinstance(x, (O.Project, O.Order)): # projections the rules left; ORDER BY
|
|
1431
|
+
wrappers.append(x)
|
|
1432
|
+
x = x.input
|
|
1433
|
+
if not isinstance(x, O.Join) or x.condition is not None:
|
|
1434
|
+
return None
|
|
1435
|
+
if any(isinstance(w, O.Order) and not all(k.column in proj.names for k in w.keys)
|
|
1436
|
+
for w in wrappers):
|
|
1437
|
+
return None
|
|
1438
|
+
leaves: List[O.Op] = []
|
|
1439
|
+
firm = [True]
|
|
1440
|
+
|
|
1441
|
+
def flat(y: O.Op) -> None:
|
|
1442
|
+
if isinstance(y, O.Join) and y.condition is None:
|
|
1443
|
+
if not all(y.left.slots[c].present is Bound.ALWAYS and
|
|
1444
|
+
y.right.slots[c].present is Bound.ALWAYS for c in y.shared):
|
|
1445
|
+
firm[0] = False
|
|
1446
|
+
flat(y.left)
|
|
1447
|
+
flat(y.right)
|
|
1448
|
+
else:
|
|
1449
|
+
leaves.append(y)
|
|
1450
|
+
flat(x)
|
|
1451
|
+
if not firm[0] or len(leaves) < 2:
|
|
1452
|
+
return None
|
|
1453
|
+
wanted = set(proj.names)
|
|
1454
|
+
counts: Dict[str, int] = {}
|
|
1455
|
+
for y in leaves:
|
|
1456
|
+
for c in y.columns:
|
|
1457
|
+
counts[c] = counts.get(c, 0) + 1
|
|
1458
|
+
joins = {c for c, n in counts.items() if n > 1}
|
|
1459
|
+
changed = False
|
|
1460
|
+
new_leaves: List[O.Op] = []
|
|
1461
|
+
for y in leaves:
|
|
1462
|
+
keep = tuple(c for c in y.columns if c in wanted or c in joins)
|
|
1463
|
+
if len(keep) < len(y.columns):
|
|
1464
|
+
new_leaves.append(O.Distinct(O.Project(y, keep)))
|
|
1465
|
+
changed = True
|
|
1466
|
+
else:
|
|
1467
|
+
new_leaves.append(y)
|
|
1468
|
+
if not changed:
|
|
1469
|
+
return None
|
|
1470
|
+
joined: O.Op = new_leaves[0]
|
|
1471
|
+
for y in new_leaves[1:]:
|
|
1472
|
+
joined = O.Join(joined, y)
|
|
1473
|
+
for w in reversed(wrappers):
|
|
1474
|
+
if isinstance(w, O.Order):
|
|
1475
|
+
joined = dataclasses.replace(w, input=joined)
|
|
1476
|
+
certs.append(Certificate("R8 distinct into a join",
|
|
1477
|
+
f"{len(leaves)} inputs keep {sorted(wanted | joins)}"))
|
|
1478
|
+
return O.Distinct(O.Project(joined, proj.names))
|
|
1479
|
+
|
|
1480
|
+
# -- R13: a joint query on the parent's key, the child's rows with the parent's existence ----------
|
|
1481
|
+
|
|
1482
|
+
def _one_text(child: O.ScanColumn, parent: O.ScanColumn) -> bool:
|
|
1483
|
+
"""Whether two columns equal by SQL equality hold one value's text: read alike (datatype, zone,
|
|
1484
|
+
digits, reader, a date-time's precision), an integer either way (INTEGER and BIGINT), a string only
|
|
1485
|
+
where equality is byte equality (a case-blind collation matches 'a' and 'A'), a date-time of one zone
|
|
1486
|
+
and one stated precision (equal local values are one wall clock, equal zoned ones one instant, its
|
|
1487
|
+
text in UTC; TIMESTAMP_NS against TIMESTAMP is not), never a float (-0.0 = 0.0), nor a column of
|
|
1488
|
+
unknown type (VARCHAR '01' and INTEGER 1 are equal on some databases, EndRef.parent_columns)."""
|
|
1489
|
+
from prograph.schema import xsd
|
|
1490
|
+
integers = xsd.INTEGER_DATATYPES | {xsd.INTEGER}
|
|
1491
|
+
if child.held in integers and parent.held in integers:
|
|
1492
|
+
return child.reader is None and parent.reader is None
|
|
1493
|
+
if child.held is None or (child.held, child.zone, child.digits, child.reader, child.precision) != \
|
|
1494
|
+
(parent.held, parent.zone, parent.digits, parent.reader, parent.precision):
|
|
1495
|
+
return False
|
|
1496
|
+
if child.held == xsd.STRING:
|
|
1497
|
+
return child.bytewise and parent.bytewise
|
|
1498
|
+
if child.held == xsd.DATETIME:
|
|
1499
|
+
return child.zone is not None
|
|
1500
|
+
return child.held in (xsd.DECIMAL, xsd.DATE, xsd.BOOLEAN)
|
|
1501
|
+
|
|
1502
|
+
|
|
1503
|
+
@home("joint-query-on-a-parent-key", "A referencing object map joined on its parent's trusted key is the "
|
|
1504
|
+
"child's rows with the parent row's existence (R13)",
|
|
1505
|
+
terms=("R13", "joint query", "referencing object map", "rr:parentTriplesMap", "rr:joinCondition",
|
|
1506
|
+
"parent key", "foreign key", "semi-join", "child scan"))
|
|
1507
|
+
def r13_joint_on_parent_key(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
1508
|
+
"""`MapScan(joint query)` -> `SemiJoin(child's scan, parent node map's scan)` (R2RML 8): each
|
|
1509
|
+
child row meets at most one parent row where the join is on a key the parent is trusted unique on,
|
|
1510
|
+
so the joint query's rows are the child rows some parent row matches; and a parent column read
|
|
1511
|
+
only where it is joined to a child column holding the same text (`_one_text`) is that child
|
|
1512
|
+
column. Fires only where every parent column the scan reads is a join column and every join
|
|
1513
|
+
column is read (the parent's subject is its key), each column in one condition, a trusted key of
|
|
1514
|
+
the parent among them, no graph column. The semi-join reads the parent's filter and NULLs as the
|
|
1515
|
+
join did (a NULL matches nothing). R3 then merges the child scan with the child's own
|
|
1516
|
+
properties, and R3e/R4b read its semi-join as for an edge on its node's table."""
|
|
1517
|
+
if not isinstance(op, O.MapScan) or op.joint is None or op.graph is not None or op.parent_map is None:
|
|
1518
|
+
return None
|
|
1519
|
+
joint, parent = op.joint, op.parent_map
|
|
1520
|
+
if parent.source != joint.parent or len(op.join_reads) != len(joint.conditions):
|
|
1521
|
+
return None
|
|
1522
|
+
children = [c for c, _ in joint.conditions]
|
|
1523
|
+
parents = [p for _, p in joint.conditions]
|
|
1524
|
+
if len(set(children)) != len(children) or len(set(parents)) != len(parents):
|
|
1525
|
+
return None
|
|
1526
|
+
read = {r.column: r for r in op.reads if r.parent}
|
|
1527
|
+
if len(read) != len([r for r in op.reads if r.parent]) or set(read) != set(parents):
|
|
1528
|
+
return None # a parent column beyond the join, or a join column the scan does not read
|
|
1529
|
+
if not any(k and set(k) <= set(parents) for k in op.parent_unique_keys):
|
|
1530
|
+
return None # not joined on a trusted key: a child row may meet many parent rows
|
|
1531
|
+
child_of = {p: c for c, p in joint.conditions}
|
|
1532
|
+
as_child = {}
|
|
1533
|
+
for jr, (c, p) in zip(op.join_reads, joint.conditions):
|
|
1534
|
+
if jr.column != c or not _one_text(jr, read[p]):
|
|
1535
|
+
return None
|
|
1536
|
+
as_child[read[p].name] = dataclasses.replace(jr, name=read[p].name)
|
|
1537
|
+
edge = op.map
|
|
1538
|
+
ends = {f: dataclasses.replace(getattr(edge, f), parent_columns=())
|
|
1539
|
+
for f in ("start", "end") if getattr(edge, f).parent_columns}
|
|
1540
|
+
child = O.MapScan(map=dataclasses.replace(edge, join=None, **ends),
|
|
1541
|
+
reads=tuple(as_child.get(r.name, r) for r in op.reads),
|
|
1542
|
+
required=op.required, unique_keys=op.unique_keys, not_null=op.not_null)
|
|
1543
|
+
rows = O.MapScan(map=parent, reads=tuple(dataclasses.replace(r, parent=False) for r in op.reads if r.parent),
|
|
1544
|
+
required=tuple(r.name for r in op.reads if r.parent),
|
|
1545
|
+
unique_keys=op.parent_unique_keys, not_null=op.parent_not_null)
|
|
1546
|
+
certs.append(Certificate("R13 joint query on a parent key",
|
|
1547
|
+
f"edge {edge.id} joined to {parent.id} on {sorted(child_of.items())}, keys "
|
|
1548
|
+
f"{[list(k) for k in op.parent_unique_keys]}"))
|
|
1549
|
+
return O.SemiJoin(child, rows)
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
# -- R12: a joint scan's child as a set, under a Distinct ------------------------------
|
|
1553
|
+
|
|
1554
|
+
def r12_joint_child_set(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
1555
|
+
"""`Distinct(F(MapScan joint))` for F filters, extensions and projections: the scan reads each
|
|
1556
|
+
child row with each parent row its condition matches, and a referencing object map's child
|
|
1557
|
+
contributes only its own columns (the subject's), so a child with many rows of one such
|
|
1558
|
+
value builds that value's pairs many times over (GTFS shapes: ~2,000 points a shape, 117
|
|
1559
|
+
million pairs for 58,540 triples). Every operator of F is a function of one row, and a
|
|
1560
|
+
Distinct keeps one row of each set of equal rows, so the answer is the same set whether F reads
|
|
1561
|
+
the bag of (child row, parent row) pairs or the pairs of (distinct child columns read, parent
|
|
1562
|
+
row): two child rows with the same columns read give the same rows of F. Where F is volatile
|
|
1563
|
+
(BNODE, RAND, UUID: a row's own fresh value) rows are not a function of the columns, so not here.
|
|
1564
|
+
Without a Distinct the bag is observable (a join's multiplicity), so not there either: the
|
|
1565
|
+
pattern's own Distinct (a graph is a set, RDF 1.1 Concepts 3) is what licenses it. The
|
|
1566
|
+
lowering reads the child as `SELECT DISTINCT` its columns, filtered and non-null."""
|
|
1567
|
+
if not isinstance(op, O.Distinct):
|
|
1568
|
+
return None
|
|
1569
|
+
from prograph.plan.ir.fresh import volatile
|
|
1570
|
+
chain: List[O.Op] = []
|
|
1571
|
+
x = op.input
|
|
1572
|
+
while isinstance(x, (O.Filter, O.Extend, O.Project)):
|
|
1573
|
+
chain.append(x)
|
|
1574
|
+
x = x.input
|
|
1575
|
+
if not isinstance(x, O.MapScan) or x.joint is None or x.child_set or x.graph is not None:
|
|
1576
|
+
return None
|
|
1577
|
+
if volatile(op.input):
|
|
1578
|
+
return None
|
|
1579
|
+
new: O.Op = dataclasses.replace(x, child_set=True)
|
|
1580
|
+
for c in reversed(chain):
|
|
1581
|
+
new = dataclasses.replace(c, input=new)
|
|
1582
|
+
certs.append(Certificate("R12 joint child as a set",
|
|
1583
|
+
"the child's columns read and joined on, distinct, before the parent's join"))
|
|
1584
|
+
return O.Distinct(new)
|
|
1585
|
+
|
|
1586
|
+
|
|
1587
|
+
# -- R11: eager aggregation, counts per node before the join -----------------------------
|
|
1588
|
+
|
|
1589
|
+
@home("eager-aggregation", "COUNTs over a join at a node both sides hold many rows of, counted per side first",
|
|
1590
|
+
terms=("eager aggregation", "R11", "SUM_COUNTS", "sum of counts", "count per side"))
|
|
1591
|
+
def r11_eager_aggregation(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
|
|
1592
|
+
"""`Aggregate(G, COUNTs)` over `Join(L, R)` at one node v both sides hold many rows of
|
|
1593
|
+
(`cost.fan_out`: friends and posts counted per person) becomes the counts per side first:
|
|
1594
|
+
L' = Aggregate(L, v + G within L, COUNT(*) + a COUNT per counted column of L), R' likewise, then
|
|
1595
|
+
Aggregate(G, sum over L' join R' of the side's count times the other's COUNT(*)). A group of the
|
|
1596
|
+
join holds, for each pair of L' and R' rows through v, their row counts multiplied, so the sum of
|
|
1597
|
+
those products is the group's count (every row of a side gathers into the L' or R' row of its own
|
|
1598
|
+
v and group columns). Fires only where every aggregate is a plain COUNT, v is bound in every row
|
|
1599
|
+
of both sides (an unbound v joins everything), and the join has no condition: the two sides then
|
|
1600
|
+
meet on equal v alone, as the groups of L' and R' do. The sums are `SUM_COUNTS`, COUNT's slot."""
|
|
1601
|
+
if not isinstance(op, O.Aggregate) or not isinstance(op.input, O.Join) or not op.aggregates:
|
|
1602
|
+
return None
|
|
1603
|
+
from prograph.plan.ir.cost import fan_out
|
|
1604
|
+
j = op.input
|
|
1605
|
+
if j.condition is not None or len(j.shared) != 1:
|
|
1606
|
+
return None
|
|
1607
|
+
v = j.shared[0]
|
|
1608
|
+
if fan_out(op) != v:
|
|
1609
|
+
return None
|
|
1610
|
+
left, right = j.left, j.right
|
|
1611
|
+
if left.slots[v].present is not Bound.ALWAYS or right.slots[v].present is not Bound.ALWAYS:
|
|
1612
|
+
return None
|
|
1613
|
+
for _, agg in op.aggregates:
|
|
1614
|
+
if agg.fn != "COUNT" or agg.distinct or agg.semantics != E.SPARQL:
|
|
1615
|
+
return None
|
|
1616
|
+
if agg.arg is not None and not (isinstance(agg.arg, E.Col) and agg.arg.name in j.columns):
|
|
1617
|
+
return None
|
|
1618
|
+
if not all(g in left.columns or g in right.columns for g in op.group):
|
|
1619
|
+
return None
|
|
1620
|
+
side_counts: Dict[str, List[Tuple[str, E.Agg]]] = {"L": [("#r11.cL", E.Agg("COUNT"))],
|
|
1621
|
+
"R": [("#r11.cR", E.Agg("COUNT"))]}
|
|
1622
|
+
counted: Dict[Tuple[str, str], str] = {}
|
|
1623
|
+
tops: List[Tuple[str, E.Agg]] = []
|
|
1624
|
+
for name, agg in op.aggregates:
|
|
1625
|
+
a, b = "#r11.cL", "#r11.cR"
|
|
1626
|
+
if agg.arg is not None and agg.arg.name != v:
|
|
1627
|
+
c = agg.arg.name
|
|
1628
|
+
side = "L" if c in left.columns else "R"
|
|
1629
|
+
if (side, c) not in counted:
|
|
1630
|
+
counted[(side, c)] = f"#r11.n{side}{len(counted)}"
|
|
1631
|
+
side_counts[side].append((counted[(side, c)], E.Agg("COUNT", E.Col(c))))
|
|
1632
|
+
if side == "L":
|
|
1633
|
+
a = counted[(side, c)]
|
|
1634
|
+
else:
|
|
1635
|
+
b = counted[(side, c)]
|
|
1636
|
+
tops.append((name, E.Agg("SUM_COUNTS", E.Call("*", (E.Col(a), E.Col(b))))))
|
|
1637
|
+
group_l = tuple(g for g in op.group if g in left.columns and g != v)
|
|
1638
|
+
group_r = tuple(g for g in op.group if g in right.columns and g != v)
|
|
1639
|
+
new_left = O.Aggregate(left, (v,) + group_l, tuple(side_counts["L"]))
|
|
1640
|
+
new_right = O.Aggregate(right, (v,) + group_r, tuple(side_counts["R"]))
|
|
1641
|
+
certs.append(Certificate("R11 eager aggregation",
|
|
1642
|
+
f"both sides hold many rows per ?{v} (declared keys, none within it), "
|
|
1643
|
+
f"?{v} bound in every row, COUNTs only, no join condition"))
|
|
1644
|
+
return O.Aggregate(O.Join(new_left, new_right), op.group, tuple(tops))
|
|
1645
|
+
|
|
1646
|
+
|
|
1647
|
+
# -- R14: a grouped sub-select restricted to the keys its join keeps ------------------
|
|
1648
|
+
|
|
1649
|
+
# The engines R14 applies on: a trade-off (where the other side keeps every key it adds a semi-join over
|
|
1650
|
+
# the aggregate's input), so it fires only where the other side holds a condition (`_filtered`), and only
|
|
1651
|
+
# on the engines measured, TPC-H SF1, all 22 queries (2026-11-13; q17 and q20 are the two it fires
|
|
1652
|
+
# on): PostgreSQL q17 12.1 -> 0.07 s, q20 1.2 -> 0.16 s (reference 0.42, 0.62); DuckDB q17 0.45 -> 0.25 s,
|
|
1653
|
+
# q20 0.07 s either way. Without `_filtered` it fired on q15 and q18 too, PostgreSQL q18 1.7 -> 10.7 s.
|
|
1654
|
+
# `None` (no engine named) keeps the rule, as the plan's own tests read it.
|
|
1655
|
+
RESTRICT_GROUPS_ENGINES = frozenset({"postgresql", "postgres", "duckdb", None})
|
|
1656
|
+
|
|
1657
|
+
|
|
1658
|
+
def _grouped(op: O.Op) -> Optional[Tuple[List[O.Op], O.Aggregate]]:
|
|
1659
|
+
"""`op` as row-wise operators (Project, Extend, Filter) over an Aggregate with GROUP BY keys:
|
|
1660
|
+
the operators, outermost first, and the Aggregate."""
|
|
1661
|
+
chain: List[O.Op] = []
|
|
1662
|
+
x = op
|
|
1663
|
+
while isinstance(x, (O.Project, O.Extend, O.Filter)):
|
|
1664
|
+
chain.append(x)
|
|
1665
|
+
x = x.input
|
|
1666
|
+
if not isinstance(x, O.Aggregate) or not x.group:
|
|
1667
|
+
return None
|
|
1668
|
+
return chain, x
|
|
1669
|
+
|
|
1670
|
+
|
|
1671
|
+
def _restricted(join: O.Join, conds: Tuple[E.Expr, ...], certs: List[Certificate]) -> Optional[O.Join]:
|
|
1672
|
+
"""`join` with each grouped side's aggregate input restricted to the keys the other side holds
|
|
1673
|
+
(`r14_grouped_join_restricted`); None where no side qualifies."""
|
|
1674
|
+
from prograph.plan.ir.fresh import holds, volatile
|
|
1675
|
+
if join.condition is not None or volatile(join):
|
|
1676
|
+
return None
|
|
1677
|
+
shared = join.shared
|
|
1678
|
+
if not shared:
|
|
1679
|
+
return None
|
|
1680
|
+
if not all(join.left.slots[c].present is Bound.ALWAYS and join.right.slots[c].present is Bound.ALWAYS
|
|
1681
|
+
for c in shared):
|
|
1682
|
+
return None # an unbound key joins every row
|
|
1683
|
+
sides = {"left": join.left, "right": join.right}
|
|
1684
|
+
changes = {}
|
|
1685
|
+
for name, g, other in (("left", join.left, join.right), ("right", join.right, join.left)):
|
|
1686
|
+
found = _grouped(g)
|
|
1687
|
+
if found is None:
|
|
1688
|
+
continue
|
|
1689
|
+
chain, agg = found
|
|
1690
|
+
if not set(shared) <= set(agg.group):
|
|
1691
|
+
continue # a group holding several values of a join column: no group is whole
|
|
1692
|
+
if isinstance(agg.input, O.SemiJoin) and isinstance(agg.input.right, O.Project) \
|
|
1693
|
+
and agg.input.right.names == shared:
|
|
1694
|
+
continue # restricted already
|
|
1695
|
+
mine = [c for c in conds
|
|
1696
|
+
if c.columns() <= set(other.columns) and not holds(c)
|
|
1697
|
+
and all(other.slots[x].present is Bound.ALWAYS for x in c.columns() if x in g.columns)]
|
|
1698
|
+
if not mine and not _filtered(other):
|
|
1699
|
+
continue # the other side keeps every key: the restriction would only add work (TPC-H q15, q18)
|
|
1700
|
+
keys: O.Op = other
|
|
1701
|
+
if mine:
|
|
1702
|
+
keys = O.Filter(other, mine[0] if len(mine) == 1 else E.And(tuple(mine)))
|
|
1703
|
+
new: O.Op = dataclasses.replace(agg, input=O.SemiJoin(agg.input, O.Project(keys, shared)))
|
|
1704
|
+
for c in reversed(chain):
|
|
1705
|
+
new = dataclasses.replace(c, input=new)
|
|
1706
|
+
changes[name] = new
|
|
1707
|
+
certs.append(Certificate("R14 grouped join restricted",
|
|
1708
|
+
f"grouped by {list(agg.group)}, joined on {list(shared)}, bound on both "
|
|
1709
|
+
f"sides; {len(mine)} of the filter's conjuncts read the other side only"))
|
|
1710
|
+
if not changes:
|
|
1711
|
+
return None
|
|
1712
|
+
return dataclasses.replace(join, **changes)
|
|
1713
|
+
|
|
1714
|
+
|
|
1715
|
+
def _filtered(op: O.Op) -> bool:
|
|
1716
|
+
"""Whether a condition of the query (a FILTER, a constant in a pattern) restricts `op`'s rows
|
|
1717
|
+
somewhere in its tree: R14's structural rule for a side that may keep fewer keys than the groups."""
|
|
1718
|
+
if isinstance(op, O.Filter):
|
|
1719
|
+
return True
|
|
1720
|
+
return any(_filtered(c) for c in op.children())
|
|
1721
|
+
|
|
1722
|
+
|
|
1723
|
+
@home("grouped-join-restricted", "A grouped sub-select joined on its group keys computes only the groups the join keeps (R14)",
|
|
1724
|
+
terms=("R14", "semi-join reduction", "sub-select", "subquery", "group by", "aggregate", "correlated",
|
|
1725
|
+
"grouped join", "restrict"))
|
|
1726
|
+
def _restrict_groups(op: O.Op, certs: List[Certificate], memo: ByIdentity) -> O.Op:
|
|
1727
|
+
"""R14, top-down: `Filter?(Join(L, G))`, G row-wise operators over `Aggregate(I, GROUP BY K)` joined
|
|
1728
|
+
on columns J within K, becomes the same with I replaced by `SemiJoin(I, Project(Filter(C_L, L), J))`,
|
|
1729
|
+
C_L the Filter's conjuncts reading only L (TPC-H q17's per-part average, q20's per-(part, supplier)
|
|
1730
|
+
sum: the reference correlates, computing only the outer side's keys). Rows of one group agree on J,
|
|
1731
|
+
so the semi-join keeps or drops a group whole, and a kept group's aggregates read the rows they did;
|
|
1732
|
+
a dropped group meets no row of L passing C_L, so it gave no row of the join passing C. Only where
|
|
1733
|
+
the join is inner with no condition (no OPTIONAL, MINUS or Cypher condition), G holds no LIMIT, ORDER
|
|
1734
|
+
BY or DISTINCT (a sub-select's LIMIT picks groups), the Aggregate has GROUP BY keys (an implicit group
|
|
1735
|
+
is one row over no rows), J is bound in every row of both sides, and a conjunct taken calls no fresh
|
|
1736
|
+
function and reads columns of L only, bound in L where G has them too. L is the same object in the
|
|
1737
|
+
join and the restriction, so the lowering writes it once."""
|
|
1738
|
+
hit = memo.get(op)
|
|
1739
|
+
if hit is not None:
|
|
1740
|
+
return hit
|
|
1741
|
+
out = op
|
|
1742
|
+
if isinstance(op, O.Filter) and isinstance(op.input, O.Join):
|
|
1743
|
+
j = _restricted(op.input, E.conjuncts(op.condition), certs)
|
|
1744
|
+
if j is not None:
|
|
1745
|
+
out = dataclasses.replace(op, input=_children(j, certs, memo))
|
|
1746
|
+
if _CHECK:
|
|
1747
|
+
_check_firing(_restrict_groups, op, out)
|
|
1748
|
+
memo.put(op, out)
|
|
1749
|
+
return out
|
|
1750
|
+
elif isinstance(op, O.Join):
|
|
1751
|
+
j = _restricted(op, (), certs)
|
|
1752
|
+
if j is not None:
|
|
1753
|
+
out = _children(j, certs, memo)
|
|
1754
|
+
if _CHECK:
|
|
1755
|
+
_check_firing(_restrict_groups, op, out)
|
|
1756
|
+
memo.put(op, out)
|
|
1757
|
+
return out
|
|
1758
|
+
out = _children(op, certs, memo)
|
|
1759
|
+
memo.put(op, out)
|
|
1760
|
+
return out
|
|
1761
|
+
|
|
1762
|
+
|
|
1763
|
+
def _children(op: O.Op, certs: List[Certificate], memo: ByIdentity) -> O.Op:
|
|
1764
|
+
changes = {}
|
|
1765
|
+
for f in dataclasses.fields(op):
|
|
1766
|
+
v = getattr(op, f.name)
|
|
1767
|
+
if isinstance(v, O.Op):
|
|
1768
|
+
nv = _restrict_groups(v, certs, memo)
|
|
1769
|
+
if nv is not v:
|
|
1770
|
+
changes[f.name] = nv
|
|
1771
|
+
elif isinstance(v, tuple) and v and all(isinstance(x, O.Op) for x in v):
|
|
1772
|
+
nv = tuple(_restrict_groups(x, certs, memo) for x in v)
|
|
1773
|
+
if any(a is not b for a, b in zip(nv, v)):
|
|
1774
|
+
changes[f.name] = nv
|
|
1775
|
+
return dataclasses.replace(op, **changes) if changes else op
|
|
1776
|
+
|
|
1777
|
+
|
|
1778
|
+
RULES: Tuple[Rule, ...] = (r13_joint_on_parent_key, r6_union_branches, r2_contained_pattern, r1_distinct_removal,
|
|
1779
|
+
r3_self_join, r3_left_join, r4_implied_type, r8_distinct_into_join,
|
|
1780
|
+
r9_union_containment, r11_eager_aggregation,
|
|
1781
|
+
r12_joint_child_set)
|
|
1782
|
+
LATE_RULES: Tuple[Rule, ...] = (r4_trusted_reference, r3_own_row_exists, r7_limit_inward)
|