prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1782 @@
1
+ """The rewrite registry (one-plan.md piece 1f; m2-performance.md 2.2).
2
+
3
+ A rewrite turns a plan into one with the same answers and less work. Each
4
+ rule reads the facts (`prograph.plan.ir.facts`) and fires only where they
5
+ prove it keeps the answers; unknown is no. The rules run bottom-up, to a
6
+ fixpoint, between the front end and the lowering.
7
+
8
+ Two classes (m2-performance.md):
9
+ - dominance: never more work where it fires (R1 a Distinct over distinct
10
+ rows, R2 a contained pattern); applied wherever proven;
11
+ - trade-off: better on some data, worse on other (where a Distinct goes);
12
+ applied only where a structural rule says so.
13
+
14
+ Each rewrite that fires leaves a certificate: the rule, and what it relied
15
+ on (`Certificate`), so a wrong answer can be traced to the fact that allowed
16
+ it. `PROGRAPH_REWRITES=off` turns the registry off, for tests that compare
17
+ the answers with and without it; it is no engine option.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import dataclasses
23
+ import os
24
+ from dataclasses import dataclass
25
+ from typing import Any, Callable, Dict, FrozenSet, List, Optional, Tuple
26
+
27
+ from prograph.plan.ir import expr as E
28
+ from prograph.plan.ir import ops as O
29
+ from prograph.plan.ir.facts import facts
30
+ from prograph.plan.ir.slot import UNKNOWN, Bound, injective_over, template_shape
31
+ from prograph.schema.source_map import EdgeMap, NodeMap, TermMap
32
+ from prograph.rules import home
33
+ from prograph.utils.helpers import ByIdentity
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class Certificate:
38
+ """A rewrite that fired: its rule and what it relied on."""
39
+ rule: str
40
+ relied_on: str
41
+
42
+ @property
43
+ def short(self) -> str:
44
+ """The rule's number: `R4b` for R4's second part, the trusted
45
+ reference; `R3o` for R3's left-join form (OPTIONAL)."""
46
+ n = self.rule.split(" ", 1)[0]
47
+ if self.rule.startswith("R3 left join"):
48
+ return n + "o"
49
+ if self.rule.startswith("R3 own row"):
50
+ return n + "e"
51
+ return n + "b" if self.rule.startswith("R4 trusted reference") else n
52
+
53
+
54
+ Rule = Callable[[O.Op, List[Certificate]], Optional[O.Op]]
55
+
56
+
57
+ def enabled() -> bool:
58
+ return os.environ.get("PROGRAPH_REWRITES", "on") != "off"
59
+
60
+
61
+ @home("plan-rewrites", "The plan's rewrites and what each must prove",
62
+ terms=("rewrite", "Distinct removal", "self-join", "contained pattern", "certificate",
63
+ "union branch", "containment pruning", "duplicate branch", "R9",
64
+ "one scan per source", "variable predicate", "R10", "R12", "joint query child", "R14"))
65
+ def rewrite(plan: O.Op, engine: Optional[str] = None) -> Tuple[O.Op, Tuple[Certificate, ...]]:
66
+ """`plan` with every rule applied where its facts allow, and the
67
+ certificates of the rewrites that fired. `engine` is the SQL dialect
68
+ the plan will run on: a trade-off rule reads it (R5)."""
69
+ if not enabled():
70
+ return plan, ()
71
+ certs: List[Certificate] = []
72
+ plan = _down(plan, certs, ByIdentity(), RULES)
73
+ # R10 after the rules that remove branches (R6's join-incompatible ones), so it merges only what
74
+ # is left, and before the late rules and the second pass, which read the merged scan.
75
+ plan = _outermost_unions(plan, certs)
76
+ late = LATE_RULES if engine in LIMIT_INWARD_ENGINES else \
77
+ tuple(r for r in LATE_RULES if r is not r7_limit_inward)
78
+ plan = _down(plan, certs, ByIdentity(), late)
79
+ plan = _down(plan, certs, ByIdentity(), RULES) # what the late rules opened up
80
+ if engine in RESTRICT_GROUPS_ENGINES:
81
+ # Top-down, once, after the bottom-up passes: a Filter is met before its join (R14 reads its
82
+ # conjuncts), and the plan's patterns are in their final, merged form.
83
+ plan = _restrict_groups(plan, certs, ByIdentity())
84
+ if engine in DISTINCT_BESIDE_LIMIT:
85
+ # A trade-off, after the rules that remove work: R2 reads the
86
+ # patterns' own Distincts.
87
+ plan = _down(plan, certs, ByIdentity(), (r5_distinct_beside_limit,))
88
+ _log(certs)
89
+ return plan, tuple(certs)
90
+
91
+
92
+ def _log(certs: List[Certificate]) -> None:
93
+ """For the score only: where `PROGRAPH_REWRITE_LOG` names a directory,
94
+ each firing is appended there with the test that caused it, so the
95
+ score can say how often each rule fired on cases with Jena's answers."""
96
+ where = os.environ.get("PROGRAPH_REWRITE_LOG")
97
+ if not where or not certs:
98
+ return
99
+ test = os.environ.get("PYTEST_CURRENT_TEST", "").rsplit(" (", 1)[0]
100
+ with open(os.path.join(where, f"fired-{os.getpid()}.tsv"), "a") as f:
101
+ for c in certs:
102
+ f.write(f"{c.short}\t{test}\n")
103
+
104
+
105
+ def _down(op: O.Op, certs: List[Certificate], memo: ByIdentity,
106
+ rules: Tuple[Rule, ...]) -> O.Op:
107
+ # Keyed by identity, holding the key: rules build and drop operators,
108
+ # and a dropped one's id may be given to a later one (rules.md, the
109
+ # memo keyed by an object's identity).
110
+ hit = memo.get(op)
111
+ if hit is not None:
112
+ return hit
113
+ changes = {}
114
+ for f in dataclasses.fields(op):
115
+ v = getattr(op, f.name)
116
+ if isinstance(v, O.Op):
117
+ nv = _down(v, certs, memo, rules)
118
+ if nv is not v:
119
+ changes[f.name] = nv
120
+ elif isinstance(v, tuple) and v and all(isinstance(x, O.Op) for x in v):
121
+ nv = tuple(_down(x, certs, memo, rules) for x in v)
122
+ if any(a is not b for a, b in zip(nv, v)):
123
+ changes[f.name] = nv
124
+ out = dataclasses.replace(op, **changes) if changes else op
125
+ from prograph.plan.ir.fresh import volatile
126
+ if volatile(out):
127
+ # A fresh value made below (fresh.py): no rule moves, copies or drops it.
128
+ memo.put(op, out)
129
+ return out
130
+ for _ in range(64):
131
+ for rule in rules:
132
+ new = rule(out, certs)
133
+ if new is not None and _CHECK:
134
+ _check_firing(rule, out, new)
135
+ if new is not None:
136
+ out = _down(new, certs, memo, rules) if new is not out else out
137
+ break
138
+ else:
139
+ break
140
+ memo.put(op, out)
141
+ return out
142
+
143
+
144
+ _CHECK = os.environ.get("PROGRAPH_REWRITE_CHECK") == "1"
145
+ _NARROWER = {(Bound.MAYBE, Bound.ALWAYS), (Bound.MAYBE, Bound.NEVER)}
146
+
147
+
148
+ def _check_firing(rule: Rule, before: O.Op, after: O.Op) -> None:
149
+ """For tests (the score sets `PROGRAPH_REWRITE_CHECK=1`): one firing
150
+ keeps the operator's columns, and no column's slot or the rows'
151
+ distinctness gets wider. A rule may narrow (a union branch gone leaves
152
+ fewer term maps); it may not widen, since what the plan above was
153
+ built on would no longer hold (m2-performance.md 2.3)."""
154
+ name = rule.__name__
155
+ if set(before.columns) != set(after.columns):
156
+ raise AssertionError(f"{name}: columns {before.columns} became {after.columns}")
157
+ if _no_rows(before) or _no_rows(after):
158
+ # Rows proven not to exist satisfy every claim: R6 removing every
159
+ # branch, or a rule firing inside a join the facts already prove
160
+ # empty (the fuzzer's cases, 2026-11-09).
161
+ return
162
+ old, new = before.slots, after.slots
163
+ for c in before.columns:
164
+ a, b = old[c], new[c]
165
+ bad = []
166
+ if not b.kinds <= a.kinds:
167
+ bad.append(f"kinds {sorted(a.kinds)} -> {sorted(b.kinds)}")
168
+ if not b.datatypes <= a.datatypes and UNKNOWN not in a.datatypes:
169
+ bad.append(f"datatypes {sorted(a.datatypes)} -> {sorted(b.datatypes)}")
170
+ for f in ("bound", "null"):
171
+ x, y = getattr(a, f), getattr(b, f)
172
+ if x is not y and (x, y) not in _NARROWER:
173
+ bad.append(f"{f} {x.value} -> {y.value}")
174
+ if a.provenance is not None and (b.provenance is None or not _same_terms(b.provenance, a.provenance)):
175
+ bad.append("provenance wider")
176
+ for f in ("language", "zone"):
177
+ if getattr(a, f) != getattr(b, f):
178
+ bad.append(f"{f} {getattr(a, f)!r} -> {getattr(b, f)!r}")
179
+ for f in ("native", "held_as"):
180
+ # None -> a form is a narrowing (one template or one datatype
181
+ # left after a union branch went); the lowering reads the form
182
+ # from the final plan. A form changed or lost is not.
183
+ if getattr(a, f) is not None and getattr(b, f) != getattr(a, f):
184
+ bad.append(f"{f} {getattr(a, f)!r} -> {getattr(b, f)!r}")
185
+ if bad:
186
+ raise AssertionError(f"{name}: column {c}: " + "; ".join(bad))
187
+ if facts(before).distinct and not facts(after).distinct:
188
+ raise AssertionError(f"{name}: distinct rows became possibly repeated")
189
+
190
+
191
+ def _no_rows(op: O.Op) -> bool:
192
+ """Whether the facts prove `op` has no row: a bound of 0, or a column
193
+ bound in every row that no kind or no map can fill."""
194
+ if facts(op).rows[1] == 0:
195
+ return True
196
+ if isinstance(op, O.Values):
197
+ return not op.rows
198
+ if isinstance(op, (O.Join, O.SemiJoin)):
199
+ return _no_rows(op.left) or _no_rows(op.right)
200
+ if isinstance(op, O.LeftJoin):
201
+ return _no_rows(op.left)
202
+ if isinstance(op, O.Union):
203
+ return all(_no_rows(x) for x in op.inputs)
204
+ if isinstance(op, (O.Distinct, O.Project, O.Filter, O.Extend, O.Order, O.Slice)):
205
+ return _no_rows(op.input)
206
+ return any(sl.bound is Bound.ALWAYS and (not sl.kinds or sl.provenance == frozenset())
207
+ for sl in op.slots.values())
208
+
209
+
210
+ def _same_terms(new: FrozenSet[TermMap], old: FrozenSet[TermMap]) -> bool:
211
+ """Each map of `new` gives only terms a map of `old` gives: the same
212
+ map, or a template of the same shape (a node's subject and an edge's
213
+ end, `template_shape`)."""
214
+ shapes = {template_shape(m) for m in old if m.template}
215
+ return all(m in old or (m.template and template_shape(m) in shapes) for m in new)
216
+
217
+
218
+ # -- R1: a Distinct over rows already distinct ----------------------------------------
219
+
220
+ def r1_distinct_removal(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
221
+ """`Distinct(X) -> X` where the facts prove X has no two equal rows: a
222
+ trusted key whose columns X always binds, through an injective term
223
+ constructor (one-plan.md 4.1); a join of distinct inputs; a union of
224
+ distinct branches no two of which can give one row; at most one row."""
225
+ if not isinstance(op, O.Distinct):
226
+ return None
227
+ f = facts(op.input)
228
+ if not f.distinct:
229
+ return None
230
+ why = (f"keys {sorted(sorted(k) for k in f.keys)}" if f.keys else "distinct input")
231
+ certs.append(Certificate("R1 distinct removal", why))
232
+ return op.input
233
+
234
+
235
+ # -- R2: a pattern another pattern of the same map contains ---------------------------
236
+
237
+ @dataclass(frozen=True)
238
+ class _Pattern:
239
+ """A pattern's rows read off one map scan: the source, the source
240
+ columns a row must have, the row conditions, and each output column's
241
+ expression over source columns."""
242
+ source: object
243
+ map_id: str
244
+ required: FrozenSet[str]
245
+ conditions: FrozenSet[E.Expr]
246
+ outputs: Tuple[Tuple[str, E.Expr], ...]
247
+ distinct: bool
248
+ scan: Optional[O.MapScan] = None
249
+ # Node maps every row's end is proven to exist in: (node map id, the
250
+ # (scan source column, node key column) pairs, the node scan).
251
+ exists: Tuple[Tuple[str, Tuple[Tuple[str, str], ...], O.MapScan], ...] = ()
252
+
253
+
254
+ def _pattern(op: O.Op) -> Optional[_Pattern]:
255
+ """`op` as one scan's pattern: Distinct? Project(Filter*(Extend*(MapScan)))."""
256
+ distinct = False
257
+ if isinstance(op, O.Distinct):
258
+ distinct, op = True, op.input
259
+ if not isinstance(op, O.Project):
260
+ return None
261
+ names = op.names
262
+ x = op.input
263
+ conditions: List[E.Expr] = []
264
+ exprs: Dict[str, E.Expr] = {}
265
+ semis: List[O.SemiJoin] = []
266
+ while True:
267
+ if isinstance(x, O.Filter):
268
+ conditions.extend(E.conjuncts(x.condition))
269
+ x = x.input
270
+ elif isinstance(x, O.Extend):
271
+ exprs.setdefault(x.name, x.expr)
272
+ x = x.input
273
+ elif isinstance(x, O.SemiJoin) and x.condition is None and isinstance(x.right, O.MapScan) \
274
+ and isinstance(x.right.map, NodeMap) and x.right.graph is None \
275
+ and x.right.joint is None:
276
+ semis.append(x)
277
+ x = x.left
278
+ else:
279
+ break
280
+ if not isinstance(x, O.MapScan) or x.graph is not None or x.joint is not None:
281
+ return None
282
+ to_source = {r.name: r.column for r in x.reads}
283
+
284
+ def source_expr(e: E.Expr) -> Optional[E.Expr]:
285
+ """`e` with every column named by its source column."""
286
+ if isinstance(e, E.Col):
287
+ if e.name in exprs:
288
+ return source_expr(exprs[e.name])
289
+ return E.Col("src:" + to_source[e.name]) if e.name in to_source else None
290
+ if isinstance(e, E.Const):
291
+ return e
292
+ if isinstance(e, E.MakeTerm):
293
+ args = []
294
+ for c, a in e.args:
295
+ sa = source_expr(a)
296
+ if sa is None:
297
+ return None
298
+ args.append((c, sa))
299
+ return E.MakeTerm(e.map, tuple(args))
300
+ if isinstance(e, E.Compare):
301
+ l, r = source_expr(e.left), source_expr(e.right)
302
+ return None if l is None or r is None else E.Compare(e.op, l, r, e.semantics)
303
+ return None
304
+
305
+ outputs = []
306
+ for n in names:
307
+ if n not in exprs:
308
+ return None
309
+ se = source_expr(exprs[n])
310
+ if se is None:
311
+ return None
312
+ outputs.append((n, se))
313
+ conds = set()
314
+ for c in conditions:
315
+ sc = source_expr(c)
316
+ if sc is None:
317
+ return None
318
+ if isinstance(sc, E.Compare) and sc.op == "=" and isinstance(sc.left, E.Const) \
319
+ and sc.left == sc.right:
320
+ continue # a constant equal to itself: every row
321
+ conds.add(sc)
322
+ required = frozenset(to_source[n] for n in x.required)
323
+ exists = []
324
+ for sj in semis:
325
+ theirs = {r.name: r.column for r in sj.right.reads}
326
+ pairs = tuple((to_source[c], theirs[c]) for c in sj.shared
327
+ if c in to_source and c in theirs)
328
+ if not (pairs and len(pairs) == len(sj.shared)):
329
+ # A semi-join this pattern cannot state on raw columns is no part of a pattern the
330
+ # rewrites may merge: refused, never dropped (M2 review R10).
331
+ return None
332
+ exists.append((sj.right.map.id, pairs, sj.right))
333
+ return _Pattern(x.map.source, x.map.id, required, frozenset(conds), tuple(outputs), distinct, x,
334
+ tuple(exists))
335
+
336
+
337
+ def _contains(big: _Pattern, small: _Pattern, any_map: bool = False) -> bool:
338
+ """Every row of `big`'s pattern, projected on `small`'s columns, is a row
339
+ of `small`'s: one map's rows (with `any_map`, one source's: two maps of
340
+ one logical table read the same rows), `small`'s requirements and
341
+ conditions among `big`'s, each of `small`'s columns built alike in `big`."""
342
+ if big.source != small.source or (big.map_id != small.map_id and not any_map):
343
+ return False
344
+ if not small.required <= big.required or not small.conditions <= big.conditions:
345
+ return False
346
+ mine = dict(big.outputs)
347
+ return all(n in mine and mine[n] == e for n, e in small.outputs)
348
+
349
+
350
+ def r2_contained_pattern(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
351
+ """`Join(L, R) -> R` (or `-> L`) where one side is distinct and the
352
+ other contains it (`_contains`): each row of the containing side
353
+ matches exactly one row of the contained, on columns both bind in every
354
+ row (m2-performance.md 2.2, R2). Needs no key: `?p a :Post . ?p :length
355
+ ?n` over one table is the second pattern alone."""
356
+ if not isinstance(op, O.Join) or op.condition is not None:
357
+ return None
358
+ for small_op, big_op in ((op.left, op.right), (op.right, op.left)):
359
+ small, big = _pattern(small_op), _pattern(big_op)
360
+ if small is None or big is None:
361
+ continue
362
+ if not (small.distinct or facts(small_op).distinct):
363
+ continue
364
+ cols = set(small_op.columns)
365
+ if not cols <= set(big_op.columns):
366
+ continue
367
+ if not all(small_op.slots[c].present is Bound.ALWAYS and
368
+ big_op.slots[c].present is Bound.ALWAYS for c in cols):
369
+ continue
370
+ if not _contains(big, small):
371
+ continue
372
+ certs.append(Certificate("R2 contained pattern",
373
+ f"map {small.map_id}: {sorted(cols)} within {sorted(big_op.columns)}"))
374
+ if big_op.columns == op.columns:
375
+ return big_op
376
+ return O.Project(big_op, op.columns)
377
+ return _join_tree_r2(op, certs)
378
+
379
+
380
+ def _join_tree_r2(op: O.Join, certs: List[Certificate]) -> Optional[O.Op]:
381
+ """R2 across a tree of joins: a pattern contained by another anywhere in
382
+ one tree of inner joins with no condition goes; the tree is rebuilt
383
+ left-deep in its order (a compatible join is associative and
384
+ commutative)."""
385
+ leaves: List[O.Op] = []
386
+
387
+ def flat(x: O.Op) -> None:
388
+ if isinstance(x, O.Join) and x.condition is None:
389
+ flat(x.left)
390
+ flat(x.right)
391
+ else:
392
+ leaves.append(x)
393
+ flat(op)
394
+ if len(leaves) < 3:
395
+ return None
396
+ pats = [_pattern(x) for x in leaves]
397
+ for i, small in enumerate(pats):
398
+ if small is None or not (small.distinct or facts(leaves[i]).distinct):
399
+ continue
400
+ cols = set(leaves[i].columns)
401
+ for j, big in enumerate(pats):
402
+ if i == j or big is None or not cols <= set(leaves[j].columns):
403
+ continue
404
+ if not all(leaves[i].slots[c].present is Bound.ALWAYS and
405
+ leaves[j].slots[c].present is Bound.ALWAYS for c in cols):
406
+ continue
407
+ if not _contains(big, small):
408
+ continue
409
+ certs.append(Certificate("R2 contained pattern",
410
+ f"map {small.map_id}: {sorted(cols)} within "
411
+ f"{sorted(leaves[j].columns)}"))
412
+ rest = leaves[:i] + leaves[i + 1:]
413
+ out = rest[0]
414
+ for x in rest[1:]:
415
+ out = O.Join(out, x)
416
+ return out if out.columns == op.columns else O.Project(out, op.columns)
417
+ return None
418
+
419
+
420
+ # -- R3: two patterns of one source row joined on a trusted key ---------------------
421
+
422
+ def _one_source(a: _Pattern, b: _Pattern) -> Optional[Tuple[_Pattern, _Pattern]]:
423
+ """`(narrow, wide)` where `a` and `b` read rows of one source: one map;
424
+ or two maps of one table (the row filter of `wide` within `narrow`'s, so
425
+ every row `narrow` reads is a row `wide` reads: a node's property
426
+ and an edge on its table, `?li :price ?p ; :part ?x`); or two maps of
427
+ one rr:sqlQuery text (one property per triples map), or a plain
428
+ select of a table (`simple_select`, a link map with its EXISTS) and a
429
+ map of that table, the scan then the query's, which must project the
430
+ columns the other reads. None otherwise."""
431
+ if a.map_id == b.map_id and a.source == b.source:
432
+ return a, b
433
+ for narrow, wide in ((a, b), (b, a)):
434
+ plain = narrow.source.query is None and wide.source.query is None
435
+ if narrow.source.rows_within(wide.source, folds=plain) and _projects(narrow, wide) \
436
+ and (plain or _folds_agree(narrow, wide)):
437
+ return narrow, wide
438
+ return None
439
+
440
+
441
+ def _projects(narrow: _Pattern, wide: _Pattern) -> bool:
442
+ """The columns `wide` reads are among those the plain select `narrow` reads projects (all of them for
443
+ `*`); two tables, or two maps of one query text, share their columns by name."""
444
+ if narrow.source.query is None or wide.source.query is not None:
445
+ return True
446
+ from prograph.schema.source_map import simple_select
447
+ shape = simple_select(narrow.source.query)
448
+ if shape is None:
449
+ return False
450
+ return shape.columns is None or {r.column for r in wide.scan.reads if not r.parent} <= set(shape.columns)
451
+
452
+
453
+ def _folds_agree(a: _Pattern, b: _Pattern) -> bool:
454
+ """The merged scan folds the columns either map writes undelimited (`Source.regular_columns`: the maps of
455
+ one query text, or a query and its table, name only their own), so each column either pattern reads must
456
+ be written in the merged set as its own map wrote it: no column read exactly by one that the other writes
457
+ undelimited."""
458
+ both = a.source.regular_columns | b.source.regular_columns
459
+ return all((r.column in p.source.regular_columns) == (r.column in both)
460
+ for p in (a, b) for r in p.scan.reads if not r.parent)
461
+
462
+
463
+ def _exists_restated(p: _Pattern) -> bool:
464
+ """Every semi-join of `p` reads only the columns it is joined on, so
465
+ it can be stated again over a merged scan (`_merged`)."""
466
+ for _, pairs, node_scan in p.exists:
467
+ if sorted(r.column for r in node_scan.reads) != sorted(k for _, k in pairs) \
468
+ or any(r.parent for r in node_scan.reads):
469
+ return False
470
+ return True
471
+
472
+
473
+ def _key_join(a: _Pattern, b: _Pattern, a_op: O.Op, b_op: O.Op) -> Optional[str]:
474
+ """The shared column on which `a` and `b` meet one row each: both read
475
+ one source's rows (`_one_source`), every shared column is built alike
476
+ on both sides, and one of them is built by an injective map from
477
+ columns holding a key the wider side's source is trusted unique on
478
+ (`MapScan.unique_keys`): equal terms there are one source row. Each
479
+ side's semi-joins (an edge's ends proven to exist) are kept on the
480
+ merged scan. None where no column proves it."""
481
+ if a.scan is None or b.scan is None or a.distinct or b.distinct:
482
+ return None
483
+ sides = _one_source(a, b)
484
+ if sides is None or not (_exists_restated(a) and _exists_restated(b)):
485
+ return None
486
+ wide = sides[1]
487
+ shared = set(a_op.columns) & set(b_op.columns)
488
+ if not shared:
489
+ return None
490
+ mine, theirs = dict(a.outputs), dict(b.outputs)
491
+ if any(mine.get(c) != theirs.get(c) for c in shared):
492
+ return None
493
+ keys = [frozenset(k) for k in wide.scan.unique_keys]
494
+ by_source = {}
495
+ for scan in (b.scan, a.scan):
496
+ held = facts(scan).slots
497
+ by_source.update({r.column: held.get(r.name) for r in scan.reads if not r.parent})
498
+ for c in sorted(shared):
499
+ e = mine[c]
500
+ if isinstance(e, E.MakeTerm) and e.args and \
501
+ all(isinstance(x, E.Col) and x.name.startswith("src:") for _, x in e.args) and \
502
+ injective_over(e.map, {t: by_source.get(x.name[4:]) for t, x in e.args}):
503
+ cols = frozenset(x.name[4:] for _, x in e.args)
504
+ if any(k <= cols for k in keys):
505
+ return c
506
+ return None
507
+
508
+
509
+ def _merged(a: _Pattern, b: _Pattern, columns: Tuple[str, ...], optional: bool = False) -> O.Op:
510
+ """One scan of the map giving both patterns' columns: its rows those
511
+ that meet both patterns' requirements and conditions. With `optional`
512
+ (`b` the right side of a left join) its rows are `a`'s, and
513
+ each column only `b` gives is its value where `b`'s requirements and
514
+ conditions hold on the row, else unbound."""
515
+ if optional:
516
+ narrow, wide = a, b
517
+ else:
518
+ narrow, wide = _one_source(a, b)
519
+ scan = narrow.scan
520
+ if narrow.source.regular_columns != wide.source.regular_columns:
521
+ # Each column is read as its own map wrote it (`_folds_agree`: no column in both differs).
522
+ both = narrow.source.regular_columns | wide.source.regular_columns
523
+ scan = dataclasses.replace(scan, map=dataclasses.replace(
524
+ scan.map, source=dataclasses.replace(scan.map.source, regular_columns=both)))
525
+ used: List[str] = []
526
+
527
+ def walk(e: E.Expr) -> None:
528
+ if isinstance(e, E.Col) and e.name.startswith("src:") and e.name[4:] not in used:
529
+ used.append(e.name[4:])
530
+ for f in dataclasses.fields(e) if dataclasses.is_dataclass(e) else ():
531
+ v = getattr(e, f.name)
532
+ if isinstance(v, E.Expr):
533
+ walk(v)
534
+ elif isinstance(v, tuple):
535
+ for x in v:
536
+ if isinstance(x, E.Expr):
537
+ walk(x)
538
+ elif isinstance(x, tuple):
539
+ for y in x:
540
+ if isinstance(y, E.Expr):
541
+ walk(y)
542
+ outputs = dict(a.outputs)
543
+ outputs.update(dict(b.outputs))
544
+ conditions = list(a.conditions | b.conditions)
545
+ for e in list(outputs.values()) + conditions:
546
+ walk(e)
547
+ exists = list(a.exists) + ([] if optional else [e for e in b.exists if e not in a.exists])
548
+ for c in sorted(a.required | b.required) + [s for _, pairs, _ in exists for s, _ in pairs]:
549
+ if c not in used:
550
+ used.append(c)
551
+ required = a.required if optional else a.required | b.required
552
+ only_b = [c for c in columns if c in dict(b.outputs) and c not in dict(a.outputs)]
553
+ if optional:
554
+ conditions = list(a.conditions)
555
+ by_column = {r.column: r for r in a.scan.reads + b.scan.reads + scan.reads if not r.parent}
556
+ names = {c: f"#r3.{i}" for i, c in enumerate(used)}
557
+
558
+ def local(e: E.Expr) -> E.Expr:
559
+ if isinstance(e, E.Col) and e.name.startswith("src:"):
560
+ return E.Col(names[e.name[4:]])
561
+ if isinstance(e, E.MakeTerm):
562
+ return E.MakeTerm(e.map, tuple((c, local(x)) for c, x in e.args))
563
+ if isinstance(e, E.Compare):
564
+ return E.Compare(e.op, local(e.left), local(e.right), e.semantics)
565
+ return e
566
+ reads = tuple(dataclasses.replace(by_column[c], name=names[c]) for c in used)
567
+ x: O.Op = O.MapScan(map=scan.map, reads=reads,
568
+ required=tuple(names[c] for c in used if c in required),
569
+ unique_keys=tuple(dict.fromkeys(scan.unique_keys + wide.scan.unique_keys)))
570
+ for _, pairs, node_scan in exists:
571
+ # The semi-join again, over the merged scan's names (`_exists_restated`).
572
+ to_name = {k: names[s] for s, k in pairs}
573
+ old = {r.name: r.column for r in node_scan.reads}
574
+ x = O.SemiJoin(x, dataclasses.replace(
575
+ node_scan, reads=tuple(dataclasses.replace(r, name=to_name[r.column]) for r in node_scan.reads),
576
+ required=tuple(to_name[old[n]] for n in node_scan.required if n in old)))
577
+ matched: Optional[E.Expr] = None
578
+ if optional:
579
+ tests = [E.IsBound(E.Col(names[c])) for c in sorted(b.required - a.required)]
580
+ tests += [local(c) for c in sorted(b.conditions - a.conditions, key=repr)]
581
+ matched = tests[0] if len(tests) == 1 else E.And(tuple(tests)) if tests else None
582
+ for c in columns:
583
+ e = local(outputs[c])
584
+ if c in only_b and matched is not None:
585
+ e = E.If(matched, e, E.Const(None))
586
+ x = O.Extend(x, c, e)
587
+ conds = [local(c) for c in conditions]
588
+ if conds:
589
+ x = O.Filter(x, conds[0] if len(conds) == 1 else E.And(tuple(conds)))
590
+ return O.Project(x, columns)
591
+
592
+
593
+ @home("self-join-on-a-key", "Two patterns of one source row on a trusted key are one scan (R3, R3o)",
594
+ terms=("self-join", "self join", "R3", "R3o", "one scan", "own row", "same row", "trusted key",
595
+ "OPTIONAL", "left join", "optional property", "_own_row", "join elimination"))
596
+ def r3_self_join(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
597
+ """Two patterns of one map joined on a column built from a trusted key
598
+ of its source (`_key_join`) are one scan of it (m2-performance.md 2.2,
599
+ R3): each row of either side meets exactly the one row of the other
600
+ read from the same source row. Without a trusted key it does not fire:
601
+ two rows of one key with names A, B and ages 1, 2 give four pairs, one
602
+ scan two."""
603
+ if not isinstance(op, O.Join) or op.condition is not None:
604
+ return None
605
+ leaves: List[O.Op] = []
606
+
607
+ def flat(x: O.Op) -> None:
608
+ if isinstance(x, O.Join) and x.condition is None:
609
+ flat(x.left)
610
+ flat(x.right)
611
+ else:
612
+ leaves.append(x)
613
+ flat(op)
614
+ pats = [_pattern(x) for x in leaves]
615
+ for i in range(len(leaves)):
616
+ for j in range(i + 1, len(leaves)):
617
+ a, b = pats[i], pats[j]
618
+ if a is None or b is None:
619
+ continue
620
+ c = _key_join(a, b, leaves[i], leaves[j])
621
+ if c is None:
622
+ continue
623
+ cols = tuple(dict.fromkeys(leaves[i].columns + leaves[j].columns))
624
+ wide = _one_source(a, b)[1]
625
+ maps = a.map_id if a.map_id == b.map_id else f"{a.map_id} and {b.map_id} (one table)"
626
+ certs.append(Certificate("R3 self-join on a trusted key",
627
+ f"map {maps}, column {c}, keys "
628
+ f"{[list(k) for k in wide.scan.unique_keys]}"))
629
+ merged = _merged(a, b, cols)
630
+ # In the first one's place: each leaf then follows at least the columns it followed, so the
631
+ # written order stays connected (rule join-order; at the front, a leaf tied only to a later
632
+ # one became a cross join, lineitem x part in TPC-H q17).
633
+ rest = [merged if k == i else x for k, x in enumerate(leaves) if k != j]
634
+ out = rest[0]
635
+ for x in rest[1:]:
636
+ out = O.Join(out, x)
637
+ return out if out.columns == op.columns else O.Project(out, op.columns)
638
+ return None
639
+
640
+
641
+ def r3_left_join(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
642
+ """R3's left-join form (pre-1.0.0's `_own_row`): an OPTIONAL
643
+ pattern `b` of the map a pattern `a` of the left side reads, joined on a
644
+ column built from a trusted key of its source (`_key_join`), is that
645
+ row's columns: each row of `a` meets at most the one row of `b` read
646
+ from the same source row, and `b` has it exactly where its requirements
647
+ and conditions hold there (`_merged` with `optional`). `b` shares no
648
+ column with the left side outside `a`, so the rest of the left side's
649
+ joins are kept around it. Not with a left join's own condition."""
650
+ if not isinstance(op, O.LeftJoin) or op.condition is not None:
651
+ return None
652
+ b = _pattern(op.right)
653
+ if b is None:
654
+ return None
655
+ leaves: List[O.Op] = []
656
+
657
+ def flat(x: O.Op) -> None:
658
+ if isinstance(x, O.Join) and x.condition is None:
659
+ flat(x.left)
660
+ flat(x.right)
661
+ else:
662
+ leaves.append(x)
663
+ flat(op.left)
664
+ shared = set(op.left.columns) & set(op.right.columns)
665
+ for i, leaf in enumerate(leaves):
666
+ a = _pattern(leaf)
667
+ if a is None or not shared <= set(leaf.columns):
668
+ continue
669
+ if a.map_id != b.map_id or a.source != b.source or a.exists or b.exists:
670
+ continue # R3o stays within one map, no semi-join (R3's wider form is for inner joins)
671
+ c = _key_join(a, b, leaf, op.right)
672
+ if c is None:
673
+ continue
674
+ cols = tuple(dict.fromkeys(leaf.columns + op.right.columns))
675
+ certs.append(Certificate("R3 left join on a trusted key",
676
+ f"map {a.map_id}, column {c}, keys "
677
+ f"{[list(k) for k in a.scan.unique_keys]}"))
678
+ rest = [_merged(a, b, cols, optional=True) if k == i else x for k, x in enumerate(leaves)]
679
+ out = rest[0]
680
+ for x in rest[1:]:
681
+ out = O.Join(out, x)
682
+ return out if out.columns == op.columns else O.Project(out, op.columns)
683
+ return None
684
+
685
+
686
+ # -- R9: a union branch another branch already gives ---------------------------------
687
+
688
+ def r9_union_containment(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
689
+ """`Distinct(Union(..., X, ..., Y, ...))` without X where every row of X is
690
+ a row of Y (`_contains` over one source): the same columns,
691
+ each built alike from the same source row, and Y requiring no more of the
692
+ row than X. An ontology's saturation makes such branches: a map giving
693
+ `ex:surname` and `ex:name` from one column gives `ex:name` twice, and two
694
+ maps of one table typed Manager and Employee give the Employee subjects
695
+ twice. Under the Distinct a row given twice is one row."""
696
+ if not isinstance(op, O.Distinct) or not isinstance(op.input, O.Union):
697
+ return None
698
+ branches = list(op.input.inputs)
699
+ pats = [_pattern(b) for b in branches]
700
+ for i, (x, px) in enumerate(zip(branches, pats)):
701
+ if px is None:
702
+ continue
703
+ for j, (y, py) in enumerate(zip(branches, pats)):
704
+ if i == j or py is None or set(x.columns) != set(y.columns):
705
+ continue
706
+ if px.exists or py.exists or not _contains(px, py, any_map=True):
707
+ continue
708
+ if _contains(py, px, any_map=True) and j > i:
709
+ continue # the same rows both ways: keep the first, drop the later one
710
+ certs.append(Certificate("R9 union branch within another",
711
+ f"map {px.map_id} within map {py.map_id}"))
712
+ rest = tuple(b for k, b in enumerate(branches) if k != i)
713
+ inner = rest[0] if len(rest) == 1 else O.Union(rest)
714
+ if inner.columns != op.input.columns:
715
+ inner = O.Project(inner, op.input.columns)
716
+ return O.Distinct(inner)
717
+ return None
718
+
719
+
720
+ # -- R10: one scan per source for a union of its branches ------------------------------
721
+
722
+ def _src_columns(e: E.Expr, out: List[str]) -> None:
723
+ """The source columns (`src:` names) `e` reads, in order of first reading."""
724
+ if isinstance(e, E.Col):
725
+ if e.name.startswith("src:") and e.name[4:] not in out:
726
+ out.append(e.name[4:])
727
+ return
728
+ for child in e.children():
729
+ _src_columns(child, out)
730
+
731
+
732
+ def _branch_of(x: O.Op):
733
+ """A union branch as (its constant columns, its pattern), or None: `Extend`s of constants (a
734
+ variable predicate's term) over one map's pattern with no existence proof and no graph map."""
735
+ consts: Dict[str, E.Expr] = {}
736
+ while isinstance(x, O.Extend) and isinstance(x.expr, E.Const):
737
+ consts.setdefault(x.name, x.expr)
738
+ x = x.input
739
+ narrowed = None
740
+ if isinstance(x, O.Project) and isinstance(x.input, O.Distinct):
741
+ # A narrowing of a distinct pattern (a typing branch keeps the subject of its pattern):
742
+ # its rows are no set, so it is not distinct; under a group's Distinct that is the same.
743
+ narrowed, x = x.names, x.input
744
+ p = _pattern(x)
745
+ if p is None or p.scan is None or p.scan.graph is not None or p.scan.joint is not None:
746
+ return None
747
+ if narrowed is not None:
748
+ if not set(narrowed) <= set(dict(p.outputs)):
749
+ return None
750
+ p = dataclasses.replace(p, outputs=tuple((n, e) for n, e in p.outputs if n in narrowed), distinct=False)
751
+ return consts, p
752
+
753
+
754
+ def _as_read(e: E.Expr, p) -> E.Expr:
755
+ """A pattern's expression (source columns, `src:`) over its scan's read names."""
756
+ names = {r.column: r.name for r in p.scan.reads if not r.parent}
757
+ if isinstance(e, E.Col) and e.name.startswith("src:"):
758
+ return E.Col(names[e.name[4:]])
759
+ if isinstance(e, E.MakeTerm):
760
+ return E.MakeTerm(e.map, tuple((c, _as_read(x, p)) for c, x in e.args))
761
+ if isinstance(e, E.Compare):
762
+ return E.Compare(e.op, _as_read(e.left, p), _as_read(e.right, p), e.semantics)
763
+ return e
764
+
765
+
766
+ def _form(r) -> Tuple:
767
+ """How a scan reads a column (held datatype, zone, digits, reader): one column read in two
768
+ forms by two members is two reads of the merged scan (Spark refused a double read as text)."""
769
+ return (r.parent, r.held, r.zone, r.digits, r.reader)
770
+
771
+
772
+ def _proof(p) -> Tuple:
773
+ """A pattern's existence proofs as a key: (node map, its column pairs) each; members with one key
774
+ share one semi-join."""
775
+ return tuple(sorted((node_id, tuple(sorted(pairs))) for node_id, pairs, _ in p.exists))
776
+
777
+
778
+ def _outermost_unions(op: O.Op, certs: List[Certificate]) -> O.Op:
779
+ """R10 at each outermost union, before the bottom-up passes: it flattens the Distinct(Union)
780
+ groups below it (a predicate's saturated map copies), which a bottom-up pass would merge one group
781
+ at a time and so never across groups (turbofan ?p: 409 scans left that way)."""
782
+ if isinstance(op, O.Union):
783
+ merged = r10_one_scan_per_source(op, certs)
784
+ if merged is not None:
785
+ if _CHECK:
786
+ _check_firing(r10_one_scan_per_source, op, merged)
787
+ return merged
788
+ changes = {}
789
+ for f in dataclasses.fields(op):
790
+ v = getattr(op, f.name)
791
+ if isinstance(v, O.Op):
792
+ nv = _outermost_unions(v, certs)
793
+ if nv is not v:
794
+ changes[f.name] = nv
795
+ elif isinstance(v, tuple) and v and all(isinstance(x, O.Op) for x in v):
796
+ nv = tuple(_outermost_unions(x, certs) for x in v)
797
+ if any(a is not b for a, b in zip(nv, v)):
798
+ changes[f.name] = nv
799
+ return dataclasses.replace(op, **changes) if changes else op
800
+
801
+
802
+ def _groups_of(op: O.Union):
803
+ """The union's inputs as groups of members, each member (its constant columns, its pattern):
804
+ an input is a group of one, or, under a Distinct, the union it holds (a predicate's map copies,
805
+ saturated). None where an input is neither; each group's Distinct flag beside it."""
806
+ groups = []
807
+ for x in op.inputs:
808
+ consts: Dict[str, E.Expr] = {}
809
+ y = x
810
+ while isinstance(y, O.Extend) and isinstance(y.expr, E.Const):
811
+ consts.setdefault(y.name, y.expr)
812
+ y = y.input
813
+ if isinstance(y, O.Distinct) and isinstance(y.input, O.Union):
814
+ members, raw = [], []
815
+
816
+ def expand(m: O.Op, outer: Dict[str, E.Expr]) -> None:
817
+ if _no_rows(m):
818
+ # A member the facts prove empty (a required column never bound) gives no row:
819
+ # left out, as the passes below would drop it, never merged into a scan where
820
+ # its term would still be built.
821
+ return
822
+ b = _branch_of(m)
823
+ if b is not None:
824
+ members.append(({**outer, **b[0]}, b[1], m, dict(outer)))
825
+ return
826
+ inner_consts = dict(outer)
827
+ z = m
828
+ while isinstance(z, O.Extend) and isinstance(z.expr, E.Const):
829
+ inner_consts.setdefault(z.name, z.expr)
830
+ z = z.input
831
+ if isinstance(z, O.Project) and isinstance(z.input, O.Distinct) and isinstance(z.input.input, O.Union):
832
+ z = z.input
833
+ if isinstance(z, O.Distinct) and isinstance(z.input, O.Union):
834
+ # A group within the group (a superclass's typing branches): the outer Distinct
835
+ # covers its own, so its members join the group.
836
+ for n in z.input.inputs:
837
+ expand(n, inner_consts)
838
+ return
839
+ raw.append((m, outer)) # kept as it is, its rows deduplicated with the group's
840
+ for m in y.input.inputs:
841
+ expand(m, dict(consts))
842
+ groups.append((members, True, x, consts, raw, False))
843
+ continue
844
+ m = None if _no_rows(y) else _branch_of(y)
845
+ if m is None:
846
+ groups.append((None, isinstance(x, O.Distinct) or facts(x).distinct, x, {}, [], False))
847
+ continue
848
+ # A set without its own Distinct (its rows keyed, as R1 reads the facts): then no Distinct is
849
+ # needed above it once merged.
850
+ below = y.input if isinstance(y, O.Distinct) else y
851
+ keyed = facts(below).distinct
852
+ # A set: its own Distinct, or rows the facts prove distinct (R1 removed that Distinct).
853
+ groups.append(([({**consts, **m[0]}, m[1], y, dict(consts))], m[1].distinct or keyed, x, {}, [], keyed))
854
+ return groups
855
+
856
+
857
+ def r10_one_scan_per_source(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
858
+ """A union's branches over one source (pre-1.0.0's
859
+ `_each_row_once`) read it once: one scan of the source, joined to a Values row per branch (its
860
+ index), each output the branch's term for its row (`If` on the index, so each branch's term
861
+ keeps its own kind, datatype and SQL type, `lower.if_cell`), the rows a branch gives kept by its
862
+ own requirements and conditions. The same rows: each source row meets each branch's Values row,
863
+ and the filter keeps the pairs that branch gave. Every input distinct (a predicate's saturated
864
+ map copies under their Distinct, or a pattern's own): one Distinct over (input, outputs) above
865
+ the merged scans keeps each input's set. `?s ?p ?o` over the turbofan ontology's saturated maps
866
+ read one scan per map copy (1270 CTEs, past DuckDB's binder); now one per source."""
867
+ if not isinstance(op, O.Union):
868
+ return None
869
+ groups = _groups_of(op)
870
+ if groups is None or not all(g[1] for g in groups):
871
+ return None # an input that is not a set: one Distinct above would change its rows
872
+ members = [(gi, consts, p, (mop, outer)) for gi, (ms, _, _, _, _, _) in enumerate(groups) if ms is not None
873
+ for consts, p, mop, outer in ms]
874
+ by_source: Dict[Any, List[int]] = {}
875
+ for k, (_, _, p, _) in enumerate(members):
876
+ by_source.setdefault((p.source, _proof(p)), []).append(k)
877
+ # A member whose own terms or conditions may raise (`lower.never_raises_over`, NEVER_RAISES, the
878
+ # check the one-plan rewrites use): kept as its own branch, since in a merged scan another member's row could reach
879
+ # them (Spark does not promise a CASE guards an error; a date's text raises outside years 1-9999).
880
+ from prograph.plan.ir.lower import never_raises_over
881
+ apart: List[int] = []
882
+ for key, ks in list(by_source.items()):
883
+ if len(ks) < 2:
884
+ continue
885
+ keep = []
886
+ # The merged scan requires only what every member requires (`_one_scan`): each member's terms
887
+ # and conditions are checked over that scan, its other columns possibly unbound (fresh review
888
+ # 2026-11-11, an IRI column one member required was built, and raised, on a row
889
+ # where it did not hold a valid IRI).
890
+ every = set.intersection(*[set(members[j][2].required) for j in ks])
891
+ for k in ks:
892
+ _, _, p, _ = members[k]
893
+ p_merged = dataclasses.replace(p, scan=dataclasses.replace(
894
+ p.scan, required=tuple(r.name for r in p.scan.reads if not r.parent and r.column in every)))
895
+ outs = dict(p.outputs)
896
+ own = [(e, op.slots.get(c)) for c, e in outs.items()
897
+ if any(dict(members[j][2].outputs).get(c) != e for j in ks if j != k)]
898
+ others = set.intersection(*[set(members[j][2].conditions) for j in ks if j != k])
899
+ conds = [c for c in p.conditions if c not in others]
900
+ if never_raises_over(p_merged.scan, [(_as_read(e, p), s) for e, s in own], [_as_read(c, p) for c in conds]):
901
+ keep.append(k)
902
+ else:
903
+ apart.append(k)
904
+ by_source[key] = keep
905
+ # Merged only where a condition narrows each member's rows (a constant subject or object):
906
+ # there one scan of the few rows beats a scan per branch (LDBC <person/14> ?p ?o, 1.02 ->
907
+ # 0.8 s on the warehouse); over a whole table it multiplies every row by the Values list and reads
908
+ # every column, where the branches each read one (COUNT(*) { ?s ?p ?o } 2.0 s as branches, 7 s
909
+ # merged), and the lowering keeps the branches in one CTE (`Lowering._Union`).
910
+ # A source left unmerged keeps each of its members as its own branch, as one kept apart.
911
+ narrowed = {key: ks for key, ks in by_source.items() if ks and all(members[k][2].conditions for k in ks)}
912
+ apart += [k for key, ks in by_source.items() if key not in narrowed for k in ks]
913
+ by_source = narrowed
914
+ if all(len(v) < 2 for v in by_source.values()):
915
+ return None # nothing merges (members kept apart alone are no reason to rebuild the union)
916
+ cols = op.columns
917
+ g = "#r10.g"
918
+ parts: List[O.Op] = []
919
+ # The columns the union binds in every row: each merged part says so (an IsBound that every row
920
+ # meets, since each member's own requirements are in its guard), so no slot gets wider.
921
+ always = [c for c in cols if op.slots[c].present is Bound.ALWAYS]
922
+ # One Values for every group (rows 0 .. the largest group's size - 1), one CTE however many
923
+ # groups read it (the lowering keeps one CTE per operator object); a group's guards keep its own.
924
+ from prograph.plan.ir.terms import LiteralTerm as _Lit
925
+ from prograph.schema import xsd as _x
926
+ size = max(len(ks) for ks in by_source.values())
927
+ shared = O.Values(("#r10.b",), tuple((_Lit(str(i), _x.INTEGER),) for i in range(size)))
928
+ for ks in by_source.values():
929
+ part = _one_scan([members[k][:3] for k in ks], cols, g, shared)
930
+ if always:
931
+ keep = [E.IsBound(E.Col(c)) for c in always]
932
+ part = O.Filter(part, keep[0] if len(keep) == 1 else E.And(tuple(keep)))
933
+ parts.append(part)
934
+ from prograph.plan.ir.terms import LiteralTerm
935
+ from prograph.schema import xsd as _xsd
936
+ for k in apart: # a member kept apart: its own branch, with its group's id and constants
937
+ gi, _, _, (mop, outer) = members[k]
938
+ for name_, e in outer.items():
939
+ if name_ not in mop.columns:
940
+ mop = O.Extend(mop, name_, e)
941
+ part = O.Project(O.Extend(mop, g, E.Const(LiteralTerm(str(gi), _xsd.INTEGER))), (g,) + tuple(cols))
942
+ parts.append(part)
943
+ for gi, (ms, _, x, consts, raw, _) in enumerate(groups):
944
+ tag = E.Const(LiteralTerm(str(gi), _xsd.INTEGER))
945
+ if ms is None: # an input no scan merges: as it is, with its id
946
+ parts.append(O.Project(O.Extend(x, g, tag), (g,) + tuple(cols)))
947
+ for m, outer in raw: # a group's member no scan merges: with the constants around it
948
+ for name, e in outer.items():
949
+ if name not in m.columns:
950
+ m = O.Extend(m, name, e)
951
+ parts.append(O.Project(O.Extend(m, g, tag), (g,) + tuple(cols)))
952
+ certs.append(Certificate("R10 one scan per source",
953
+ f"{len(members)} branches over {len(by_source)} sources"))
954
+ body = parts[0] if len(parts) == 1 else O.Union(tuple(parts))
955
+ if facts(op).distinct:
956
+ # The union's rows were a set (its branches apart, a predicate each): one Distinct over its
957
+ # columns removes no row and keeps that fact for the plan above.
958
+ return O.Distinct(O.Project(body, tuple(cols)))
959
+ if all(gr[5] for gr in groups):
960
+ # Every input a single pattern whose rows are a set by their key: the union of their rows as
961
+ # it was, no Distinct (R1's facts must still hold after the merge).
962
+ return O.Project(body, tuple(cols))
963
+ return O.Project(O.Distinct(O.Project(body, (g,) + tuple(cols))), tuple(cols))
964
+
965
+
966
+ def _one_scan(members, cols: Tuple[str, ...], g: str, values: Optional[O.Op] = None) -> O.Op:
967
+ """One scan of the members' source giving each member's rows, with its group's id in `g`."""
968
+ forms = [{r.column: _form(r) for r in p.scan.reads if not r.parent} for _, _, p in members]
969
+ used: List[Tuple[str, Tuple]] = []
970
+ for k, (_, consts, p) in enumerate(members):
971
+ cols_k: List[str] = []
972
+ for _, e in p.outputs:
973
+ _src_columns(e, cols_k)
974
+ for c in sorted(p.conditions, key=repr):
975
+ _src_columns(c, cols_k)
976
+ for c in sorted(p.required):
977
+ if c not in cols_k:
978
+ cols_k.append(c)
979
+ for _, pairs, _ in p.exists:
980
+ for src, _ in pairs:
981
+ if src not in cols_k:
982
+ cols_k.append(src)
983
+ for c in cols_k:
984
+ key = (c, forms[k].get(c))
985
+ if key not in used:
986
+ used.append(key)
987
+ by_key = {}
988
+ for k, (_, _, p) in enumerate(members):
989
+ for r in p.scan.reads:
990
+ if not r.parent:
991
+ by_key.setdefault((r.column, _form(r)), r)
992
+ names = {key: f"#r10.{i}" for i, key in enumerate(used)}
993
+
994
+ def local(e: E.Expr, k: int = 0) -> E.Expr:
995
+ if isinstance(e, E.Col) and e.name.startswith("src:"):
996
+ c = e.name[4:]
997
+ return E.Col(names[(c, forms[k].get(c))])
998
+ if isinstance(e, E.MakeTerm):
999
+ return E.MakeTerm(e.map, tuple((c, local(x, k)) for c, x in e.args))
1000
+ if isinstance(e, E.Compare):
1001
+ return E.Compare(e.op, local(e.left, k), local(e.right, k), e.semantics)
1002
+ return e
1003
+
1004
+ def name(c: str, k: int) -> str:
1005
+ return names[(c, forms[k].get(c))]
1006
+ from prograph.plan.ir.terms import LiteralTerm
1007
+ from prograph.schema import xsd as _xsd
1008
+
1009
+ def number(i: int) -> E.Const:
1010
+ return E.Const(LiteralTerm(str(i), _xsd.INTEGER))
1011
+ first = members[0][2].scan
1012
+ # Required: the columns every member requires (in one form), so their slots are bound in every
1013
+ # row, as in each member's own scan.
1014
+ every_req = set.intersection(*[set(p.required) for _, _, p in members])
1015
+ x: O.Op = O.MapScan(map=first.map, reads=tuple(dataclasses.replace(by_key[key], name=names[key]) for key in used),
1016
+ required=tuple(names[key] for key in used if key[0] in every_req
1017
+ and all(f.get(key[0]) == key[1] for f in forms)),
1018
+ unique_keys=first.unique_keys)
1019
+ # The members' one existence proof (they share it: `_proof`): a semi-join into the node map on
1020
+ # the source's columns, as each member had it.
1021
+ for node_id, pairs, node_scan in members[0][2].exists:
1022
+ theirs = {r.column: r for r in node_scan.reads}
1023
+ right_reads = tuple(dataclasses.replace(theirs[key], name=name(src, 0)) for src, key in pairs)
1024
+ right = O.MapScan(map=node_scan.map, reads=right_reads, required=tuple(r.name for r in right_reads),
1025
+ unique_keys=node_scan.unique_keys)
1026
+ x = O.SemiJoin(x, right)
1027
+ if len(members) == 1:
1028
+ gi, consts, p = members[0]
1029
+ pre = [E.IsBound(E.Col(name(c, 0))) for c in sorted(p.required)] + [local(c, 0) for c in sorted(p.conditions, key=repr)]
1030
+ if pre:
1031
+ x = O.Filter(x, pre[0] if len(pre) == 1 else E.And(tuple(pre)))
1032
+ x = O.Extend(x, g, number(gi))
1033
+ outs = dict(p.outputs)
1034
+ for c in cols:
1035
+ x = O.Extend(x, c, consts[c] if c in consts else local(outs[c], 0) if c in outs else E.Const(None))
1036
+ return O.Project(x, (g,) + tuple(cols))
1037
+ # What every member requires and every condition they share filter the scan itself (a constant
1038
+ # subject is pushed into it, as each branch had it).
1039
+ one_form = {c for c in set.intersection(*[set(f) for f in forms]) if len({f[c] for f in forms}) == 1}
1040
+
1041
+ def reads_one_form(e: E.Expr) -> bool:
1042
+ cs: List[str] = []
1043
+ _src_columns(e, cs)
1044
+ return all(c in one_form for c in cs)
1045
+ shared_req = {c for c in set.intersection(*[set(p.required) for _, _, p in members]) if c in one_form}
1046
+ shared_cond = {c for c in set.intersection(*[set(p.conditions) for _, _, p in members]) if reads_one_form(c)}
1047
+ pre = [E.IsBound(E.Col(name(c, 0))) for c in sorted(shared_req)] + [local(c, 0) for c in sorted(shared_cond, key=repr)]
1048
+ if pre:
1049
+ x = O.Filter(x, pre[0] if len(pre) == 1 else E.And(tuple(pre)))
1050
+ b = "#r10.b"
1051
+ x = O.Join(x, values if values is not None else
1052
+ O.Values((b,), tuple((LiteralTerm(str(i), _xsd.INTEGER),) for i in range(len(members)))))
1053
+
1054
+ def is_member(i: int) -> E.Expr:
1055
+ return E.Compare("=", E.Col(b), number(i), E.TERM)
1056
+ guards = []
1057
+ for i, (_, _, p) in enumerate(members):
1058
+ own = [E.IsBound(E.Col(name(c, i))) for c in sorted(set(p.required) - shared_req)]
1059
+ own += [local(c, i) for c in sorted(p.conditions - shared_cond, key=repr)]
1060
+ guards.append(E.And((is_member(i), *own)) if own else is_member(i))
1061
+ x = O.Filter(x, E.Or(tuple(guards)))
1062
+
1063
+ def chain(pick) -> E.Expr:
1064
+ # The last member's value is the chain's end: the filter above keeps only rows of some
1065
+ # member, so no row reaches an unbound end, and a column bound in every member stays so.
1066
+ out: E.Expr = pick(len(members) - 1)
1067
+ for i in reversed(range(len(members) - 1)):
1068
+ out = E.If(is_member(i), pick(i), out)
1069
+ return out
1070
+ x = O.Extend(x, g, chain(lambda i: number(members[i][0])))
1071
+ for c in cols:
1072
+ def pick(i: int, c=c) -> E.Expr:
1073
+ _, consts, p = members[i]
1074
+ outs = dict(p.outputs)
1075
+ return consts[c] if c in consts else local(outs[c], i) if c in outs else E.Const(None)
1076
+ x = O.Extend(x, c, chain(pick))
1077
+ return O.Project(x, (g,) + tuple(cols))
1078
+
1079
+
1080
+ # -- R4: a type pattern an existence proof implies ----------------------------------
1081
+
1082
+ def _subject_only(t: _Pattern) -> Optional[Tuple[str, E.MakeTerm]]:
1083
+ """`t` as a node map's subjects and nothing else: one output, its
1084
+ subject term built from the map's key columns, no condition, nothing
1085
+ required beyond those columns (a type pattern, `?x a :C`)."""
1086
+ if t.scan is None or not isinstance(t.scan.map, NodeMap) or len(t.outputs) != 1 or t.conditions:
1087
+ return None
1088
+ name, e = t.outputs[0]
1089
+ m = t.scan.map
1090
+ if not isinstance(e, E.MakeTerm) or e.map != m.subject:
1091
+ return None
1092
+ cols = [x.name[4:] for _, x in e.args if isinstance(x, E.Col) and x.name.startswith("src:")]
1093
+ if len(cols) != len(e.args) or set(cols) != set(m.key.columns) or not t.required <= set(cols):
1094
+ return None
1095
+ return name, e
1096
+
1097
+
1098
+ def _implied(t: _Pattern, t_op: O.Op, x: _Pattern, x_op: O.Op) -> Optional[str]:
1099
+ """Every row of `x` meets exactly one row of `t` (`t` the subjects of a
1100
+ node map N, distinct): `x` builds the shared column from columns it
1101
+ has proven to exist among N's keys (a semi-join into N's scan, on
1102
+ columns of the same types), by a template of the same shape as N's
1103
+ subject, the same values in the same places. Then the join adds
1104
+ nothing."""
1105
+ sub = _subject_only(t)
1106
+ if sub is None or x.scan is None:
1107
+ return None
1108
+ name, t_expr = sub
1109
+ if not (t.distinct or facts(t_op).distinct) or name not in x_op.columns:
1110
+ return None
1111
+ if x_op.slots[name].present is not Bound.ALWAYS:
1112
+ return None
1113
+ x_expr = dict(x.outputs).get(name)
1114
+ if not isinstance(x_expr, E.MakeTerm) or x_expr.map.template is None:
1115
+ return None
1116
+ if template_shape(x_expr.map) != template_shape(t_expr.map):
1117
+ return None
1118
+ x_cols = [a.name[4:] for _, a in x_expr.args if isinstance(a, E.Col) and a.name.startswith("src:")]
1119
+ t_cols = [a.name[4:] for _, a in t_expr.args]
1120
+ if len(x_cols) != len(x_expr.args):
1121
+ return None
1122
+ # the same value in each place of the template
1123
+ order_x = _value_order(x_expr.map)
1124
+ order_t = _value_order(t_expr.map)
1125
+ by_place_x = dict(zip(order_x, x_cols)) if len(order_x) == len(x_cols) else None
1126
+ by_place_t = dict(zip(order_t, t_cols)) if len(order_t) == len(t_cols) else None
1127
+ if by_place_x is None or by_place_t is None:
1128
+ return None
1129
+ held_x = {r.column: r.held for r in x.scan.reads}
1130
+ held_t = {r.column: r.held for r in t.scan.reads}
1131
+ node = t.scan.map
1132
+ for node_id, pairs, node_scan in x.exists:
1133
+ if node_id != node.id:
1134
+ continue
1135
+ to_key = dict(pairs)
1136
+ held_k = {r.column: r.held for r in node_scan.reads}
1137
+ if all(to_key.get(by_place_x[i]) == by_place_t[i] and held_x.get(by_place_x[i]) is not None
1138
+ and held_x.get(by_place_x[i]) == held_k.get(by_place_t[i]) == held_t.get(by_place_t[i])
1139
+ for i in by_place_t):
1140
+ return name
1141
+ return None
1142
+
1143
+
1144
+ def _value_order(m: TermMap) -> List[int]:
1145
+ """The places of a template's values, in order of first reading."""
1146
+ out: List[int] = []
1147
+ seen: List[str] = []
1148
+ for run, is_column in m.template:
1149
+ if is_column and run not in seen:
1150
+ seen.append(run)
1151
+ out.append(len(seen) - 1)
1152
+ return out
1153
+
1154
+
1155
+ def r4_implied_type(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
1156
+ """`Join(T, X) -> X` where T is a node map's subjects (a type pattern)
1157
+ and X's rows are proven to exist among them (`_implied`): `?c :replyOf
1158
+ ?p . ?p a :Post` is the first pattern alone where the edge's end is
1159
+ semi-joined into the post table (m2-performance.md 2.2, R4)."""
1160
+ if not isinstance(op, O.Join) or op.condition is not None:
1161
+ return None
1162
+ leaves: List[O.Op] = []
1163
+
1164
+ def flat(x: O.Op) -> None:
1165
+ if isinstance(x, O.Join) and x.condition is None:
1166
+ flat(x.left)
1167
+ flat(x.right)
1168
+ else:
1169
+ leaves.append(x)
1170
+ flat(op)
1171
+ pats = [_pattern(x) for x in leaves]
1172
+ for i, t in enumerate(pats):
1173
+ if t is None:
1174
+ continue
1175
+ for j, x in enumerate(pats):
1176
+ if i == j or x is None:
1177
+ continue
1178
+ name = _implied(t, leaves[i], x, leaves[j])
1179
+ if name is None:
1180
+ continue
1181
+ certs.append(Certificate("R4 implied type pattern",
1182
+ f"map {t.map_id}: {name} exists among its keys in {x.map_id}"))
1183
+ rest = [y for k, y in enumerate(leaves) if k != i]
1184
+ out = rest[0]
1185
+ for y in rest[1:]:
1186
+ out = O.Join(out, y)
1187
+ return out if out.columns == op.columns else O.Project(out, op.columns)
1188
+ return None
1189
+
1190
+
1191
+ def r4_trusted_reference(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
1192
+ """`SemiJoin(edge scan, node scan) -> edge scan` where the edge map
1193
+ asserts its ends exist (`referential_integrity`, the mapping's
1194
+ statement, trusted as a key is: one-plan.md 2.4) and the node map reads
1195
+ all its table's rows (no row filter, no query). Run after the other
1196
+ rules, which read the semi-join as the proof R4 needs."""
1197
+ if not isinstance(op, O.SemiJoin) or op.condition is not None:
1198
+ return None
1199
+ left = op.left
1200
+ while isinstance(left, O.SemiJoin):
1201
+ left = left.left
1202
+ if not isinstance(left, O.MapScan) or not isinstance(left.map, EdgeMap):
1203
+ return None
1204
+ # A split relationship (its end label's maps give several subjects) checks each end in its own maps:
1205
+ # every key has a node of the label, not of each branch's maps, so no check is dropped.
1206
+ if not left.map.referential_integrity or getattr(left.map, "split", False) or not isinstance(op.right, O.MapScan):
1207
+ return None
1208
+ source = op.right.map.source
1209
+ if source.filter or source.query is not None:
1210
+ # The assertion is that each end is a row of the node's table; a
1211
+ # node map holding only some of its rows (a row filter, a view)
1212
+ # still needs the check.
1213
+ return None
1214
+ certs.append(Certificate("R4 trusted reference",
1215
+ f"edge map {left.map.id} asserts its ends exist in {op.right.map.id}"))
1216
+ return op.left
1217
+
1218
+
1219
+ @home("own-row-exists", "A row's existence in its own table's node map on that map's key is the row itself (R3e)",
1220
+ terms=("own row", "start exists", "semi-join", "R3e", "existence check", "same table", "edge on its node's table"))
1221
+ def r3_own_row_exists(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
1222
+ """`SemiJoin(scan of T, node map N's scan of T) -> scan of T`:
1223
+ an edge's start proven to exist among the node's keys, the edge read
1224
+ from the node's own table. Fires only where the left scan's source is
1225
+ N's (one table, no query), its row filter holds N's (every row it reads
1226
+ is a row of N), it is joined on exactly a trusted key of N, each key
1227
+ column paired with itself and required on the left: the edge row is
1228
+ the node row. The end's semi-join (another table) stays, unless R4b.
1229
+ Run after the other rules, which read the semi-join as the proof R4
1230
+ needs."""
1231
+ if not isinstance(op, O.SemiJoin) or op.condition is not None:
1232
+ return None
1233
+ right = op.right
1234
+ if not isinstance(right, O.MapScan) or not isinstance(right.map, NodeMap) or right.graph is not None \
1235
+ or right.joint is not None:
1236
+ return None
1237
+ left = op.left
1238
+ while isinstance(left, O.SemiJoin):
1239
+ left = left.left
1240
+ if not isinstance(left, O.MapScan) or left.graph is not None or left.joint is not None:
1241
+ return None
1242
+ if not left.map.source.rows_within(right.map.source, folds=False):
1243
+ return None # another table, a query, or a node map holding only some of the rows the left reads
1244
+ # The columns folded may differ (a merged scan names the columns of both its maps undelimited):
1245
+ # the node's own columns must be written as it wrote them.
1246
+ if any((r.column in left.map.source.regular_columns) != (r.column in right.map.source.regular_columns)
1247
+ for r in right.reads if not r.parent):
1248
+ return None
1249
+ mine = {r.name: r.column for r in left.reads if not r.parent}
1250
+ theirs = {r.name: r.column for r in right.reads if not r.parent}
1251
+ shared = op.shared
1252
+ if not shared or any(n not in mine or n not in theirs or mine[n] != theirs[n] for n in shared):
1253
+ return None # each key column joined with itself
1254
+ if not set(right.required) <= set(shared) or not set(shared) <= set(left.required):
1255
+ return None
1256
+ cols = frozenset(mine[n] for n in shared)
1257
+ if not any(frozenset(k) == cols for k in right.unique_keys):
1258
+ return None
1259
+ certs.append(Certificate("R3 own row exists",
1260
+ f"map {left.map.id} reads rows of {right.map.id}'s table on its key {sorted(cols)}"))
1261
+ return op.left
1262
+
1263
+
1264
+ # -- R6: a union branch no row of the join can meet --------------------------------
1265
+
1266
+ def r6_union_branches(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
1267
+ """In a tree of joins, a union branch whose terms in a shared column,
1268
+ bound in every row, come from maps none of which may give a term the
1269
+ other pattern's column holds (`TermMap.may_equal`) joins no row and
1270
+ goes: `?c :replyOf ?p . ?p a :Post` keeps only the reply-to-a-post
1271
+ branch. The front end does this for two neighbours (`sparql_front.
1272
+ meeting`); this does it across the whole tree."""
1273
+ if not isinstance(op, O.Join) or op.condition is not None:
1274
+ return None
1275
+ from prograph.plan.ir.sparql_front import meeting
1276
+ leaves: List[O.Op] = []
1277
+
1278
+ def flat(x: O.Op) -> None:
1279
+ if isinstance(x, O.Join) and x.condition is None:
1280
+ flat(x.left)
1281
+ flat(x.right)
1282
+ else:
1283
+ leaves.append(x)
1284
+ flat(op)
1285
+ if len(leaves) < 3:
1286
+ return None
1287
+ for i, x in enumerate(leaves):
1288
+ inner = x.input if isinstance(x, O.Distinct) else x
1289
+ if not isinstance(inner, O.Union):
1290
+ continue
1291
+ for j, other in enumerate(leaves):
1292
+ if i == j:
1293
+ continue
1294
+ pruned = meeting(x, other)
1295
+ if pruned is x:
1296
+ continue
1297
+ certs.append(Certificate("R6 union branches",
1298
+ f"{len(inner.inputs)} branches against {sorted(other.columns)}"))
1299
+ rest = list(leaves)
1300
+ rest[i] = pruned
1301
+ out = rest[0]
1302
+ for y in rest[1:]:
1303
+ out = O.Join(out, y)
1304
+ return out if out.columns == op.columns else O.Project(out, op.columns)
1305
+ return None
1306
+
1307
+
1308
+ # -- R5: one Distinct beside the LIMIT (a trade-off: engine-aware) --------------------
1309
+
1310
+ # Engines measured to stop early on `SELECT DISTINCT ... LIMIT n` and to lose
1311
+ # on a Distinct per pattern (m2-performance.md 6: on 41 LDBC queries with no
1312
+ # keys, PostgreSQL 2.4 s with one Distinct beside the LIMIT against 25.8 s with
1313
+ # one per pattern; DuckDB 2.6 s against 1.4 s, so not DuckDB). Spark and
1314
+ # Databricks join when measured.
1315
+ DISTINCT_BESIDE_LIMIT = frozenset({"postgresql", "postgres"})
1316
+
1317
+
1318
+ def r5_distinct_beside_limit(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
1319
+ """`Slice(limit)` over Project, Filter and Extend over a tree of inner
1320
+ joins whose leaves are Distinct: the leaves' Distincts become one over
1321
+ the join, beside the LIMIT, so the database can stop at the first rows.
1322
+ Sound without keys: a join of inputs without equal rows has none, so
1323
+ Distinct(A) join Distinct(B) and Distinct(A join B) are the same rows.
1324
+ The one Distinct sits directly over the join, beneath any Project, so a
1325
+ projection dropping columns above it changes nothing. Not through an
1326
+ aggregate, ORDER BY, OPTIONAL or UNION (m2-performance.md 2.2, R5)."""
1327
+ if not isinstance(op, O.Slice) or op.limit is None:
1328
+ return None
1329
+ chain: List[O.Op] = []
1330
+ x = op.input
1331
+ while isinstance(x, (O.Project, O.Filter, O.Extend)):
1332
+ chain.append(x)
1333
+ x = x.input
1334
+ if not isinstance(x, O.Join) or x.condition is not None:
1335
+ return None
1336
+ leaves: List[O.Op] = []
1337
+ firm = [True]
1338
+
1339
+ def flat(y: O.Op) -> None:
1340
+ if isinstance(y, O.Join) and y.condition is None:
1341
+ if not all(y.left.slots[c].present is Bound.ALWAYS and
1342
+ y.right.slots[c].present is Bound.ALWAYS for c in y.shared):
1343
+ firm[0] = False # a shared column unbound: pairs may merge
1344
+ flat(y.left)
1345
+ flat(y.right)
1346
+ else:
1347
+ leaves.append(y)
1348
+ flat(x)
1349
+ if not firm[0]:
1350
+ return None
1351
+ if sum(isinstance(y, O.Distinct) for y in leaves) < 2:
1352
+ return None
1353
+ if not all(isinstance(y, O.Distinct) or facts(y).distinct for y in leaves):
1354
+ return None
1355
+ bare = [y.input if isinstance(y, O.Distinct) else y for y in leaves]
1356
+ joined = bare[0]
1357
+ for y in bare[1:]:
1358
+ joined = O.Join(joined, y)
1359
+ if joined.columns != x.columns:
1360
+ joined = O.Project(joined, x.columns)
1361
+ out: O.Op = O.Distinct(joined)
1362
+ for c in reversed(chain):
1363
+ out = dataclasses.replace(c, input=out)
1364
+ certs.append(Certificate("R5 one Distinct beside the LIMIT",
1365
+ f"{len(leaves)} patterns, LIMIT {op.limit}"))
1366
+ return dataclasses.replace(op, input=out)
1367
+
1368
+
1369
+ # -- R7: a LIMIT into the rows that decide it ---------------------------------
1370
+
1371
+ # The engines R7 applies on: a trade-off (m2-performance.md, the two classes), applied only where measured
1372
+ # to help, on the 11 LDBC SF1 queries it fires on (medians of 5, 2026-11-12, totals R7 on / off):
1373
+ # DuckDB 296 / 351 ms (S77 0.77x, S80 0.73x, S82 0.81x; a few small queries 1-3 ms slower); PostgreSQL
1374
+ # 58.3 / 17.0 s (S77 3.05x, S80 97x slower with it; S82 0.13x, S86 0.31x); local Spark 8 of 11 slower
1375
+ # (S77 1.29x); a Databricks warehouse (FEVM) all five measured slower (S77 1.31x, S81 1.45x). Not measured
1376
+ # elsewhere, so not applied. `None` (no engine named) keeps the rule, as the plan's own tests read it.
1377
+ LIMIT_INWARD_ENGINES = frozenset({"duckdb", None})
1378
+
1379
+ def r7_limit_inward(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
1380
+ """`Slice(offset k, limit n)` with no ORDER BY, over Project and Extend,
1381
+ over a LeftJoin: the left side limited to k + n rows first. Each left
1382
+ row gives at least one row of the left join, so the first k + n rows
1383
+ come from at most k + n left rows, and SPARQL's LIMIT without ORDER BY
1384
+ may answer any n solutions (15.4). Over a Union, each branch limited to
1385
+ k + n rows the same way. Not through Filter, Distinct, an inner join
1386
+ or an aggregate, which may drop or merge rows (m2-performance.md 3)."""
1387
+ if not isinstance(op, O.Slice) or op.limit is None:
1388
+ return None
1389
+ cap = op.offset + op.limit
1390
+ chain: List[O.Op] = []
1391
+ x = op.input
1392
+ while isinstance(x, (O.Project, O.Extend)):
1393
+ chain.append(x)
1394
+ x = x.input
1395
+ if isinstance(x, O.LeftJoin):
1396
+ if isinstance(x.left, O.Slice) and x.left.offset == 0 and x.left.limit is not None \
1397
+ and x.left.limit <= cap:
1398
+ return None
1399
+ new: O.Op = dataclasses.replace(x, left=O.Slice(x.left, 0, cap))
1400
+ elif isinstance(x, O.Union):
1401
+ if all(isinstance(b, O.Slice) and b.offset == 0 and b.limit is not None and b.limit <= cap
1402
+ for b in x.inputs):
1403
+ return None
1404
+ new = dataclasses.replace(x, inputs=tuple(
1405
+ b if isinstance(b, O.Slice) and b.offset == 0 and b.limit is not None and b.limit <= cap
1406
+ else O.Slice(b, 0, cap) for b in x.inputs))
1407
+ else:
1408
+ return None
1409
+ for c in reversed(chain):
1410
+ new = dataclasses.replace(c, input=new)
1411
+ certs.append(Certificate("R7 limit inward", f"LIMIT {cap} into the {type(x).__name__}"))
1412
+ return dataclasses.replace(op, input=new)
1413
+
1414
+
1415
+ # -- R8: a projected Distinct into a join's inputs ------------------------------------
1416
+
1417
+ def r8_distinct_into_join(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
1418
+ """`Distinct(Project(X, names))` over a tree of inner joins whose shared
1419
+ columns are bound on both sides: each input keeps only the columns the
1420
+ answer or the joins read, once each. Distinct(Project_X(L join R)) =
1421
+ Distinct(Project_X(Distinct(Project_{X,J}(L)) join Distinct(Project_{X,J}(R))))
1422
+ for J the join columns: a join of sets, then a projection to a set,
1423
+ reaches the same set (S50: `SELECT DISTINCT ?tagName` over post authors,
1424
+ friends and interests built 967 million rows to keep 9,489)."""
1425
+ if not isinstance(op, O.Distinct) or not isinstance(op.input, O.Project):
1426
+ return None
1427
+ proj = op.input
1428
+ x = proj.input
1429
+ wrappers: List[O.Op] = []
1430
+ while isinstance(x, (O.Project, O.Order)): # projections the rules left; ORDER BY
1431
+ wrappers.append(x)
1432
+ x = x.input
1433
+ if not isinstance(x, O.Join) or x.condition is not None:
1434
+ return None
1435
+ if any(isinstance(w, O.Order) and not all(k.column in proj.names for k in w.keys)
1436
+ for w in wrappers):
1437
+ return None
1438
+ leaves: List[O.Op] = []
1439
+ firm = [True]
1440
+
1441
+ def flat(y: O.Op) -> None:
1442
+ if isinstance(y, O.Join) and y.condition is None:
1443
+ if not all(y.left.slots[c].present is Bound.ALWAYS and
1444
+ y.right.slots[c].present is Bound.ALWAYS for c in y.shared):
1445
+ firm[0] = False
1446
+ flat(y.left)
1447
+ flat(y.right)
1448
+ else:
1449
+ leaves.append(y)
1450
+ flat(x)
1451
+ if not firm[0] or len(leaves) < 2:
1452
+ return None
1453
+ wanted = set(proj.names)
1454
+ counts: Dict[str, int] = {}
1455
+ for y in leaves:
1456
+ for c in y.columns:
1457
+ counts[c] = counts.get(c, 0) + 1
1458
+ joins = {c for c, n in counts.items() if n > 1}
1459
+ changed = False
1460
+ new_leaves: List[O.Op] = []
1461
+ for y in leaves:
1462
+ keep = tuple(c for c in y.columns if c in wanted or c in joins)
1463
+ if len(keep) < len(y.columns):
1464
+ new_leaves.append(O.Distinct(O.Project(y, keep)))
1465
+ changed = True
1466
+ else:
1467
+ new_leaves.append(y)
1468
+ if not changed:
1469
+ return None
1470
+ joined: O.Op = new_leaves[0]
1471
+ for y in new_leaves[1:]:
1472
+ joined = O.Join(joined, y)
1473
+ for w in reversed(wrappers):
1474
+ if isinstance(w, O.Order):
1475
+ joined = dataclasses.replace(w, input=joined)
1476
+ certs.append(Certificate("R8 distinct into a join",
1477
+ f"{len(leaves)} inputs keep {sorted(wanted | joins)}"))
1478
+ return O.Distinct(O.Project(joined, proj.names))
1479
+
1480
+ # -- R13: a joint query on the parent's key, the child's rows with the parent's existence ----------
1481
+
1482
+ def _one_text(child: O.ScanColumn, parent: O.ScanColumn) -> bool:
1483
+ """Whether two columns equal by SQL equality hold one value's text: read alike (datatype, zone,
1484
+ digits, reader, a date-time's precision), an integer either way (INTEGER and BIGINT), a string only
1485
+ where equality is byte equality (a case-blind collation matches 'a' and 'A'), a date-time of one zone
1486
+ and one stated precision (equal local values are one wall clock, equal zoned ones one instant, its
1487
+ text in UTC; TIMESTAMP_NS against TIMESTAMP is not), never a float (-0.0 = 0.0), nor a column of
1488
+ unknown type (VARCHAR '01' and INTEGER 1 are equal on some databases, EndRef.parent_columns)."""
1489
+ from prograph.schema import xsd
1490
+ integers = xsd.INTEGER_DATATYPES | {xsd.INTEGER}
1491
+ if child.held in integers and parent.held in integers:
1492
+ return child.reader is None and parent.reader is None
1493
+ if child.held is None or (child.held, child.zone, child.digits, child.reader, child.precision) != \
1494
+ (parent.held, parent.zone, parent.digits, parent.reader, parent.precision):
1495
+ return False
1496
+ if child.held == xsd.STRING:
1497
+ return child.bytewise and parent.bytewise
1498
+ if child.held == xsd.DATETIME:
1499
+ return child.zone is not None
1500
+ return child.held in (xsd.DECIMAL, xsd.DATE, xsd.BOOLEAN)
1501
+
1502
+
1503
+ @home("joint-query-on-a-parent-key", "A referencing object map joined on its parent's trusted key is the "
1504
+ "child's rows with the parent row's existence (R13)",
1505
+ terms=("R13", "joint query", "referencing object map", "rr:parentTriplesMap", "rr:joinCondition",
1506
+ "parent key", "foreign key", "semi-join", "child scan"))
1507
+ def r13_joint_on_parent_key(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
1508
+ """`MapScan(joint query)` -> `SemiJoin(child's scan, parent node map's scan)` (R2RML 8): each
1509
+ child row meets at most one parent row where the join is on a key the parent is trusted unique on,
1510
+ so the joint query's rows are the child rows some parent row matches; and a parent column read
1511
+ only where it is joined to a child column holding the same text (`_one_text`) is that child
1512
+ column. Fires only where every parent column the scan reads is a join column and every join
1513
+ column is read (the parent's subject is its key), each column in one condition, a trusted key of
1514
+ the parent among them, no graph column. The semi-join reads the parent's filter and NULLs as the
1515
+ join did (a NULL matches nothing). R3 then merges the child scan with the child's own
1516
+ properties, and R3e/R4b read its semi-join as for an edge on its node's table."""
1517
+ if not isinstance(op, O.MapScan) or op.joint is None or op.graph is not None or op.parent_map is None:
1518
+ return None
1519
+ joint, parent = op.joint, op.parent_map
1520
+ if parent.source != joint.parent or len(op.join_reads) != len(joint.conditions):
1521
+ return None
1522
+ children = [c for c, _ in joint.conditions]
1523
+ parents = [p for _, p in joint.conditions]
1524
+ if len(set(children)) != len(children) or len(set(parents)) != len(parents):
1525
+ return None
1526
+ read = {r.column: r for r in op.reads if r.parent}
1527
+ if len(read) != len([r for r in op.reads if r.parent]) or set(read) != set(parents):
1528
+ return None # a parent column beyond the join, or a join column the scan does not read
1529
+ if not any(k and set(k) <= set(parents) for k in op.parent_unique_keys):
1530
+ return None # not joined on a trusted key: a child row may meet many parent rows
1531
+ child_of = {p: c for c, p in joint.conditions}
1532
+ as_child = {}
1533
+ for jr, (c, p) in zip(op.join_reads, joint.conditions):
1534
+ if jr.column != c or not _one_text(jr, read[p]):
1535
+ return None
1536
+ as_child[read[p].name] = dataclasses.replace(jr, name=read[p].name)
1537
+ edge = op.map
1538
+ ends = {f: dataclasses.replace(getattr(edge, f), parent_columns=())
1539
+ for f in ("start", "end") if getattr(edge, f).parent_columns}
1540
+ child = O.MapScan(map=dataclasses.replace(edge, join=None, **ends),
1541
+ reads=tuple(as_child.get(r.name, r) for r in op.reads),
1542
+ required=op.required, unique_keys=op.unique_keys, not_null=op.not_null)
1543
+ rows = O.MapScan(map=parent, reads=tuple(dataclasses.replace(r, parent=False) for r in op.reads if r.parent),
1544
+ required=tuple(r.name for r in op.reads if r.parent),
1545
+ unique_keys=op.parent_unique_keys, not_null=op.parent_not_null)
1546
+ certs.append(Certificate("R13 joint query on a parent key",
1547
+ f"edge {edge.id} joined to {parent.id} on {sorted(child_of.items())}, keys "
1548
+ f"{[list(k) for k in op.parent_unique_keys]}"))
1549
+ return O.SemiJoin(child, rows)
1550
+
1551
+
1552
+ # -- R12: a joint scan's child as a set, under a Distinct ------------------------------
1553
+
1554
+ def r12_joint_child_set(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
1555
+ """`Distinct(F(MapScan joint))` for F filters, extensions and projections: the scan reads each
1556
+ child row with each parent row its condition matches, and a referencing object map's child
1557
+ contributes only its own columns (the subject's), so a child with many rows of one such
1558
+ value builds that value's pairs many times over (GTFS shapes: ~2,000 points a shape, 117
1559
+ million pairs for 58,540 triples). Every operator of F is a function of one row, and a
1560
+ Distinct keeps one row of each set of equal rows, so the answer is the same set whether F reads
1561
+ the bag of (child row, parent row) pairs or the pairs of (distinct child columns read, parent
1562
+ row): two child rows with the same columns read give the same rows of F. Where F is volatile
1563
+ (BNODE, RAND, UUID: a row's own fresh value) rows are not a function of the columns, so not here.
1564
+ Without a Distinct the bag is observable (a join's multiplicity), so not there either: the
1565
+ pattern's own Distinct (a graph is a set, RDF 1.1 Concepts 3) is what licenses it. The
1566
+ lowering reads the child as `SELECT DISTINCT` its columns, filtered and non-null."""
1567
+ if not isinstance(op, O.Distinct):
1568
+ return None
1569
+ from prograph.plan.ir.fresh import volatile
1570
+ chain: List[O.Op] = []
1571
+ x = op.input
1572
+ while isinstance(x, (O.Filter, O.Extend, O.Project)):
1573
+ chain.append(x)
1574
+ x = x.input
1575
+ if not isinstance(x, O.MapScan) or x.joint is None or x.child_set or x.graph is not None:
1576
+ return None
1577
+ if volatile(op.input):
1578
+ return None
1579
+ new: O.Op = dataclasses.replace(x, child_set=True)
1580
+ for c in reversed(chain):
1581
+ new = dataclasses.replace(c, input=new)
1582
+ certs.append(Certificate("R12 joint child as a set",
1583
+ "the child's columns read and joined on, distinct, before the parent's join"))
1584
+ return O.Distinct(new)
1585
+
1586
+
1587
+ # -- R11: eager aggregation, counts per node before the join -----------------------------
1588
+
1589
+ @home("eager-aggregation", "COUNTs over a join at a node both sides hold many rows of, counted per side first",
1590
+ terms=("eager aggregation", "R11", "SUM_COUNTS", "sum of counts", "count per side"))
1591
+ def r11_eager_aggregation(op: O.Op, certs: List[Certificate]) -> Optional[O.Op]:
1592
+ """`Aggregate(G, COUNTs)` over `Join(L, R)` at one node v both sides hold many rows of
1593
+ (`cost.fan_out`: friends and posts counted per person) becomes the counts per side first:
1594
+ L' = Aggregate(L, v + G within L, COUNT(*) + a COUNT per counted column of L), R' likewise, then
1595
+ Aggregate(G, sum over L' join R' of the side's count times the other's COUNT(*)). A group of the
1596
+ join holds, for each pair of L' and R' rows through v, their row counts multiplied, so the sum of
1597
+ those products is the group's count (every row of a side gathers into the L' or R' row of its own
1598
+ v and group columns). Fires only where every aggregate is a plain COUNT, v is bound in every row
1599
+ of both sides (an unbound v joins everything), and the join has no condition: the two sides then
1600
+ meet on equal v alone, as the groups of L' and R' do. The sums are `SUM_COUNTS`, COUNT's slot."""
1601
+ if not isinstance(op, O.Aggregate) or not isinstance(op.input, O.Join) or not op.aggregates:
1602
+ return None
1603
+ from prograph.plan.ir.cost import fan_out
1604
+ j = op.input
1605
+ if j.condition is not None or len(j.shared) != 1:
1606
+ return None
1607
+ v = j.shared[0]
1608
+ if fan_out(op) != v:
1609
+ return None
1610
+ left, right = j.left, j.right
1611
+ if left.slots[v].present is not Bound.ALWAYS or right.slots[v].present is not Bound.ALWAYS:
1612
+ return None
1613
+ for _, agg in op.aggregates:
1614
+ if agg.fn != "COUNT" or agg.distinct or agg.semantics != E.SPARQL:
1615
+ return None
1616
+ if agg.arg is not None and not (isinstance(agg.arg, E.Col) and agg.arg.name in j.columns):
1617
+ return None
1618
+ if not all(g in left.columns or g in right.columns for g in op.group):
1619
+ return None
1620
+ side_counts: Dict[str, List[Tuple[str, E.Agg]]] = {"L": [("#r11.cL", E.Agg("COUNT"))],
1621
+ "R": [("#r11.cR", E.Agg("COUNT"))]}
1622
+ counted: Dict[Tuple[str, str], str] = {}
1623
+ tops: List[Tuple[str, E.Agg]] = []
1624
+ for name, agg in op.aggregates:
1625
+ a, b = "#r11.cL", "#r11.cR"
1626
+ if agg.arg is not None and agg.arg.name != v:
1627
+ c = agg.arg.name
1628
+ side = "L" if c in left.columns else "R"
1629
+ if (side, c) not in counted:
1630
+ counted[(side, c)] = f"#r11.n{side}{len(counted)}"
1631
+ side_counts[side].append((counted[(side, c)], E.Agg("COUNT", E.Col(c))))
1632
+ if side == "L":
1633
+ a = counted[(side, c)]
1634
+ else:
1635
+ b = counted[(side, c)]
1636
+ tops.append((name, E.Agg("SUM_COUNTS", E.Call("*", (E.Col(a), E.Col(b))))))
1637
+ group_l = tuple(g for g in op.group if g in left.columns and g != v)
1638
+ group_r = tuple(g for g in op.group if g in right.columns and g != v)
1639
+ new_left = O.Aggregate(left, (v,) + group_l, tuple(side_counts["L"]))
1640
+ new_right = O.Aggregate(right, (v,) + group_r, tuple(side_counts["R"]))
1641
+ certs.append(Certificate("R11 eager aggregation",
1642
+ f"both sides hold many rows per ?{v} (declared keys, none within it), "
1643
+ f"?{v} bound in every row, COUNTs only, no join condition"))
1644
+ return O.Aggregate(O.Join(new_left, new_right), op.group, tuple(tops))
1645
+
1646
+
1647
+ # -- R14: a grouped sub-select restricted to the keys its join keeps ------------------
1648
+
1649
+ # The engines R14 applies on: a trade-off (where the other side keeps every key it adds a semi-join over
1650
+ # the aggregate's input), so it fires only where the other side holds a condition (`_filtered`), and only
1651
+ # on the engines measured, TPC-H SF1, all 22 queries (2026-11-13; q17 and q20 are the two it fires
1652
+ # on): PostgreSQL q17 12.1 -> 0.07 s, q20 1.2 -> 0.16 s (reference 0.42, 0.62); DuckDB q17 0.45 -> 0.25 s,
1653
+ # q20 0.07 s either way. Without `_filtered` it fired on q15 and q18 too, PostgreSQL q18 1.7 -> 10.7 s.
1654
+ # `None` (no engine named) keeps the rule, as the plan's own tests read it.
1655
+ RESTRICT_GROUPS_ENGINES = frozenset({"postgresql", "postgres", "duckdb", None})
1656
+
1657
+
1658
+ def _grouped(op: O.Op) -> Optional[Tuple[List[O.Op], O.Aggregate]]:
1659
+ """`op` as row-wise operators (Project, Extend, Filter) over an Aggregate with GROUP BY keys:
1660
+ the operators, outermost first, and the Aggregate."""
1661
+ chain: List[O.Op] = []
1662
+ x = op
1663
+ while isinstance(x, (O.Project, O.Extend, O.Filter)):
1664
+ chain.append(x)
1665
+ x = x.input
1666
+ if not isinstance(x, O.Aggregate) or not x.group:
1667
+ return None
1668
+ return chain, x
1669
+
1670
+
1671
+ def _restricted(join: O.Join, conds: Tuple[E.Expr, ...], certs: List[Certificate]) -> Optional[O.Join]:
1672
+ """`join` with each grouped side's aggregate input restricted to the keys the other side holds
1673
+ (`r14_grouped_join_restricted`); None where no side qualifies."""
1674
+ from prograph.plan.ir.fresh import holds, volatile
1675
+ if join.condition is not None or volatile(join):
1676
+ return None
1677
+ shared = join.shared
1678
+ if not shared:
1679
+ return None
1680
+ if not all(join.left.slots[c].present is Bound.ALWAYS and join.right.slots[c].present is Bound.ALWAYS
1681
+ for c in shared):
1682
+ return None # an unbound key joins every row
1683
+ sides = {"left": join.left, "right": join.right}
1684
+ changes = {}
1685
+ for name, g, other in (("left", join.left, join.right), ("right", join.right, join.left)):
1686
+ found = _grouped(g)
1687
+ if found is None:
1688
+ continue
1689
+ chain, agg = found
1690
+ if not set(shared) <= set(agg.group):
1691
+ continue # a group holding several values of a join column: no group is whole
1692
+ if isinstance(agg.input, O.SemiJoin) and isinstance(agg.input.right, O.Project) \
1693
+ and agg.input.right.names == shared:
1694
+ continue # restricted already
1695
+ mine = [c for c in conds
1696
+ if c.columns() <= set(other.columns) and not holds(c)
1697
+ and all(other.slots[x].present is Bound.ALWAYS for x in c.columns() if x in g.columns)]
1698
+ if not mine and not _filtered(other):
1699
+ continue # the other side keeps every key: the restriction would only add work (TPC-H q15, q18)
1700
+ keys: O.Op = other
1701
+ if mine:
1702
+ keys = O.Filter(other, mine[0] if len(mine) == 1 else E.And(tuple(mine)))
1703
+ new: O.Op = dataclasses.replace(agg, input=O.SemiJoin(agg.input, O.Project(keys, shared)))
1704
+ for c in reversed(chain):
1705
+ new = dataclasses.replace(c, input=new)
1706
+ changes[name] = new
1707
+ certs.append(Certificate("R14 grouped join restricted",
1708
+ f"grouped by {list(agg.group)}, joined on {list(shared)}, bound on both "
1709
+ f"sides; {len(mine)} of the filter's conjuncts read the other side only"))
1710
+ if not changes:
1711
+ return None
1712
+ return dataclasses.replace(join, **changes)
1713
+
1714
+
1715
+ def _filtered(op: O.Op) -> bool:
1716
+ """Whether a condition of the query (a FILTER, a constant in a pattern) restricts `op`'s rows
1717
+ somewhere in its tree: R14's structural rule for a side that may keep fewer keys than the groups."""
1718
+ if isinstance(op, O.Filter):
1719
+ return True
1720
+ return any(_filtered(c) for c in op.children())
1721
+
1722
+
1723
+ @home("grouped-join-restricted", "A grouped sub-select joined on its group keys computes only the groups the join keeps (R14)",
1724
+ terms=("R14", "semi-join reduction", "sub-select", "subquery", "group by", "aggregate", "correlated",
1725
+ "grouped join", "restrict"))
1726
+ def _restrict_groups(op: O.Op, certs: List[Certificate], memo: ByIdentity) -> O.Op:
1727
+ """R14, top-down: `Filter?(Join(L, G))`, G row-wise operators over `Aggregate(I, GROUP BY K)` joined
1728
+ on columns J within K, becomes the same with I replaced by `SemiJoin(I, Project(Filter(C_L, L), J))`,
1729
+ C_L the Filter's conjuncts reading only L (TPC-H q17's per-part average, q20's per-(part, supplier)
1730
+ sum: the reference correlates, computing only the outer side's keys). Rows of one group agree on J,
1731
+ so the semi-join keeps or drops a group whole, and a kept group's aggregates read the rows they did;
1732
+ a dropped group meets no row of L passing C_L, so it gave no row of the join passing C. Only where
1733
+ the join is inner with no condition (no OPTIONAL, MINUS or Cypher condition), G holds no LIMIT, ORDER
1734
+ BY or DISTINCT (a sub-select's LIMIT picks groups), the Aggregate has GROUP BY keys (an implicit group
1735
+ is one row over no rows), J is bound in every row of both sides, and a conjunct taken calls no fresh
1736
+ function and reads columns of L only, bound in L where G has them too. L is the same object in the
1737
+ join and the restriction, so the lowering writes it once."""
1738
+ hit = memo.get(op)
1739
+ if hit is not None:
1740
+ return hit
1741
+ out = op
1742
+ if isinstance(op, O.Filter) and isinstance(op.input, O.Join):
1743
+ j = _restricted(op.input, E.conjuncts(op.condition), certs)
1744
+ if j is not None:
1745
+ out = dataclasses.replace(op, input=_children(j, certs, memo))
1746
+ if _CHECK:
1747
+ _check_firing(_restrict_groups, op, out)
1748
+ memo.put(op, out)
1749
+ return out
1750
+ elif isinstance(op, O.Join):
1751
+ j = _restricted(op, (), certs)
1752
+ if j is not None:
1753
+ out = _children(j, certs, memo)
1754
+ if _CHECK:
1755
+ _check_firing(_restrict_groups, op, out)
1756
+ memo.put(op, out)
1757
+ return out
1758
+ out = _children(op, certs, memo)
1759
+ memo.put(op, out)
1760
+ return out
1761
+
1762
+
1763
+ def _children(op: O.Op, certs: List[Certificate], memo: ByIdentity) -> O.Op:
1764
+ changes = {}
1765
+ for f in dataclasses.fields(op):
1766
+ v = getattr(op, f.name)
1767
+ if isinstance(v, O.Op):
1768
+ nv = _restrict_groups(v, certs, memo)
1769
+ if nv is not v:
1770
+ changes[f.name] = nv
1771
+ elif isinstance(v, tuple) and v and all(isinstance(x, O.Op) for x in v):
1772
+ nv = tuple(_restrict_groups(x, certs, memo) for x in v)
1773
+ if any(a is not b for a, b in zip(nv, v)):
1774
+ changes[f.name] = nv
1775
+ return dataclasses.replace(op, **changes) if changes else op
1776
+
1777
+
1778
+ RULES: Tuple[Rule, ...] = (r13_joint_on_parent_key, r6_union_branches, r2_contained_pattern, r1_distinct_removal,
1779
+ r3_self_join, r3_left_join, r4_implied_type, r8_distinct_into_join,
1780
+ r9_union_containment, r11_eager_aggregation,
1781
+ r12_joint_child_set)
1782
+ LATE_RULES: Tuple[Rule, ...] = (r4_trusted_reference, r3_own_row_exists, r7_limit_inward)