prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/plan/ir/keys.py
ADDED
|
@@ -0,0 +1,454 @@
|
|
|
1
|
+
"""Which keys a plan may rely on as unique (one-plan.md 2.4, decided with
|
|
2
|
+
decided 2026-11-08): a key is trusted only when asserted or checked. An
|
|
3
|
+
enforced constraint, a Databricks RELY primary key, or a mapping's
|
|
4
|
+
`keys: trusted`; `keys: check`, verified for the data version, is not built
|
|
5
|
+
yet and relies on nothing until it is. Otherwise the plan keeps its
|
|
6
|
+
Distincts. A stated key is identity, not uniqueness; an R2RML subject
|
|
7
|
+
template's columns state no key; `on_duplicate` is Cypher's policy and says
|
|
8
|
+
nothing to a plan.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import re
|
|
14
|
+
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
|
15
|
+
|
|
16
|
+
from prograph.rules import home
|
|
17
|
+
|
|
18
|
+
KEY_TRUST = ("trusted", "check")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def key_trust_name(value: Optional[str], where: str) -> Optional[str]:
|
|
22
|
+
"""A mapping's `keys` setting, checked: 'trusted', 'check' or None."""
|
|
23
|
+
if value is None:
|
|
24
|
+
return None
|
|
25
|
+
if value not in KEY_TRUST:
|
|
26
|
+
raise ValueError(f"{where}: {value!r} is not one of {', '.join(KEY_TRUST)}")
|
|
27
|
+
return value
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@home("trusted-keys", "When a plan may rely on a key being unique",
|
|
31
|
+
terms=("trusted key", "unique_keys", "RELY", "PRIMARY KEY", "UNIQUE", "keys: trusted"))
|
|
32
|
+
def trusted_keys(graph: Any, source_map: Any, maps: Any = None) -> Tuple[Tuple[str, ...], ...]:
|
|
33
|
+
"""The column sets of `source_map`'s source a plan may rely on as unique:
|
|
34
|
+
the mapping's own key where it says `keys: trusted`; every PRIMARY KEY
|
|
35
|
+
and UNIQUE constraint the database enforces (DuckDB, PostgreSQL); a
|
|
36
|
+
Databricks primary key declared RELY on exactly the key; and, for an edge
|
|
37
|
+
map, a node map's trusted key on the same table that is exactly the edge's
|
|
38
|
+
start columns (`maps`: a parent pointer declared `keys: trusted` on its
|
|
39
|
+
node). Read once per graph and table, cached on the graph."""
|
|
40
|
+
return tuple(columns for columns, _ in key_sources(graph, source_map, maps))
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
ASSERTED = "asserted by the mapping (keys: trusted), not checked"
|
|
44
|
+
ASSERTED_ID = "asserted by the mapping (a relationship's id_column), not checked"
|
|
45
|
+
CHECKED = "checked for this version of the data (keys: check)"
|
|
46
|
+
ENFORCED = "a constraint the database enforces"
|
|
47
|
+
RELY = "a Databricks primary key declared RELY"
|
|
48
|
+
STORE_VIEW = "one row per created element, by the owned store's view"
|
|
49
|
+
SAME_TABLE = "the key of node map {node} on the same table ({why})"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def key_sources(graph: Any, source_map: Any, maps: Any = None) -> Tuple[Tuple[Tuple[str, ...], str], ...]:
|
|
53
|
+
"""`trusted_keys`, each with what makes it trusted (`ASSERTED`,
|
|
54
|
+
`ENFORCED`, `RELY`), for EXPLAIN: an asserted key is never shown as a
|
|
55
|
+
checked one (decided 2026-11-09)."""
|
|
56
|
+
source = source_map.source
|
|
57
|
+
from prograph.schema.store_view import is_store_map
|
|
58
|
+
if is_store_map(source_map):
|
|
59
|
+
# The owned store's generic map: one row per created element (its query groups by it).
|
|
60
|
+
key = getattr(source_map, "key", None)
|
|
61
|
+
return ((tuple(key.columns) if key is not None else tuple(source_map.row_key), STORE_VIEW),)
|
|
62
|
+
if source.query is not None:
|
|
63
|
+
return query_keys(graph, source)
|
|
64
|
+
key = getattr(source_map, "key", None)
|
|
65
|
+
out: List[Tuple[Tuple[str, ...], str]] = []
|
|
66
|
+
if key is not None and key.stated and key.trust == "trusted":
|
|
67
|
+
out.append((tuple(key.columns), ASSERTED))
|
|
68
|
+
if key is not None and key.stated and key.trust == "check":
|
|
69
|
+
if checked_key(graph, source_map):
|
|
70
|
+
out.append((tuple(key.columns), CHECKED))
|
|
71
|
+
row_key = getattr(source_map, "row_key", None)
|
|
72
|
+
if row_key:
|
|
73
|
+
# An edge map's id_column: a key the mapping states, not checked (as `keys: trusted`).
|
|
74
|
+
out.append((tuple(row_key), ASSERTED_ID))
|
|
75
|
+
if getattr(source_map, "trust", None) == "trusted" and hasattr(source_map, "start"):
|
|
76
|
+
# An edge map's asserted key: its start and end columns.
|
|
77
|
+
edge_key = tuple(dict.fromkeys(tuple(source_map.start.columns) + tuple(source_map.end.columns)))
|
|
78
|
+
if edge_key:
|
|
79
|
+
out.append((edge_key, ASSERTED))
|
|
80
|
+
if maps is not None and hasattr(source_map, "start") and source.query is None \
|
|
81
|
+
and getattr(source_map, "join", None) is None:
|
|
82
|
+
# An edge map reading a node map's table, its start columns that node map's trusted key (a parent
|
|
83
|
+
# pointer: `unit.parent` on the unit table, the node keyed `keys: trusted` on `id`): the edge has one
|
|
84
|
+
# row per table row, so it too is unique on its start (rule trusted-keys).
|
|
85
|
+
start = tuple(source_map.start.columns)
|
|
86
|
+
for node in getattr(maps, "nodes", ()):
|
|
87
|
+
# The node's key holds over its filtered rows only: the edge must read no row outside them (its
|
|
88
|
+
# own filter holds every (column, value) of the node's).
|
|
89
|
+
if source.rows_within(node.source):
|
|
90
|
+
for columns, why in key_sources(graph, node):
|
|
91
|
+
if set(columns) == set(start) and columns not in [c for c, _ in out]:
|
|
92
|
+
out.append((columns, SAME_TABLE.format(node=node.id, why=why)))
|
|
93
|
+
rely = _dialect(graph) in ("databricks_sql", "databricks")
|
|
94
|
+
# On a Databricks warehouse, where the mapping asserts its key, no catalog read is needed to trust it:
|
|
95
|
+
# the constraint probe (a DESCRIBE TABLE EXTENDED, a round trip of about a second) is skipped.
|
|
96
|
+
# Only on a warehouse, where the read is a round trip; elsewhere the database's own keys are read too
|
|
97
|
+
# (DuckDB and PostgreSQL enforce them, and a plan may rely on one beside the asserted key).
|
|
98
|
+
asserted = rely and [c for c, why in out if why == ASSERTED]
|
|
99
|
+
for k in (() if asserted else _constraint_keys(graph, source.qualified, key)):
|
|
100
|
+
if k not in [c for c, _ in out]:
|
|
101
|
+
out.append((k, RELY if rely else ENFORCED))
|
|
102
|
+
for k, why in out:
|
|
103
|
+
if why in (ASSERTED, ASSERTED_ID, RELY):
|
|
104
|
+
warn_if_repeating(graph, source.qualified, k, why)
|
|
105
|
+
return tuple(out)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
QUERY_TABLE = "{why} of the table a plain rr:sqlQuery selects from"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def query_keys(graph: Any, source: Any) -> Tuple[Tuple[Tuple[str, ...], str], ...]:
|
|
112
|
+
"""The keys an rr:sqlQuery source holds: where the query is a plain select of one table (`simple_select`:
|
|
113
|
+
`SELECT * | t.* | columns FROM table WHERE ...`, no join, grouping, DISTINCT, window, LIMIT, set operation,
|
|
114
|
+
expression or renamed column), its rows are rows of that table, each once, so a key the database
|
|
115
|
+
enforces (or Databricks RELY) on the table holds on them where the query projects all its columns. A
|
|
116
|
+
mapping cannot assert a key on a query; any other query has none (rule trusted-keys)."""
|
|
117
|
+
from prograph.schema.source_map import simple_select
|
|
118
|
+
shape = simple_select(source.query)
|
|
119
|
+
if shape is None:
|
|
120
|
+
return ()
|
|
121
|
+
rely = _dialect(graph) in ("databricks_sql", "databricks")
|
|
122
|
+
out: List[Tuple[Tuple[str, ...], str]] = []
|
|
123
|
+
for k in _constraint_keys(graph, shape.table, None):
|
|
124
|
+
if (shape.columns is None or set(k) <= set(shape.columns)) and k not in [c for c, _ in out]:
|
|
125
|
+
out.append((k, QUERY_TABLE.format(why=RELY if rely else ENFORCED)))
|
|
126
|
+
return tuple(out)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _dialect(graph: Any) -> Optional[str]:
|
|
130
|
+
try:
|
|
131
|
+
return graph.sql_dialect() if graph is not None and hasattr(graph, "sql_dialect") else None
|
|
132
|
+
except Exception: # noqa: BLE001
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class AssertedKeyWarning(UserWarning):
|
|
137
|
+
"""A mapping's key is asserted (`keys: trusted`) or asked to be checked
|
|
138
|
+
(`keys: check`), not enforced or checked: see `warn_asserted_keys`."""
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def warn_asserted_keys(schema: Any) -> None:
|
|
142
|
+
"""At mapping load, one warning per mapping whose keys are asserted
|
|
143
|
+
(`keys: trusted`) or asked to be checked (`keys: check`, not built: it
|
|
144
|
+
relies on nothing yet). An asserted key is the user's promise, an
|
|
145
|
+
escape hatch where the database cannot enforce one; plans rely on it,
|
|
146
|
+
and a key the data breaks gives wrong answers (decided 2026-11-09:
|
|
147
|
+
m2-performance.md 2.4)."""
|
|
148
|
+
import warnings
|
|
149
|
+
from prograph.plan.ir.entry import source_maps
|
|
150
|
+
try:
|
|
151
|
+
maps = source_maps(schema)
|
|
152
|
+
except Exception: # noqa: BLE001 - a schema the one plan does not read asserts no key here
|
|
153
|
+
return
|
|
154
|
+
# The source maps, not the authoring objects: a mapping generated as source maps (no YAML) warns
|
|
155
|
+
# alike, one warning per map (one per YAML mapping: the compiler makes one map of each).
|
|
156
|
+
for kind, found in (("node", [(m, m.key.trust, m.label or m.id, m.key.columns) for m in maps.nodes]),
|
|
157
|
+
("relationship", [(m, m.trust, m.rel_type or m.id, m.start.columns + m.end.columns)
|
|
158
|
+
for m in maps.edges])):
|
|
159
|
+
for m, trust, name, cols in found:
|
|
160
|
+
if trust is None:
|
|
161
|
+
continue
|
|
162
|
+
columns = ", ".join(str(c) for c in cols if c)
|
|
163
|
+
if trust == "trusted":
|
|
164
|
+
message = (f"The key of {kind} mapping {name!r} ({m.source.table}: {columns}) is "
|
|
165
|
+
f"asserted (keys: trusted), not checked. Plans rely on it to drop "
|
|
166
|
+
f"duplicate removal and joins; if two rows share a key value, SPARQL "
|
|
167
|
+
f"answers are wrong. Where the database can enforce the key (a PRIMARY "
|
|
168
|
+
f"KEY, UNIQUE, or a Databricks PRIMARY KEY ... RELY), declare it there.")
|
|
169
|
+
else:
|
|
170
|
+
continue # checked per data version (`checked_key`), refused without one
|
|
171
|
+
warnings.warn(message, AssertedKeyWarning, stacklevel=3)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _constraint_keys(graph: Any, qualified: str, key: Any) -> Tuple[Tuple[str, ...], ...]:
|
|
175
|
+
if graph is None or not hasattr(graph, "sql_dialect") or not hasattr(graph, "_execute_sql"):
|
|
176
|
+
return ()
|
|
177
|
+
cache = graph.__dict__.setdefault("_trusted_key_cache", {})
|
|
178
|
+
if qualified not in cache:
|
|
179
|
+
try:
|
|
180
|
+
cache[qualified] = _read_constraints(graph, qualified, key)
|
|
181
|
+
except Exception: # noqa: BLE001 - a catalog that cannot be read trusts nothing
|
|
182
|
+
cache[qualified] = ()
|
|
183
|
+
return cache[qualified]
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _read_constraints(graph: Any, qualified: str, key: Any) -> Tuple[Tuple[str, ...], ...]:
|
|
187
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
188
|
+
name = graph.sql_dialect()
|
|
189
|
+
lit = get_dialect_instance(name).literal
|
|
190
|
+
parts = [p.strip('`"') for p in qualified.split(".")]
|
|
191
|
+
table, schema = parts[-1], (parts[-2] if len(parts) > 1 else None)
|
|
192
|
+
database = parts[-3] if len(parts) > 2 else None
|
|
193
|
+
if name == "duckdb":
|
|
194
|
+
# The table the name resolves to: its database and schema the
|
|
195
|
+
# current ones where the name leaves them out (fresh review,
|
|
196
|
+
# 2026-11-09: a same-named table elsewhere is another table).
|
|
197
|
+
sql = ("SELECT constraint_column_names AS cols FROM duckdb_constraints() "
|
|
198
|
+
f"WHERE constraint_type IN ('PRIMARY KEY', 'UNIQUE') AND table_name = {lit(table)}"
|
|
199
|
+
f" AND schema_name = {lit(schema) if schema else 'current_schema()'}"
|
|
200
|
+
f" AND database_name = {lit(database) if database else 'current_database()'}")
|
|
201
|
+
return tuple(tuple(r["cols"]) for r in graph._execute_sql(sql))
|
|
202
|
+
if name in ("postgresql", "postgres"):
|
|
203
|
+
sql = ("SELECT tc.constraint_name AS name, kcu.column_name AS col "
|
|
204
|
+
"FROM information_schema.table_constraints tc "
|
|
205
|
+
"JOIN information_schema.key_column_usage kcu "
|
|
206
|
+
"ON kcu.constraint_schema = tc.constraint_schema "
|
|
207
|
+
"AND kcu.constraint_name = tc.constraint_name "
|
|
208
|
+
f"WHERE tc.constraint_type IN ('PRIMARY KEY', 'UNIQUE') AND tc.table_name = {lit(table)}"
|
|
209
|
+
" AND tc.is_deferrable = 'NO'")
|
|
210
|
+
sql += f" AND tc.table_schema = {lit(schema) if schema else 'current_schema()'}"
|
|
211
|
+
sql += " ORDER BY tc.constraint_name, kcu.ordinal_position"
|
|
212
|
+
by_name: Dict[str, List[str]] = {}
|
|
213
|
+
for r in graph._execute_sql(sql):
|
|
214
|
+
by_name.setdefault(r["name"], []).append(r["col"])
|
|
215
|
+
return tuple(tuple(v) for v in by_name.values())
|
|
216
|
+
if name in ("databricks_sql", "databricks"):
|
|
217
|
+
# Every RELY primary key the table declares, as PostgreSQL's and DuckDB's are read
|
|
218
|
+
# whether or not the mapping states the key (LDBC's edge tables). A table the engine's
|
|
219
|
+
# catalog read found no primary key for has none to read.
|
|
220
|
+
if graph.__dict__.get("_warehouse_keyed", {}).get(qualified) is False:
|
|
221
|
+
return ()
|
|
222
|
+
rows = _described_extended(graph, qualified)
|
|
223
|
+
found = (rely_primary_key_columns(" ".join(str(v) for v in row.values() if v is not None))
|
|
224
|
+
for row in rows)
|
|
225
|
+
return tuple(dict.fromkeys(cols for cols in found if cols))
|
|
226
|
+
return ()
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def rely_primary_key_columns(text: str) -> Optional[Tuple[str, ...]]:
|
|
230
|
+
"""The columns of a RELY primary key a constraint's text declares
|
|
231
|
+
(``PRIMARY KEY (`Forum.id`, `Person.id`) RELY``, any case, names quoted
|
|
232
|
+
or not), or None."""
|
|
233
|
+
import re
|
|
234
|
+
match = re.search(r"PRIMARY\s+KEY\s*\(([^)]*)\)(.*)", text, re.IGNORECASE)
|
|
235
|
+
if match is None or not re.search(r"\bRELY\b", match.group(2), re.IGNORECASE) \
|
|
236
|
+
or re.search(r"\bNORELY\b", match.group(2), re.IGNORECASE):
|
|
237
|
+
return None
|
|
238
|
+
names = re.findall(r"`((?:[^`]|``)*)`|\"((?:[^\"]|\"\")*)\"|([^,\s]+)", match.group(1))
|
|
239
|
+
return tuple((a.replace("``", "`") or b.replace('""', '"') or c) for a, b, c in names) or None
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def rely_primary_key(text: str, columns: Iterable[str]) -> bool:
|
|
243
|
+
"""Whether a constraint's text declares a RELY primary key on exactly `columns`."""
|
|
244
|
+
found = rely_primary_key_columns(text)
|
|
245
|
+
return found is not None and {c.lower() for c in found} == {c.lower() for c in columns}
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
# -- keys: check (docs/architecture/keys-and-identity.md) -------------------------------
|
|
249
|
+
|
|
250
|
+
import contextvars # noqa: E402
|
|
251
|
+
|
|
252
|
+
# The pinned reads (`key_checks.Pins`) the keys checked while a query is planned ask for: the
|
|
253
|
+
# statement reads each checked table at the version checked (`key_checks.pinned_name`, `lower._pinned`).
|
|
254
|
+
PLANNING_PINS: contextvars.ContextVar = contextvars.ContextVar("prograph_planning_pins", default=None)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
class KeyCheckUnavailableError(ValueError):
|
|
258
|
+
"""`keys: check` on a map whose source has no version to check it at (a PostgreSQL or DuckDB
|
|
259
|
+
table, a Spark frame that is not Delta): the check and the query would be two statements over
|
|
260
|
+
data that may change between them, so the setting is refused rather than quietly not acted on
|
|
261
|
+
(decided 2026-11-10)."""
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
class RepeatedKeyWarning(UserWarning):
|
|
265
|
+
"""A `keys: check` key that two of the map's rows share: not relied on, the plan keeps its
|
|
266
|
+
Distincts (the answers stay right)."""
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _checked_mapping(graph: Any, source_map: Any) -> Any:
|
|
270
|
+
"""What `key_checks.KeyChecks` reads of a map, one object per map (its verdicts are kept by
|
|
271
|
+
the object): the map's rows (its source, row filter or query) and its key."""
|
|
272
|
+
from types import SimpleNamespace
|
|
273
|
+
held = graph.__dict__.setdefault("_key_check_maps", {})
|
|
274
|
+
source = source_map.source
|
|
275
|
+
ident = (source_map.id, source.table, source.qualified, source.query, source.filter,
|
|
276
|
+
tuple(source_map.key.columns))
|
|
277
|
+
if ident not in held:
|
|
278
|
+
held[ident] = SimpleNamespace(
|
|
279
|
+
label=getattr(source_map, "label", None) or source_map.id, source_table=source.table,
|
|
280
|
+
qualified_source_table=source.qualified, source_query=source.query,
|
|
281
|
+
id_columns=list(source_map.key.columns), filter_predicate=dict(source.filter or ()))
|
|
282
|
+
return held[ident]
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _key_source(graph: Any) -> Any:
|
|
286
|
+
held = graph.__dict__.get("_key_check_source")
|
|
287
|
+
if held is None:
|
|
288
|
+
from prograph.plan.ir.key_checks import KeyChecks, source_of
|
|
289
|
+
held = (source_of(graph), KeyChecks())
|
|
290
|
+
graph.__dict__["_key_check_source"] = held
|
|
291
|
+
return held
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
@home("keys-check", "keys: check, a key verified for the data's version",
|
|
295
|
+
terms=("keys: check", "KeyCheckUnavailableError", "RepeatedKeyWarning", "VERSION AS OF",
|
|
296
|
+
"data version", "checked key"))
|
|
297
|
+
def checked_key(graph: Any, source_map: Any) -> bool:
|
|
298
|
+
"""Whether a `keys: check` map's key is unique over the map's rows (no two rows share it, a
|
|
299
|
+
row with a missing key part left out) at the data's current version, read once per version
|
|
300
|
+
(`key_checks.KeyChecks`); the statement then reads the version checked (`PLANNING_PINS`). A
|
|
301
|
+
source with no version is refused when the engine starts (`refuse_unversioned_checks`); a key
|
|
302
|
+
two rows share is not relied on, with a warning naming one of its values."""
|
|
303
|
+
if graph is None:
|
|
304
|
+
return False
|
|
305
|
+
source, checks = _key_source(graph)
|
|
306
|
+
if source is None:
|
|
307
|
+
return False
|
|
308
|
+
mapping = _checked_mapping(graph, source_map)
|
|
309
|
+
if source.version(mapping) is None:
|
|
310
|
+
return False
|
|
311
|
+
found, pin = checks.verdict(source, mapping)
|
|
312
|
+
if found is not None:
|
|
313
|
+
import warnings
|
|
314
|
+
warned = graph.__dict__.setdefault("_key_check_warned", set())
|
|
315
|
+
if (id(mapping), found) not in warned:
|
|
316
|
+
warned.add((id(mapping), found))
|
|
317
|
+
warnings.warn(f"keys: check on {mapping.label} ({mapping.source_table}): two rows share the key "
|
|
318
|
+
f"{', '.join(mapping.id_columns)} = {found!r}, so the plan does not rely on it",
|
|
319
|
+
RepeatedKeyWarning, stacklevel=4)
|
|
320
|
+
return False
|
|
321
|
+
from prograph.plan.ir.key_checks import FrameSource
|
|
322
|
+
if pin is None and not isinstance(source, FrameSource):
|
|
323
|
+
# Checked but not pinned: the statement would read the source as it is then, not the
|
|
324
|
+
# version checked (a Spark frame derived from Delta tables; a table with no history, or
|
|
325
|
+
# one that cannot be described). Not relied on (M2 review R2, R3); a pandas frame's
|
|
326
|
+
# version is the frame itself, which ProGraph's own writes replace.
|
|
327
|
+
return False
|
|
328
|
+
pins = PLANNING_PINS.get()
|
|
329
|
+
if pins is not None and pin is not None:
|
|
330
|
+
pins[mapping.qualified_source_table] = pin
|
|
331
|
+
return True
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def refuse_unversioned_checks(engine: Any) -> None:
|
|
335
|
+
"""At engine start: `keys: check` on a map whose source gives no version is refused, naming the
|
|
336
|
+
ways out for that source."""
|
|
337
|
+
schema = getattr(engine, "_schema", None)
|
|
338
|
+
graph = getattr(getattr(engine, "executor", None), "graph", None)
|
|
339
|
+
if schema is None or graph is None:
|
|
340
|
+
return
|
|
341
|
+
try:
|
|
342
|
+
from prograph.plan.ir.entry import source_maps
|
|
343
|
+
maps = source_maps(schema)
|
|
344
|
+
except Exception: # noqa: BLE001 - a schema the one plan does not read checks nothing here
|
|
345
|
+
return
|
|
346
|
+
checked = [m for m in maps.nodes if m.key is not None and m.key.stated and m.key.trust == "check"]
|
|
347
|
+
if not checked:
|
|
348
|
+
return
|
|
349
|
+
source, _ = _key_source(graph)
|
|
350
|
+
backend = getattr(engine, "backend", None)
|
|
351
|
+
for m in checked:
|
|
352
|
+
mapping = _checked_mapping(graph, m)
|
|
353
|
+
if source is not None and source.version(mapping) is not None:
|
|
354
|
+
continue
|
|
355
|
+
if backend == "spark":
|
|
356
|
+
ways = "make the table a Delta table, write keys: trusted, or remove keys: check"
|
|
357
|
+
what = "a Spark frame that is not a Delta table"
|
|
358
|
+
elif backend in ("postgresql", "duckdb"):
|
|
359
|
+
ways = ("declare a PRIMARY KEY or UNIQUE constraint on the key (the database enforces it and "
|
|
360
|
+
"the plan relies on it), write keys: trusted, or remove keys: check (the plan keeps "
|
|
361
|
+
"its Distincts: the same answers, slower)")
|
|
362
|
+
what = f"a {backend} table, which has no version"
|
|
363
|
+
else:
|
|
364
|
+
ways = "write keys: trusted, or remove keys: check"
|
|
365
|
+
what = "a source with no version"
|
|
366
|
+
raise KeyCheckUnavailableError(
|
|
367
|
+
f"keys: check on {mapping.label} ({m.source.table}) cannot be kept: its source is {what}, so "
|
|
368
|
+
f"the check and the query would read data that may change between them. To fix: {ways}.")
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
# -- the load warning (one-plan.md 2.4; keys-and-identity.md, point 3) ----------------------------
|
|
372
|
+
|
|
373
|
+
class RepeatingKeyWarning(UserWarning):
|
|
374
|
+
"""A key the plan relies on because it is asserted (`keys: trusted`, a Databricks RELY), whose
|
|
375
|
+
table's statistics count fewer distinct values than 0.9 of its rows: a repeat gives wrong
|
|
376
|
+
answers. 0.9 catches gross repeats only; for a relied-on key, a 1% repeat is wrong answers too."""
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
@home("key-load-warning", "Warning that an asserted key repeats, from the engine's statistics",
|
|
380
|
+
terms=("RepeatingKeyWarning", "n_distinct", "distinct_count", "statistics", "0.9"))
|
|
381
|
+
def warn_if_repeating(graph: Any, qualified: str, columns: Tuple[str, ...], why: str) -> None:
|
|
382
|
+
"""Once per table and key: warn where the engine's own statistics (PostgreSQL `pg_stats`, a
|
|
383
|
+
Databricks or Spark table's column statistics) give the key fewer distinct values than 0.9 of
|
|
384
|
+
the rows. Never a scan: no statistics, no warning. A single-column key only."""
|
|
385
|
+
if graph is None or len(columns) != 1:
|
|
386
|
+
return
|
|
387
|
+
seen = graph.__dict__.setdefault("_repeating_key_warned", set())
|
|
388
|
+
if (qualified, columns) in seen:
|
|
389
|
+
return
|
|
390
|
+
seen.add((qualified, columns))
|
|
391
|
+
try:
|
|
392
|
+
counted = _statistics(graph, qualified, columns[0])
|
|
393
|
+
except Exception: # noqa: BLE001 - statistics that cannot be read warn nothing
|
|
394
|
+
return
|
|
395
|
+
if counted is None:
|
|
396
|
+
return
|
|
397
|
+
distinct, rows = counted
|
|
398
|
+
if rows > 0 and distinct < 0.9 * rows:
|
|
399
|
+
import warnings
|
|
400
|
+
warnings.warn(
|
|
401
|
+
f"the key {columns[0]} of {qualified} is relied on ({why}), but the table's statistics count "
|
|
402
|
+
f"about {int(distinct)} distinct values in {int(rows)} rows: where two rows share a key value, "
|
|
403
|
+
f"SPARQL answers are wrong. Use a fuller key, a current-rows filter, or keys: check.",
|
|
404
|
+
RepeatingKeyWarning, stacklevel=4)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _described_extended(graph: Any, qualified: str) -> List[Dict[str, Any]]:
|
|
408
|
+
"""`DESCRIBE TABLE EXTENDED` of a table, read once per graph: the RELY constraints and the table's
|
|
409
|
+
statistics come from one statement."""
|
|
410
|
+
cache = graph.__dict__.setdefault("_described_extended", {})
|
|
411
|
+
if qualified not in cache:
|
|
412
|
+
cache[qualified] = list(graph._execute_sql(f"DESCRIBE TABLE EXTENDED {qualified}"))
|
|
413
|
+
return cache[qualified]
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _statistics(graph: Any, qualified: str, column: str) -> Optional[Tuple[float, float]]:
|
|
417
|
+
"""(distinct values, rows) from the engine's statistics, or None where it keeps none."""
|
|
418
|
+
if not hasattr(graph, "_execute_sql"):
|
|
419
|
+
return None
|
|
420
|
+
name = _dialect(graph)
|
|
421
|
+
parts = [p.strip('`"') for p in qualified.split(".")]
|
|
422
|
+
if name in ("postgresql", "postgres"):
|
|
423
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
424
|
+
lit = get_dialect_instance(name).literal
|
|
425
|
+
table, schema = parts[-1], (parts[-2] if len(parts) > 1 else None)
|
|
426
|
+
where = f"tablename = {lit(table)} AND attname = {lit(column)}" + \
|
|
427
|
+
(f" AND schemaname = {lit(schema)}" if schema else " AND schemaname = current_schema()")
|
|
428
|
+
rows = graph._execute_sql(
|
|
429
|
+
f"SELECT s.n_distinct AS d, c.reltuples AS r FROM pg_stats s JOIN pg_class c ON c.relname = s.tablename "
|
|
430
|
+
f"JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = s.schemaname WHERE {where}")
|
|
431
|
+
if not rows or rows[0]["r"] is None or float(rows[0]["r"]) <= 0:
|
|
432
|
+
return None
|
|
433
|
+
d, r = float(rows[0]["d"]), float(rows[0]["r"])
|
|
434
|
+
return (-d * r if d < 0 else d), r
|
|
435
|
+
if name == "databricks_sql":
|
|
436
|
+
# On a SQL warehouse each statistics read is a round trip of about a second before the first answer;
|
|
437
|
+
# the warning is a hint, not a check, so it reads nothing there.
|
|
438
|
+
return None
|
|
439
|
+
if name in ("databricks", "spark"):
|
|
440
|
+
import re
|
|
441
|
+
table_rows = None
|
|
442
|
+
for row in _described_extended(graph, qualified):
|
|
443
|
+
text = " ".join(str(v) for v in row.values() if v is not None)
|
|
444
|
+
m = re.search(r"Statistics\s+[\d,]+ bytes,\s*([\d,]+) rows", text)
|
|
445
|
+
if m:
|
|
446
|
+
table_rows = float(m.group(1).replace(",", ""))
|
|
447
|
+
if not table_rows:
|
|
448
|
+
return None
|
|
449
|
+
for row in graph._execute_sql(f"DESCRIBE TABLE EXTENDED {qualified} {column}"):
|
|
450
|
+
values = [str(v) for v in row.values() if v is not None]
|
|
451
|
+
if values and values[0] == "distinct_count" and len(values) > 1 and values[1] not in ("NULL", ""):
|
|
452
|
+
return float(values[1]), table_rows
|
|
453
|
+
return None
|
|
454
|
+
return None
|