prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""The order a front end writes a group's joins in (the Cypher half follows the same rule).
|
|
2
|
+
|
|
3
|
+
The lowering fuses inner joins into one SELECT so the database orders them (`lower.region`), but under a
|
|
4
|
+
condition across the joins it keeps the written order (`lower._across`: DuckDB takes such a condition for a
|
|
5
|
+
join edge and may join on it first). So the written order must already be a good one: connected, each leaf
|
|
6
|
+
joined where a condition ties it to what is joined, a cross join only where the group is disconnected, and a
|
|
7
|
+
part started from its most selective leaf.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from typing import FrozenSet, List, Sequence, Set, Tuple
|
|
13
|
+
|
|
14
|
+
from prograph.rules import home
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@home("join-order", "A group's joins written in connected order, from its most selective leaf",
|
|
18
|
+
terms=("join order", "connected order", "cross join", "pattern order", "selective leaf", "seed",
|
|
19
|
+
"most selective"))
|
|
20
|
+
def connected_order(leaves: Sequence[Tuple[str, FrozenSet[str], int]], have: Set[str],
|
|
21
|
+
conditions: Sequence[FrozenSet[str]]) -> List[str]:
|
|
22
|
+
"""The names of `leaves` (name, the columns it gives, how selective it is: higher first) in the order to
|
|
23
|
+
join them after the columns `have`: next, a leaf a condition ties to what is joined (a condition reading
|
|
24
|
+
its columns and some joined ones, all present once it is), the most selective of those, then the first
|
|
25
|
+
written; with none, a new part from the most selective leaf left. Each condition is the column set it
|
|
26
|
+
reads."""
|
|
27
|
+
have = set(have)
|
|
28
|
+
left = list(leaves)
|
|
29
|
+
out: List[str] = []
|
|
30
|
+
while left:
|
|
31
|
+
def tied(leaf: Tuple[str, FrozenSet[str], int]) -> bool:
|
|
32
|
+
cols = leaf[1]
|
|
33
|
+
return any(c & have and c & cols and c <= have | cols for c in conditions)
|
|
34
|
+
candidates = [x for x in left if tied(x)] or left
|
|
35
|
+
best = max(candidates, key=lambda x: x[2]) # max keeps the first written of equals
|
|
36
|
+
left.remove(best)
|
|
37
|
+
have |= best[1]
|
|
38
|
+
out.append(best[0])
|
|
39
|
+
return out
|
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
"""Reading a mapping's rows at a version of the data, and checking its key there (docs/architecture/keys-and-identity.md): a source's version (a pandas frame by object, a Spark
|
|
2
|
+
frame of fixed rows or a Delta table, a Databricks Delta table by its history), one key value two
|
|
3
|
+
of a map's rows share, read once per version (`KeyChecks`), and how a statement reads the version
|
|
4
|
+
checked (`PinnedRead`, `pinned_name`). `keys.checked_key` asks it; it moved here
|
|
5
|
+
from `prograph.node_keys`, whose Cypher policy stays there."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import contextvars
|
|
10
|
+
import time
|
|
11
|
+
import weakref
|
|
12
|
+
from typing import Any, Dict, Optional, Tuple
|
|
13
|
+
|
|
14
|
+
from prograph.plan.ir.keys import rely_primary_key # noqa: F401 (the home)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# -- reading what was checked ----------------------------------------------
|
|
18
|
+
|
|
19
|
+
class PinnedRead:
|
|
20
|
+
"""How a statement reads a checked table at the checked version: the
|
|
21
|
+
text that stands for the table in its SQL (``t VERSION AS OF n``), and
|
|
22
|
+
on Spark the frame that stands for it on the DataFrame path."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, sql: str, frame: Any = None):
|
|
25
|
+
self.sql = sql
|
|
26
|
+
self.frame = frame
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Pins(dict):
|
|
30
|
+
"""A statement's pinned reads, by table name."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
_PINNED: contextvars.ContextVar = contextvars.ContextVar("prograph_pinned_reads", default=None)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def pinned_name(table: str, written: str) -> str:
|
|
37
|
+
"""A scan's table as the statement reads it: `written` (the table's name
|
|
38
|
+
as the dialect writes it), or the pinned read of a checked table."""
|
|
39
|
+
pins = _PINNED.get()
|
|
40
|
+
if pins:
|
|
41
|
+
pin = pins.get(table)
|
|
42
|
+
if pin is not None:
|
|
43
|
+
return pin.sql
|
|
44
|
+
return written
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# -- the sources: a mapping's version, and a key value two rows share ------
|
|
48
|
+
|
|
49
|
+
def _example(values) -> Tuple[Any, ...]:
|
|
50
|
+
out = []
|
|
51
|
+
for v in values:
|
|
52
|
+
item = getattr(v, "item", None)
|
|
53
|
+
out.append(item() if callable(item) and type(v).__module__ == "numpy" else v)
|
|
54
|
+
return tuple(out)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class FrameSource:
|
|
58
|
+
"""Pandas frames (``backend='duckdb'`` over frames).
|
|
59
|
+
A frame's version is the frame object and its arrays: a write that
|
|
60
|
+
replaces either is a new version."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, graph: Any):
|
|
63
|
+
self.graph = graph
|
|
64
|
+
|
|
65
|
+
def _frame(self, mapping):
|
|
66
|
+
return self.graph.dataframes.get(mapping.source_table)
|
|
67
|
+
|
|
68
|
+
def version(self, mapping):
|
|
69
|
+
from prograph.graph.pandas_graph import _frame_state
|
|
70
|
+
frame = self._frame(mapping)
|
|
71
|
+
if frame is None:
|
|
72
|
+
return None
|
|
73
|
+
state = _frame_state(frame)
|
|
74
|
+
if state[2] is None:
|
|
75
|
+
return None
|
|
76
|
+
return (weakref.ref(frame),) + state
|
|
77
|
+
|
|
78
|
+
@staticmethod
|
|
79
|
+
def same(held, now) -> bool:
|
|
80
|
+
from prograph.graph.pandas_graph import _same_state
|
|
81
|
+
return held[0]() is not None and held[0]() is now[0]() and _same_state(held[1:], now[1:])
|
|
82
|
+
|
|
83
|
+
def describe(self, mapping):
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
def pin(self, mapping, version):
|
|
87
|
+
return None
|
|
88
|
+
|
|
89
|
+
def duplicate(self, mapping, version=None):
|
|
90
|
+
from prograph.schema.rows import owned_rows
|
|
91
|
+
from prograph.schema.types import missing_mask
|
|
92
|
+
frame = owned_rows(self._frame(mapping), mapping)
|
|
93
|
+
if frame is None or len(frame) == 0:
|
|
94
|
+
return None
|
|
95
|
+
cols = list(mapping.id_columns)
|
|
96
|
+
if any(c not in frame.columns for c in cols):
|
|
97
|
+
return None
|
|
98
|
+
keys = frame[cols]
|
|
99
|
+
present = ~missing_mask(keys).any(axis=1) if len(cols) > 1 else ~missing_mask(keys[cols[0]])
|
|
100
|
+
keys = keys[present]
|
|
101
|
+
repeated = keys.duplicated()
|
|
102
|
+
if not repeated.any():
|
|
103
|
+
return None
|
|
104
|
+
return _example(keys[repeated].iloc[0].tolist())
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _spark_leaves(frame) -> Optional[list]:
|
|
108
|
+
jdf = getattr(frame, "_jdf", None)
|
|
109
|
+
if jdf is None:
|
|
110
|
+
return None
|
|
111
|
+
leaves, stack = [], [jdf.queryExecution().analyzed()]
|
|
112
|
+
while stack:
|
|
113
|
+
node = stack.pop()
|
|
114
|
+
children = node.children()
|
|
115
|
+
if children.isEmpty():
|
|
116
|
+
leaves.append(node)
|
|
117
|
+
continue
|
|
118
|
+
it = children.iterator()
|
|
119
|
+
while it.hasNext():
|
|
120
|
+
stack.append(it.next())
|
|
121
|
+
return leaves
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# Leaves of a Spark plan whose rows are fixed with the frame: a frame over
|
|
125
|
+
# only these is one version for as long as it is the same object.
|
|
126
|
+
_FIXED_SPARK_LEAVES = ("LocalRelation", "LogicalRDD", "Range", "OneRowRelation")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class SparkFrameSource:
|
|
130
|
+
"""Spark DataFrames. A frame over Delta tables is versioned by each
|
|
131
|
+
table's version (DESCRIBE HISTORY); a frame over rows held in memory by
|
|
132
|
+
the frame itself; one over anything else (a parquet table, a view of
|
|
133
|
+
another kind) has no version and is checked on each statement. A frame
|
|
134
|
+
that reads one Delta table as it is (`_plain_delta`) is checked and read
|
|
135
|
+
at the version looked up; a frame derived from Delta tables (a filter, a
|
|
136
|
+
join) cannot be pinned, since its plan is the caller's, and is checked at
|
|
137
|
+
the version looked up and read at the table's latest."""
|
|
138
|
+
|
|
139
|
+
def __init__(self, graph: Any):
|
|
140
|
+
self.graph = graph
|
|
141
|
+
|
|
142
|
+
def _frame(self, mapping):
|
|
143
|
+
return self.graph.dataframes.get(mapping.source_table)
|
|
144
|
+
|
|
145
|
+
def version(self, mapping):
|
|
146
|
+
frame = self._frame(mapping)
|
|
147
|
+
if frame is None:
|
|
148
|
+
return None
|
|
149
|
+
try:
|
|
150
|
+
leaves = _spark_leaves(frame)
|
|
151
|
+
except Exception: # noqa: BLE001 - a plan that cannot be read has no version
|
|
152
|
+
return None
|
|
153
|
+
if leaves is None:
|
|
154
|
+
return None
|
|
155
|
+
tables = []
|
|
156
|
+
for leaf in leaves:
|
|
157
|
+
kind = leaf.getClass().getSimpleName()
|
|
158
|
+
if kind in _FIXED_SPARK_LEAVES:
|
|
159
|
+
continue
|
|
160
|
+
name = _delta_name(leaf)
|
|
161
|
+
if name is None:
|
|
162
|
+
return None
|
|
163
|
+
tables.append((name, self._delta_version(frame, name)))
|
|
164
|
+
return (weakref.ref(frame), tuple(tables))
|
|
165
|
+
|
|
166
|
+
@staticmethod
|
|
167
|
+
def _delta_version(frame, name):
|
|
168
|
+
row = frame.sparkSession.sql(f"DESCRIBE HISTORY {name} LIMIT 1").select("version").first()
|
|
169
|
+
return None if row is None else row[0]
|
|
170
|
+
|
|
171
|
+
@staticmethod
|
|
172
|
+
def same(held, now) -> bool:
|
|
173
|
+
return held[0]() is not None and held[0]() is now[0]() and held[1] == now[1]
|
|
174
|
+
|
|
175
|
+
def describe(self, mapping):
|
|
176
|
+
return None
|
|
177
|
+
|
|
178
|
+
def _pinned(self, mapping, version):
|
|
179
|
+
"""The Delta table `mapping`'s frame reads as it is, and the version
|
|
180
|
+
to read it at; None for a frame that is not one table read as it
|
|
181
|
+
is, or no version."""
|
|
182
|
+
if version is None or len(version[1]) != 1 or version[1][0][1] is None:
|
|
183
|
+
return None
|
|
184
|
+
frame = self._frame(mapping)
|
|
185
|
+
name = _plain_delta(frame)
|
|
186
|
+
if name is None or name != version[1][0][0]:
|
|
187
|
+
return None
|
|
188
|
+
return name, int(version[1][0][1])
|
|
189
|
+
|
|
190
|
+
def pin(self, mapping, version):
|
|
191
|
+
pinned = self._pinned(mapping, version)
|
|
192
|
+
if pinned is None:
|
|
193
|
+
return None
|
|
194
|
+
name, number = pinned
|
|
195
|
+
frame = self._frame(mapping)
|
|
196
|
+
return PinnedRead(f"{name} VERSION AS OF {number}",
|
|
197
|
+
frame.sparkSession.sql(f"SELECT * FROM {name} VERSION AS OF {number}"))
|
|
198
|
+
|
|
199
|
+
def duplicate(self, mapping, version=None):
|
|
200
|
+
from pyspark.sql import functions as F
|
|
201
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
202
|
+
from prograph.schema.rows import owned_rows
|
|
203
|
+
pinned = self.pin(mapping, version)
|
|
204
|
+
frame = owned_rows(pinned.frame if pinned is not None else self._frame(mapping), mapping)
|
|
205
|
+
if frame is None:
|
|
206
|
+
return None
|
|
207
|
+
quote = get_dialect_instance("spark").quote
|
|
208
|
+
cols = [F.col(quote(c)) for c in mapping.id_columns]
|
|
209
|
+
present = cols[0].isNotNull()
|
|
210
|
+
for c in cols[1:]:
|
|
211
|
+
present = present & c.isNotNull()
|
|
212
|
+
row = (frame.where(present).groupBy(*cols).count()
|
|
213
|
+
.where(F.col("count") > 1).limit(1).collect())
|
|
214
|
+
return tuple(row[0][:len(cols)]) if row else None
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _delta_name(leaf) -> Optional[str]:
|
|
218
|
+
"""The name a Spark plan's leaf reads a Delta table by (its catalog
|
|
219
|
+
name, quoted, or ``delta.`path` ``), or None for a leaf of another
|
|
220
|
+
kind."""
|
|
221
|
+
if leaf.getClass().getSimpleName() != "LogicalRelation":
|
|
222
|
+
return None
|
|
223
|
+
relation = leaf.relation()
|
|
224
|
+
location = relation.location() if hasattr(relation, "location") else None
|
|
225
|
+
if location is None or "Tahoe" not in location.getClass().getName():
|
|
226
|
+
return None
|
|
227
|
+
table = leaf.catalogTable()
|
|
228
|
+
if table.isDefined():
|
|
229
|
+
return table.get().identifier().quotedString()
|
|
230
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
231
|
+
return "delta." + get_dialect_instance("spark").quote(location.rootPaths().head().toString())
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _plain_delta(frame) -> Optional[str]:
|
|
235
|
+
"""The name of the Delta table `frame` reads as it is, every row and
|
|
236
|
+
column (``spark.table(name)``, or a ``select('*')`` of it, read directly
|
|
237
|
+
or through a temp view of one, as ``spark.table(name)`` is once an engine
|
|
238
|
+
has made a temp view under the table's name), or None."""
|
|
239
|
+
jdf = getattr(frame, "_jdf", None)
|
|
240
|
+
if jdf is None:
|
|
241
|
+
return None
|
|
242
|
+
try:
|
|
243
|
+
root = node = jdf.queryExecution().analyzed()
|
|
244
|
+
while node.getClass().getSimpleName() in ("SubqueryAlias", "Project", "View"):
|
|
245
|
+
if node.getClass().getSimpleName() == "Project" and \
|
|
246
|
+
node.output().toString() != node.child().output().toString():
|
|
247
|
+
return None
|
|
248
|
+
node = node.child()
|
|
249
|
+
if root.output().toString() != node.output().toString():
|
|
250
|
+
return None
|
|
251
|
+
return _delta_name(node)
|
|
252
|
+
except Exception: # noqa: BLE001 - a plan that cannot be read is not pinned
|
|
253
|
+
return None
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
class SQLSource:
|
|
257
|
+
"""A database through a connection: a DuckDB connection, a Databricks SQL
|
|
258
|
+
warehouse, and the other SQL databases. A Databricks table is versioned
|
|
259
|
+
by its Delta version and trusted by a RELY primary key on its key; any
|
|
260
|
+
other source has no version and is checked on each statement."""
|
|
261
|
+
|
|
262
|
+
def __init__(self, graph: Any):
|
|
263
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
264
|
+
self.graph = graph
|
|
265
|
+
self.dialect_name = graph.sql_dialect()
|
|
266
|
+
self.dialect = get_dialect_instance(self.dialect_name)
|
|
267
|
+
|
|
268
|
+
@property
|
|
269
|
+
def _databricks(self) -> bool:
|
|
270
|
+
return self.dialect_name in ("databricks_sql", "databricks")
|
|
271
|
+
|
|
272
|
+
def _table(self, mapping) -> str:
|
|
273
|
+
return self.dialect.table_name(mapping.qualified_source_table)
|
|
274
|
+
|
|
275
|
+
def describe(self, mapping) -> Optional[Dict[str, Any]]:
|
|
276
|
+
"""On Databricks SQL, what a mapping's source is, read once
|
|
277
|
+
(DESCRIBE TABLE EXTENDED): whether it has a Delta history to pin
|
|
278
|
+
(``history``), why not (``what``), and whether its key is a RELY
|
|
279
|
+
primary key (``rely``). None on another database."""
|
|
280
|
+
if not self._databricks:
|
|
281
|
+
return None
|
|
282
|
+
if getattr(mapping, "source_query", None):
|
|
283
|
+
return {"history": False, "what": "a SQL query", "rely": False}
|
|
284
|
+
try:
|
|
285
|
+
rows = self.graph._execute_sql(f"DESCRIBE TABLE EXTENDED {self._table(mapping)}")
|
|
286
|
+
except Exception: # noqa: BLE001 - no description, no history to pin
|
|
287
|
+
return {"history": False, "what": "a source that cannot be described", "rely": False}
|
|
288
|
+
fields = {str(r.get("col_name") or "").strip().lower(): str(r.get("data_type") or "").strip()
|
|
289
|
+
for r in rows}
|
|
290
|
+
kind, provider = fields.get("type", "").upper(), fields.get("provider", "").lower()
|
|
291
|
+
rely = any(rely_primary_key(" ".join(str(v) for v in row.values() if v is not None),
|
|
292
|
+
mapping.id_columns)
|
|
293
|
+
for row in rows)
|
|
294
|
+
if kind not in ("MANAGED", "EXTERNAL") or provider != "delta":
|
|
295
|
+
what = f"a {kind.lower().replace('_', ' ')}" if kind not in ("MANAGED", "EXTERNAL", "") \
|
|
296
|
+
else f"a {provider or 'non-Delta'} table"
|
|
297
|
+
return {"history": False, "what": what, "rely": rely}
|
|
298
|
+
return {"history": True, "what": "a Delta table", "rely": rely}
|
|
299
|
+
|
|
300
|
+
# How long a table's version is taken as current without asking again: a DESCRIBE HISTORY is a
|
|
301
|
+
# round trip per statement on a warehouse. A write within the window is read at the version checked
|
|
302
|
+
# (the statement pins it), so answers stay consistent; the newer rows show after the window.
|
|
303
|
+
VERSION_TTL_S = 60.0
|
|
304
|
+
|
|
305
|
+
def version(self, mapping):
|
|
306
|
+
if not self._databricks or getattr(mapping, "source_query", None):
|
|
307
|
+
return None
|
|
308
|
+
import time
|
|
309
|
+
table = self._table(mapping)
|
|
310
|
+
cache = self.graph.__dict__.setdefault("_table_versions", {})
|
|
311
|
+
held = cache.get(table)
|
|
312
|
+
if held is not None and time.monotonic() - held[0] < self.VERSION_TTL_S:
|
|
313
|
+
return held[1]
|
|
314
|
+
try:
|
|
315
|
+
rows = self.graph._execute_sql(f"DESCRIBE HISTORY {table} LIMIT 1")
|
|
316
|
+
except Exception: # noqa: BLE001 - no history read: checked on each statement
|
|
317
|
+
return None
|
|
318
|
+
if not rows:
|
|
319
|
+
return None
|
|
320
|
+
found = ("delta", table, rows[0].get("version"))
|
|
321
|
+
cache[table] = (time.monotonic(), found)
|
|
322
|
+
return found
|
|
323
|
+
|
|
324
|
+
@staticmethod
|
|
325
|
+
def same(held, now) -> bool:
|
|
326
|
+
return held == now
|
|
327
|
+
|
|
328
|
+
def pin(self, mapping, version):
|
|
329
|
+
if not self._databricks or version is None or version[2] is None:
|
|
330
|
+
return None
|
|
331
|
+
return PinnedRead(f"{self._table(mapping)} VERSION AS OF {int(version[2])}")
|
|
332
|
+
|
|
333
|
+
def duplicate(self, mapping, version=None):
|
|
334
|
+
from prograph.schema.rows import owned_conditions
|
|
335
|
+
q = self.dialect.quote
|
|
336
|
+
cols = ", ".join(q(c) for c in mapping.id_columns)
|
|
337
|
+
conditions = [f"{q(c)} IS NOT NULL" for c in mapping.id_columns]
|
|
338
|
+
conditions += owned_conditions(mapping, self.dialect)
|
|
339
|
+
pinned = self.pin(mapping, version)
|
|
340
|
+
table = pinned.sql if pinned is not None else self._table(mapping)
|
|
341
|
+
from prograph.plan.sql import serializer_for
|
|
342
|
+
sql = serializer_for(self.dialect_name).write_select(
|
|
343
|
+
table, "_k", [cols], where=conditions, group_by=[cols],
|
|
344
|
+
having=["COUNT(*) > 1"], limit=1)
|
|
345
|
+
try:
|
|
346
|
+
rows = self.graph._execute_sql(sql)
|
|
347
|
+
except Exception as exc: # noqa: BLE001 - the driver's own error types
|
|
348
|
+
# Generated SQL the database rejected: a runtime error that carries the SQL,
|
|
349
|
+
# as for the statement itself.
|
|
350
|
+
from prograph.utils.exceptions import SQLRuntimeError
|
|
351
|
+
raise SQLRuntimeError(f"SQL execution failed: {exc}",
|
|
352
|
+
reason="execution_error", sql=sql) from exc
|
|
353
|
+
if not rows:
|
|
354
|
+
return None
|
|
355
|
+
row = rows[0]
|
|
356
|
+
return tuple(row[k] for k in list(row)[:len(mapping.id_columns)])
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def source_of(graph: Any) -> Optional[Any]:
|
|
360
|
+
"""The source a graph reads its mappings' rows from, or None for a graph
|
|
361
|
+
that maps no table (the memory backend's own graph)."""
|
|
362
|
+
from prograph.graph.pandas_graph import PandasGraph
|
|
363
|
+
from prograph.graph.sql_graph import SQLGraph
|
|
364
|
+
if isinstance(graph, PandasGraph):
|
|
365
|
+
return FrameSource(graph)
|
|
366
|
+
if isinstance(graph, SQLGraph):
|
|
367
|
+
return SQLSource(graph)
|
|
368
|
+
try:
|
|
369
|
+
from prograph.graph.spark_graph import SparkGraph
|
|
370
|
+
except ImportError: # pragma: no cover - no pyspark
|
|
371
|
+
return None
|
|
372
|
+
if isinstance(graph, SparkGraph):
|
|
373
|
+
return SparkFrameSource(graph)
|
|
374
|
+
return None
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
# -- once per version -------------------------------------------------------
|
|
378
|
+
|
|
379
|
+
class KeyChecks:
|
|
380
|
+
"""One engine's verdicts on its mappings' keys, each kept for the
|
|
381
|
+
version of the data it was made on. ``scans`` counts the times the data
|
|
382
|
+
was read to check a key."""
|
|
383
|
+
|
|
384
|
+
def __init__(self):
|
|
385
|
+
self._verdicts: Dict[Any, Tuple[Any, Any]] = {}
|
|
386
|
+
self._descriptions: Dict[Any, Optional[Dict[str, Any]]] = {}
|
|
387
|
+
self.scans = 0
|
|
388
|
+
|
|
389
|
+
@staticmethod
|
|
390
|
+
def _identity(mapping) -> Tuple[Any, ...]:
|
|
391
|
+
return (id(mapping), mapping.source_table, getattr(mapping, "source_query", None),
|
|
392
|
+
tuple(mapping.id_columns),
|
|
393
|
+
tuple(sorted((mapping.filter_predicate or {}).items(), key=lambda kv: kv[0])))
|
|
394
|
+
|
|
395
|
+
def _described(self, source: Any, mapping: Any) -> Optional[Dict[str, Any]]:
|
|
396
|
+
identity = self._identity(mapping)
|
|
397
|
+
if identity not in self._descriptions:
|
|
398
|
+
described = source.describe(mapping)
|
|
399
|
+
self._descriptions[identity] = described
|
|
400
|
+
if described is not None and not described["history"] and not described["rely"]:
|
|
401
|
+
from prograph.logging import get_plan_logger
|
|
402
|
+
get_plan_logger().warning(
|
|
403
|
+
"node_key_trusted",
|
|
404
|
+
message=(f"The key of node mapping {mapping.label!r} ({mapping.source_table}, "
|
|
405
|
+
f"key {', '.join(mapping.id_columns)}) is trusted, not checked: its "
|
|
406
|
+
f"source is {described['what']}, which has no version to read the "
|
|
407
|
+
f"answer at. If two rows share a key value, the answer is undefined."),
|
|
408
|
+
label=mapping.label, table=mapping.source_table, source=described["what"])
|
|
409
|
+
return self._descriptions[identity]
|
|
410
|
+
|
|
411
|
+
def verdict(self, source: Any, mapping: Any) -> Tuple[Optional[Tuple[Any, ...]], Optional[PinnedRead]]:
|
|
412
|
+
"""One key value two of `mapping`'s rows share, or None, read from
|
|
413
|
+
the data once per version; and how the statement reads the version
|
|
414
|
+
checked (None: as the source reads now)."""
|
|
415
|
+
from prograph.logging import get_plan_logger
|
|
416
|
+
described = self._described(source, mapping)
|
|
417
|
+
if described is not None and (described["rely"] or not described["history"]):
|
|
418
|
+
return None, None
|
|
419
|
+
identity = self._identity(mapping)
|
|
420
|
+
version = source.version(mapping)
|
|
421
|
+
held = self._verdicts.get(identity)
|
|
422
|
+
if version is not None and held is not None and held[0] is not None \
|
|
423
|
+
and source.same(held[0], version):
|
|
424
|
+
return held[1], source.pin(mapping, version)
|
|
425
|
+
started = time.perf_counter()
|
|
426
|
+
found = source.duplicate(mapping, version)
|
|
427
|
+
self.scans += 1
|
|
428
|
+
get_plan_logger().debug("node_key_check", label=mapping.label,
|
|
429
|
+
table=mapping.source_table, versioned=version is not None,
|
|
430
|
+
duplicate=found is not None,
|
|
431
|
+
elapsed_ms=round((time.perf_counter() - started) * 1000, 3))
|
|
432
|
+
self._verdicts[identity] = (version, found)
|
|
433
|
+
return found, source.pin(mapping, version)
|