prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,625 @@
|
|
|
1
|
+
"""Whether stored OWL RL facts are provably current (docs/reference/owl-rl.md, "Freshness").
|
|
2
|
+
|
|
3
|
+
A run records, beside its cache key, each source table's token as the run read it (`read_run`), and the store's
|
|
4
|
+
own seq at its step. A query of an engine of profile='rl' reads the stored facts only where every source is
|
|
5
|
+
provably unchanged since that run, which only these prove:
|
|
6
|
+
|
|
7
|
+
- **a declared version column** (`version_columns={table: column}`): a column the user's writes keep
|
|
8
|
+
monotonic (an insert or update sets a value above every value before); its maximum and the row count (a
|
|
9
|
+
deleted row lowers it) are compared. Declared, it is used before anything else;
|
|
10
|
+
- **a Delta table version**: a Spark frame that reads Delta tables only (each table's latest version), a Delta
|
|
11
|
+
table on Databricks SQL (DESCRIBE HISTORY). No checksum is taken of a Delta table;
|
|
12
|
+
- **a full content checksum** of a small table: the row count and two sums of a 64-bit hash of every row,
|
|
13
|
+
taken per plan only where the table's size, read from the backend's metadata (`size_of`), is at most the
|
|
14
|
+
backend's threshold (`SMALL`), so that it costs at most about 20 ms.
|
|
15
|
+
|
|
16
|
+
Row counts alone, file times and cached versions prove nothing and are never used; the size read only decides
|
|
17
|
+
whether a checksum is taken.
|
|
18
|
+
|
|
19
|
+
Once per plan, where the dataset home reads the store (`dataset.store_now`), each source's token is read and
|
|
20
|
+
timed (`Check`, which `explain()` shows). Where a source changed (or the store took a write since the run), the
|
|
21
|
+
engine refreshes first (`refresh`: the closure computed again, its difference written). A source no token proves
|
|
22
|
+
(a table past its threshold with no declared column, not Delta; a view), and a refresh that cannot
|
|
23
|
+
write (a read-only store), refuse by name (`StoredFactsNotCurrentError`), naming `version_columns=` and
|
|
24
|
+
`freshness='last_run'`, which reads the stored facts of the last run with no check (`explain()` says when that
|
|
25
|
+
run was, and under which key).
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import json
|
|
31
|
+
from dataclasses import dataclass
|
|
32
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
33
|
+
|
|
34
|
+
from prograph.rules import module_home
|
|
35
|
+
from prograph.utils.exceptions import StoredFactsNotCurrentError
|
|
36
|
+
|
|
37
|
+
module_home("stored-facts-freshness", "Stored OWL RL facts used only where provably current: each source's "
|
|
38
|
+
"version recorded at the run and compared once per plan",
|
|
39
|
+
terms=("freshness", "last_run", "provably current", "source version", "version column",
|
|
40
|
+
"version_columns", "Delta version", "content checksum", "StoredFactsNotCurrentError",
|
|
41
|
+
"refresh", "stale facts", "stale-by-choice"))
|
|
42
|
+
|
|
43
|
+
# The engine's `freshness=`: None (stored facts used only where provably current) or 'last_run'.
|
|
44
|
+
FRESHNESS = (None, "last_run", "strict")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def engine_freshness(freshness: Optional[str], profile: Optional[str]) -> Optional[str]:
|
|
48
|
+
"""`freshness=` checked: None, 'last_run' or 'strict', under profile='rl' (the stored facts) or 'ql' (the
|
|
49
|
+
consistency verdict, `consistency.recheck`). Under 'rl' 'strict' is the default's rule. Anything else
|
|
50
|
+
raises ValueError."""
|
|
51
|
+
if freshness not in FRESHNESS:
|
|
52
|
+
raise ValueError(f"freshness is None (the default), 'last_run' or 'strict', not {freshness!r}")
|
|
53
|
+
if freshness is not None and profile not in ("rl", "ql"):
|
|
54
|
+
raise ValueError(f"freshness={freshness!r} chooses how the ontology's stored facts or consistency verdict "
|
|
55
|
+
f"are trusted: build the engine with profile='rl' or profile='ql' too")
|
|
56
|
+
return None if freshness == "strict" and profile == "rl" else freshness
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class Source:
|
|
61
|
+
"""One logical table the engine's own maps read: `name` as mapped (`Source.table`), `sql` the relation a
|
|
62
|
+
statement reads (a table name or a query in parentheses), the declared version `column` or None, `label`
|
|
63
|
+
the source in the user's words (the table, or a query's map and first line), and for a query the
|
|
64
|
+
(table, column) of each table it reads that `version_columns` declares (`reads`, empty where not all are)."""
|
|
65
|
+
name: str
|
|
66
|
+
sql: str
|
|
67
|
+
column: Optional[str]
|
|
68
|
+
label: str = ""
|
|
69
|
+
reads: Tuple[Tuple[str, str], ...] = ()
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def query(self) -> bool:
|
|
73
|
+
return self.sql.startswith("(")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def sources(engine: Any, maps: Any) -> Tuple[Source, ...]:
|
|
77
|
+
"""The logical tables `maps` (the engine's own, `dataset.own_maps`) read, with their declared version
|
|
78
|
+
columns (`version_columns`): a mapped table's own, and for an rr:sqlQuery those of the tables it reads (a
|
|
79
|
+
query is proven by them where every table it reads has one). A declared table no map reads, nor any
|
|
80
|
+
query, raises ValueError."""
|
|
81
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
82
|
+
graph = getattr(getattr(engine, "executor", None), "graph", None)
|
|
83
|
+
dialect = graph.sql_dialect() if hasattr(graph, "sql_dialect") else None
|
|
84
|
+
d = get_dialect_instance(dialect or "duckdb")
|
|
85
|
+
declared = dict(getattr(engine, "_version_columns", None) or {})
|
|
86
|
+
maps_of: Dict[str, Any] = {}
|
|
87
|
+
for m in maps.nodes + maps.edges:
|
|
88
|
+
maps_of.setdefault(m.source.table, m)
|
|
89
|
+
out, used = [], set()
|
|
90
|
+
matched: Dict[str, set] = {} # a declared name: the relations of queries it names
|
|
91
|
+
for s in maps.tables():
|
|
92
|
+
if s.query is None:
|
|
93
|
+
used.add(s.table) if s.table in declared else None
|
|
94
|
+
out.append(Source(s.table, d.table_name(s.qualified), declared.get(s.table), s.table))
|
|
95
|
+
continue
|
|
96
|
+
m = maps_of.get(s.table)
|
|
97
|
+
first = s.query.strip().splitlines()[0][:80] if s.query.strip() else ""
|
|
98
|
+
label = f"the rr:sqlQuery of {getattr(m, 'id', s.table)} ({first})"
|
|
99
|
+
reads: Tuple[Tuple[str, str], ...] = ()
|
|
100
|
+
store = _reader(engine) if _kind(engine) == "sql" else None
|
|
101
|
+
tables = _query_tables(store, f"({s.query})") if store is not None and _kind(engine) == "sql" else None
|
|
102
|
+
if tables and declared:
|
|
103
|
+
found = [(t, _declared_for(t, declared, store.dialect)) for t in tables]
|
|
104
|
+
for t, c in found:
|
|
105
|
+
if c is not None:
|
|
106
|
+
matched.setdefault(c[0], set()).add(".".join(p for p, _ in name_parts(t)))
|
|
107
|
+
if all(c is not None for _, c in found):
|
|
108
|
+
reads = tuple((".".join(d.quote(p) for p, _ in name_parts(t)), c[1]) for t, c in found)
|
|
109
|
+
used.update(c[0] for _, c in found)
|
|
110
|
+
out.append(Source(s.table, f"({s.query})", None, label, reads))
|
|
111
|
+
for name, relations in matched.items():
|
|
112
|
+
if len(relations) > 1:
|
|
113
|
+
raise ValueError(f"version_columns names {name!r}, which is each of {sorted(relations)} that the "
|
|
114
|
+
f"rr:sqlQuery maps read: name the one you mean with its schema")
|
|
115
|
+
unused = sorted(set(declared) - used)
|
|
116
|
+
if unused:
|
|
117
|
+
raise ValueError(f"version_columns names {unused}, which no map of the engine reads, nor any rr:sqlQuery; "
|
|
118
|
+
f"the mapped tables are {sorted(x.name for x in out if not x.query)}")
|
|
119
|
+
return tuple(out)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def name_parts(name: str) -> List[Tuple[str, bool]]:
|
|
123
|
+
"""A dotted name's parts, each (text, quoted): `"a"."b"`, `` `a`.`b` `` and `a.b` alike."""
|
|
124
|
+
import re
|
|
125
|
+
return [(m.group(1) if m.group(1) is not None else m.group(2) if m.group(2) is not None else m.group(3),
|
|
126
|
+
m.group(3) is None)
|
|
127
|
+
for m in re.finditer(r'"((?:[^"]|"")*)"|`((?:[^`]|``)*)`|([^."`]+)', name)]
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _same_part(declared: Tuple[str, bool], found: Tuple[str, bool], dialect: str) -> bool:
|
|
131
|
+
"""Whether a declared name's part names a relation's part: on PostgreSQL an unquoted part folds to lower
|
|
132
|
+
case and a quoted one is exact; elsewhere (DuckDB, Spark, Databricks SQL) case does not matter."""
|
|
133
|
+
if dialect == "postgresql":
|
|
134
|
+
text = declared[0] if declared[1] else declared[0].lower()
|
|
135
|
+
return text == found[0]
|
|
136
|
+
return declared[0].lower() == found[0].lower()
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _declared_for(table: str, declared: Dict[str, str], dialect: str) -> Optional[Tuple[str, str]]:
|
|
140
|
+
"""The (declared name, column) of `version_columns` that names the relation `table` (as the database names
|
|
141
|
+
it, qualified, maybe quoted): the declared name's parts are the relation's trailing parts (`t`,
|
|
142
|
+
`schema.t`, `catalog.schema.t`)."""
|
|
143
|
+
found = name_parts(table)
|
|
144
|
+
for name, column in declared.items():
|
|
145
|
+
parts = name_parts(name)
|
|
146
|
+
if 0 < len(parts) <= len(found) and all(_same_part(a, b, dialect)
|
|
147
|
+
for a, b in zip(parts, found[len(found) - len(parts):])):
|
|
148
|
+
return name, column
|
|
149
|
+
return None
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _reader(engine: Any) -> Any:
|
|
153
|
+
"""What reads the sources' versions and checksums: the owned store's connection, or with no store (profile
|
|
154
|
+
'ql') the engine's own connection, wrapped as the store wraps it (`store.store_connection`: no table made)."""
|
|
155
|
+
store = getattr(engine, "_store", None)
|
|
156
|
+
if store is not None:
|
|
157
|
+
return store
|
|
158
|
+
held = engine.__dict__.get("_freshness_reader")
|
|
159
|
+
if held is None:
|
|
160
|
+
from types import SimpleNamespace
|
|
161
|
+
from prograph.store import store_connection
|
|
162
|
+
db = store_connection(engine._connection)
|
|
163
|
+
held = SimpleNamespace(db=db, dialect=db.dialect)
|
|
164
|
+
engine.__dict__["_freshness_reader"] = held
|
|
165
|
+
return held
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _graph(engine: Any) -> Any:
|
|
169
|
+
return getattr(getattr(engine, "executor", None), "graph", None)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _kind(engine: Any) -> str:
|
|
173
|
+
"""How the engine reads its sources: 'sql' (a database connection), 'spark' (Spark frames) or 'frames'
|
|
174
|
+
(pandas frames on DuckDB)."""
|
|
175
|
+
from prograph.graph.sql_graph import SQLGraph
|
|
176
|
+
graph = _graph(engine)
|
|
177
|
+
if isinstance(graph, SQLGraph):
|
|
178
|
+
return "sql"
|
|
179
|
+
if getattr(engine, "backend", None) == "spark":
|
|
180
|
+
return "spark"
|
|
181
|
+
return "frames"
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
# -- how big a table may be for a checksum per plan ------------------------------------------------------------
|
|
185
|
+
|
|
186
|
+
# The largest table whose full checksum a plan takes, per backend, in the unit its metadata keeps (`size_of`).
|
|
187
|
+
# The target: a checksum per plan costs at most about 20 ms on DuckDB and PostgreSQL; on Spark a checksum is a
|
|
188
|
+
# job, whose floor is about 40 ms at any size, so there the target is that floor. Measured 2026-11-15 on a
|
|
189
|
+
# 16-core workstation: tables of three text columns (`s`, `o`, `label`, about 30 to 45 bytes each, like a mapped
|
|
190
|
+
# table's IRIs and labels), the median of 7 runs of `checksum_sql` / `_spark_checksum` per size:
|
|
191
|
+
# - DuckDB (rows, `duckdb_tables().estimated_size`): 100,000 rows 6.0 ms, 1,000,000 12.5 ms, 2,000,000
|
|
192
|
+
# 17.5 ms, 3,000,000 25.0 ms: 2,000,000 rows.
|
|
193
|
+
# - PostgreSQL (the test container; bytes, `pg_table_size`): 10,000 rows (1.6 MB) 6.2 ms, 30,000 (3.7 MB) 18.4 ms, 100,000
|
|
194
|
+
# (12 MB) 35.7 ms, 300,000 (35 MB) 69.7 ms: 4 MB.
|
|
195
|
+
# - Spark local[2] (bytes, the optimized plan's `sizeInBytes`): 3.4 KB 43 ms, 340 KB 39 ms, 3.4 MB 44 ms:
|
|
196
|
+
# 4 MB. Rows a session holds in memory (`createDataFrame`) have no size estimate (Long.MAX_VALUE): no
|
|
197
|
+
# checksum, so such a frame needs a declared column. A Delta table is proven by its version, never summed.
|
|
198
|
+
# - pandas frames (rows), summed on the engine's own DuckDB, which types each object column from every row
|
|
199
|
+
# (`pandas_graph.frames_connection`) on every statement: 10,000 rows 8.8 ms, 30,000 30.6 ms, 100,000
|
|
200
|
+
# 84.1 ms: 20,000 rows.
|
|
201
|
+
SMALL = {"duckdb": 2_000_000, "postgresql": 4_000_000, "spark": 4_000_000, "frames": 20_000}
|
|
202
|
+
SIZE_UNIT = {"duckdb": "rows", "postgresql": "bytes", "spark": "bytes", "frames": "rows"}
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def size_of(engine: Any, s: Source) -> Optional[int]:
|
|
206
|
+
"""A table's size from the backend's metadata, read now, cheaply: DuckDB `duckdb_tables().estimated_size`
|
|
207
|
+
(rows), PostgreSQL `pg_table_size` (bytes of a table or materialized view), Spark the optimized plan's
|
|
208
|
+
`sizeInBytes` (an estimate), a pandas frame its rows; a query or a view the sum of the tables it reads
|
|
209
|
+
(`_query_tables`). None where none is kept, so no checksum is taken."""
|
|
210
|
+
kind = _kind(engine)
|
|
211
|
+
if kind == "frames":
|
|
212
|
+
frame = (getattr(_graph(engine), "dataframes", None) or {}).get(s.name)
|
|
213
|
+
return None if frame is None else len(frame)
|
|
214
|
+
if kind == "spark":
|
|
215
|
+
frame = _spark_frame(engine, s)
|
|
216
|
+
try:
|
|
217
|
+
size = frame._jdf.queryExecution().optimizedPlan().stats().sizeInBytes() # a Scala BigInt
|
|
218
|
+
return int(size) if isinstance(size, int) else int(size.toString())
|
|
219
|
+
except Exception: # noqa: BLE001 - no estimate kept
|
|
220
|
+
return None
|
|
221
|
+
store = _reader(engine)
|
|
222
|
+
if store.dialect not in ("duckdb", "postgresql"):
|
|
223
|
+
return None
|
|
224
|
+
if not s.query:
|
|
225
|
+
size = _table_size(store, s.sql)
|
|
226
|
+
if size is not None:
|
|
227
|
+
return size
|
|
228
|
+
# A query, or a view: the sizes of the tables it reads, added, where each is known.
|
|
229
|
+
tables = _query_tables(store, s.sql if s.query else f"(SELECT * FROM {s.sql})")
|
|
230
|
+
if not tables:
|
|
231
|
+
return None
|
|
232
|
+
sizes = [_table_size(store, t) for t in tables]
|
|
233
|
+
return None if any(x is None for x in sizes) else sum(sizes)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _table_size(store: Any, name: str) -> Optional[int]:
|
|
237
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
238
|
+
literal = get_dialect_instance(store.dialect).literal
|
|
239
|
+
if store.dialect == "duckdb":
|
|
240
|
+
schema, _, table = name.replace('"', "").rpartition(".")
|
|
241
|
+
where = f"table_name = {literal(table)}" + (f" AND schema_name = {literal(schema.split('.')[-1])}" if schema else "")
|
|
242
|
+
row = store.db.execute(f"SELECT max(estimated_size) FROM duckdb_tables() WHERE {where}").fetchone()
|
|
243
|
+
else:
|
|
244
|
+
row = store.db.execute(f"SELECT CASE WHEN c.relkind IN ('r', 'm') THEN pg_table_size(c.oid) END FROM pg_class "
|
|
245
|
+
f"AS c WHERE c.oid = to_regclass({literal(name)})").fetchone()
|
|
246
|
+
return None if row is None or row[0] is None else int(row[0])
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _query_tables(store: Any, sql: str) -> Optional[List[str]]:
|
|
250
|
+
"""The base tables a query (`sql` in parentheses) reads, as the database names them, views read through:
|
|
251
|
+
DuckDB's parser (`duckdb.get_table_names`, a view by its own text in `duckdb_views()`), PostgreSQL's plan
|
|
252
|
+
(`StoreConnection.plan_json`: each scan's schema and relation; no row is read), Databricks SQL's analyzed
|
|
253
|
+
plan (`analyzed_relations` of `EXPLAIN EXTENDED`). None where they cannot be told."""
|
|
254
|
+
if store is None:
|
|
255
|
+
return None
|
|
256
|
+
try:
|
|
257
|
+
if store.dialect == "duckdb":
|
|
258
|
+
return _duckdb_tables(store, f"SELECT * FROM {sql} AS _q", 0)
|
|
259
|
+
if store.dialect in ("databricks_sql", "databricks"):
|
|
260
|
+
rows = store.db.execute(f"EXPLAIN EXTENDED SELECT * FROM {sql} AS _q").fetchall()
|
|
261
|
+
return analyzed_relations(str(rows[0][0])) if rows else None
|
|
262
|
+
if store.dialect != "postgresql":
|
|
263
|
+
return None
|
|
264
|
+
plan = store.db.plan_json(f"SELECT * FROM {sql} AS _q")
|
|
265
|
+
except Exception: # noqa: BLE001 - a query not told apart has no size
|
|
266
|
+
return None
|
|
267
|
+
if plan is None:
|
|
268
|
+
return None
|
|
269
|
+
plan = json.loads(plan) if isinstance(plan, str) else plan
|
|
270
|
+
found, stack = set(), list(plan)
|
|
271
|
+
while stack:
|
|
272
|
+
node = stack.pop()
|
|
273
|
+
node = node.get("Plan", node)
|
|
274
|
+
if node.get("Node Type") in ("Function Scan", "ProjectSet", "Table Function Scan"):
|
|
275
|
+
return None # rows a function makes: no table's size counts them
|
|
276
|
+
if "Relation Name" in node:
|
|
277
|
+
found.add(f'"{node.get("Schema", "public")}"."{node["Relation Name"]}"')
|
|
278
|
+
stack.extend(node.get("Plans", ()))
|
|
279
|
+
return sorted(found) or None
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _duckdb_tables(store: Any, sql: str, depth: int) -> Optional[List[str]]:
|
|
283
|
+
import duckdb
|
|
284
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
285
|
+
literal = get_dialect_instance("duckdb").literal
|
|
286
|
+
out: set = set()
|
|
287
|
+
serialized = store.db.execute(f"SELECT json_serialize_sql({literal(sql)})").fetchone()[0]
|
|
288
|
+
if '"TABLE_FUNCTION"' in serialized or any(f'"function_name":"{f}"' in serialized
|
|
289
|
+
for f in ("unnest", "generate_series", "range")):
|
|
290
|
+
return None # rows a function makes: no table's size counts them
|
|
291
|
+
for name in duckdb.get_table_names(sql, qualified=True):
|
|
292
|
+
parts = name.replace('"', "").split(".")
|
|
293
|
+
bare, schema = parts[-1], parts[-2] if len(parts) > 1 else None
|
|
294
|
+
where = f"view_name = {literal(bare)}" + (f" AND schema_name = {literal(schema)}" if schema else
|
|
295
|
+
" AND schema_name = current_schema()")
|
|
296
|
+
row = store.db.execute(f"SELECT sql FROM duckdb_views() WHERE {where} AND NOT internal").fetchone()
|
|
297
|
+
if row is None:
|
|
298
|
+
out.add(name)
|
|
299
|
+
continue
|
|
300
|
+
if depth >= 8:
|
|
301
|
+
return None
|
|
302
|
+
text = (row[0].split(" AS ", 1)[1] if " AS " in row[0] else row[0]).strip().rstrip(";")
|
|
303
|
+
inner = _duckdb_tables(store, text, depth + 1)
|
|
304
|
+
if inner is None:
|
|
305
|
+
return None
|
|
306
|
+
out.update(inner)
|
|
307
|
+
return sorted(out)
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def small(engine: Any, s: Source) -> bool:
|
|
311
|
+
"""Whether a plan takes `s`'s checksum: its size is known and at most its backend's threshold."""
|
|
312
|
+
kind = _kind(engine)
|
|
313
|
+
backend = kind if kind in ("spark", "frames") else _reader(engine).dialect
|
|
314
|
+
limit = SMALL.get(backend)
|
|
315
|
+
if limit is None:
|
|
316
|
+
return False
|
|
317
|
+
size = size_of(engine, s)
|
|
318
|
+
return size is not None and size <= limit
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
# -- the SQL of a token (DuckDB, PostgreSQL, Databricks SQL) ---------------------------------------------------
|
|
322
|
+
|
|
323
|
+
def _text(d: Any, expr: str) -> str:
|
|
324
|
+
return f"CAST({expr} AS {d.type_name('STRING')})"
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def column_sql(d: Any, source: Source) -> str:
|
|
328
|
+
"""One row, one text: the declared column's maximum and the row count (`column:<max>/<count>`)."""
|
|
329
|
+
col = d.quote(source.column)
|
|
330
|
+
top = f"CASE WHEN max(_t.{col}) IS NULL THEN '' ELSE {_text(d, f'max(_t.{col})')} END"
|
|
331
|
+
return f"SELECT 'column:' || {top} || '/' || {_text(d, 'count(*)')} AS a FROM {source.sql} AS _t"
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def checksum_sql(d: Any, dialect: str, source: Source) -> str:
|
|
335
|
+
"""One row, one text: the row count and two sums of a 64-bit hash of each whole row (order-free, so a
|
|
336
|
+
table's row order does not matter; a changed value changes its row's hash)."""
|
|
337
|
+
if dialect == "postgresql":
|
|
338
|
+
one, two = "hashtextextended(_t::text, 0)", "hashtextextended(_t::text, 1)"
|
|
339
|
+
elif dialect == "duckdb":
|
|
340
|
+
one, two = "hash(_t)", "hash(_t, 1)"
|
|
341
|
+
else: # Spark SQL, Databricks SQL
|
|
342
|
+
one, two = "xxhash64(*)", "hash(*)"
|
|
343
|
+
num = "DECIMAL(38, 0)"
|
|
344
|
+
return (f"SELECT 'checksum:' || {_text(d, 'count(*)')} || ':' || "
|
|
345
|
+
f"{_text(d, f'coalesce(sum(CAST({one} AS {num})), 0)')} || ':' || "
|
|
346
|
+
f"{_text(d, f'coalesce(sum(CAST({two} AS {num})), 0)')} AS a FROM {source.sql} AS _t")
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
# -- reading the tokens ---------------------------------------------------------------------------------------
|
|
350
|
+
|
|
351
|
+
@dataclass(frozen=True)
|
|
352
|
+
class Check:
|
|
353
|
+
"""How one source was read for a plan: `how` ('declared version column', 'Delta version', 'checksum', or
|
|
354
|
+
why none), and the milliseconds it took."""
|
|
355
|
+
how: str
|
|
356
|
+
ms: float
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def read_now(engine: Any, srcs: Tuple[Source, ...], checks: Optional[Dict[str, Check]] = None) \
|
|
360
|
+
-> Dict[str, Optional[str]]:
|
|
361
|
+
"""Each source's token now: by its declared column, else its Delta version, else, for a small table
|
|
362
|
+
(`small`), its checksum; None for a source none of them proves. `checks` gets how each was read, timed."""
|
|
363
|
+
import time
|
|
364
|
+
out: Dict[str, Optional[str]] = {}
|
|
365
|
+
for s in srcs:
|
|
366
|
+
started = time.perf_counter()
|
|
367
|
+
token = _proof(engine, s)
|
|
368
|
+
if token is None and small(engine, s):
|
|
369
|
+
token = _checksum(engine, s)
|
|
370
|
+
out[s.name] = token
|
|
371
|
+
if checks is not None:
|
|
372
|
+
how = proof_name(token) if token is not None else "no version, too big for a checksum per query"
|
|
373
|
+
checks[s.name] = Check(how, (time.perf_counter() - started) * 1000)
|
|
374
|
+
return out
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def read_run(engine: Any, srcs: Tuple[Source, ...]) -> Dict[str, List[str]]:
|
|
378
|
+
"""What a run records of each source: its token (`read_now`), and for a small table its checksum too, so
|
|
379
|
+
an engine that declares no column over the same store compares as well. A Delta table: its version alone."""
|
|
380
|
+
now = read_now(engine, srcs)
|
|
381
|
+
out: Dict[str, List[str]] = {}
|
|
382
|
+
for s in srcs:
|
|
383
|
+
held = [now[s.name]] if now[s.name] is not None else []
|
|
384
|
+
if (s.column is not None or s.reads) and small(engine, s):
|
|
385
|
+
held.append(_checksum(engine, s))
|
|
386
|
+
out[s.name] = held
|
|
387
|
+
return out
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _proof(engine: Any, s: Source) -> Optional[str]:
|
|
391
|
+
"""A declared column's token, else a Delta version; None for neither."""
|
|
392
|
+
kind = _kind(engine)
|
|
393
|
+
if kind == "spark":
|
|
394
|
+
from pyspark.sql import functions as F
|
|
395
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
396
|
+
frame = _spark_frame(engine, s)
|
|
397
|
+
if s.column is not None:
|
|
398
|
+
col = F.col(get_dialect_instance("spark").quote(s.column))
|
|
399
|
+
row = frame.agg(F.max(col).alias("m"), F.count(F.lit(1)).alias("n")).first()
|
|
400
|
+
return f"column:{'' if row['m'] is None else row['m']}/{row['n']}"
|
|
401
|
+
return _spark_delta(frame)
|
|
402
|
+
if kind == "sql":
|
|
403
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
404
|
+
dialect = _graph(engine).sql_dialect()
|
|
405
|
+
d = get_dialect_instance(dialect)
|
|
406
|
+
if s.column is not None:
|
|
407
|
+
return _reader(engine).db.execute(column_sql(d, s)).fetchone()[0]
|
|
408
|
+
if s.reads: # a query: the declared columns of the tables it reads
|
|
409
|
+
return "column:" + ";".join(
|
|
410
|
+
_reader(engine).db.execute(column_sql(d, Source(t, t, c))).fetchone()[0][len("column:"):]
|
|
411
|
+
for t, c in s.reads)
|
|
412
|
+
if dialect in ("databricks_sql", "databricks"):
|
|
413
|
+
return _databricks_query_version(engine, s) if s.query else _databricks_version(engine, s)
|
|
414
|
+
return None
|
|
415
|
+
frame = (getattr(_graph(engine), "dataframes", None) or {}).get(s.name)
|
|
416
|
+
if frame is None or s.column is None:
|
|
417
|
+
return None
|
|
418
|
+
top = frame[s.column].max() if len(frame) else None
|
|
419
|
+
return f"column:{'' if top is None else top}/{len(frame)}"
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def _checksum(engine: Any, s: Source) -> Optional[str]:
|
|
423
|
+
kind = _kind(engine)
|
|
424
|
+
if kind == "spark":
|
|
425
|
+
return _spark_checksum(_spark_frame(engine, s))
|
|
426
|
+
if kind == "sql":
|
|
427
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
428
|
+
dialect = _graph(engine).sql_dialect()
|
|
429
|
+
return _reader(engine).db.execute(checksum_sql(get_dialect_instance(dialect), dialect, s)).fetchone()[0]
|
|
430
|
+
# A pandas frame: its view on the engine's own DuckDB, summed there as a DuckDB table is.
|
|
431
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
432
|
+
graph = _graph(engine)
|
|
433
|
+
if (getattr(graph, "dataframes", None) or {}).get(s.name) is None or graph.sql_dialect() != "duckdb":
|
|
434
|
+
return None
|
|
435
|
+
d = get_dialect_instance("duckdb")
|
|
436
|
+
quoted = ".".join(d.quote(p) for p in s.name.split(".")) # its view, named as `register_frame` names it
|
|
437
|
+
return graph._run_local_sql(checksum_sql(d, "duckdb", Source(s.name, quoted, None))).fetchone()[0]
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _databricks_version(engine: Any, s: Source) -> Optional[str]:
|
|
441
|
+
"""A Delta table's latest version on Databricks SQL, read now (never a cached one); None where the table
|
|
442
|
+
has no Delta history."""
|
|
443
|
+
try:
|
|
444
|
+
rows = _graph(engine)._execute_sql(f"DESCRIBE HISTORY {s.sql} LIMIT 1")
|
|
445
|
+
except Exception: # noqa: BLE001 - no history: not a Delta table
|
|
446
|
+
return None
|
|
447
|
+
if not rows or rows[0].get("version") is None:
|
|
448
|
+
return None
|
|
449
|
+
return f"delta:{s.sql}@{rows[0]['version']}"
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _databricks_query_version(engine: Any, s: Source) -> Optional[str]:
|
|
453
|
+
"""An rr:sqlQuery on Databricks SQL: the Delta versions of the tables it reads, each read now. The tables
|
|
454
|
+
are its analyzed plan's leaves (`EXPLAIN EXTENDED`, which reads no row), views read through; None where a
|
|
455
|
+
leaf is not a table's `Relation` (rows of the query's own, a range, a function) or a table has no Delta
|
|
456
|
+
history: not proven."""
|
|
457
|
+
try:
|
|
458
|
+
rows = _graph(engine)._execute_sql(f"EXPLAIN EXTENDED SELECT * FROM {s.sql} AS _q")
|
|
459
|
+
except Exception: # noqa: BLE001 - no plan: not proven
|
|
460
|
+
return None
|
|
461
|
+
text = str(next(iter(rows[0].values()))) if rows else ""
|
|
462
|
+
tables = analyzed_relations(text)
|
|
463
|
+
if not tables:
|
|
464
|
+
return None
|
|
465
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
466
|
+
quote = get_dialect_instance("spark").quote
|
|
467
|
+
found = []
|
|
468
|
+
for name in tables:
|
|
469
|
+
token = _databricks_version(engine, Source(name, ".".join(quote(p) for p in name.split(".")), None))
|
|
470
|
+
if token is None:
|
|
471
|
+
return None
|
|
472
|
+
found.append(token[len("delta:"):])
|
|
473
|
+
return "delta:" + ";".join(sorted(found))
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def analyzed_relations(plan: str) -> Optional[List[str]]:
|
|
477
|
+
"""The tables a Spark SQL plan's text reads (its '== Analyzed Logical Plan ==' section): the name of each
|
|
478
|
+
leaf `Relation <name>[...]`; None where a leaf is anything else, or there is no such section."""
|
|
479
|
+
import re
|
|
480
|
+
if "== Analyzed Logical Plan ==" not in plan:
|
|
481
|
+
return None
|
|
482
|
+
section = plan.split("== Analyzed Logical Plan ==", 1)[1].split("\n== ", 1)[0]
|
|
483
|
+
lines = [x for x in section.splitlines() if x.strip()][1:] # the first line is the output's schema
|
|
484
|
+
nodes = []
|
|
485
|
+
for line in lines:
|
|
486
|
+
found = re.search(r"[A-Za-z'`]", line)
|
|
487
|
+
if found is not None:
|
|
488
|
+
nodes.append((found.start(), line[found.start():]))
|
|
489
|
+
out = set()
|
|
490
|
+
for i, (depth, text) in enumerate(nodes):
|
|
491
|
+
leaf = i + 1 == len(nodes) or nodes[i + 1][0] <= depth
|
|
492
|
+
if not leaf:
|
|
493
|
+
continue
|
|
494
|
+
match = re.match(r"Relation ([^\[\s]+)\[", text)
|
|
495
|
+
if match is None:
|
|
496
|
+
return None
|
|
497
|
+
out.add(match.group(1).replace("`", ""))
|
|
498
|
+
return sorted(out) or None
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _spark_frame(engine: Any, s: Source) -> Any:
|
|
502
|
+
frame = _graph(engine).dataframes.get(s.name)
|
|
503
|
+
if frame is None:
|
|
504
|
+
frame = engine._spark.sql(f"SELECT * FROM {s.sql}") if s.sql.startswith("(") else engine._spark.table(s.sql)
|
|
505
|
+
return frame
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def _spark_checksum(frame: Any) -> str:
|
|
509
|
+
from pyspark.sql import functions as F
|
|
510
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
511
|
+
quote = get_dialect_instance("spark").quote
|
|
512
|
+
cols = [F.col(quote(c)) for c in frame.columns]
|
|
513
|
+
num = "decimal(38,0)"
|
|
514
|
+
row = frame.agg(F.count(F.lit(1)).alias("n"),
|
|
515
|
+
F.coalesce(F.sum(F.xxhash64(*cols).cast(num)), F.lit(0).cast(num)).alias("a"),
|
|
516
|
+
F.coalesce(F.sum(F.hash(*cols).cast(num)), F.lit(0).cast(num)).alias("b")).first()
|
|
517
|
+
return f"checksum:{row['n']}:{row['a']}:{row['b']}"
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def _spark_delta(frame: Any) -> Optional[str]:
|
|
521
|
+
"""The Delta versions of the tables a Spark frame reads, each read now (DESCRIBE HISTORY); None where a
|
|
522
|
+
leaf of its plan is not a Delta table (rows held in memory included: their frame is not named in a run's
|
|
523
|
+
record, so nothing proves it the same)."""
|
|
524
|
+
from prograph.plan.ir.key_checks import _delta_name, _spark_leaves
|
|
525
|
+
try:
|
|
526
|
+
leaves = _spark_leaves(frame)
|
|
527
|
+
except Exception: # noqa: BLE001 - a plan that cannot be read proves nothing
|
|
528
|
+
return None
|
|
529
|
+
if not leaves:
|
|
530
|
+
return None
|
|
531
|
+
found = []
|
|
532
|
+
for leaf in leaves:
|
|
533
|
+
name = _delta_name(leaf)
|
|
534
|
+
if name is None:
|
|
535
|
+
return None
|
|
536
|
+
row = frame.sparkSession.sql(f"DESCRIBE HISTORY {name} LIMIT 1").select("version").first()
|
|
537
|
+
if row is None:
|
|
538
|
+
return None
|
|
539
|
+
found.append(f"{name}@{row[0]}")
|
|
540
|
+
return "delta:" + ";".join(sorted(found))
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
# -- the record, and the comparison --------------------------------------------------------------------------
|
|
544
|
+
|
|
545
|
+
def recorded(text: Optional[str]) -> Dict[str, List[str]]:
|
|
546
|
+
"""The source tokens a run's record holds (`owl_rl.RUN_SOURCES`), {} for none."""
|
|
547
|
+
if not text:
|
|
548
|
+
return {}
|
|
549
|
+
try:
|
|
550
|
+
held = json.loads(text)
|
|
551
|
+
except ValueError:
|
|
552
|
+
return {}
|
|
553
|
+
if not isinstance(held, dict):
|
|
554
|
+
return {}
|
|
555
|
+
return {k: (v if isinstance(v, list) else [v]) for k, v in held.items()}
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
def proof_name(token: Optional[str]) -> str:
|
|
559
|
+
"""What proved a token's source unchanged, in words."""
|
|
560
|
+
kind = (token or "").split(":", 1)[0]
|
|
561
|
+
return {"column": "declared version column", "delta": "Delta version", "checksum": "checksum"}.get(kind, kind)
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def as_record(tokens: Dict[str, List[str]]) -> str:
|
|
565
|
+
return json.dumps(tokens, sort_keys=True)
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
@dataclass(frozen=True)
|
|
569
|
+
class Verdict:
|
|
570
|
+
"""A comparison of the sources now with a run's record: `changed` the sources (or 'the store') not as the
|
|
571
|
+
run read them, `unproven` those no token proves per plan (`read_now`)."""
|
|
572
|
+
changed: Tuple[str, ...] = ()
|
|
573
|
+
unproven: Tuple[str, ...] = ()
|
|
574
|
+
|
|
575
|
+
@property
|
|
576
|
+
def current(self) -> bool:
|
|
577
|
+
return not self.changed and not self.unproven
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def compare(record: Dict[str, List[str]], now: Dict[str, Optional[str]], store_moved: bool) -> Verdict:
|
|
581
|
+
changed = [n for n, t in now.items() if t is not None and t not in record.get(n, ())]
|
|
582
|
+
if store_moved:
|
|
583
|
+
changed.append("the owned store")
|
|
584
|
+
unproven = [n for n, t in now.items() if t is None]
|
|
585
|
+
return Verdict(tuple(changed), tuple(unproven))
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
def refuse_unproven(srcs: Tuple[Source, ...]) -> None:
|
|
589
|
+
raise StoredFactsNotCurrentError(unproven_reason(srcs))
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def unproven_reason(srcs: Tuple[Source, ...]) -> str:
|
|
593
|
+
"""Why the stored facts cannot be proven current, naming each source as the user wrote it, and the ways out."""
|
|
594
|
+
listed = ", ".join(s.label or s.name for s in srcs)
|
|
595
|
+
one = len(srcs) == 1
|
|
596
|
+
tables = next((s for s in srcs if not s.query), None)
|
|
597
|
+
query = next((s for s in srcs if s.query), None)
|
|
598
|
+
ways = []
|
|
599
|
+
if tables is not None:
|
|
600
|
+
ways.append(f"declare a column your writes keep increasing (ProGraph(..., version_columns="
|
|
601
|
+
f"{{'{tables.name}': '<column>'}}))")
|
|
602
|
+
if query is not None:
|
|
603
|
+
ways.append("for an rr:sqlQuery, declare one on every table it reads (version_columns= takes those "
|
|
604
|
+
"tables' names)")
|
|
605
|
+
ways.append("or read the facts of the last run with freshness='last_run' (call materialize_entailments() "
|
|
606
|
+
"after your tables change)")
|
|
607
|
+
return (f"the stored OWL RL facts cannot be proven current for this query: {listed} {'has' if one else 'have'} "
|
|
608
|
+
f"no version a query can read (no declared version column, not a Delta table) and "
|
|
609
|
+
f"{'is' if one else 'are'} too big, or of unknown size, for a checksum per query. To fix: "
|
|
610
|
+
+ "; ".join(ways))
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def refresh(engine: Any, changed: Tuple[str, ...]) -> None:
|
|
614
|
+
"""The stored facts made current: the closure computed again and its difference written
|
|
615
|
+
(`materialize_entailments`). A store that cannot be written refuses by name."""
|
|
616
|
+
from prograph.utils.exceptions import StoreReadOnlyError
|
|
617
|
+
try:
|
|
618
|
+
if engine._store.db.read_only():
|
|
619
|
+
raise StoreReadOnlyError("the connection is read-only")
|
|
620
|
+
engine._materialize_run()
|
|
621
|
+
except StoreReadOnlyError as exc:
|
|
622
|
+
raise StoredFactsNotCurrentError(
|
|
623
|
+
f"the stored OWL RL facts are not current ({', '.join(changed)} changed since the last run) and the "
|
|
624
|
+
f"owned store cannot be written to refresh them ({exc}). Run materialize_entailments() on a writable "
|
|
625
|
+
f"connection, or read the facts of the last run with freshness='last_run'") from exc
|