prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/store.py
ADDED
|
@@ -0,0 +1,1737 @@
|
|
|
1
|
+
"""The owned store (one-plan 2.10): the statements ProGraph writes, in tables it owns, in the mapped data's
|
|
2
|
+
engine. This module holds the table, its effective view and the write step; the generated mapping that lets
|
|
3
|
+
queries read it, and Cypher's writes, are built on it.
|
|
4
|
+
|
|
5
|
+
- **One append-only statement table.** Each row is one statement: `graph`, `subject`, `predicate`, a tagged
|
|
6
|
+
object (`o_kind` iri or literal, `o_text`, `o_datatype`, `o_lang`), `edge_id` (keeps parallel relationships
|
|
7
|
+
and a relationship's properties apart), `op` (assert or retract), and provenance (`writer`, `step`, `written_at`).
|
|
8
|
+
Rows are never rewritten in place.
|
|
9
|
+
- **One seq per write step.** A step reads `max(seq)` and writes `max + 1` on all its rows, `pos` its place
|
|
10
|
+
in the step. For each statement the op of the highest (seq, pos) wins (`effective_sql`).
|
|
11
|
+
- **Idempotent steps.** A step carries a key (the caller's, else the run id); a key already applied appends
|
|
12
|
+
nothing, and a created element's IRI is derived from the key and its position (`mint`), so a retry mints
|
|
13
|
+
nothing twice.
|
|
14
|
+
- **Per-store namespace.** Minted IRIs are `{base}_/{store}/n/{digest}`, `store` a UUID kept with the store.
|
|
15
|
+
|
|
16
|
+
On DuckDB and PostgreSQL (one database transaction per step, through the store's connection seam,
|
|
17
|
+
`store_connection`; on DuckDB the statements table gets one INSERT per step, `OWN_ROWS`). On Delta (Spark, Databricks) a statement commits alone, so a step is optimistic: its
|
|
18
|
+
phases append under an `attempt`, a marker row (`op = 'commit'`, `seq` = the highest marked seq read at the
|
|
19
|
+
step's start + 1) is appended last and only if no other marker landed since, and the effective view shows
|
|
20
|
+
only marked attempts, each statement's seq taken from its attempt's marker. A step that lost the race redoes
|
|
21
|
+
whole (`Store.REDO_LIMIT`), so seq order is commit order and no update is lost.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import re
|
|
27
|
+
import uuid
|
|
28
|
+
from dataclasses import dataclass
|
|
29
|
+
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
|
|
30
|
+
|
|
31
|
+
from prograph.rules import home, module_home
|
|
32
|
+
|
|
33
|
+
module_home("owned-store", "The owned store: ProGraph's statement table, its effective view and its write step",
|
|
34
|
+
terms=("owned store", "statement table", "effective view", "write step", "minted IRI",
|
|
35
|
+
"idempotency key", "retract", "commit marker", "optimistic step", "attempt", "redo"))
|
|
36
|
+
|
|
37
|
+
ASSERT, RETRACT = "assert", "retract"
|
|
38
|
+
COMMIT = "commit" # a marker's op (Delta): the step's attempt is committed, at this row's seq
|
|
39
|
+
# The seq an attempt's own rows read inside their step (no marker yet): after every marked seq.
|
|
40
|
+
OWN_SEQ = 10 ** 9
|
|
41
|
+
# The Delta temporary view holding the running step's attempt (one row; empty outside a step). A view, not a
|
|
42
|
+
# session variable: the engine runs each statement in a clone of the session (`ansi_session`), which carries
|
|
43
|
+
# temporary views and not variables.
|
|
44
|
+
OWN_ATTEMPT = "prograph_own_attempt"
|
|
45
|
+
# DuckDB: the prefix of the temporary view of a running step's own rows (`Store.own_view`) and of its phases'
|
|
46
|
+
# tables. DuckDB 1.4-1.5 loses the rows of an INSERT of a full row group (122,880 rows) after an INSERT of none
|
|
47
|
+
# into one table in one transaction, and 2.0 reads such a table wrong in the transaction after its second
|
|
48
|
+
# INSERT (duckdb/duckdb#28839), so a step writes the statements table with one INSERT (`Store.write_select`).
|
|
49
|
+
OWN_ROWS = "_prograph_own"
|
|
50
|
+
# Delta: the `pos` a phase starts at is its index times this (a step's phases are Delta commits of their own,
|
|
51
|
+
# and an INSERT says how many rows it wrote nowhere), so `pos` keeps the phases' order and a statement's
|
|
52
|
+
# rows within one.
|
|
53
|
+
PHASE_STRIDE = 1 << 32
|
|
54
|
+
IRI, LITERAL = "iri", "literal"
|
|
55
|
+
# The writers' graphs (one-plan 2.10: one graph per writer).
|
|
56
|
+
CYPHER_GRAPH = "urn:prograph:store:cypher"
|
|
57
|
+
RL_GRAPH = "urn:prograph:store:rl"
|
|
58
|
+
PROACT_GRAPH = "urn:prograph:store:proact"
|
|
59
|
+
# The store's own predicates (`INTERNAL` their prefix): what the property-graph view needs and RDF has no
|
|
60
|
+
# word for. They are statements like any other but stay out of the generated mapping, so SPARQL never sees them.
|
|
61
|
+
INTERNAL = "urn:prograph:store:"
|
|
62
|
+
ELEMENT = INTERNAL + "element" # (a created node, ELEMENT, "node"): the node exists, labelled or not
|
|
63
|
+
REL_TYPE = INTERNAL + "type" # (a relationship's edge id, REL_TYPE, its type's name as Cypher wrote it)
|
|
64
|
+
RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
|
|
65
|
+
# A list property's datatype: `LIST_DATATYPE` + its elements' XSD local name (`list_datatype`), the text a
|
|
66
|
+
# JSON array of the elements. One datatype per element type, so a key's values keep one type (a column's).
|
|
67
|
+
LIST_DATATYPE = INTERNAL + "list:"
|
|
68
|
+
# A temporal property's datatype (rule cypher-writes): a date's xsd:date, its text the ISO 8601 date; a local
|
|
69
|
+
# time's, time's, local date-time's, date-time's and duration's `TEMPORAL_DATATYPE` + its kind, its text a JSON
|
|
70
|
+
# object of the integer parts a source struct of them has (`cypher_temporal.stored_text`), exactly, in one form on every engine.
|
|
71
|
+
TEMPORAL_DATATYPE = INTERNAL + "temporal:"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def list_datatype(element: str) -> str:
|
|
75
|
+
"""The datatype of a list property whose elements are of datatype `element`: an XSD datatype, or a
|
|
76
|
+
temporal's (`TEMPORAL_DATATYPE`)."""
|
|
77
|
+
from prograph.schema.xsd import XSD
|
|
78
|
+
if element.startswith(TEMPORAL_DATATYPE):
|
|
79
|
+
return LIST_DATATYPE + element[len(INTERNAL):]
|
|
80
|
+
return LIST_DATATYPE + element[len(XSD):]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def list_element(datatype: Optional[str]) -> Optional[str]:
|
|
84
|
+
"""A list property's elements' datatype (`list_datatype`'s `element`); None where `datatype` is no list's."""
|
|
85
|
+
if datatype is None or not datatype.startswith(LIST_DATATYPE):
|
|
86
|
+
return None
|
|
87
|
+
from prograph.schema.xsd import XSD
|
|
88
|
+
rest = datatype[len(LIST_DATATYPE):]
|
|
89
|
+
return INTERNAL + rest if (INTERNAL + rest).startswith(TEMPORAL_DATATYPE) else XSD + rest
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def temporal_datatype(cypher_datatype: str) -> str:
|
|
93
|
+
"""The datatype the store holds a Cypher temporal of slot datatype `cypher_datatype` in (`TEMPORAL_DATATYPE`;
|
|
94
|
+
a date xsd:date)."""
|
|
95
|
+
from prograph.schema.xsd import DATE
|
|
96
|
+
kind = cypher_datatype.rsplit(":", 1)[-1]
|
|
97
|
+
return DATE if kind == "date" else TEMPORAL_DATATYPE + kind
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# The columns of a write step's statements (`write_select`), in the table's order.
|
|
101
|
+
STATEMENT_COLUMNS = ("graph", "subject", "predicate", "o_kind", "o_text", "o_datatype", "o_lang", "edge_id")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclass(frozen=True)
|
|
105
|
+
class Statement:
|
|
106
|
+
"""One statement to write: the object an IRI (`kind` iri) or a literal's text with its datatype or
|
|
107
|
+
language; `op` assert or retract."""
|
|
108
|
+
graph: str
|
|
109
|
+
subject: str
|
|
110
|
+
predicate: str
|
|
111
|
+
object: str
|
|
112
|
+
kind: str = LITERAL
|
|
113
|
+
datatype: Optional[str] = None
|
|
114
|
+
lang: Optional[str] = None
|
|
115
|
+
edge_id: Optional[str] = None
|
|
116
|
+
op: str = ASSERT
|
|
117
|
+
|
|
118
|
+
def __post_init__(self) -> None:
|
|
119
|
+
if self.kind not in (IRI, LITERAL):
|
|
120
|
+
raise ValueError(f"an object kind is {IRI!r} or {LITERAL!r}, not {self.kind!r}")
|
|
121
|
+
if self.op not in (ASSERT, RETRACT):
|
|
122
|
+
raise ValueError(f"an op is {ASSERT!r} or {RETRACT!r}, not {self.op!r}")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class StepLostRace(Exception):
|
|
126
|
+
"""A write step on Delta lost its race (another writer's marker landed after it read the store) and the
|
|
127
|
+
caller plans again (`Store.write_select(replan=True)`): the SQL it was given was lowered against the store as
|
|
128
|
+
it was (labels, types), which the other writer may have changed. Caught by the query's run, which plans and
|
|
129
|
+
writes again; never reaches a user."""
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class Store:
|
|
133
|
+
"""The owned store on a DuckDB, an ADBC PostgreSQL, a Spark session or a Databricks SQL connection
|
|
134
|
+
(`store_connection`; the engine's own): `table` the statements, `table`_steps the applied step keys, `table`_meta the store's UUID."""
|
|
135
|
+
|
|
136
|
+
def __init__(self, connection: Any, table: str = "prograph_statements",
|
|
137
|
+
base_iri: str = "http://example.org/data/") -> None:
|
|
138
|
+
# `connection` the caller's (the engine's own); `db` the store's side of it (`store_connection`).
|
|
139
|
+
self.connection, self.table, self.base_iri = connection, table, base_iri
|
|
140
|
+
self.db = store_connection(connection)
|
|
141
|
+
self.dialect = self.db.dialect
|
|
142
|
+
self._ensured: set = set() # the `_answers` and `_digests` tables made (`_ensure`)
|
|
143
|
+
self._schema: Any = None # the schema the store is read beside (`attach`)
|
|
144
|
+
self._create()
|
|
145
|
+
self.store_id = self._store_id()
|
|
146
|
+
# The generated maps' version: it changes when a write step changes what the store holds
|
|
147
|
+
# (`Profile`: a predicate, object kind, datatype or language, a class, a type first held, or last
|
|
148
|
+
# retracted).
|
|
149
|
+
self._version, self._maps = 0, None
|
|
150
|
+
# A premise view (`premises`): the generated maps read another relation for the length of a block.
|
|
151
|
+
self._premises: Optional[Tuple[str, Tuple[tuple, ...], Tuple[tuple, ...], int]] = None
|
|
152
|
+
self._derived_maps: Any = None # (version, maps): `derived_maps`
|
|
153
|
+
self._joined: Optional[str] = None # the key of a step that joined a transaction the caller holds
|
|
154
|
+
# DuckDB (`StoreConnection.one_insert`): the temporary view of the running step's own rows, which its
|
|
155
|
+
# phases write to tables of their own and the statements table gets in one INSERT at the step's end;
|
|
156
|
+
# `_own` while a query that writes is planned and run (`writing`, `read_own`), so its reads after a
|
|
157
|
+
# write see the step's rows through it.
|
|
158
|
+
import hashlib
|
|
159
|
+
self.own_view = f"{OWN_ROWS}_{hashlib.sha1(table.encode('utf-8')).hexdigest()[:12]}"
|
|
160
|
+
self._writing, self._own = False, False
|
|
161
|
+
self._profile = self._read_profile(f"({self.effective_sql()})")
|
|
162
|
+
self._seen_seq = self._max_seq()
|
|
163
|
+
# The alias table's current generation (owl:sameAs, `plan.ir.identity`): a plan reads it; a new one
|
|
164
|
+
# changes the store's version, so every plan is made again.
|
|
165
|
+
from prograph.plan.ir.identity import read_state
|
|
166
|
+
self._alias = read_state(self)
|
|
167
|
+
|
|
168
|
+
# How many times a step on Delta reads the store again and redoes itself after losing its race to another
|
|
169
|
+
# writer's marker, before `StoreWriteConflictError`.
|
|
170
|
+
REDO_LIMIT = 20
|
|
171
|
+
|
|
172
|
+
def max_seq_sql(self) -> str:
|
|
173
|
+
"""SQL of the highest committed seq (a marker's, on Delta; every row's elsewhere), one row `m`."""
|
|
174
|
+
where = f" WHERE op = '{COMMIT}'" if self.db.optimistic else ""
|
|
175
|
+
return f"SELECT coalesce(max(seq), 0) AS m FROM {self.table}{where}"
|
|
176
|
+
|
|
177
|
+
def _max_seq(self) -> int:
|
|
178
|
+
return self.db.execute(self.max_seq_sql()).fetchone()[0]
|
|
179
|
+
|
|
180
|
+
def _visible(self) -> str:
|
|
181
|
+
"""The statements a reader sees, as a relation of the table's columns (seq as committed): the table
|
|
182
|
+
itself, or on Delta the rows of marked attempts (one snapshot of the table: a marker is the last
|
|
183
|
+
row of its step, so a snapshot holding it holds the whole step) and the running step's own."""
|
|
184
|
+
t = self.table
|
|
185
|
+
if not self.db.optimistic:
|
|
186
|
+
# DuckDB, in a query that writes: the statements and the step's own rows (`read_own`).
|
|
187
|
+
return f"(SELECT * FROM {t} UNION ALL SELECT * FROM {self.own_view})" if self._own else t
|
|
188
|
+
cols = ", ".join(f"p.{c}" for c in ("pos", "graph", "subject", "predicate", "o_kind", "o_text",
|
|
189
|
+
"o_datatype", "o_lang", "edge_id", "op", "writer", "step",
|
|
190
|
+
"written_at"))
|
|
191
|
+
return (f"(WITH _pg_s AS (SELECT * FROM {t}) SELECT coalesce(m.seq, {OWN_SEQ}) AS seq, p.attempt, {cols} "
|
|
192
|
+
f"FROM _pg_s AS p LEFT JOIN (SELECT attempt, seq FROM _pg_s WHERE op = '{COMMIT}') AS m "
|
|
193
|
+
f"ON p.attempt = m.attempt WHERE p.op <> '{COMMIT}' AND (m.attempt IS NOT NULL "
|
|
194
|
+
f"OR p.attempt IN (SELECT attempt FROM {OWN_ATTEMPT})))")
|
|
195
|
+
|
|
196
|
+
def attach(self, schema: Any) -> None:
|
|
197
|
+
"""Read this store beside `schema`'s own maps: its generated maps join the schema's source maps
|
|
198
|
+
(`prograph.r2rml.schema.with_included`), after them. The schema's engine must run on this
|
|
199
|
+
store's connection."""
|
|
200
|
+
schema.__dict__["owned_store"] = self
|
|
201
|
+
self._schema = schema
|
|
202
|
+
|
|
203
|
+
@property
|
|
204
|
+
def merge_keys(self) -> Dict[str, Tuple[str, ...]]:
|
|
205
|
+
"""The key of each label the store holds that Cypher's MERGE may create (its properties, whose values
|
|
206
|
+
give the node's IRI), as the schema it is read beside declares them (YAML `store: labels: <L>:
|
|
207
|
+
{key: [...]}`, `GraphSchema.store_keys`): every writer reads the one mapping, so two engines merging
|
|
208
|
+
one key meet."""
|
|
209
|
+
return dict(getattr(self._schema, "store_keys", None) or {})
|
|
210
|
+
|
|
211
|
+
def sync(self) -> None:
|
|
212
|
+
"""Read what the store holds again where another writer (another `Store` over the table, another
|
|
213
|
+
engine) appended a step since this one last read it: one read of the highest seq."""
|
|
214
|
+
self.saw(self._max_seq())
|
|
215
|
+
|
|
216
|
+
def saw(self, seq: int) -> None:
|
|
217
|
+
"""The highest committed seq as just read (`sync`, `dataset.store_now`): read what the store holds
|
|
218
|
+
again where it moved."""
|
|
219
|
+
if seq != self._seen_seq:
|
|
220
|
+
self._refresh()
|
|
221
|
+
|
|
222
|
+
def source_maps(self) -> Any:
|
|
223
|
+
"""The generated mapping's source maps of its asserted statements (`r2rml`, compiled), None while the
|
|
224
|
+
store holds none; compiled once per version. In a `premises` block, once per premise view: the
|
|
225
|
+
premises' asserted statements and the run's own derivations."""
|
|
226
|
+
from prograph.r2rml.compile import compile_r2rml
|
|
227
|
+
from prograph.r2rml.parser import R2RMLParser
|
|
228
|
+
if self._premises is not None:
|
|
229
|
+
source, kinds, derived_kinds, _ = self._premises
|
|
230
|
+
if not kinds and not derived_kinds:
|
|
231
|
+
return None
|
|
232
|
+
return compile_r2rml(R2RMLParser().parse(self.r2rml(kinds, derived_kinds, source, stored=True)),
|
|
233
|
+
self.base_iri)
|
|
234
|
+
if not self._kinds:
|
|
235
|
+
return None
|
|
236
|
+
if self._maps is None or self._maps[0] != self._version:
|
|
237
|
+
self._maps = (self._version, compile_r2rml(R2RMLParser().parse(self.r2rml()), self.base_iri))
|
|
238
|
+
return self._maps[1]
|
|
239
|
+
|
|
240
|
+
def derived_maps(self) -> Any:
|
|
241
|
+
"""The generated maps of the stored derived facts, in the default graph alone (`dataset.store_mapping`;
|
|
242
|
+
which reader sees them is `dataset.maps_seen`'s to say), None where the store holds none, or in a
|
|
243
|
+
`premises` block (the run's own are in `source_maps`)."""
|
|
244
|
+
from prograph.r2rml.compile import compile_r2rml
|
|
245
|
+
from prograph.r2rml.parser import R2RMLParser
|
|
246
|
+
if self._premises is not None or not self._profile.derived:
|
|
247
|
+
return None
|
|
248
|
+
held = self._derived_maps
|
|
249
|
+
if held is None or held[0] != self._version:
|
|
250
|
+
text = self.r2rml((), self._profile.derived, stored=True)
|
|
251
|
+
held = (self._version, compile_r2rml(R2RMLParser().parse(text), self.base_iri))
|
|
252
|
+
self._derived_maps = held
|
|
253
|
+
return held[1]
|
|
254
|
+
|
|
255
|
+
@property
|
|
256
|
+
def view_key(self) -> Any:
|
|
257
|
+
"""What the generated maps read besides the store's version: None, the premise view's number, or
|
|
258
|
+
("own", that) while they read the running step's own rows too (`read_own`)."""
|
|
259
|
+
key = None if self._premises is None else self._premises[3]
|
|
260
|
+
return ("own", key) if self._own else key
|
|
261
|
+
|
|
262
|
+
def writing(self) -> Any:
|
|
263
|
+
"""A block in which a query that may write is planned and run: in it, `read_own` makes the store's
|
|
264
|
+
reads see the running step's own rows (DuckDB)."""
|
|
265
|
+
import contextlib
|
|
266
|
+
|
|
267
|
+
@contextlib.contextmanager
|
|
268
|
+
def block():
|
|
269
|
+
held = self._writing, self._own
|
|
270
|
+
self._writing = True
|
|
271
|
+
try:
|
|
272
|
+
yield self
|
|
273
|
+
finally:
|
|
274
|
+
if self._own and not held[1]: # the view `read_own` made, left on no connection
|
|
275
|
+
try:
|
|
276
|
+
self.db.execute(f"DROP VIEW IF EXISTS {self.own_view}")
|
|
277
|
+
except Exception: # noqa: BLE001 - an aborted transaction the caller holds drops it on rollback
|
|
278
|
+
pass
|
|
279
|
+
self._writing, self._own = held
|
|
280
|
+
return block()
|
|
281
|
+
|
|
282
|
+
def read_own(self) -> None:
|
|
283
|
+
"""From here on in the `writing` block, the store is read with the running step's own rows (its
|
|
284
|
+
earlier phases': a MATCH or MERGE after a CREATE, a retraction after an assert), where its engine
|
|
285
|
+
writes a step with one INSERT (`StoreConnection.one_insert`): those rows are not in the statements
|
|
286
|
+
table until the step's end. Elsewhere the statements table holds them already."""
|
|
287
|
+
if self._writing and self.db.one_insert and not self._own:
|
|
288
|
+
t = self.table
|
|
289
|
+
self.db.execute(f"CREATE OR REPLACE TEMP VIEW {self.own_view} AS SELECT * FROM {t} WHERE false")
|
|
290
|
+
self._own = True
|
|
291
|
+
|
|
292
|
+
def premises(self, source: str, kinds: Iterable[tuple], derived_kinds: Iterable[tuple] = ()) -> Any:
|
|
293
|
+
"""A block in which the generated maps read `source` (SQL of the columns `graph`, `subject`,
|
|
294
|
+
`predicate`, `o_kind`, `o_text`, `o_datatype`, `o_lang`) in place of the effective view, one map per
|
|
295
|
+
(predicate, kind, datatype, language) of `kinds` over its asserted statements and of `derived_kinds`
|
|
296
|
+
over its derived ones: what OWL RL's rules read (`owl_rl`, `dataset.premise_statements`)."""
|
|
297
|
+
import contextlib
|
|
298
|
+
|
|
299
|
+
@contextlib.contextmanager
|
|
300
|
+
def block():
|
|
301
|
+
held = self._premises
|
|
302
|
+
Store._views += 1
|
|
303
|
+
self._premises = (source, tuple(sorted(set(kinds), key=_sort_key)),
|
|
304
|
+
tuple(sorted(set(derived_kinds), key=_sort_key)), Store._views)
|
|
305
|
+
try:
|
|
306
|
+
yield self
|
|
307
|
+
finally:
|
|
308
|
+
self._premises = held
|
|
309
|
+
return block()
|
|
310
|
+
|
|
311
|
+
_views = 0
|
|
312
|
+
|
|
313
|
+
@property
|
|
314
|
+
def _kinds(self) -> Tuple[tuple, ...]:
|
|
315
|
+
return self._profile.kinds
|
|
316
|
+
|
|
317
|
+
def _read_profile(self, rows: str, params: Sequence[Any] = (), step: bool = False) -> "Profile":
|
|
318
|
+
"""What the statements `rows` (SQL of the table's columns) hold (`Profile`), in one statement."""
|
|
319
|
+
from prograph.plan.ir.dataset import derived_graphs
|
|
320
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
321
|
+
prefix = self.minted_prefix()
|
|
322
|
+
sw = self.db.starts_with
|
|
323
|
+
lit, own = f"o_kind = '{LITERAL}'", f"NOT {sw}(predicate, '{INTERNAL}')"
|
|
324
|
+
derived = f"graph IN ({', '.join(get_dialect_instance(self.dialect).literal(g) for g in derived_graphs())})"
|
|
325
|
+
sql = (f"WITH r AS (SELECT * FROM {rows} AS _r) "
|
|
326
|
+
f"SELECT 'kind', predicate, o_kind, o_datatype, o_lang FROM r WHERE {own} AND NOT {derived} "
|
|
327
|
+
f"UNION SELECT 'derived', predicate, o_kind, o_datatype, o_lang FROM r WHERE {own} AND {derived} "
|
|
328
|
+
f"UNION SELECT 'class', o_text, NULL, NULL, NULL FROM r WHERE predicate = '{RDF_TYPE}' "
|
|
329
|
+
f"AND o_kind = '{IRI}' AND {sw}(subject, '{prefix}') "
|
|
330
|
+
f"UNION SELECT 'type', o_text, NULL, NULL, NULL FROM r WHERE predicate = '{REL_TYPE}' "
|
|
331
|
+
f"UNION SELECT 'any', NULL, NULL, NULL, NULL FROM r WHERE predicate = '{ELEMENT}' "
|
|
332
|
+
f"UNION SELECT 'node', c.o_text, p.predicate, p.o_datatype, NULL FROM r AS p "
|
|
333
|
+
f"JOIN r AS e ON e.subject = p.subject AND e.predicate = '{ELEMENT}' "
|
|
334
|
+
f"LEFT JOIN r AS c ON c.subject = p.subject AND c.predicate = '{RDF_TYPE}' "
|
|
335
|
+
f"WHERE p.{lit} AND NOT {sw}(p.predicate, '{INTERNAL}') "
|
|
336
|
+
f"UNION SELECT 'rel', t.o_text, p.predicate, p.o_datatype, NULL FROM r AS p "
|
|
337
|
+
f"JOIN r AS t ON t.subject = p.subject AND t.predicate = '{REL_TYPE}' "
|
|
338
|
+
f"WHERE p.{lit} AND NOT {sw}(p.predicate, '{INTERNAL}')")
|
|
339
|
+
rows_ = (self.db.hashed if step else self.db.execute)(sql, list(params)).fetchall()
|
|
340
|
+
return Profile(kinds=tuple(sorted((r[1:] for r in rows_ if r[0] == "kind"), key=_sort_key)),
|
|
341
|
+
derived=tuple(sorted((r[1:] for r in rows_ if r[0] == "derived"), key=_sort_key)),
|
|
342
|
+
classes=tuple(sorted(r[1] for r in rows_ if r[0] == "class")),
|
|
343
|
+
types=tuple(sorted(r[1] for r in rows_ if r[0] == "type")),
|
|
344
|
+
nodes=any(r[0] == "any" for r in rows_),
|
|
345
|
+
node_props=frozenset((r[1], r[2], r[3]) for r in rows_ if r[0] == "node"),
|
|
346
|
+
rel_props=frozenset((r[1], r[2], r[3]) for r in rows_ if r[0] == "rel"))
|
|
347
|
+
|
|
348
|
+
def profile(self) -> "Profile":
|
|
349
|
+
"""What the store holds at its `version`."""
|
|
350
|
+
self._settle()
|
|
351
|
+
return self._profile
|
|
352
|
+
|
|
353
|
+
@property
|
|
354
|
+
def version(self) -> int:
|
|
355
|
+
"""Changes when what the store holds changes (`Profile`)."""
|
|
356
|
+
self._settle()
|
|
357
|
+
return self._version
|
|
358
|
+
|
|
359
|
+
def _settle(self) -> None:
|
|
360
|
+
"""After a step joined a transaction the caller held open: once that transaction has ended, or its
|
|
361
|
+
step key is no longer there (the caller rolled back; on a connection out of autocommit the caller is
|
|
362
|
+
always in a transaction), read what the store holds again, as it may not hold the step."""
|
|
363
|
+
if self._joined is None:
|
|
364
|
+
return
|
|
365
|
+
if self._in_transaction():
|
|
366
|
+
try:
|
|
367
|
+
if self.applied(self._joined):
|
|
368
|
+
return
|
|
369
|
+
except Exception: # noqa: BLE001 - an aborted transaction refuses reads until the caller rolls back
|
|
370
|
+
return
|
|
371
|
+
self._joined = None
|
|
372
|
+
self._refresh()
|
|
373
|
+
|
|
374
|
+
def _in_transaction(self) -> bool:
|
|
375
|
+
return self.db.in_transaction()
|
|
376
|
+
|
|
377
|
+
def _refresh(self, seq: Optional[int] = None) -> None:
|
|
378
|
+
"""Read what the store holds again: the whole effective view, or where `seq` (a step that only
|
|
379
|
+
asserts) is given, that step's own rows added to what it held."""
|
|
380
|
+
self._seen_seq = self._max_seq()
|
|
381
|
+
if seq is None:
|
|
382
|
+
profile = self._read_profile(f"({self.effective_sql()})")
|
|
383
|
+
else:
|
|
384
|
+
profile = self._profile.with_(self._read_profile(f"(SELECT * FROM {self._visible()} AS _v WHERE seq = ?)", [seq],
|
|
385
|
+
step=True))
|
|
386
|
+
if profile != self._profile:
|
|
387
|
+
self._profile, self._version = profile, self._version + 1
|
|
388
|
+
from prograph.plan.ir.identity import read_state
|
|
389
|
+
alias = read_state(self)
|
|
390
|
+
if alias != getattr(self, "_alias", None):
|
|
391
|
+
self._alias, self._version = alias, self._version + 1
|
|
392
|
+
|
|
393
|
+
@property
|
|
394
|
+
def alias(self) -> Any:
|
|
395
|
+
"""The alias table's current generation (`identity.AliasState`), None where none holds an alias."""
|
|
396
|
+
self._settle()
|
|
397
|
+
return self._alias
|
|
398
|
+
|
|
399
|
+
def minted_prefix(self) -> str:
|
|
400
|
+
"""What every IRI this store mints starts with: `{base}_/{store}/n/`."""
|
|
401
|
+
return f"{self.base_iri}_/{self.store_id}/n/"
|
|
402
|
+
|
|
403
|
+
def _create(self) -> None:
|
|
404
|
+
c, t = self.db, self.table
|
|
405
|
+
x, stamp = c.text, c.stamp # the engine's text and timestamp types
|
|
406
|
+
attempt = f"attempt {x} NOT NULL, " if c.optimistic else ""
|
|
407
|
+
pos_type = "BIGINT" if c.optimistic else "INTEGER"
|
|
408
|
+
c.create(t, f"CREATE TABLE IF NOT EXISTS {t} (seq BIGINT NOT NULL, {attempt}pos {pos_type} NOT NULL, "
|
|
409
|
+
f"graph {x} NOT NULL, subject {x} NOT NULL, predicate {x} NOT NULL, o_kind {x} NOT NULL, "
|
|
410
|
+
f"o_text {x}, o_datatype {x}, o_lang {x}, edge_id {x}, op {x} NOT NULL, writer {x} NOT NULL, "
|
|
411
|
+
f"step {x} NOT NULL, written_at {stamp} NOT NULL)")
|
|
412
|
+
if c.optimistic:
|
|
413
|
+
c.has_column(t, "attempt") # a store made before the marker is refused by name
|
|
414
|
+
else:
|
|
415
|
+
c.create(f"{t}_steps", f"CREATE TABLE IF NOT EXISTS {t}_steps (step {x} PRIMARY KEY, seq BIGINT NOT NULL)")
|
|
416
|
+
c.create(f"{t}_meta", f"CREATE TABLE IF NOT EXISTS {t}_meta (store_id {x} NOT NULL)")
|
|
417
|
+
# `{t}_answers` and `{t}_digests` are made by the first step that keeps one (`_ensure`), never here: a
|
|
418
|
+
# read-only connection opens a store made before them, and a missing table reads as nothing kept.
|
|
419
|
+
c.index(t, "seq") # a step's own rows, and max(seq), read without a scan of the table
|
|
420
|
+
|
|
421
|
+
def _store_id(self) -> str:
|
|
422
|
+
meta = f"{self.table}_meta"
|
|
423
|
+
row = self.db.execute(f"SELECT min(store_id) FROM {meta}").fetchone()
|
|
424
|
+
if row is not None and row[0] is not None:
|
|
425
|
+
return row[0]
|
|
426
|
+
made = uuid.uuid4().hex
|
|
427
|
+
if not self.db.optimistic:
|
|
428
|
+
self.db.execute(f"INSERT INTO {meta} VALUES (?)", [made])
|
|
429
|
+
return made
|
|
430
|
+
# Two first writers: each appends only where the table is empty (an append that read the table
|
|
431
|
+
# conflicts with the other's), and the least id read afterwards is the store's.
|
|
432
|
+
try:
|
|
433
|
+
self.db.execute(f"INSERT INTO {meta} SELECT ? WHERE NOT EXISTS (SELECT 1 FROM {meta})", [made])
|
|
434
|
+
except Exception as exc: # noqa: BLE001
|
|
435
|
+
if not self.db.is_conflict(exc):
|
|
436
|
+
raise
|
|
437
|
+
return self.db.execute(f"SELECT min(store_id) FROM {meta}").fetchone()[0]
|
|
438
|
+
|
|
439
|
+
def mint(self, step: str, position: int) -> str:
|
|
440
|
+
"""A created element's IRI, from the write step's key and the element's position in it: a retried
|
|
441
|
+
step mints the same IRI (`fresh.mint_iri`, rule store-mint)."""
|
|
442
|
+
from prograph.plan.ir.fresh import mint_iri
|
|
443
|
+
return mint_iri(self.minted_prefix(), self.store_id, step, str(position))
|
|
444
|
+
|
|
445
|
+
def write(self, statements: Sequence[Statement], writer: str, step: Optional[str] = None) -> Optional[int]:
|
|
446
|
+
"""Append one write step: all of `statements` under one seq (`max(seq) + 1`), in one transaction, or
|
|
447
|
+
nothing where `step` (the idempotency key; a new one by default) was applied already. The step's seq,
|
|
448
|
+
None where it was applied before."""
|
|
449
|
+
step = step or uuid.uuid4().hex
|
|
450
|
+
c, t = self.db, self.table
|
|
451
|
+
if c.optimistic:
|
|
452
|
+
return self._write_values(statements, writer, step)
|
|
453
|
+
with c.step_settings(not self._in_transaction()):
|
|
454
|
+
return self._write_rows(statements, writer, step)
|
|
455
|
+
|
|
456
|
+
def _write_rows(self, statements: Sequence[Statement], writer: str, step: str) -> Optional[int]:
|
|
457
|
+
"""`write` in one transaction (DuckDB, PostgreSQL): the statements in one INSERT."""
|
|
458
|
+
c, t = self.db, self.table
|
|
459
|
+
own = self._begin()
|
|
460
|
+
try:
|
|
461
|
+
c.lock(f"{t}_steps")
|
|
462
|
+
if c.execute(f"SELECT 1 FROM {t}_steps WHERE step = ?", [step]).fetchone() is not None:
|
|
463
|
+
self._end(own, "COMMIT")
|
|
464
|
+
return None
|
|
465
|
+
seq = c.execute(f"SELECT coalesce(max(seq), 0) + 1 FROM {t}").fetchone()[0]
|
|
466
|
+
rows = [(seq, i, s.graph, s.subject, s.predicate, s.kind, s.object, s.datatype, s.lang, s.edge_id, s.op,
|
|
467
|
+
writer, step) for i, s in enumerate(statements)]
|
|
468
|
+
if rows:
|
|
469
|
+
c.insert_rows(t, rows)
|
|
470
|
+
c.execute(f"INSERT INTO {t}_steps VALUES (?, ?)", [step, seq])
|
|
471
|
+
c.inserted(t, seq)
|
|
472
|
+
self._end(own, "COMMIT")
|
|
473
|
+
except Exception:
|
|
474
|
+
self._end(own, "ROLLBACK")
|
|
475
|
+
self._joined = self._joined if own else step
|
|
476
|
+
raise
|
|
477
|
+
self._joined = self._joined if own else step
|
|
478
|
+
self._refresh()
|
|
479
|
+
return seq
|
|
480
|
+
|
|
481
|
+
def _write_values(self, statements: Sequence[Statement], writer: str, step: str) -> Optional[int]:
|
|
482
|
+
"""`write` on Delta: the statements as one VALUES select, one phase of an optimistic step."""
|
|
483
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
484
|
+
lit = get_dialect_instance(self.dialect).literal
|
|
485
|
+
rows = ", ".join("(" + ", ".join(lit(v) for v in (s.graph, s.subject, s.predicate, s.kind, s.object,
|
|
486
|
+
s.datatype, s.lang, s.edge_id, s.op, i)) + ")"
|
|
487
|
+
for i, s in enumerate(statements))
|
|
488
|
+
names = "graph, subject, predicate, o_kind, o_text, o_datatype, o_lang, edge_id, op, _o"
|
|
489
|
+
selects = [f"SELECT * FROM VALUES {rows} AS _v({names})"] if statements else []
|
|
490
|
+
return self.write_select(selects, writer, step)[0]
|
|
491
|
+
|
|
492
|
+
def _begin(self) -> bool:
|
|
493
|
+
"""Begin the step's transaction; False inside a transaction the caller holds open, which the step
|
|
494
|
+
then joins (an error leaves it for the caller to roll back)."""
|
|
495
|
+
if self._in_transaction():
|
|
496
|
+
return False
|
|
497
|
+
self.db.begin()
|
|
498
|
+
return True
|
|
499
|
+
|
|
500
|
+
def _end(self, own: bool, how: str) -> None:
|
|
501
|
+
if own:
|
|
502
|
+
self.db.commit() if how == "COMMIT" else self.db.rollback()
|
|
503
|
+
|
|
504
|
+
# The most an applied step's answer keeps (bytes of its Arrow IPC stream); a bigger one is not kept and a
|
|
505
|
+
# retry of the step says so (`StoreAnswerNotKeptError`).
|
|
506
|
+
ANSWER_CAP = 1024 * 1024 # 1 MiB of IPC bytes: about 1.4 MB of text, inlined in one statement on Delta
|
|
507
|
+
|
|
508
|
+
def _ensure(self, suffix: str) -> None:
|
|
509
|
+
"""Make `{table}_answers` (the answer each applied step gave: its Arrow table as base64 of an IPC stream,
|
|
510
|
+
NULL beyond `ANSWER_CAP`; `attempt` is the step's attempt on Delta, the marker's, and '' elsewhere) or
|
|
511
|
+
`{table}_digests` (the digest of the query a step ran, what a retry of its key is checked against), where
|
|
512
|
+
the step that keeps one finds it missing."""
|
|
513
|
+
if suffix in self._ensured:
|
|
514
|
+
return
|
|
515
|
+
c, t, x = self.db, self.table, self.db.text
|
|
516
|
+
ddl = {"answers": f"(step {x} NOT NULL, attempt {x} NOT NULL, kept {x}, size BIGINT NOT NULL)",
|
|
517
|
+
"digests": f"(step {x} NOT NULL, attempt {x} NOT NULL, digest {x} NOT NULL)"}[suffix]
|
|
518
|
+
c.create(f"{t}_{suffix}", f"CREATE TABLE IF NOT EXISTS {t}_{suffix} {ddl}")
|
|
519
|
+
if not self._in_transaction(): # made inside the caller's transaction, it may be rolled back
|
|
520
|
+
self._ensured.add(suffix)
|
|
521
|
+
|
|
522
|
+
def keep_answer(self, step: str, attempt: str, table: Any) -> None:
|
|
523
|
+
"""Keep the answer `table` of the step `step` beside its rows (rule store-step-answer), in the step's
|
|
524
|
+
own transaction (on Delta, before its marker, under its `attempt`: the marker makes it the step's).
|
|
525
|
+
An answer over `ANSWER_CAP` is recorded as not kept."""
|
|
526
|
+
import base64
|
|
527
|
+
import pyarrow as pa
|
|
528
|
+
sink = pa.BufferOutputStream()
|
|
529
|
+
with pa.ipc.new_stream(sink, table.schema) as writer:
|
|
530
|
+
writer.write_table(table)
|
|
531
|
+
raw = sink.getvalue().to_pybytes()
|
|
532
|
+
self._ensure("answers")
|
|
533
|
+
kept = base64.b64encode(raw).decode("ascii") if len(raw) <= self.ANSWER_CAP else None
|
|
534
|
+
self.db.execute(f"INSERT INTO {self.table}_answers VALUES (?, ?, ?, ?)", [step, attempt, kept, len(raw)])
|
|
535
|
+
|
|
536
|
+
@staticmethod
|
|
537
|
+
def digest(query: str, parameters: Optional[dict] = None) -> str:
|
|
538
|
+
"""The digest of a step's query: SHA-256 of the query text with its hint block removed, as the engine
|
|
539
|
+
reads it (a changed space is a different query: the safest reading) and its parameters in a canonical JSON (keys sorted, each value
|
|
540
|
+
tagged with its type, so 1, 1.0, True and '1' differ)."""
|
|
541
|
+
import hashlib
|
|
542
|
+
import json
|
|
543
|
+
|
|
544
|
+
def tag(v: Any) -> Any:
|
|
545
|
+
if hasattr(v, "tolist") and not isinstance(v, (list, tuple, dict, str, bytes)):
|
|
546
|
+
v = v.tolist() # a numpy scalar or array: the Python value it holds
|
|
547
|
+
elif hasattr(v, "to_pydatetime"):
|
|
548
|
+
v = v.to_pydatetime() # a pandas Timestamp
|
|
549
|
+
if v is None:
|
|
550
|
+
return ["NoneType", None]
|
|
551
|
+
for kind in (bool, int, float, str): # a subclass (numpy's str_, float64) by its base
|
|
552
|
+
if isinstance(v, kind):
|
|
553
|
+
return [kind.__name__, repr(float(v)) if kind is float else kind(v)]
|
|
554
|
+
if isinstance(v, (list, tuple)):
|
|
555
|
+
return ["list", [tag(x) for x in v]]
|
|
556
|
+
if isinstance(v, dict):
|
|
557
|
+
return ["map", [[str(k), tag(x)] for k, x in sorted(v.items(), key=lambda kv: str(kv[0]))]]
|
|
558
|
+
return [type(v).__name__, repr(v)]
|
|
559
|
+
text = json.dumps([query, tag(dict(parameters or {}))], sort_keys=True, ensure_ascii=True)
|
|
560
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
561
|
+
|
|
562
|
+
def keep_digest(self, step: str, attempt: str, digest: str) -> None:
|
|
563
|
+
"""Record the digest of the query of step `step`, in the step's own transaction (on Delta under its
|
|
564
|
+
`attempt`: the marker makes it the step's)."""
|
|
565
|
+
self._ensure("digests")
|
|
566
|
+
self.db.execute(f"INSERT INTO {self.table}_digests VALUES (?, ?, ?)", [step, attempt, digest])
|
|
567
|
+
|
|
568
|
+
def check_digest(self, step: str, digest: Optional[str]) -> None:
|
|
569
|
+
"""Raise `StoreStepKeyReusedError` where the applied step `step` ran another query (or the same query
|
|
570
|
+
with other parameters) than the one with `digest`, or where it recorded no digest (applied before the
|
|
571
|
+
store did). None `digest`: nothing to check."""
|
|
572
|
+
if digest is None:
|
|
573
|
+
return
|
|
574
|
+
t = self.table
|
|
575
|
+
if not self.db.has_table(f"{t}_digests"):
|
|
576
|
+
row = None
|
|
577
|
+
elif self.db.optimistic:
|
|
578
|
+
row = self.db.execute(
|
|
579
|
+
f"SELECT a.digest FROM {t}_digests AS a JOIN {t} AS m ON m.step = a.step AND "
|
|
580
|
+
f"m.attempt = a.attempt WHERE m.op = '{COMMIT}' AND m.step = ?", [step]).fetchone()
|
|
581
|
+
else:
|
|
582
|
+
row = self.db.execute(f"SELECT digest FROM {t}_digests WHERE step = ?", [step]).fetchone()
|
|
583
|
+
if row is not None and row[0] == digest:
|
|
584
|
+
return
|
|
585
|
+
from prograph.utils.exceptions import StoreStepKeyReusedError
|
|
586
|
+
why = ("it ran another query, or this query with other parameters" if row is not None
|
|
587
|
+
else "it was applied before the store recorded the query of a step, so this query cannot be "
|
|
588
|
+
"checked to be the one it ran")
|
|
589
|
+
raise StoreStepKeyReusedError(
|
|
590
|
+
f"the step key {step!r} was applied before and {why}: nothing was written. Use a new step key for "
|
|
591
|
+
f"a new query, or the same query and parameters to read the answer the step kept", step)
|
|
592
|
+
|
|
593
|
+
def kept_answer(self, step: str) -> Any:
|
|
594
|
+
"""The answer the applied step `step` gave: its Arrow table (call only for a step that answers).
|
|
595
|
+
Raises `StoreAnswerNotKeptError` where it was not kept (over `ANSWER_CAP`, or the step was applied
|
|
596
|
+
before answers were kept), never an answer read again from the store as it is now."""
|
|
597
|
+
import base64
|
|
598
|
+
import pyarrow as pa
|
|
599
|
+
t = self.table
|
|
600
|
+
if not self.db.has_table(f"{t}_answers"):
|
|
601
|
+
row = None
|
|
602
|
+
elif self.db.optimistic:
|
|
603
|
+
row = self.db.execute(
|
|
604
|
+
f"SELECT a.kept, a.size FROM {t}_answers AS a JOIN {t} AS m ON m.step = a.step AND "
|
|
605
|
+
f"m.attempt = a.attempt WHERE m.op = '{COMMIT}' AND m.step = ?", [step]).fetchone()
|
|
606
|
+
else:
|
|
607
|
+
row = self.db.execute(f"SELECT kept, size FROM {t}_answers WHERE step = ?", [step]).fetchone()
|
|
608
|
+
if row is None or row[0] is None:
|
|
609
|
+
from prograph.utils.exceptions import StoreAnswerNotKeptError
|
|
610
|
+
why = (f"it is {row[1]} bytes and the store keeps at most {self.ANSWER_CAP}" if row is not None
|
|
611
|
+
else "the step was applied before the store kept answers")
|
|
612
|
+
raise StoreAnswerNotKeptError(
|
|
613
|
+
f"the write step {step!r} was applied before and its answer was not kept ({why}): the write is "
|
|
614
|
+
f"done and visible, and nothing was written again. Read what it wrote with a query of its own",
|
|
615
|
+
step)
|
|
616
|
+
return pa.ipc.open_stream(base64.b64decode(row[0])).read_all()
|
|
617
|
+
|
|
618
|
+
def applied(self, step: str) -> bool:
|
|
619
|
+
"""Whether the step key `step` was applied already (on Delta: a marker of that step key)."""
|
|
620
|
+
if self.db.optimistic:
|
|
621
|
+
return self.db.execute(f"SELECT 1 FROM {self.table} WHERE op = '{COMMIT}' AND step = ?",
|
|
622
|
+
[step]).fetchone() is not None
|
|
623
|
+
return self.db.execute(f"SELECT 1 FROM {self.table}_steps WHERE step = ?", [step]).fetchone() \
|
|
624
|
+
is not None
|
|
625
|
+
|
|
626
|
+
def write_select(self, selects: Sequence[Any], writer: str, step: str, before: Sequence[str] = (),
|
|
627
|
+
answer: Optional[str] = None, after: Sequence[str] = (),
|
|
628
|
+
digits: Sequence[Any] = (), replan: bool = False,
|
|
629
|
+
digest: Optional[str] = None, writers: bool = False) -> Tuple[Optional[int], Any]:
|
|
630
|
+
"""Append one write step whose statements are the rows of `selects` (each of the columns
|
|
631
|
+
`STATEMENT_COLUMNS`, then `op` and `_o`, its order within the select), set-based in the engine: an
|
|
632
|
+
`INSERT INTO ... SELECT` per select (on DuckDB one INSERT of them all, `_write_step`), in order, all under one seq (`pos` after the ones before, so a
|
|
633
|
+
later one's op wins), in one transaction, or nothing where `step` was applied already (as `write`).
|
|
634
|
+
In the
|
|
635
|
+
transaction: the statements `before` first (the step's rows written to temporary tables), and after
|
|
636
|
+
the insert `answer`, read whole (the write's answer: where it fails, the step is rolled back), then
|
|
637
|
+
`after` (the temporary tables dropped) however it ends. The step's seq and the answer's Arrow table
|
|
638
|
+
(read as the engine reads an answer, `digits` its decimal columns'); (None, None) where the step was
|
|
639
|
+
applied before. On PostgreSQL the step holds the step lock (`StoreConnection.lock`) from its start. On Delta
|
|
640
|
+
a step that lost its race redoes the SQL it was given (`_write_optimistic`), or with `replan` raises
|
|
641
|
+
`StepLostRace` for the caller to plan again. With `writers`, each select carries a `_writer` column after
|
|
642
|
+
`op`, each statement's writer (OWL RL's rule ids); `writer` is then the step's (a Delta marker's)."""
|
|
643
|
+
c = self.db
|
|
644
|
+
if c.optimistic:
|
|
645
|
+
return self._write_optimistic(selects, writer, step, before, answer, after, digits, replan, digest,
|
|
646
|
+
writers)
|
|
647
|
+
with c.step_settings(not self._in_transaction()):
|
|
648
|
+
return self._write_step(selects, writer, step, before, answer, after, digits, digest, writers)
|
|
649
|
+
|
|
650
|
+
def _write_step(self, selects: Sequence[Any], writer: str, step: str, before: Sequence[str],
|
|
651
|
+
answer: Optional[str], after: Sequence[str], digits: Sequence[Any], digest: Optional[str],
|
|
652
|
+
writers: bool) -> Tuple[Optional[int], Any]:
|
|
653
|
+
"""`write_select` in one transaction (DuckDB, PostgreSQL). On PostgreSQL each select is an INSERT of
|
|
654
|
+
the statements table in turn, a later one reading the earlier ones' rows there. On DuckDB
|
|
655
|
+
(`StoreConnection.one_insert`) the table gets one INSERT at the end (duckdb/duckdb#28839, `OWN_ROWS`):
|
|
656
|
+
each select's rows are a temporary table of their own (one CREATE ... AS, `pos` after the rows before),
|
|
657
|
+
the store's reads see them through `own_view` (`read_own`), redefined after each, and one INSERT
|
|
658
|
+
writes them all; `own_view` is then empty again and the tables dropped."""
|
|
659
|
+
from prograph.execution import StatementGuard, checkpoint, committing
|
|
660
|
+
c, t = self.db, self.table
|
|
661
|
+
one = c.one_insert
|
|
662
|
+
|
|
663
|
+
def run(sql: str, parameters: Any = None) -> Any:
|
|
664
|
+
# Each phase starts only while the query is not cancelled.
|
|
665
|
+
checkpoint()
|
|
666
|
+
return c.execute(sql, parameters) if parameters is not None else c.execute(sql)
|
|
667
|
+
# A cancel interrupts the running phase (`StatementGuard`) and the step rolls back; the guard
|
|
668
|
+
# ends before the commit, so a step committed is never reported cancelled.
|
|
669
|
+
guard = StatementGuard(stop=c.interrupt)
|
|
670
|
+
try:
|
|
671
|
+
own = self._begin()
|
|
672
|
+
except BaseException as error:
|
|
673
|
+
guard.stop()
|
|
674
|
+
if isinstance(error, Exception):
|
|
675
|
+
guard.check() # an interrupted begin is the cancel (or the timeout), not DuckDB's error
|
|
676
|
+
raise
|
|
677
|
+
try:
|
|
678
|
+
checkpoint()
|
|
679
|
+
c.lock(f"{t}_steps")
|
|
680
|
+
if run(f"SELECT 1 FROM {t}_steps WHERE step = ?", [step]).fetchone() is not None:
|
|
681
|
+
guard.stop()
|
|
682
|
+
self._end(own, "COMMIT")
|
|
683
|
+
return None, None
|
|
684
|
+
names = ", ".join(STATEMENT_COLUMNS)
|
|
685
|
+
columns = f"seq, pos, {names}, op, writer, step, written_at"
|
|
686
|
+
empty = f"SELECT {columns} FROM {t} WHERE false"
|
|
687
|
+
if one: # the step's own rows: none yet
|
|
688
|
+
run(f"CREATE OR REPLACE TEMP VIEW {self.own_view} AS {empty}")
|
|
689
|
+
for sql in before:
|
|
690
|
+
run(sql)
|
|
691
|
+
seq = run(f"SELECT coalesce(max(seq), 0) + 1 FROM {t}").fetchone()[0]
|
|
692
|
+
tag, phases, written = uuid.uuid4().hex[:12], [], 0 # DuckDB: the phases' tables, their rows
|
|
693
|
+
for select_sql in selects:
|
|
694
|
+
# A program step: ("run", sql) a statement (a temporary table of the step's rows), ("insert",
|
|
695
|
+
# sql) or a plain select an insert of statements.
|
|
696
|
+
if isinstance(select_sql, tuple):
|
|
697
|
+
kind, select_sql = select_sql
|
|
698
|
+
if kind == "run":
|
|
699
|
+
run(select_sql)
|
|
700
|
+
continue
|
|
701
|
+
if not one:
|
|
702
|
+
run(f"INSERT INTO {t} ({columns}) "
|
|
703
|
+
f"SELECT ?, CAST((SELECT count(*) FROM {t} WHERE seq = ?) + ROW_NUMBER() OVER (ORDER BY _o) "
|
|
704
|
+
f"- 1 AS INTEGER), {names}, op, {'_writer' if writers else '?'}, ?, now() "
|
|
705
|
+
f"FROM ({select_sql}) AS _statements",
|
|
706
|
+
[seq, seq, step] if writers else [seq, seq, writer, step])
|
|
707
|
+
continue
|
|
708
|
+
# The phase's rows as the statements table types them, `pos` after the step's rows so far.
|
|
709
|
+
phase = f"{OWN_ROWS}_{tag}_{len(phases)}"
|
|
710
|
+
text = ", ".join(f"CAST({n} AS {c.text}) AS {n}" for n in STATEMENT_COLUMNS)
|
|
711
|
+
run(f"CREATE TEMP TABLE {phase} AS SELECT CAST(? AS BIGINT) AS seq, CAST({written} + "
|
|
712
|
+
f"ROW_NUMBER() OVER (ORDER BY _o) - 1 AS INTEGER) AS pos, {text}, CAST(op AS {c.text}) AS op, "
|
|
713
|
+
f"CAST({'_writer' if writers else '?'} AS {c.text}) AS writer, CAST(? AS {c.text}) AS step, "
|
|
714
|
+
f"CAST(now() AS {c.stamp}) AS written_at FROM ({select_sql}) AS _statements",
|
|
715
|
+
[seq, step] if writers else [seq, writer, step])
|
|
716
|
+
phases.append(phase)
|
|
717
|
+
written += run(f"SELECT count(*) FROM {phase}").fetchone()[0]
|
|
718
|
+
held = " UNION ALL ".join(f"SELECT {columns} FROM {p}" for p in phases)
|
|
719
|
+
run(f"CREATE OR REPLACE TEMP VIEW {self.own_view} AS {held}")
|
|
720
|
+
if one:
|
|
721
|
+
if written: # one INSERT of the step's rows; none where it has none
|
|
722
|
+
run(f"INSERT INTO {t} ({columns}) " + " UNION ALL ".join(f"SELECT {columns} FROM {p}"
|
|
723
|
+
for p in phases))
|
|
724
|
+
run(f"CREATE OR REPLACE TEMP VIEW {self.own_view} AS {empty}") # they are the table's now
|
|
725
|
+
for p in phases:
|
|
726
|
+
run(f"DROP TABLE {p}")
|
|
727
|
+
run(f"INSERT INTO {t}_steps VALUES (?, ?)", [step, seq])
|
|
728
|
+
c.inserted(t, seq)
|
|
729
|
+
if digest is not None:
|
|
730
|
+
self.keep_digest(step, "", digest)
|
|
731
|
+
if answer is not None:
|
|
732
|
+
checkpoint()
|
|
733
|
+
answered = c.answer(answer, digits) if answer is not None else None
|
|
734
|
+
if answered is not None:
|
|
735
|
+
self.keep_answer(step, "", answered)
|
|
736
|
+
for sql in after:
|
|
737
|
+
run(sql)
|
|
738
|
+
guard.stop()
|
|
739
|
+
committing() # cancelled: rolled back; else a cancel from here on changes nothing
|
|
740
|
+
self._end(own, "COMMIT")
|
|
741
|
+
except Exception:
|
|
742
|
+
guard.stop()
|
|
743
|
+
self._end(own, "ROLLBACK")
|
|
744
|
+
if not own: # the caller's transaction is aborted: it rolls back, its temporary tables with it
|
|
745
|
+
self._joined = step
|
|
746
|
+
guard.check() # cancelled, or past the deadline: that, not the interrupted statement's error
|
|
747
|
+
raise
|
|
748
|
+
self._joined = self._joined if own else step
|
|
749
|
+
self._after_step(seq)
|
|
750
|
+
return seq, answered
|
|
751
|
+
|
|
752
|
+
def _write_optimistic(self, selects: Sequence[Any], writer: str, step: str, before: Sequence[str],
|
|
753
|
+
answer: Optional[str], after: Sequence[str], digits: Sequence[Any],
|
|
754
|
+
replan: bool = False, digest: Optional[str] = None,
|
|
755
|
+
writers: bool = False) -> Tuple[Optional[int], Any]:
|
|
756
|
+
"""`write_select` on Delta, where every statement commits alone. Take the highest marked seq the caller has
|
|
757
|
+
read (`seen`: the store's `_seen_seq`, as of the plan's `sync`);
|
|
758
|
+
append the step's phases under a fresh `attempt` (each phase its own Delta commit, invisible until the
|
|
759
|
+
marker); read the answer from them; then append the marker at seq `seen + 1`, only where no marker of a
|
|
760
|
+
higher seq or of this step key is there (`COMMIT`). A marker that did not land (another writer's came
|
|
761
|
+
first, or Delta refused the append as concurrent) means the step read a store that has changed: it
|
|
762
|
+
redoes whole, on the store as it is now, up to `REDO_LIMIT` times, then raises
|
|
763
|
+
`StoreWriteConflictError` (nothing visible); with `replan` it raises `StepLostRace` at once, as the SQL it
|
|
764
|
+
was given may not be the SQL of a plan made on the store as it is now. Every step is thus serialised against the marked ones: seq
|
|
765
|
+
order is commit order. A step key already marked writes nothing: (None, None)."""
|
|
766
|
+
import random
|
|
767
|
+
import time
|
|
768
|
+
from prograph.execution import StatementGuard, checkpoint, committing
|
|
769
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
770
|
+
from prograph.utils.exceptions import StoreWriteConflictError
|
|
771
|
+
c, t = self.db, self.table
|
|
772
|
+
lit = get_dialect_instance(self.dialect).literal
|
|
773
|
+
names = ", ".join(STATEMENT_COLUMNS)
|
|
774
|
+
|
|
775
|
+
def run(sql: str) -> Any:
|
|
776
|
+
checkpoint() # each phase starts only while the query is not cancelled
|
|
777
|
+
return c.execute(sql)
|
|
778
|
+
guard = None
|
|
779
|
+
try:
|
|
780
|
+
for tries in range(self.REDO_LIMIT):
|
|
781
|
+
checkpoint()
|
|
782
|
+
if self.applied(step):
|
|
783
|
+
return None, None
|
|
784
|
+
attempt = uuid.uuid4().hex
|
|
785
|
+
if not replan:
|
|
786
|
+
self.sync() # the store as it is: the caller's SQL was not planned on it
|
|
787
|
+
# The store as the caller read it: a plan is made on what `sync` last read, so a marker that
|
|
788
|
+
# landed since (even before this attempt began) fails this step's marker and the query is
|
|
789
|
+
# planned again, not run on a plan that cannot see what that writer wrote.
|
|
790
|
+
seen = self._seen_seq
|
|
791
|
+
guard = StatementGuard(stop=c.interrupt)
|
|
792
|
+
c.scope(attempt)
|
|
793
|
+
won, answered = False, None
|
|
794
|
+
try:
|
|
795
|
+
for sql in before:
|
|
796
|
+
run(sql)
|
|
797
|
+
phase = 0 # a phase's `pos` starts at its index * PHASE_STRIDE: an INSERT tells no count
|
|
798
|
+
for select_sql in selects:
|
|
799
|
+
if isinstance(select_sql, tuple):
|
|
800
|
+
kind, select_sql = select_sql
|
|
801
|
+
if kind == "run":
|
|
802
|
+
run(select_sql)
|
|
803
|
+
continue
|
|
804
|
+
run(f"INSERT INTO {t} (seq, attempt, pos, {names}, op, writer, step, written_at) "
|
|
805
|
+
f"SELECT 0, {lit(attempt)}, CAST({phase * PHASE_STRIDE} + ROW_NUMBER() OVER "
|
|
806
|
+
f"(ORDER BY _o) - 1 AS BIGINT), {names}, op, {'_writer' if writers else lit(writer)}, "
|
|
807
|
+
f"{lit(step)}, now() "
|
|
808
|
+
f"FROM ({select_sql}) AS _statements")
|
|
809
|
+
phase += 1
|
|
810
|
+
if digest is not None:
|
|
811
|
+
self.keep_digest(step, attempt, digest)
|
|
812
|
+
if answer is not None:
|
|
813
|
+
checkpoint()
|
|
814
|
+
answered = c.answer(answer, digits)
|
|
815
|
+
self.keep_answer(step, attempt, answered)
|
|
816
|
+
guard.stop()
|
|
817
|
+
committing() # cancelled: nothing is marked; else a cancel from here on changes nothing
|
|
818
|
+
c.execute(
|
|
819
|
+
f"INSERT INTO {t} (seq, attempt, pos, graph, subject, predicate, o_kind, op, writer, step, "
|
|
820
|
+
f"written_at) SELECT {seen + 1}, {lit(attempt)}, 0, '', '', '', '', '{COMMIT}', "
|
|
821
|
+
f"{lit(writer)}, {lit(step)}, now() WHERE NOT EXISTS (SELECT 1 FROM {t} WHERE "
|
|
822
|
+
f"op = '{COMMIT}' AND (seq > {seen} OR step = {lit(step)}))")
|
|
823
|
+
# An INSERT tells no count: the marker is read back (its seq and op let Delta skip every
|
|
824
|
+
# file but the marker's).
|
|
825
|
+
won = c.execute(f"SELECT 1 FROM {t} WHERE op = '{COMMIT}' AND seq = {seen + 1} "
|
|
826
|
+
f"AND attempt = {lit(attempt)}").fetchone() is not None
|
|
827
|
+
except Exception as exc: # noqa: BLE001 - a lost race is a redo; anything else is the step's error
|
|
828
|
+
if not c.is_conflict(exc):
|
|
829
|
+
raise
|
|
830
|
+
finally:
|
|
831
|
+
guard.stop()
|
|
832
|
+
c.scope("")
|
|
833
|
+
if (answered is not None or digest is not None) and not won:
|
|
834
|
+
for kept in ("answers", "digests"): # kept for an attempt that did not mark
|
|
835
|
+
try:
|
|
836
|
+
c.execute(f"DELETE FROM {t}_{kept} WHERE attempt = {lit(attempt)}")
|
|
837
|
+
except Exception: # noqa: BLE001 - never read: a marker of its attempt makes one the step's
|
|
838
|
+
pass
|
|
839
|
+
for sql in after: # the step's tables, whether it won, lost or failed
|
|
840
|
+
try:
|
|
841
|
+
c.execute(sql)
|
|
842
|
+
except Exception: # noqa: BLE001 - dropped again by the next step's `IF EXISTS`
|
|
843
|
+
pass
|
|
844
|
+
if won:
|
|
845
|
+
try:
|
|
846
|
+
self._after_step(seen + 1)
|
|
847
|
+
except Exception as exc: # noqa: BLE001 - the step is committed: say so, never "nothing happened"
|
|
848
|
+
from prograph.utils.exceptions import StoreStepCommittedError
|
|
849
|
+
raise StoreStepCommittedError(
|
|
850
|
+
f"the write step {step!r} committed (its marker is at seq {seen + 1}) and a statement "
|
|
851
|
+
f"after the commit failed, so its answer could not be read: {type(exc).__name__}. The "
|
|
852
|
+
f"write is done and visible; call the query again with step_key={step!r} to read the "
|
|
853
|
+
f"answer the step kept, without writing again", step) from exc
|
|
854
|
+
return seen + 1, answered
|
|
855
|
+
if self.applied(step): # another attempt of this key marked first: its step wrote it
|
|
856
|
+
return None, None
|
|
857
|
+
if replan:
|
|
858
|
+
raise StepLostRace()
|
|
859
|
+
time.sleep(random.uniform(0, 0.05 * (tries + 1)))
|
|
860
|
+
raise StoreWriteConflictError(
|
|
861
|
+
f"write contention: the write lost its race to other writers {self.REDO_LIMIT} times (each redo "
|
|
862
|
+
f"reads the store again and runs the step whole); nothing was written. Call the query again, or "
|
|
863
|
+
f"write from fewer sessions at once")
|
|
864
|
+
except Exception:
|
|
865
|
+
if guard is not None:
|
|
866
|
+
guard.check() # cancelled, or past the deadline: that, not the interrupted statement's error
|
|
867
|
+
raise
|
|
868
|
+
|
|
869
|
+
def _after_step(self, seq: int) -> None:
|
|
870
|
+
"""Read what the store holds again after step `seq` committed."""
|
|
871
|
+
t, v = self.table, self._visible()
|
|
872
|
+
# The step's rows alone tell what it added where it retracted nothing and wrote only about elements
|
|
873
|
+
# it created (their labels and types are among its rows). A retraction may take away what the store
|
|
874
|
+
# held, and a statement about an element made before (a SET) is joined to that element's labels and
|
|
875
|
+
# type, which only the store holds: then the store is read again whole.
|
|
876
|
+
# Two counts over the step's rows alone (one CTE of them), no LIMIT and no OR: the second an anti join
|
|
877
|
+
# written as a LEFT JOIN, a hash join on every engine (PostgreSQL ran an OR's NOT EXISTS once per row
|
|
878
|
+
# of the step). A correlated NOT EXISTS over this view fails a Databricks warehouse's optimizer
|
|
879
|
+
# ([INTERNAL_ERROR], SQLSTATE XX000), which the join does not.
|
|
880
|
+
touched = self.db.hashed(
|
|
881
|
+
f"WITH _step AS (SELECT * FROM {v} AS _v WHERE seq = ?) "
|
|
882
|
+
f"SELECT (SELECT count(*) FROM _step WHERE op = '{RETRACT}') + (SELECT count(*) FROM _step AS s "
|
|
883
|
+
f"LEFT JOIN (SELECT DISTINCT subject FROM _step WHERE predicate IN ('{ELEMENT}', '{REL_TYPE}')) AS x "
|
|
884
|
+
f"ON x.subject = s.subject WHERE s.edge_id IS NULL AND x.subject IS NULL)",
|
|
885
|
+
[seq]).fetchone()[0]
|
|
886
|
+
self._refresh(None if touched else seq)
|
|
887
|
+
|
|
888
|
+
def effective_sql(self, graphs: Optional[Iterable[str]] = None, but: Optional[Iterable[str]] = None,
|
|
889
|
+
about: Optional[Tuple[str, str]] = None) -> str:
|
|
890
|
+
"""The effective statements as SQL: for each statement (graph, subject, predicate, object, edge_id)
|
|
891
|
+
the op of the highest (seq, pos), kept where it is an assert; only `graphs` where given, none of
|
|
892
|
+
`but` where given; only those of (subject, predicate) `about` where given (read before grouping)."""
|
|
893
|
+
t = self.table
|
|
894
|
+
where = ""
|
|
895
|
+
if graphs is not None or but is not None or about is not None:
|
|
896
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
897
|
+
literal = get_dialect_instance(self.dialect).literal
|
|
898
|
+
tests = []
|
|
899
|
+
if graphs is not None:
|
|
900
|
+
tests.append(f"graph IN ({', '.join(literal(g) for g in graphs)})")
|
|
901
|
+
elif but is not None:
|
|
902
|
+
tests.append(f"graph NOT IN ({', '.join(literal(g) for g in but)})")
|
|
903
|
+
if about is not None:
|
|
904
|
+
tests.append(f"subject = {literal(about[0])} AND predicate = {literal(about[1])}")
|
|
905
|
+
where = " WHERE " + " AND ".join(tests)
|
|
906
|
+
# One group per statement, kept where its last assert (seq, pos as one number) is after its last
|
|
907
|
+
# retract. The filter an IS NOT NULL, which PostgreSQL estimates as most rows (a window's `rn = 1`
|
|
908
|
+
# it took for one in two hundred, and every join over the view became a nested loop).
|
|
909
|
+
last = "seq * 2147483648 + pos" if not self.db.optimistic else \
|
|
910
|
+
f"(CAST(seq AS DECIMAL(38, 0)) * {1 << 50} + pos)"
|
|
911
|
+
source = f"{self._visible()} AS _v" if self.db.optimistic or self._own else t
|
|
912
|
+
return (f"SELECT graph, subject, predicate, o_kind, o_text, o_datatype, o_lang, edge_id FROM {source}{where} "
|
|
913
|
+
f"GROUP BY graph, subject, predicate, o_kind, o_text, o_datatype, o_lang, edge_id "
|
|
914
|
+
f"HAVING CASE WHEN max(CASE WHEN op = '{ASSERT}' THEN {last} END) > "
|
|
915
|
+
f"coalesce(max(CASE WHEN op = '{RETRACT}' THEN {last} END), -1) THEN 1 END IS NOT NULL")
|
|
916
|
+
|
|
917
|
+
def effective(self, graphs: Optional[Iterable[str]] = None) -> List[tuple]:
|
|
918
|
+
"""The effective statements' rows, for tests and inspection."""
|
|
919
|
+
return self.db.execute(self.effective_sql(graphs) + " ORDER BY 1, 2, 3, 5").fetchall()
|
|
920
|
+
|
|
921
|
+
def r2rml(self, kinds: Optional[Sequence[tuple]] = None, derived_kinds: Sequence[tuple] = (),
|
|
922
|
+
source: Optional[str] = None, stored: bool = False) -> str:
|
|
923
|
+
"""The generated mapping (one-plan 2.10) as R2RML Turtle (`dataset.store_mapping`): one triples map per
|
|
924
|
+
predicate and kind of object the store holds (IRIs; literals of each datatype; literals of each
|
|
925
|
+
language), each over the effective view (an `rr:sqlQuery`); with `stored`, one per `derived_kinds` of its
|
|
926
|
+
stored derived facts too. Regenerated after a write step: a datatype first written then gets its map.
|
|
927
|
+
`kinds` and `source` (a `premises` view): the maps of those kinds over that relation instead."""
|
|
928
|
+
from prograph.plan.ir.dataset import store_mapping
|
|
929
|
+
return store_mapping(self, self._kinds if kinds is None else kinds, derived_kinds,
|
|
930
|
+
self.effective_sql() if source is None else source,
|
|
931
|
+
apart=bool(derived_kinds) if source is not None else bool(self._profile.derived),
|
|
932
|
+
stored=stored)
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
@dataclass(frozen=True)
|
|
936
|
+
class Profile:
|
|
937
|
+
"""What the store's statements hold, for its generated maps and its property-graph view: each
|
|
938
|
+
(predicate, object kind, datatype, language) of its asserted statements (`kinds`, the store's own
|
|
939
|
+
predicates aside) and of its stored derived facts (`derived`, `dataset.derived_graphs`); its created
|
|
940
|
+
nodes' classes and its relationships' types; each (class, predicate, datatype) of a created node's
|
|
941
|
+
literal (`node_props`, class None; `nodes` whether it holds any created node for a node of no label) and each (type, predicate, datatype) of a
|
|
942
|
+
relationship's (`rel_props`)."""
|
|
943
|
+
kinds: Tuple[tuple, ...] = ()
|
|
944
|
+
derived: Tuple[tuple, ...] = ()
|
|
945
|
+
classes: Tuple[str, ...] = ()
|
|
946
|
+
types: Tuple[str, ...] = ()
|
|
947
|
+
nodes: bool = False
|
|
948
|
+
node_props: frozenset = frozenset()
|
|
949
|
+
rel_props: frozenset = frozenset()
|
|
950
|
+
|
|
951
|
+
def with_(self, more: "Profile") -> "Profile":
|
|
952
|
+
"""This and `more`, held together (a step that only asserts adds to what the store held)."""
|
|
953
|
+
return Profile(kinds=tuple(sorted(set(self.kinds) | set(more.kinds), key=_sort_key)),
|
|
954
|
+
derived=tuple(sorted(set(self.derived) | set(more.derived), key=_sort_key)),
|
|
955
|
+
classes=tuple(sorted(set(self.classes) | set(more.classes))),
|
|
956
|
+
types=tuple(sorted(set(self.types) | set(more.types))),
|
|
957
|
+
nodes=self.nodes or more.nodes, node_props=self.node_props | more.node_props, rel_props=self.rel_props | more.rel_props)
|
|
958
|
+
|
|
959
|
+
|
|
960
|
+
def _sort_key(row: tuple) -> tuple:
|
|
961
|
+
return tuple((x is not None, x or "") for x in row)
|
|
962
|
+
|
|
963
|
+
|
|
964
|
+
@home("store-connection", "The owned store's connection: its statements, transaction, step lock and cancel on each engine",
|
|
965
|
+
terms=("store connection", "step transaction", "step lock", "LOCK TABLE", "EXCLUSIVE MODE", "ADBC",
|
|
966
|
+
"PostgreSQL", "DuckDB", "placeholder", "read-only transaction", "interrupt", "adbc_cancel", "Delta",
|
|
967
|
+
"SparkStore", "DatabricksStore"))
|
|
968
|
+
def store_connection(connection: Any) -> "StoreConnection":
|
|
969
|
+
"""The owned store's side of `connection`: a DuckDB connection (`DuckDBStore`) or an ADBC PostgreSQL
|
|
970
|
+
one (`PostgreSQLStore`, the connection a PostgreSQL engine reads on: one driver, one type mapping).
|
|
971
|
+
A Spark session (`SparkStore`) or a Databricks SQL connection (`DatabricksStore`) is Delta: the store's
|
|
972
|
+
tables are Delta tables and a step is optimistic. Anything else is refused by name."""
|
|
973
|
+
if isinstance(connection, StoreConnection):
|
|
974
|
+
return connection
|
|
975
|
+
module = type(connection).__module__.split(".")[0]
|
|
976
|
+
if module == "pyspark":
|
|
977
|
+
return SparkStore(connection)
|
|
978
|
+
if module == "databricks":
|
|
979
|
+
return DatabricksStore(connection)
|
|
980
|
+
if hasattr(connection, "adbc_get_info"):
|
|
981
|
+
vendor = str((connection.adbc_get_info() or {}).get("vendor_name", "")).lower()
|
|
982
|
+
if "postgres" in vendor:
|
|
983
|
+
return PostgreSQLStore(connection)
|
|
984
|
+
raise TypeError(f"the owned store runs on DuckDB or PostgreSQL, not on an ADBC connection to {vendor!r}")
|
|
985
|
+
if type(connection).__module__.split(".")[0] in ("duckdb", "_duckdb"):
|
|
986
|
+
return DuckDBStore(connection)
|
|
987
|
+
raise TypeError(f"the owned store takes a DuckDB connection, an ADBC PostgreSQL connection "
|
|
988
|
+
f"(adbc-driver-postgresql), a Spark session or a Databricks SQL connection, not a "
|
|
989
|
+
f"{type(connection).__module__}.{type(connection).__name__}")
|
|
990
|
+
|
|
991
|
+
|
|
992
|
+
def engine_connection(store: Any, connection: Any, frames: bool) -> Any:
|
|
993
|
+
"""The DuckDB connection an engine over a DuckDB reads on, beside the owned store `store`: the store's own,
|
|
994
|
+
since a query reads the mapped tables and the store's statements in one statement. Over `frames` (pandas
|
|
995
|
+
frames, `connection` None) the frames are registered on the store's connection, which is returned; over a
|
|
996
|
+
caller's `connection` it must be the store's. A store on another connection, or not on DuckDB, raises
|
|
997
|
+
`StoreConnectionError`, saying what to pass."""
|
|
998
|
+
from prograph.utils.exceptions import StoreConnectionError
|
|
999
|
+
if frames and store.db.dialect != "duckdb":
|
|
1000
|
+
raise StoreConnectionError(
|
|
1001
|
+
f"an owned store beside pandas frames must be on a DuckDB connection (the frames are read on one); "
|
|
1002
|
+
f"this store is on {store.db.dialect}. Pass Store(duckdb.connect()) to the engine, or put the frames "
|
|
1003
|
+
f"in tables of that database and pass connection= and store= on the same connection.")
|
|
1004
|
+
if connection is not None and store.connection is not connection:
|
|
1005
|
+
raise StoreConnectionError(
|
|
1006
|
+
"store= is on another connection than connection=: a query reads the mapped tables and the store's "
|
|
1007
|
+
"statements in one statement, so both must be on one connection. Pass Store(connection) built on the "
|
|
1008
|
+
"connection the engine reads, or leave connection= out when the data is pandas frames (the engine "
|
|
1009
|
+
"then reads them on the store's connection).")
|
|
1010
|
+
return store.connection if connection is None else connection
|
|
1011
|
+
|
|
1012
|
+
|
|
1013
|
+
def check_frame_name(con: Any, name: str) -> None:
|
|
1014
|
+
"""A pandas frame's source name `name`, read on the store's DuckDB connection `con` (`engine_connection`):
|
|
1015
|
+
its view is a TEMP view, which cannot be in a schema, so a name with a schema is refused, and a name with a
|
|
1016
|
+
catalog needs one that is not a database of the connection (it is attached in memory); either would
|
|
1017
|
+
write into the user's own database. Raises `StoreConnectionError`."""
|
|
1018
|
+
from prograph.utils.exceptions import StoreConnectionError
|
|
1019
|
+
parts = name.split(".")
|
|
1020
|
+
if len(parts) == 2:
|
|
1021
|
+
raise StoreConnectionError(
|
|
1022
|
+
f"the frame {name!r} is named with a schema, which would be made in the database of the "
|
|
1023
|
+
f"connection the owned store is on. Name the frame with no schema (a table of the mapping), or "
|
|
1024
|
+
f"with a catalog, schema and table (`catalog.schema.table`), whose catalog is made in memory.")
|
|
1025
|
+
if len(parts) == 3:
|
|
1026
|
+
held = con.execute("SELECT path, database_name = current_database() FROM duckdb_databases() "
|
|
1027
|
+
"WHERE database_name = ?", [parts[0]]).fetchall()
|
|
1028
|
+
if held and (held[0][0] is not None or held[0][1]):
|
|
1029
|
+
raise StoreConnectionError(
|
|
1030
|
+
f"the frame {name!r} is named in the catalog {parts[0]!r}, a database of the connection the "
|
|
1031
|
+
f"owned store is on: a view of the frame would be written there. Name the frame in a catalog "
|
|
1032
|
+
f"that is not attached to it.")
|
|
1033
|
+
|
|
1034
|
+
|
|
1035
|
+
class StoreConnection:
|
|
1036
|
+
"""What the store asks of its engine's connection (`store_connection`): a statement with `?`
|
|
1037
|
+
parameters (`execute`, `executemany`: a result's `fetchone`, `fetchall`, `fetch_arrow_table`), the
|
|
1038
|
+
write's answer read as the engine's reads read one (`answer`), the step's transaction (`begin`, `commit`,
|
|
1039
|
+
`rollback`, `in_transaction`), the step lock (`lock`), and `interrupt` for a cancel. `dialect` names
|
|
1040
|
+
and quotes as the engine's SQL does; `text` and `stamp` are the DDL's types."""
|
|
1041
|
+
dialect = ""
|
|
1042
|
+
text = "VARCHAR"
|
|
1043
|
+
stamp = "TIMESTAMP"
|
|
1044
|
+
# Delta: no transaction, no lock; a step is optimistic (`Store._write_optimistic`).
|
|
1045
|
+
optimistic = False
|
|
1046
|
+
# DuckDB: a step writes the statements table with one INSERT (`Store._write_step`, `OWN_ROWS`).
|
|
1047
|
+
one_insert = False
|
|
1048
|
+
starts_with = "starts_with" # the engine's function name
|
|
1049
|
+
|
|
1050
|
+
def __init__(self, raw: Any) -> None:
|
|
1051
|
+
self.raw = raw
|
|
1052
|
+
|
|
1053
|
+
def step_table(self, name: str, rows: str) -> str:
|
|
1054
|
+
"""The statement that writes a step's rows `rows` (a select) to a table of its own, `name`."""
|
|
1055
|
+
return f"CREATE TEMP TABLE {name} AS {rows}"
|
|
1056
|
+
|
|
1057
|
+
def drop_step_table(self, name: str) -> str:
|
|
1058
|
+
return f"DROP TABLE {name}"
|
|
1059
|
+
|
|
1060
|
+
def stage(self, store: str, key: str, index: int, name: str, rows: str) -> Tuple[str, str, str]:
|
|
1061
|
+
"""How a step stages the rows `rows` of its stage `index`: (what a statement reads them as, the statement
|
|
1062
|
+
that writes them, the statement that removes them). Here a temporary table `name` of the session, dropped
|
|
1063
|
+
after the step."""
|
|
1064
|
+
return name, self.step_table(name, rows), self.drop_step_table(name)
|
|
1065
|
+
|
|
1066
|
+
def scope(self, attempt: str) -> None:
|
|
1067
|
+
"""Delta: the running step's attempt, which its own reads see unmarked (empty: none)."""
|
|
1068
|
+
|
|
1069
|
+
def is_conflict(self, error: BaseException) -> bool:
|
|
1070
|
+
"""Whether `error` is the engine refusing an append as concurrent with another writer's (Delta)."""
|
|
1071
|
+
return False
|
|
1072
|
+
|
|
1073
|
+
def has_table(self, table: str) -> bool:
|
|
1074
|
+
"""Whether `table` exists (a read of a store made before a table was added says it holds nothing)."""
|
|
1075
|
+
return self.execute("SELECT count(*) FROM information_schema.tables WHERE table_name = ?",
|
|
1076
|
+
[table]).fetchone()[0] > 0
|
|
1077
|
+
|
|
1078
|
+
def has_column(self, table: str, column: str) -> None:
|
|
1079
|
+
"""Raise where `table` lacks `column` (Delta: a store made before the commit marker)."""
|
|
1080
|
+
|
|
1081
|
+
def execute(self, sql: str, parameters: Sequence[Any] = ()) -> Any:
|
|
1082
|
+
raise NotImplementedError
|
|
1083
|
+
|
|
1084
|
+
def hashed(self, sql: str, parameters: Sequence[Any] = ()) -> Any:
|
|
1085
|
+
"""`execute` for a read that joins one step's rows with themselves on the subject (what a step added,
|
|
1086
|
+
`Store._after_step` and `_refresh`): a hash join is right whatever the planner estimates."""
|
|
1087
|
+
return self.execute(sql, parameters)
|
|
1088
|
+
|
|
1089
|
+
def executemany(self, sql: str, rows: Sequence[Sequence[Any]]) -> None:
|
|
1090
|
+
raise NotImplementedError
|
|
1091
|
+
|
|
1092
|
+
def answer(self, sql: str, digits: Sequence[Any] = ()) -> Any:
|
|
1093
|
+
return self.execute(sql).fetch_arrow_table()
|
|
1094
|
+
|
|
1095
|
+
def in_transaction(self) -> bool:
|
|
1096
|
+
# DuckDB says no more than this: in autocommit each statement is a transaction of its own, so two
|
|
1097
|
+
# reads of the id differ; inside an open one they are equal. (A BEGIN there would abort it.)
|
|
1098
|
+
ids = [self.execute("SELECT txid_current()").fetchone()[0] for _ in range(2)]
|
|
1099
|
+
return ids[0] == ids[1]
|
|
1100
|
+
|
|
1101
|
+
def begin(self) -> None:
|
|
1102
|
+
self.execute("BEGIN TRANSACTION")
|
|
1103
|
+
|
|
1104
|
+
def commit(self) -> None:
|
|
1105
|
+
self.execute("COMMIT")
|
|
1106
|
+
|
|
1107
|
+
def rollback(self) -> None:
|
|
1108
|
+
self.execute("ROLLBACK")
|
|
1109
|
+
|
|
1110
|
+
def step_settings(self, own: bool = True) -> Any:
|
|
1111
|
+
"""A block holding the settings a write step runs under (`own`: the step is its own transaction, not
|
|
1112
|
+
one the caller holds), the connection's own put back after it."""
|
|
1113
|
+
import contextlib
|
|
1114
|
+
return contextlib.nullcontext()
|
|
1115
|
+
|
|
1116
|
+
def insert_rows(self, table: str, rows: Sequence[Sequence[Any]]) -> None:
|
|
1117
|
+
"""`rows` (seq, pos, the statement's columns, op, writer, step) appended to the statements `table`,
|
|
1118
|
+
written now."""
|
|
1119
|
+
self.executemany(f"INSERT INTO {table} VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, now())", rows)
|
|
1120
|
+
|
|
1121
|
+
def plan_json(self, sql: str) -> Optional[str]:
|
|
1122
|
+
"""The database's plan of `sql` as JSON text, reading no row (PostgreSQL's `EXPLAIN (VERBOSE, FORMAT
|
|
1123
|
+
JSON)`); None where the connection gives none."""
|
|
1124
|
+
return None
|
|
1125
|
+
|
|
1126
|
+
def read_only(self) -> bool:
|
|
1127
|
+
"""Whether the connection refuses writes (a read-only PostgreSQL transaction or session); False where
|
|
1128
|
+
that cannot be told before a write."""
|
|
1129
|
+
return False
|
|
1130
|
+
|
|
1131
|
+
def lock(self, table: str) -> None:
|
|
1132
|
+
"""Hold the step lock on `table` (the steps table) until the step's transaction ends, so two
|
|
1133
|
+
writers never read one `max(seq)`; raises `StoreReadOnlyError` where the transaction is read-only."""
|
|
1134
|
+
|
|
1135
|
+
def create(self, table: str, ddl: str) -> None:
|
|
1136
|
+
"""`ddl` (a CREATE TABLE IF NOT EXISTS of `table`), where `table` is missing."""
|
|
1137
|
+
self.execute(ddl)
|
|
1138
|
+
|
|
1139
|
+
def index(self, table: str, column: str) -> None:
|
|
1140
|
+
"""An index on `table`'s `column`, where the engine needs one to read a step's rows."""
|
|
1141
|
+
|
|
1142
|
+
def inserted(self, table: str, seq: int) -> None:
|
|
1143
|
+
"""After step `seq`'s inserts into `table`, in its transaction."""
|
|
1144
|
+
|
|
1145
|
+
def interrupt(self) -> None:
|
|
1146
|
+
"""Stop the running statement (from another thread: a cancel or the deadline)."""
|
|
1147
|
+
|
|
1148
|
+
|
|
1149
|
+
class DuckDBStore(StoreConnection):
|
|
1150
|
+
"""The store on a DuckDB connection: its own `?` parameters, one writer at a time (DuckDB serializes
|
|
1151
|
+
them), `connection.interrupt` for a cancel. A step writes the statements table with one INSERT
|
|
1152
|
+
(`one_insert`) under `preserve_insertion_order = false` (`step_settings`), the two ways round DuckDB's
|
|
1153
|
+
lost INSERT (duckdb/duckdb#28839)."""
|
|
1154
|
+
dialect = "duckdb"
|
|
1155
|
+
one_insert = True
|
|
1156
|
+
|
|
1157
|
+
def step_settings(self, own: bool = True) -> Any:
|
|
1158
|
+
"""`preserve_insertion_order` false for the step, in this connection's session alone (a plain SET is
|
|
1159
|
+
DuckDB's global setting, every connection to the database's), its value (it may be the caller's) set back
|
|
1160
|
+
after the step however it ends. Only where the step is its own transaction (`own`): in a transaction the
|
|
1161
|
+
caller holds, a failed step leaves it aborted, where no SET runs, so the setting stays as it is (the
|
|
1162
|
+
one INSERT is the fix; the setting a second guard). Putting it back never hides the step's error."""
|
|
1163
|
+
import contextlib
|
|
1164
|
+
read = "SELECT current_setting('preserve_insertion_order')"
|
|
1165
|
+
|
|
1166
|
+
@contextlib.contextmanager
|
|
1167
|
+
def block():
|
|
1168
|
+
if not own:
|
|
1169
|
+
yield
|
|
1170
|
+
return
|
|
1171
|
+
held = self.raw.execute(read).fetchone()[0]
|
|
1172
|
+
self.raw.execute("SET SESSION preserve_insertion_order = false")
|
|
1173
|
+
failed = False
|
|
1174
|
+
try:
|
|
1175
|
+
yield
|
|
1176
|
+
except BaseException:
|
|
1177
|
+
failed = True
|
|
1178
|
+
raise
|
|
1179
|
+
finally:
|
|
1180
|
+
try:
|
|
1181
|
+
# The global value again, unless the session held its own before.
|
|
1182
|
+
self.raw.execute("RESET SESSION preserve_insertion_order")
|
|
1183
|
+
if self.raw.execute(read).fetchone()[0] != held:
|
|
1184
|
+
self.raw.execute(f"SET SESSION preserve_insertion_order = {'true' if held else 'false'}")
|
|
1185
|
+
except Exception: # noqa: BLE001 - the step's own error, where there is one, is the one raised
|
|
1186
|
+
if not failed:
|
|
1187
|
+
raise
|
|
1188
|
+
return block()
|
|
1189
|
+
|
|
1190
|
+
def insert_rows(self, table: str, rows: Sequence[Sequence[Any]]) -> None:
|
|
1191
|
+
"""One INSERT of all `rows`, from an Arrow table registered on the connection for it."""
|
|
1192
|
+
import pyarrow as pa
|
|
1193
|
+
names = ("seq", "pos") + STATEMENT_COLUMNS + ("op", "writer", "step")
|
|
1194
|
+
types = [pa.int64(), pa.int32()] + [pa.string()] * (len(names) - 2)
|
|
1195
|
+
data = pa.table([pa.array([r[i] for r in rows], type=ty) for i, ty in enumerate(types)], names=list(names))
|
|
1196
|
+
view = f"{OWN_ROWS}_{uuid.uuid4().hex[:12]}"
|
|
1197
|
+
self.raw.register(view, data)
|
|
1198
|
+
try:
|
|
1199
|
+
self.raw.execute(f"INSERT INTO {table} ({', '.join(names)}, written_at) "
|
|
1200
|
+
f"SELECT {', '.join(names)}, now() FROM {view}")
|
|
1201
|
+
finally:
|
|
1202
|
+
self.raw.unregister(view)
|
|
1203
|
+
|
|
1204
|
+
def execute(self, sql: str, parameters: Sequence[Any] = ()) -> Any:
|
|
1205
|
+
return self.raw.execute(sql, list(parameters)) if parameters else self.raw.execute(sql)
|
|
1206
|
+
|
|
1207
|
+
def executemany(self, sql: str, rows: Sequence[Sequence[Any]]) -> None:
|
|
1208
|
+
self.raw.executemany(sql, rows)
|
|
1209
|
+
|
|
1210
|
+
def answer(self, sql: str, digits: Sequence[Any] = ()) -> Any:
|
|
1211
|
+
from prograph.graph.drivers import duckdb_arrow_table
|
|
1212
|
+
return duckdb_arrow_table(self.raw, sql)
|
|
1213
|
+
|
|
1214
|
+
def interrupt(self) -> None:
|
|
1215
|
+
self.raw.interrupt()
|
|
1216
|
+
|
|
1217
|
+
|
|
1218
|
+
class _Rows:
|
|
1219
|
+
"""A PostgreSQL statement's result, read whole (the driver's Arrow), the cursor closed."""
|
|
1220
|
+
|
|
1221
|
+
def __init__(self, table: Any) -> None:
|
|
1222
|
+
self.table = table
|
|
1223
|
+
|
|
1224
|
+
def fetch_arrow_table(self) -> Any:
|
|
1225
|
+
return self.table
|
|
1226
|
+
|
|
1227
|
+
def fetchall(self) -> List[tuple]:
|
|
1228
|
+
if self.table is None:
|
|
1229
|
+
return []
|
|
1230
|
+
columns = [c.to_pylist() for c in self.table.columns]
|
|
1231
|
+
return list(zip(*columns)) if columns else []
|
|
1232
|
+
|
|
1233
|
+
def fetchone(self) -> Optional[tuple]:
|
|
1234
|
+
rows = self.fetchall()
|
|
1235
|
+
return rows[0] if rows else None
|
|
1236
|
+
|
|
1237
|
+
|
|
1238
|
+
class PostgreSQLStore(StoreConnection):
|
|
1239
|
+
"""The store on an ADBC PostgreSQL connection. `?` parameters become `$n`; each statement on a
|
|
1240
|
+
cursor of its own, cancelled through `cursor.adbc_cancel`. On a connection in autocommit (one a URI
|
|
1241
|
+
opened) the step is a `BEGIN` ... `COMMIT` of its own; one not in autocommit is the caller's
|
|
1242
|
+
transaction, which the step joins, and after a failed statement the driver refuses every later one
|
|
1243
|
+
until the caller's `connection.rollback()`, so the step runs nothing more there. The step lock is
|
|
1244
|
+
`LOCK TABLE <steps> IN EXCLUSIVE MODE` (readers go on reading what was committed): PostgreSQL's
|
|
1245
|
+
single-writer rule, which DuckDB gives for free."""
|
|
1246
|
+
dialect = "postgresql"
|
|
1247
|
+
text = "TEXT"
|
|
1248
|
+
stamp = "TIMESTAMPTZ"
|
|
1249
|
+
|
|
1250
|
+
def __init__(self, raw: Any) -> None:
|
|
1251
|
+
super().__init__(raw)
|
|
1252
|
+
self._running: Any = None
|
|
1253
|
+
|
|
1254
|
+
def _autocommit(self) -> bool:
|
|
1255
|
+
try:
|
|
1256
|
+
return str(self.raw.adbc_connection.get_option("adbc.connection.autocommit")).lower() == "true"
|
|
1257
|
+
except Exception: # noqa: BLE001 - a driver that does not say: take it as the caller's transaction
|
|
1258
|
+
return False
|
|
1259
|
+
|
|
1260
|
+
def _run(self, sql: str, parameters: Sequence[Any] = (), many: bool = False) -> Any:
|
|
1261
|
+
cursor = self.raw.cursor()
|
|
1262
|
+
self._running = cursor
|
|
1263
|
+
try:
|
|
1264
|
+
if parameters:
|
|
1265
|
+
sql = _numbered(sql, len(parameters[0]) if many else len(parameters))
|
|
1266
|
+
if many:
|
|
1267
|
+
cursor.executemany(sql, [tuple(r) for r in parameters])
|
|
1268
|
+
return _Rows(None)
|
|
1269
|
+
cursor.execute(sql, tuple(parameters)) if parameters else cursor.execute(sql)
|
|
1270
|
+
return _Rows(cursor.fetch_arrow_table() if cursor.description else None)
|
|
1271
|
+
finally:
|
|
1272
|
+
self._running = None
|
|
1273
|
+
cursor.close()
|
|
1274
|
+
|
|
1275
|
+
def execute(self, sql: str, parameters: Sequence[Any] = ()) -> Any:
|
|
1276
|
+
return self._run(sql, parameters)
|
|
1277
|
+
|
|
1278
|
+
def plan_json(self, sql: str) -> Optional[str]:
|
|
1279
|
+
# ADBC reads a result through COPY (... TO STDOUT), which an EXPLAIN cannot be inside: this one
|
|
1280
|
+
# statement is read without it.
|
|
1281
|
+
# In a transaction the caller holds, none: a failed statement there leaves the transaction in error,
|
|
1282
|
+
# and this driver refuses every statement after it, a ROLLBACK TO SAVEPOINT included (only the
|
|
1283
|
+
# connection's rollback, the caller's whole transaction, clears it). No plan: the query is not sized.
|
|
1284
|
+
if not self._autocommit():
|
|
1285
|
+
return None
|
|
1286
|
+
cursor = self.raw.cursor()
|
|
1287
|
+
try:
|
|
1288
|
+
cursor.adbc_statement.set_options(**{"adbc.postgresql.use_copy": "false"})
|
|
1289
|
+
cursor.execute(f"EXPLAIN (VERBOSE, FORMAT JSON) {sql}")
|
|
1290
|
+
row = cursor.fetchone()
|
|
1291
|
+
finally:
|
|
1292
|
+
cursor.close()
|
|
1293
|
+
return None if row is None else row[0]
|
|
1294
|
+
|
|
1295
|
+
def hashed(self, sql: str, parameters: Sequence[Any] = ()) -> Any:
|
|
1296
|
+
"""A step's rows filtered by their seq are estimated from the table's statistics, which a step made
|
|
1297
|
+
since the last ANALYZE is not in: about 160 rows for a step of 30,000, and PostgreSQL then joins them
|
|
1298
|
+
with themselves in nested loops over a spilled CTE (90 s a step once the store held about a dozen
|
|
1299
|
+
steps, against 0.06 s hashed). Nested loops are off for the one statement, by `SET LOCAL`, which its
|
|
1300
|
+
transaction's end undoes whatever happens: the session's own setting is never changed. In the
|
|
1301
|
+
caller's transaction the value it had is set back after the statement; on a connection in
|
|
1302
|
+
autocommit the statement runs in a transaction of its own."""
|
|
1303
|
+
if self.in_transaction():
|
|
1304
|
+
before = self._run("SELECT current_setting('enable_nestloop')").fetchone()[0]
|
|
1305
|
+
self._run("SET LOCAL enable_nestloop = off")
|
|
1306
|
+
try:
|
|
1307
|
+
return self._run(sql, parameters)
|
|
1308
|
+
finally:
|
|
1309
|
+
try:
|
|
1310
|
+
self._run(f"SET LOCAL enable_nestloop = {'on' if before == 'on' else 'off'}")
|
|
1311
|
+
except Exception: # noqa: BLE001 - an aborted transaction: its end undoes the SET LOCAL
|
|
1312
|
+
pass
|
|
1313
|
+
self._run("BEGIN")
|
|
1314
|
+
try:
|
|
1315
|
+
self._run("SET LOCAL enable_nestloop = off")
|
|
1316
|
+
rows = self._run(sql, parameters)
|
|
1317
|
+
except BaseException:
|
|
1318
|
+
try:
|
|
1319
|
+
self._run("ROLLBACK")
|
|
1320
|
+
except Exception: # noqa: BLE001 - the original error is the one to raise
|
|
1321
|
+
pass
|
|
1322
|
+
raise
|
|
1323
|
+
self._run("COMMIT")
|
|
1324
|
+
return rows
|
|
1325
|
+
|
|
1326
|
+
def executemany(self, sql: str, rows: Sequence[Sequence[Any]]) -> None:
|
|
1327
|
+
if rows:
|
|
1328
|
+
self._run(sql, rows, many=True)
|
|
1329
|
+
|
|
1330
|
+
def answer(self, sql: str, digits: Sequence[Any] = ()) -> Any:
|
|
1331
|
+
"""The write's answer as a PostgreSQL engine reads any (`postgresql_graph.typed_cursor`: numerics as
|
|
1332
|
+
exact decimals, a term's four columns one struct)."""
|
|
1333
|
+
from prograph.graph.postgresql_graph import typed_cursor
|
|
1334
|
+
cursor = self.raw.cursor()
|
|
1335
|
+
self._running = cursor
|
|
1336
|
+
try:
|
|
1337
|
+
cursor, numerics = typed_cursor(self.raw, sql, digits=digits, cursor=cursor)
|
|
1338
|
+
return numerics.convert(cursor.fetch_arrow_table())
|
|
1339
|
+
finally:
|
|
1340
|
+
self._running = None
|
|
1341
|
+
cursor.close()
|
|
1342
|
+
|
|
1343
|
+
def in_transaction(self) -> bool:
|
|
1344
|
+
# Read from the driver, no statement: a connection out of autocommit is always in the caller's
|
|
1345
|
+
# transaction (the DB-API way to hold one); one in autocommit is in none (a `BEGIN` the caller sends
|
|
1346
|
+
# as a statement itself is not seen). No transaction id is taken: a standby assigns none.
|
|
1347
|
+
return not self._autocommit()
|
|
1348
|
+
|
|
1349
|
+
def begin(self) -> None:
|
|
1350
|
+
self.execute("BEGIN")
|
|
1351
|
+
|
|
1352
|
+
def read_only(self) -> bool:
|
|
1353
|
+
return self.execute("SELECT current_setting('transaction_read_only')").fetchone()[0] == "on"
|
|
1354
|
+
|
|
1355
|
+
def lock(self, table: str) -> None:
|
|
1356
|
+
# Read-only first: nothing before it in the step writes or takes a transaction id.
|
|
1357
|
+
if self.read_only():
|
|
1358
|
+
from prograph.utils.exceptions import StoreReadOnlyError
|
|
1359
|
+
raise StoreReadOnlyError(
|
|
1360
|
+
"the owned store's PostgreSQL connection is read-only (a read-only transaction or session, "
|
|
1361
|
+
"default_transaction_read_only): a write step needs a writable connection; nothing was written")
|
|
1362
|
+
self.execute(f"LOCK TABLE {table} IN EXCLUSIVE MODE")
|
|
1363
|
+
|
|
1364
|
+
def has_table(self, table: str) -> bool:
|
|
1365
|
+
return not self.execute("SELECT to_regclass(?) IS NULL", [table]).fetchone()[0]
|
|
1366
|
+
|
|
1367
|
+
def create(self, table: str, ddl: str) -> None:
|
|
1368
|
+
# A read-only connection reads a store made before: CREATE (even IF NOT EXISTS) is refused there.
|
|
1369
|
+
if self.execute("SELECT to_regclass(?) IS NULL", [table]).fetchone()[0]:
|
|
1370
|
+
self.execute(ddl)
|
|
1371
|
+
|
|
1372
|
+
def index(self, table: str, column: str) -> None:
|
|
1373
|
+
# DuckDB's zone maps skip what a monotonic seq does not hold; PostgreSQL reads a step's rows (the
|
|
1374
|
+
# INSERT's count of them, a step's profile) by an index, else by a scan of the whole table.
|
|
1375
|
+
self.create(f"{table}_{column}", f"CREATE INDEX IF NOT EXISTS {table}_{column} ON {table} ({column})")
|
|
1376
|
+
|
|
1377
|
+
def inserted(self, table: str, seq: int) -> None:
|
|
1378
|
+
"""The table's statistics read again (ANALYZE) where the step grew it by more than a tenth of what
|
|
1379
|
+
PostgreSQL last counted, or it never counted: autovacuum's rule, now rather than a minute later, so
|
|
1380
|
+
the step's own reads and the next ones are planned on what the table holds (planned on no statistics,
|
|
1381
|
+
a step's self-joins run as nested loops: 86 s for 10,000 created nodes)."""
|
|
1382
|
+
rows = self.execute(f"SELECT count(*) FROM {table} WHERE seq = ?", [seq]).fetchone()[0]
|
|
1383
|
+
known = self.execute("SELECT reltuples FROM pg_class WHERE oid = to_regclass(?)", [table]).fetchone()[0]
|
|
1384
|
+
if known is None or known < 0 or rows > 0.1 * known:
|
|
1385
|
+
self.execute(f"ANALYZE {table}")
|
|
1386
|
+
|
|
1387
|
+
def interrupt(self) -> None:
|
|
1388
|
+
cursor = self._running
|
|
1389
|
+
if cursor is not None:
|
|
1390
|
+
cursor.adbc_cancel()
|
|
1391
|
+
|
|
1392
|
+
|
|
1393
|
+
class _SparkRows:
|
|
1394
|
+
"""A Spark statement's result, collected inside the statement's session (`SparkStore.statement`)."""
|
|
1395
|
+
|
|
1396
|
+
def __init__(self, rows: List[tuple], table: Any = None) -> None:
|
|
1397
|
+
self.rows, self.table = rows, table
|
|
1398
|
+
|
|
1399
|
+
def fetchall(self) -> List[tuple]:
|
|
1400
|
+
return list(self.rows)
|
|
1401
|
+
|
|
1402
|
+
def fetchone(self) -> Optional[tuple]:
|
|
1403
|
+
return self.rows[0] if self.rows else None
|
|
1404
|
+
|
|
1405
|
+
def fetch_arrow_table(self) -> Any:
|
|
1406
|
+
return self.table
|
|
1407
|
+
|
|
1408
|
+
|
|
1409
|
+
class DeltaStore(StoreConnection):
|
|
1410
|
+
"""What the two Delta connections share: the store's tables are Delta tables (`USING delta`), a statement
|
|
1411
|
+
commits alone, `?` parameters are written into the SQL as the dialect's literals (Spark binds neither a
|
|
1412
|
+
positional nor a named parameter inside the subquery of an `INSERT ... SELECT`), a step's rows go to a
|
|
1413
|
+
Delta table of their own (a temporary view would read the changing store again), and the running step's
|
|
1414
|
+
attempt is a temporary view the effective view reads."""
|
|
1415
|
+
optimistic = True
|
|
1416
|
+
starts_with = "startswith"
|
|
1417
|
+
text = "STRING"
|
|
1418
|
+
stamp = "TIMESTAMP"
|
|
1419
|
+
|
|
1420
|
+
def __init__(self, raw: Any) -> None:
|
|
1421
|
+
super().__init__(raw)
|
|
1422
|
+
self.scope("")
|
|
1423
|
+
|
|
1424
|
+
def statement(self, sql: str) -> Any:
|
|
1425
|
+
raise NotImplementedError
|
|
1426
|
+
|
|
1427
|
+
def execute(self, sql: str, parameters: Sequence[Any] = ()) -> Any:
|
|
1428
|
+
if parameters:
|
|
1429
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
1430
|
+
sql = _inlined(sql, parameters, get_dialect_instance(self.dialect).literal)
|
|
1431
|
+
return self.statement(sql)
|
|
1432
|
+
|
|
1433
|
+
def executemany(self, sql: str, rows: Sequence[Sequence[Any]]) -> None:
|
|
1434
|
+
for row in rows:
|
|
1435
|
+
self.execute(sql, row)
|
|
1436
|
+
|
|
1437
|
+
def in_transaction(self) -> bool:
|
|
1438
|
+
return False
|
|
1439
|
+
|
|
1440
|
+
def begin(self) -> None:
|
|
1441
|
+
pass
|
|
1442
|
+
|
|
1443
|
+
def commit(self) -> None:
|
|
1444
|
+
pass
|
|
1445
|
+
|
|
1446
|
+
def rollback(self) -> None:
|
|
1447
|
+
pass
|
|
1448
|
+
|
|
1449
|
+
def create(self, table: str, ddl: str) -> None:
|
|
1450
|
+
self.execute(f"{ddl} USING delta")
|
|
1451
|
+
|
|
1452
|
+
def has_column(self, table: str, column: str) -> None:
|
|
1453
|
+
if column not in self.columns(table):
|
|
1454
|
+
raise ValueError(f"the owned store's table {table} was made before the commit marker (it has no "
|
|
1455
|
+
f"{column} column) and a Delta store is not altered: write to a new table name "
|
|
1456
|
+
f"(Store(connection, table=...))")
|
|
1457
|
+
|
|
1458
|
+
def has_table(self, table: str) -> bool:
|
|
1459
|
+
try:
|
|
1460
|
+
self.columns(table)
|
|
1461
|
+
except Exception: # noqa: BLE001 - the engine's own "table not found"
|
|
1462
|
+
return False
|
|
1463
|
+
return True
|
|
1464
|
+
|
|
1465
|
+
def columns(self, table: str) -> List[str]:
|
|
1466
|
+
raise NotImplementedError
|
|
1467
|
+
|
|
1468
|
+
STAGE_KEEP = 86400 # seconds a staged row may stay before the first step of a connection removes it (a step killed before its cleanup)
|
|
1469
|
+
|
|
1470
|
+
@staticmethod
|
|
1471
|
+
def _ddl_type(text: str) -> str:
|
|
1472
|
+
"""`text`, a type as `DESCRIBE QUERY` writes it, as DDL: a struct field's name quoted with backticks. A
|
|
1473
|
+
name holding a character Delta refuses in a column name (` ,;{}()=`, a tab or a newline: a map key with
|
|
1474
|
+
a space), one that cannot be told from the type's own syntax (`:` or `<`), and an empty struct raise
|
|
1475
|
+
`ValueError` naming it, before anything is retried. A NULL-only type (`void`, which a table column
|
|
1476
|
+
cannot hold, nested or not) is `string`: the value read back is the same null."""
|
|
1477
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
1478
|
+
quote = get_dialect_instance("spark").quote
|
|
1479
|
+
|
|
1480
|
+
def parse(i: int) -> Tuple[str, int]:
|
|
1481
|
+
j = i
|
|
1482
|
+
while j < len(text) and text[j] not in "<,>(":
|
|
1483
|
+
j += 1
|
|
1484
|
+
word = text[i:j]
|
|
1485
|
+
if j < len(text) and text[j] == "(": # decimal(10,2), varchar(3)
|
|
1486
|
+
k = text.index(")", j)
|
|
1487
|
+
return text[i:k + 1], k + 1
|
|
1488
|
+
if j < len(text) and text[j] == "<":
|
|
1489
|
+
parts, j = [], j + 1
|
|
1490
|
+
if word.lower() == "struct" and text[j] == ">":
|
|
1491
|
+
raise ValueError("an empty map")
|
|
1492
|
+
while True:
|
|
1493
|
+
if word.lower() == "struct":
|
|
1494
|
+
k = text.index(":", j)
|
|
1495
|
+
name = text[j:k]
|
|
1496
|
+
if any(c in name for c in " ,;{}()=\n\t<>"):
|
|
1497
|
+
raise ValueError(name)
|
|
1498
|
+
inner, j = parse(k + 1)
|
|
1499
|
+
parts.append(quote(name.strip("`")) + ":" + inner)
|
|
1500
|
+
else:
|
|
1501
|
+
inner, j = parse(j)
|
|
1502
|
+
parts.append(inner)
|
|
1503
|
+
if text[j] == ">":
|
|
1504
|
+
return f"{word}<{','.join(parts)}>", j + 1
|
|
1505
|
+
j += 1 # the comma
|
|
1506
|
+
if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*(?: [A-Za-z_][A-Za-z0-9_.]*)*", word):
|
|
1507
|
+
# A leaf is a type's name, words of letters, digits and '_' (timestamp_ntz, `string collate
|
|
1508
|
+
# UTF8_LCASE`, `interval day to second`): `b:bigint` is a field name holding a colon.
|
|
1509
|
+
raise ValueError(text)
|
|
1510
|
+
return ("string" if word.lower() in ("void", "null") else word), j
|
|
1511
|
+
try:
|
|
1512
|
+
return parse(0)[0]
|
|
1513
|
+
except (IndexError, ValueError) as exc:
|
|
1514
|
+
raise ValueError(str(exc) if isinstance(exc, ValueError) else text) from exc
|
|
1515
|
+
|
|
1516
|
+
def _stage_columns(self, table: str, slots: Dict[str, str]) -> None:
|
|
1517
|
+
"""Make `table`'s slot columns `slots` (name -> type) exist: those it lacks added by one ALTER. Two
|
|
1518
|
+
writers adding at once: one's ALTER fails (the column exists, or Delta's metadata changed); read the
|
|
1519
|
+
columns again and add what is still missing, a bounded number of times."""
|
|
1520
|
+
import time
|
|
1521
|
+
for tries in range(8):
|
|
1522
|
+
have = set(self.columns(table))
|
|
1523
|
+
missing = {n: t for n, t in slots.items() if n not in have}
|
|
1524
|
+
if not missing:
|
|
1525
|
+
return
|
|
1526
|
+
try:
|
|
1527
|
+
self.execute(f"ALTER TABLE {table} ADD COLUMNS ({', '.join(f'{n} {t}' for n, t in missing.items())})")
|
|
1528
|
+
except Exception: # noqa: BLE001 - another writer's ALTER landed first: look again
|
|
1529
|
+
if tries == 7:
|
|
1530
|
+
raise
|
|
1531
|
+
time.sleep(0.1 * (tries + 1))
|
|
1532
|
+
|
|
1533
|
+
def stage(self, store: str, key: str, index: int, name: str, rows: str) -> Tuple[str, str, str]:
|
|
1534
|
+
"""Delta has no temporary table, and a managed table made and dropped for every step counts against the
|
|
1535
|
+
schema's table quota for days (Unity Catalog keeps a dropped table about a week), so a step stages its
|
|
1536
|
+
rows in the one table `{store}_stage`, never in one of its own. Its columns are `_key`, `_stage`, `_at`
|
|
1537
|
+
and typed slots added as steps need them (`s_<hash of the exact type>_<n>`, the n-th column of that
|
|
1538
|
+
type in a stage's rows: a decimal(10,2) and a decimal(12,2) never share a column, and nothing is
|
|
1539
|
+
cast; the slots a stage does not use are null, which Delta stores for nothing). A step's rows are
|
|
1540
|
+
written once under its `_key` and `_stage` and read back by them (the step reads its own rows as they
|
|
1541
|
+
were written, whatever the store does after, and never another writer's), then deleted by `_key`.
|
|
1542
|
+
Delta keeps a deleted row's file until `VACUUM`; rows a killed step left are deleted by
|
|
1543
|
+
`clear_stale_stage` when a store opens."""
|
|
1544
|
+
import hashlib
|
|
1545
|
+
import time
|
|
1546
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
1547
|
+
quote = get_dialect_instance(self.dialect).quote
|
|
1548
|
+
literal = get_dialect_instance(self.dialect).literal
|
|
1549
|
+
if not getattr(self, "_stale_cleared", False): # the first step of a connection, which can write
|
|
1550
|
+
self._stale_cleared = True
|
|
1551
|
+
self.clear_stale_stage(store)
|
|
1552
|
+
described = [(n, t) for n, t, *_ in self.execute(f"DESCRIBE QUERY {rows}").fetchall()]
|
|
1553
|
+
table = f"{store}_stage"
|
|
1554
|
+
picks, slots, used, wanted = [], [], {}, {}
|
|
1555
|
+
for n, t in described:
|
|
1556
|
+
try:
|
|
1557
|
+
self._ddl_type(t)
|
|
1558
|
+
except ValueError as exc:
|
|
1559
|
+
from prograph.plan.ir.cypher_write import CypherWriteRefusedError
|
|
1560
|
+
raise CypherWriteRefusedError(
|
|
1561
|
+
f"{str(exc)!r}, a map key or field name or an empty map, cannot be held in a Delta write "
|
|
1562
|
+
f"step's rows (Delta refuses an empty map and a name with a space or any of , ; {{ }} ( ) = "
|
|
1563
|
+
f": < >): rename the key, or write the value in a query of its own. Nothing was written") from exc
|
|
1564
|
+
# A column holding only NULL has no type for a table: held as text (a null of any type reads as one).
|
|
1565
|
+
void = t.lower() in ("void", "null")
|
|
1566
|
+
ddl = "string" if void else self._ddl_type(t)
|
|
1567
|
+
tag = hashlib.sha256(ddl.encode()).hexdigest()[:8]
|
|
1568
|
+
used[tag] = used.get(tag, 0) + 1
|
|
1569
|
+
slot = f"s_{tag}_{used[tag]}"
|
|
1570
|
+
wanted[slot] = ddl
|
|
1571
|
+
slots.append(slot)
|
|
1572
|
+
picks.append(f"CAST({quote(n)} AS STRING)" if void else quote(n))
|
|
1573
|
+
for tries in range(20): # two writers making the table at once: one's create fails until the other's lands
|
|
1574
|
+
try:
|
|
1575
|
+
self.execute(f"CREATE TABLE IF NOT EXISTS {table} (_key STRING NOT NULL, _stage INT NOT NULL, "
|
|
1576
|
+
f"_at TIMESTAMP NOT NULL) USING delta")
|
|
1577
|
+
break
|
|
1578
|
+
except Exception: # noqa: BLE001 - made by another writer meanwhile, or a real failure after the last try
|
|
1579
|
+
time.sleep(0.25)
|
|
1580
|
+
if self.has_table(table):
|
|
1581
|
+
break
|
|
1582
|
+
if tries == 19:
|
|
1583
|
+
raise
|
|
1584
|
+
self._stage_columns(table, wanted)
|
|
1585
|
+
k = literal(key)
|
|
1586
|
+
write = (f"INSERT INTO {table} (_key, _stage, _at{''.join(', ' + c for c in slots)}) "
|
|
1587
|
+
f"SELECT {k}, {index}, now(){''.join(', ' + c for c in picks)} FROM ({rows}) AS _r")
|
|
1588
|
+
read = (f"(SELECT {', '.join(f'{c} AS {quote(n)}' for c, (n, _) in zip(slots, described))} FROM {table} "
|
|
1589
|
+
f"WHERE _key = {k} AND _stage = {index})")
|
|
1590
|
+
return read, write, f"DELETE FROM {table} WHERE _key = {k}"
|
|
1591
|
+
|
|
1592
|
+
def clear_stale_stage(self, store: str) -> None:
|
|
1593
|
+
"""Delete the staged rows of `store` older than `STAGE_KEEP` (a step killed before its cleanup), where
|
|
1594
|
+
its staging table exists. Never fatal: a read-only connection or a concurrent writer leaves them."""
|
|
1595
|
+
try:
|
|
1596
|
+
if self.has_table(f"{store}_stage"):
|
|
1597
|
+
self.execute(f"DELETE FROM {store}_stage WHERE _at < now() - INTERVAL '{self.STAGE_KEEP}' SECOND")
|
|
1598
|
+
except Exception: # noqa: BLE001 - the next opening tries again
|
|
1599
|
+
pass
|
|
1600
|
+
|
|
1601
|
+
def scope(self, attempt: str) -> None:
|
|
1602
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
1603
|
+
literal = get_dialect_instance(self.dialect).literal
|
|
1604
|
+
self.session_statement(
|
|
1605
|
+
f"CREATE OR REPLACE TEMPORARY VIEW {OWN_ATTEMPT} AS SELECT {literal(attempt)} AS attempt")
|
|
1606
|
+
|
|
1607
|
+
def session_statement(self, sql: str) -> None:
|
|
1608
|
+
"""`sql` on the connection's own session (a temporary view made there is the one statements read)."""
|
|
1609
|
+
self.statement(sql)
|
|
1610
|
+
|
|
1611
|
+
def is_conflict(self, error: BaseException) -> bool:
|
|
1612
|
+
text = f"{type(error).__name__} {error}"
|
|
1613
|
+
return ("Concurrent" in text or "DELTA_CONCURRENT" in text or "ConcurrentModification" in text
|
|
1614
|
+
or "MetadataChanged" in text or "DELTA_METADATA_CHANGED" in text)
|
|
1615
|
+
|
|
1616
|
+
|
|
1617
|
+
class SparkStore(DeltaStore):
|
|
1618
|
+
"""The store on a Spark session (`spark.sql`; Delta Lake configured on it): one session per writer, as the
|
|
1619
|
+
own-attempt view is the session's. Each statement runs as the engine's own do (`statement_session`: a
|
|
1620
|
+
clone with ANSI mode on), so a step's rows are computed as a read of them is; a cancel is
|
|
1621
|
+
`cancelJobGroup` of the group the statements run in."""
|
|
1622
|
+
dialect = "spark"
|
|
1623
|
+
|
|
1624
|
+
def __init__(self, raw: Any) -> None:
|
|
1625
|
+
self._group = "prograph-store-" + uuid.uuid4().hex[:12]
|
|
1626
|
+
super().__init__(raw)
|
|
1627
|
+
|
|
1628
|
+
def statement(self, sql: str, arrow: bool = False) -> Any:
|
|
1629
|
+
from prograph.graph.ansi_session import statement_session
|
|
1630
|
+
from prograph.graph.spark_graph import _spark_arrow
|
|
1631
|
+
self.raw.sparkContext.setJobGroup(self._group, "prograph owned-store step", True)
|
|
1632
|
+
with statement_session(self.raw) as session:
|
|
1633
|
+
frame = session.sql(sql)
|
|
1634
|
+
if arrow:
|
|
1635
|
+
return _SparkRows([], _spark_arrow(frame))
|
|
1636
|
+
return _SparkRows([tuple(r) for r in frame.collect()] if frame.columns else [])
|
|
1637
|
+
|
|
1638
|
+
def answer(self, sql: str, digits: Sequence[Any] = ()) -> Any:
|
|
1639
|
+
return self.statement(sql, arrow=True).fetch_arrow_table()
|
|
1640
|
+
|
|
1641
|
+
def session_statement(self, sql: str) -> None:
|
|
1642
|
+
self.raw.sql(sql) # the session itself: each statement's clone copies its temporary views
|
|
1643
|
+
|
|
1644
|
+
def columns(self, table: str) -> List[str]:
|
|
1645
|
+
return [c.lower() for c in self.raw.sql(f"SELECT * FROM {table} LIMIT 0").columns]
|
|
1646
|
+
|
|
1647
|
+
def interrupt(self) -> None:
|
|
1648
|
+
self.raw.sparkContext.cancelJobGroup(self._group)
|
|
1649
|
+
|
|
1650
|
+
|
|
1651
|
+
class DatabricksStore(DeltaStore):
|
|
1652
|
+
"""The store on a Databricks SQL warehouse connection (`databricks.sql`), a cursor per statement, cancelled
|
|
1653
|
+
by `cursor.cancel`. Written to the same seam as `SparkStore` and run on a warehouse (the temporary view, the
|
|
1654
|
+
literal inlining, the conflict errors); the cancel is unverified there. A step takes about 8 to 11 seconds
|
|
1655
|
+
(each statement is a round trip), so writes suit a few steps, not a stream."""
|
|
1656
|
+
dialect = "databricks_sql" # the engine's name for a SQL warehouse
|
|
1657
|
+
|
|
1658
|
+
def __init__(self, raw: Any) -> None:
|
|
1659
|
+
self._running: Any = None
|
|
1660
|
+
super().__init__(raw)
|
|
1661
|
+
|
|
1662
|
+
def statement(self, sql: str) -> Any:
|
|
1663
|
+
cursor = self.raw.cursor()
|
|
1664
|
+
self._running = cursor
|
|
1665
|
+
try:
|
|
1666
|
+
cursor.execute(sql)
|
|
1667
|
+
return _Rows(cursor.fetchall_arrow()) if cursor.description else _Rows(None)
|
|
1668
|
+
finally:
|
|
1669
|
+
self._running = None
|
|
1670
|
+
cursor.close()
|
|
1671
|
+
|
|
1672
|
+
def answer(self, sql: str, digits: Sequence[Any] = ()) -> Any:
|
|
1673
|
+
return self.statement(sql).fetch_arrow_table()
|
|
1674
|
+
|
|
1675
|
+
def columns(self, table: str) -> List[str]:
|
|
1676
|
+
return [c.lower() for c in self.statement(f"SELECT * FROM {table} LIMIT 0").fetch_arrow_table().column_names]
|
|
1677
|
+
|
|
1678
|
+
def interrupt(self) -> None:
|
|
1679
|
+
cursor = self._running
|
|
1680
|
+
if cursor is not None:
|
|
1681
|
+
cursor.cancel()
|
|
1682
|
+
|
|
1683
|
+
|
|
1684
|
+
def _inlined(sql: str, parameters: Sequence[Any], literal: Any) -> str:
|
|
1685
|
+
"""`sql` with each `?` outside a quoted string written as `literal` of its parameter. Only the store's own
|
|
1686
|
+
SQL comes here ('' quoting, no comment, no `?` operator); one whose `?` count is not the parameters' is
|
|
1687
|
+
refused rather than filled wrong."""
|
|
1688
|
+
out, n, quote = [], 0, None
|
|
1689
|
+
for ch in sql:
|
|
1690
|
+
if quote:
|
|
1691
|
+
if ch == quote:
|
|
1692
|
+
quote = None
|
|
1693
|
+
elif ch in ("'", '"', "`"):
|
|
1694
|
+
quote = ch
|
|
1695
|
+
elif ch == "?":
|
|
1696
|
+
if n >= len(parameters):
|
|
1697
|
+
raise ValueError(f"the owned store's SQL has more ? than its {len(parameters)} parameters: {sql[:80]!r}")
|
|
1698
|
+
out.append(literal(parameters[n]))
|
|
1699
|
+
n += 1
|
|
1700
|
+
continue
|
|
1701
|
+
out.append(ch)
|
|
1702
|
+
if n != len(parameters):
|
|
1703
|
+
raise ValueError(f"the owned store's SQL has {n} ? outside quotes for {len(parameters)} parameters: {sql[:80]!r}")
|
|
1704
|
+
return "".join(out)
|
|
1705
|
+
|
|
1706
|
+
|
|
1707
|
+
def _numbered(sql: str, count: int) -> str:
|
|
1708
|
+
"""`sql` with each `?` parameter outside a quoted string or name written `$1`, `$2`, ... (PostgreSQL's).
|
|
1709
|
+
Only the store's own SQL comes here: strings quoted with '' (an E'' string's backslash escapes too, as the
|
|
1710
|
+
PostgreSQL dialect writes every string), and outside them no comment, no dollar quote and no `?`
|
|
1711
|
+
operator. SQL it cannot read that way, or whose `?` are not the `count` parameters, is refused rather
|
|
1712
|
+
than numbered wrong."""
|
|
1713
|
+
out, n, quote, before, escaped, skip = [], 0, None, " ", False, False
|
|
1714
|
+
for i, ch in enumerate(sql):
|
|
1715
|
+
if quote:
|
|
1716
|
+
if skip:
|
|
1717
|
+
skip = False
|
|
1718
|
+
elif escaped and ch == "\\":
|
|
1719
|
+
skip = True # in an E'' string a backslash escapes the next character
|
|
1720
|
+
elif ch == quote:
|
|
1721
|
+
quote, escaped = None, False
|
|
1722
|
+
elif ch in ("'", '"'):
|
|
1723
|
+
escaped = ch == "'" and before in "eE" and (i < 2 or not (sql[i - 2].isalnum() or sql[i - 2] == "_"))
|
|
1724
|
+
quote = ch
|
|
1725
|
+
elif ch == "$" or sql.startswith("--", i) or sql.startswith("/*", i):
|
|
1726
|
+
raise ValueError("the owned store numbers ? parameters only in its own SQL (no comment or dollar "
|
|
1727
|
+
f"quote): {sql[:80]!r}")
|
|
1728
|
+
elif ch == "?":
|
|
1729
|
+
n += 1
|
|
1730
|
+
out.append(f"${n}")
|
|
1731
|
+
before = ch
|
|
1732
|
+
continue
|
|
1733
|
+
out.append(ch)
|
|
1734
|
+
before = ch
|
|
1735
|
+
if n != count:
|
|
1736
|
+
raise ValueError(f"the owned store's SQL has {n} ? outside quotes for {count} parameters: {sql[:80]!r}")
|
|
1737
|
+
return "".join(out)
|