prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/store.py ADDED
@@ -0,0 +1,1737 @@
1
+ """The owned store (one-plan 2.10): the statements ProGraph writes, in tables it owns, in the mapped data's
2
+ engine. This module holds the table, its effective view and the write step; the generated mapping that lets
3
+ queries read it, and Cypher's writes, are built on it.
4
+
5
+ - **One append-only statement table.** Each row is one statement: `graph`, `subject`, `predicate`, a tagged
6
+ object (`o_kind` iri or literal, `o_text`, `o_datatype`, `o_lang`), `edge_id` (keeps parallel relationships
7
+ and a relationship's properties apart), `op` (assert or retract), and provenance (`writer`, `step`, `written_at`).
8
+ Rows are never rewritten in place.
9
+ - **One seq per write step.** A step reads `max(seq)` and writes `max + 1` on all its rows, `pos` its place
10
+ in the step. For each statement the op of the highest (seq, pos) wins (`effective_sql`).
11
+ - **Idempotent steps.** A step carries a key (the caller's, else the run id); a key already applied appends
12
+ nothing, and a created element's IRI is derived from the key and its position (`mint`), so a retry mints
13
+ nothing twice.
14
+ - **Per-store namespace.** Minted IRIs are `{base}_/{store}/n/{digest}`, `store` a UUID kept with the store.
15
+
16
+ On DuckDB and PostgreSQL (one database transaction per step, through the store's connection seam,
17
+ `store_connection`; on DuckDB the statements table gets one INSERT per step, `OWN_ROWS`). On Delta (Spark, Databricks) a statement commits alone, so a step is optimistic: its
18
+ phases append under an `attempt`, a marker row (`op = 'commit'`, `seq` = the highest marked seq read at the
19
+ step's start + 1) is appended last and only if no other marker landed since, and the effective view shows
20
+ only marked attempts, each statement's seq taken from its attempt's marker. A step that lost the race redoes
21
+ whole (`Store.REDO_LIMIT`), so seq order is commit order and no update is lost.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import re
27
+ import uuid
28
+ from dataclasses import dataclass
29
+ from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
30
+
31
+ from prograph.rules import home, module_home
32
+
33
+ module_home("owned-store", "The owned store: ProGraph's statement table, its effective view and its write step",
34
+ terms=("owned store", "statement table", "effective view", "write step", "minted IRI",
35
+ "idempotency key", "retract", "commit marker", "optimistic step", "attempt", "redo"))
36
+
37
+ ASSERT, RETRACT = "assert", "retract"
38
+ COMMIT = "commit" # a marker's op (Delta): the step's attempt is committed, at this row's seq
39
+ # The seq an attempt's own rows read inside their step (no marker yet): after every marked seq.
40
+ OWN_SEQ = 10 ** 9
41
+ # The Delta temporary view holding the running step's attempt (one row; empty outside a step). A view, not a
42
+ # session variable: the engine runs each statement in a clone of the session (`ansi_session`), which carries
43
+ # temporary views and not variables.
44
+ OWN_ATTEMPT = "prograph_own_attempt"
45
+ # DuckDB: the prefix of the temporary view of a running step's own rows (`Store.own_view`) and of its phases'
46
+ # tables. DuckDB 1.4-1.5 loses the rows of an INSERT of a full row group (122,880 rows) after an INSERT of none
47
+ # into one table in one transaction, and 2.0 reads such a table wrong in the transaction after its second
48
+ # INSERT (duckdb/duckdb#28839), so a step writes the statements table with one INSERT (`Store.write_select`).
49
+ OWN_ROWS = "_prograph_own"
50
+ # Delta: the `pos` a phase starts at is its index times this (a step's phases are Delta commits of their own,
51
+ # and an INSERT says how many rows it wrote nowhere), so `pos` keeps the phases' order and a statement's
52
+ # rows within one.
53
+ PHASE_STRIDE = 1 << 32
54
+ IRI, LITERAL = "iri", "literal"
55
+ # The writers' graphs (one-plan 2.10: one graph per writer).
56
+ CYPHER_GRAPH = "urn:prograph:store:cypher"
57
+ RL_GRAPH = "urn:prograph:store:rl"
58
+ PROACT_GRAPH = "urn:prograph:store:proact"
59
+ # The store's own predicates (`INTERNAL` their prefix): what the property-graph view needs and RDF has no
60
+ # word for. They are statements like any other but stay out of the generated mapping, so SPARQL never sees them.
61
+ INTERNAL = "urn:prograph:store:"
62
+ ELEMENT = INTERNAL + "element" # (a created node, ELEMENT, "node"): the node exists, labelled or not
63
+ REL_TYPE = INTERNAL + "type" # (a relationship's edge id, REL_TYPE, its type's name as Cypher wrote it)
64
+ RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
65
+ # A list property's datatype: `LIST_DATATYPE` + its elements' XSD local name (`list_datatype`), the text a
66
+ # JSON array of the elements. One datatype per element type, so a key's values keep one type (a column's).
67
+ LIST_DATATYPE = INTERNAL + "list:"
68
+ # A temporal property's datatype (rule cypher-writes): a date's xsd:date, its text the ISO 8601 date; a local
69
+ # time's, time's, local date-time's, date-time's and duration's `TEMPORAL_DATATYPE` + its kind, its text a JSON
70
+ # object of the integer parts a source struct of them has (`cypher_temporal.stored_text`), exactly, in one form on every engine.
71
+ TEMPORAL_DATATYPE = INTERNAL + "temporal:"
72
+
73
+
74
+ def list_datatype(element: str) -> str:
75
+ """The datatype of a list property whose elements are of datatype `element`: an XSD datatype, or a
76
+ temporal's (`TEMPORAL_DATATYPE`)."""
77
+ from prograph.schema.xsd import XSD
78
+ if element.startswith(TEMPORAL_DATATYPE):
79
+ return LIST_DATATYPE + element[len(INTERNAL):]
80
+ return LIST_DATATYPE + element[len(XSD):]
81
+
82
+
83
+ def list_element(datatype: Optional[str]) -> Optional[str]:
84
+ """A list property's elements' datatype (`list_datatype`'s `element`); None where `datatype` is no list's."""
85
+ if datatype is None or not datatype.startswith(LIST_DATATYPE):
86
+ return None
87
+ from prograph.schema.xsd import XSD
88
+ rest = datatype[len(LIST_DATATYPE):]
89
+ return INTERNAL + rest if (INTERNAL + rest).startswith(TEMPORAL_DATATYPE) else XSD + rest
90
+
91
+
92
+ def temporal_datatype(cypher_datatype: str) -> str:
93
+ """The datatype the store holds a Cypher temporal of slot datatype `cypher_datatype` in (`TEMPORAL_DATATYPE`;
94
+ a date xsd:date)."""
95
+ from prograph.schema.xsd import DATE
96
+ kind = cypher_datatype.rsplit(":", 1)[-1]
97
+ return DATE if kind == "date" else TEMPORAL_DATATYPE + kind
98
+
99
+
100
+ # The columns of a write step's statements (`write_select`), in the table's order.
101
+ STATEMENT_COLUMNS = ("graph", "subject", "predicate", "o_kind", "o_text", "o_datatype", "o_lang", "edge_id")
102
+
103
+
104
+ @dataclass(frozen=True)
105
+ class Statement:
106
+ """One statement to write: the object an IRI (`kind` iri) or a literal's text with its datatype or
107
+ language; `op` assert or retract."""
108
+ graph: str
109
+ subject: str
110
+ predicate: str
111
+ object: str
112
+ kind: str = LITERAL
113
+ datatype: Optional[str] = None
114
+ lang: Optional[str] = None
115
+ edge_id: Optional[str] = None
116
+ op: str = ASSERT
117
+
118
+ def __post_init__(self) -> None:
119
+ if self.kind not in (IRI, LITERAL):
120
+ raise ValueError(f"an object kind is {IRI!r} or {LITERAL!r}, not {self.kind!r}")
121
+ if self.op not in (ASSERT, RETRACT):
122
+ raise ValueError(f"an op is {ASSERT!r} or {RETRACT!r}, not {self.op!r}")
123
+
124
+
125
+ class StepLostRace(Exception):
126
+ """A write step on Delta lost its race (another writer's marker landed after it read the store) and the
127
+ caller plans again (`Store.write_select(replan=True)`): the SQL it was given was lowered against the store as
128
+ it was (labels, types), which the other writer may have changed. Caught by the query's run, which plans and
129
+ writes again; never reaches a user."""
130
+
131
+
132
+ class Store:
133
+ """The owned store on a DuckDB, an ADBC PostgreSQL, a Spark session or a Databricks SQL connection
134
+ (`store_connection`; the engine's own): `table` the statements, `table`_steps the applied step keys, `table`_meta the store's UUID."""
135
+
136
+ def __init__(self, connection: Any, table: str = "prograph_statements",
137
+ base_iri: str = "http://example.org/data/") -> None:
138
+ # `connection` the caller's (the engine's own); `db` the store's side of it (`store_connection`).
139
+ self.connection, self.table, self.base_iri = connection, table, base_iri
140
+ self.db = store_connection(connection)
141
+ self.dialect = self.db.dialect
142
+ self._ensured: set = set() # the `_answers` and `_digests` tables made (`_ensure`)
143
+ self._schema: Any = None # the schema the store is read beside (`attach`)
144
+ self._create()
145
+ self.store_id = self._store_id()
146
+ # The generated maps' version: it changes when a write step changes what the store holds
147
+ # (`Profile`: a predicate, object kind, datatype or language, a class, a type first held, or last
148
+ # retracted).
149
+ self._version, self._maps = 0, None
150
+ # A premise view (`premises`): the generated maps read another relation for the length of a block.
151
+ self._premises: Optional[Tuple[str, Tuple[tuple, ...], Tuple[tuple, ...], int]] = None
152
+ self._derived_maps: Any = None # (version, maps): `derived_maps`
153
+ self._joined: Optional[str] = None # the key of a step that joined a transaction the caller holds
154
+ # DuckDB (`StoreConnection.one_insert`): the temporary view of the running step's own rows, which its
155
+ # phases write to tables of their own and the statements table gets in one INSERT at the step's end;
156
+ # `_own` while a query that writes is planned and run (`writing`, `read_own`), so its reads after a
157
+ # write see the step's rows through it.
158
+ import hashlib
159
+ self.own_view = f"{OWN_ROWS}_{hashlib.sha1(table.encode('utf-8')).hexdigest()[:12]}"
160
+ self._writing, self._own = False, False
161
+ self._profile = self._read_profile(f"({self.effective_sql()})")
162
+ self._seen_seq = self._max_seq()
163
+ # The alias table's current generation (owl:sameAs, `plan.ir.identity`): a plan reads it; a new one
164
+ # changes the store's version, so every plan is made again.
165
+ from prograph.plan.ir.identity import read_state
166
+ self._alias = read_state(self)
167
+
168
+ # How many times a step on Delta reads the store again and redoes itself after losing its race to another
169
+ # writer's marker, before `StoreWriteConflictError`.
170
+ REDO_LIMIT = 20
171
+
172
+ def max_seq_sql(self) -> str:
173
+ """SQL of the highest committed seq (a marker's, on Delta; every row's elsewhere), one row `m`."""
174
+ where = f" WHERE op = '{COMMIT}'" if self.db.optimistic else ""
175
+ return f"SELECT coalesce(max(seq), 0) AS m FROM {self.table}{where}"
176
+
177
+ def _max_seq(self) -> int:
178
+ return self.db.execute(self.max_seq_sql()).fetchone()[0]
179
+
180
+ def _visible(self) -> str:
181
+ """The statements a reader sees, as a relation of the table's columns (seq as committed): the table
182
+ itself, or on Delta the rows of marked attempts (one snapshot of the table: a marker is the last
183
+ row of its step, so a snapshot holding it holds the whole step) and the running step's own."""
184
+ t = self.table
185
+ if not self.db.optimistic:
186
+ # DuckDB, in a query that writes: the statements and the step's own rows (`read_own`).
187
+ return f"(SELECT * FROM {t} UNION ALL SELECT * FROM {self.own_view})" if self._own else t
188
+ cols = ", ".join(f"p.{c}" for c in ("pos", "graph", "subject", "predicate", "o_kind", "o_text",
189
+ "o_datatype", "o_lang", "edge_id", "op", "writer", "step",
190
+ "written_at"))
191
+ return (f"(WITH _pg_s AS (SELECT * FROM {t}) SELECT coalesce(m.seq, {OWN_SEQ}) AS seq, p.attempt, {cols} "
192
+ f"FROM _pg_s AS p LEFT JOIN (SELECT attempt, seq FROM _pg_s WHERE op = '{COMMIT}') AS m "
193
+ f"ON p.attempt = m.attempt WHERE p.op <> '{COMMIT}' AND (m.attempt IS NOT NULL "
194
+ f"OR p.attempt IN (SELECT attempt FROM {OWN_ATTEMPT})))")
195
+
196
+ def attach(self, schema: Any) -> None:
197
+ """Read this store beside `schema`'s own maps: its generated maps join the schema's source maps
198
+ (`prograph.r2rml.schema.with_included`), after them. The schema's engine must run on this
199
+ store's connection."""
200
+ schema.__dict__["owned_store"] = self
201
+ self._schema = schema
202
+
203
+ @property
204
+ def merge_keys(self) -> Dict[str, Tuple[str, ...]]:
205
+ """The key of each label the store holds that Cypher's MERGE may create (its properties, whose values
206
+ give the node's IRI), as the schema it is read beside declares them (YAML `store: labels: <L>:
207
+ {key: [...]}`, `GraphSchema.store_keys`): every writer reads the one mapping, so two engines merging
208
+ one key meet."""
209
+ return dict(getattr(self._schema, "store_keys", None) or {})
210
+
211
+ def sync(self) -> None:
212
+ """Read what the store holds again where another writer (another `Store` over the table, another
213
+ engine) appended a step since this one last read it: one read of the highest seq."""
214
+ self.saw(self._max_seq())
215
+
216
+ def saw(self, seq: int) -> None:
217
+ """The highest committed seq as just read (`sync`, `dataset.store_now`): read what the store holds
218
+ again where it moved."""
219
+ if seq != self._seen_seq:
220
+ self._refresh()
221
+
222
+ def source_maps(self) -> Any:
223
+ """The generated mapping's source maps of its asserted statements (`r2rml`, compiled), None while the
224
+ store holds none; compiled once per version. In a `premises` block, once per premise view: the
225
+ premises' asserted statements and the run's own derivations."""
226
+ from prograph.r2rml.compile import compile_r2rml
227
+ from prograph.r2rml.parser import R2RMLParser
228
+ if self._premises is not None:
229
+ source, kinds, derived_kinds, _ = self._premises
230
+ if not kinds and not derived_kinds:
231
+ return None
232
+ return compile_r2rml(R2RMLParser().parse(self.r2rml(kinds, derived_kinds, source, stored=True)),
233
+ self.base_iri)
234
+ if not self._kinds:
235
+ return None
236
+ if self._maps is None or self._maps[0] != self._version:
237
+ self._maps = (self._version, compile_r2rml(R2RMLParser().parse(self.r2rml()), self.base_iri))
238
+ return self._maps[1]
239
+
240
+ def derived_maps(self) -> Any:
241
+ """The generated maps of the stored derived facts, in the default graph alone (`dataset.store_mapping`;
242
+ which reader sees them is `dataset.maps_seen`'s to say), None where the store holds none, or in a
243
+ `premises` block (the run's own are in `source_maps`)."""
244
+ from prograph.r2rml.compile import compile_r2rml
245
+ from prograph.r2rml.parser import R2RMLParser
246
+ if self._premises is not None or not self._profile.derived:
247
+ return None
248
+ held = self._derived_maps
249
+ if held is None or held[0] != self._version:
250
+ text = self.r2rml((), self._profile.derived, stored=True)
251
+ held = (self._version, compile_r2rml(R2RMLParser().parse(text), self.base_iri))
252
+ self._derived_maps = held
253
+ return held[1]
254
+
255
+ @property
256
+ def view_key(self) -> Any:
257
+ """What the generated maps read besides the store's version: None, the premise view's number, or
258
+ ("own", that) while they read the running step's own rows too (`read_own`)."""
259
+ key = None if self._premises is None else self._premises[3]
260
+ return ("own", key) if self._own else key
261
+
262
+ def writing(self) -> Any:
263
+ """A block in which a query that may write is planned and run: in it, `read_own` makes the store's
264
+ reads see the running step's own rows (DuckDB)."""
265
+ import contextlib
266
+
267
+ @contextlib.contextmanager
268
+ def block():
269
+ held = self._writing, self._own
270
+ self._writing = True
271
+ try:
272
+ yield self
273
+ finally:
274
+ if self._own and not held[1]: # the view `read_own` made, left on no connection
275
+ try:
276
+ self.db.execute(f"DROP VIEW IF EXISTS {self.own_view}")
277
+ except Exception: # noqa: BLE001 - an aborted transaction the caller holds drops it on rollback
278
+ pass
279
+ self._writing, self._own = held
280
+ return block()
281
+
282
+ def read_own(self) -> None:
283
+ """From here on in the `writing` block, the store is read with the running step's own rows (its
284
+ earlier phases': a MATCH or MERGE after a CREATE, a retraction after an assert), where its engine
285
+ writes a step with one INSERT (`StoreConnection.one_insert`): those rows are not in the statements
286
+ table until the step's end. Elsewhere the statements table holds them already."""
287
+ if self._writing and self.db.one_insert and not self._own:
288
+ t = self.table
289
+ self.db.execute(f"CREATE OR REPLACE TEMP VIEW {self.own_view} AS SELECT * FROM {t} WHERE false")
290
+ self._own = True
291
+
292
+ def premises(self, source: str, kinds: Iterable[tuple], derived_kinds: Iterable[tuple] = ()) -> Any:
293
+ """A block in which the generated maps read `source` (SQL of the columns `graph`, `subject`,
294
+ `predicate`, `o_kind`, `o_text`, `o_datatype`, `o_lang`) in place of the effective view, one map per
295
+ (predicate, kind, datatype, language) of `kinds` over its asserted statements and of `derived_kinds`
296
+ over its derived ones: what OWL RL's rules read (`owl_rl`, `dataset.premise_statements`)."""
297
+ import contextlib
298
+
299
+ @contextlib.contextmanager
300
+ def block():
301
+ held = self._premises
302
+ Store._views += 1
303
+ self._premises = (source, tuple(sorted(set(kinds), key=_sort_key)),
304
+ tuple(sorted(set(derived_kinds), key=_sort_key)), Store._views)
305
+ try:
306
+ yield self
307
+ finally:
308
+ self._premises = held
309
+ return block()
310
+
311
+ _views = 0
312
+
313
+ @property
314
+ def _kinds(self) -> Tuple[tuple, ...]:
315
+ return self._profile.kinds
316
+
317
+ def _read_profile(self, rows: str, params: Sequence[Any] = (), step: bool = False) -> "Profile":
318
+ """What the statements `rows` (SQL of the table's columns) hold (`Profile`), in one statement."""
319
+ from prograph.plan.ir.dataset import derived_graphs
320
+ from prograph.plan.sql.dialect import get_dialect_instance
321
+ prefix = self.minted_prefix()
322
+ sw = self.db.starts_with
323
+ lit, own = f"o_kind = '{LITERAL}'", f"NOT {sw}(predicate, '{INTERNAL}')"
324
+ derived = f"graph IN ({', '.join(get_dialect_instance(self.dialect).literal(g) for g in derived_graphs())})"
325
+ sql = (f"WITH r AS (SELECT * FROM {rows} AS _r) "
326
+ f"SELECT 'kind', predicate, o_kind, o_datatype, o_lang FROM r WHERE {own} AND NOT {derived} "
327
+ f"UNION SELECT 'derived', predicate, o_kind, o_datatype, o_lang FROM r WHERE {own} AND {derived} "
328
+ f"UNION SELECT 'class', o_text, NULL, NULL, NULL FROM r WHERE predicate = '{RDF_TYPE}' "
329
+ f"AND o_kind = '{IRI}' AND {sw}(subject, '{prefix}') "
330
+ f"UNION SELECT 'type', o_text, NULL, NULL, NULL FROM r WHERE predicate = '{REL_TYPE}' "
331
+ f"UNION SELECT 'any', NULL, NULL, NULL, NULL FROM r WHERE predicate = '{ELEMENT}' "
332
+ f"UNION SELECT 'node', c.o_text, p.predicate, p.o_datatype, NULL FROM r AS p "
333
+ f"JOIN r AS e ON e.subject = p.subject AND e.predicate = '{ELEMENT}' "
334
+ f"LEFT JOIN r AS c ON c.subject = p.subject AND c.predicate = '{RDF_TYPE}' "
335
+ f"WHERE p.{lit} AND NOT {sw}(p.predicate, '{INTERNAL}') "
336
+ f"UNION SELECT 'rel', t.o_text, p.predicate, p.o_datatype, NULL FROM r AS p "
337
+ f"JOIN r AS t ON t.subject = p.subject AND t.predicate = '{REL_TYPE}' "
338
+ f"WHERE p.{lit} AND NOT {sw}(p.predicate, '{INTERNAL}')")
339
+ rows_ = (self.db.hashed if step else self.db.execute)(sql, list(params)).fetchall()
340
+ return Profile(kinds=tuple(sorted((r[1:] for r in rows_ if r[0] == "kind"), key=_sort_key)),
341
+ derived=tuple(sorted((r[1:] for r in rows_ if r[0] == "derived"), key=_sort_key)),
342
+ classes=tuple(sorted(r[1] for r in rows_ if r[0] == "class")),
343
+ types=tuple(sorted(r[1] for r in rows_ if r[0] == "type")),
344
+ nodes=any(r[0] == "any" for r in rows_),
345
+ node_props=frozenset((r[1], r[2], r[3]) for r in rows_ if r[0] == "node"),
346
+ rel_props=frozenset((r[1], r[2], r[3]) for r in rows_ if r[0] == "rel"))
347
+
348
+ def profile(self) -> "Profile":
349
+ """What the store holds at its `version`."""
350
+ self._settle()
351
+ return self._profile
352
+
353
+ @property
354
+ def version(self) -> int:
355
+ """Changes when what the store holds changes (`Profile`)."""
356
+ self._settle()
357
+ return self._version
358
+
359
+ def _settle(self) -> None:
360
+ """After a step joined a transaction the caller held open: once that transaction has ended, or its
361
+ step key is no longer there (the caller rolled back; on a connection out of autocommit the caller is
362
+ always in a transaction), read what the store holds again, as it may not hold the step."""
363
+ if self._joined is None:
364
+ return
365
+ if self._in_transaction():
366
+ try:
367
+ if self.applied(self._joined):
368
+ return
369
+ except Exception: # noqa: BLE001 - an aborted transaction refuses reads until the caller rolls back
370
+ return
371
+ self._joined = None
372
+ self._refresh()
373
+
374
+ def _in_transaction(self) -> bool:
375
+ return self.db.in_transaction()
376
+
377
+ def _refresh(self, seq: Optional[int] = None) -> None:
378
+ """Read what the store holds again: the whole effective view, or where `seq` (a step that only
379
+ asserts) is given, that step's own rows added to what it held."""
380
+ self._seen_seq = self._max_seq()
381
+ if seq is None:
382
+ profile = self._read_profile(f"({self.effective_sql()})")
383
+ else:
384
+ profile = self._profile.with_(self._read_profile(f"(SELECT * FROM {self._visible()} AS _v WHERE seq = ?)", [seq],
385
+ step=True))
386
+ if profile != self._profile:
387
+ self._profile, self._version = profile, self._version + 1
388
+ from prograph.plan.ir.identity import read_state
389
+ alias = read_state(self)
390
+ if alias != getattr(self, "_alias", None):
391
+ self._alias, self._version = alias, self._version + 1
392
+
393
+ @property
394
+ def alias(self) -> Any:
395
+ """The alias table's current generation (`identity.AliasState`), None where none holds an alias."""
396
+ self._settle()
397
+ return self._alias
398
+
399
+ def minted_prefix(self) -> str:
400
+ """What every IRI this store mints starts with: `{base}_/{store}/n/`."""
401
+ return f"{self.base_iri}_/{self.store_id}/n/"
402
+
403
+ def _create(self) -> None:
404
+ c, t = self.db, self.table
405
+ x, stamp = c.text, c.stamp # the engine's text and timestamp types
406
+ attempt = f"attempt {x} NOT NULL, " if c.optimistic else ""
407
+ pos_type = "BIGINT" if c.optimistic else "INTEGER"
408
+ c.create(t, f"CREATE TABLE IF NOT EXISTS {t} (seq BIGINT NOT NULL, {attempt}pos {pos_type} NOT NULL, "
409
+ f"graph {x} NOT NULL, subject {x} NOT NULL, predicate {x} NOT NULL, o_kind {x} NOT NULL, "
410
+ f"o_text {x}, o_datatype {x}, o_lang {x}, edge_id {x}, op {x} NOT NULL, writer {x} NOT NULL, "
411
+ f"step {x} NOT NULL, written_at {stamp} NOT NULL)")
412
+ if c.optimistic:
413
+ c.has_column(t, "attempt") # a store made before the marker is refused by name
414
+ else:
415
+ c.create(f"{t}_steps", f"CREATE TABLE IF NOT EXISTS {t}_steps (step {x} PRIMARY KEY, seq BIGINT NOT NULL)")
416
+ c.create(f"{t}_meta", f"CREATE TABLE IF NOT EXISTS {t}_meta (store_id {x} NOT NULL)")
417
+ # `{t}_answers` and `{t}_digests` are made by the first step that keeps one (`_ensure`), never here: a
418
+ # read-only connection opens a store made before them, and a missing table reads as nothing kept.
419
+ c.index(t, "seq") # a step's own rows, and max(seq), read without a scan of the table
420
+
421
+ def _store_id(self) -> str:
422
+ meta = f"{self.table}_meta"
423
+ row = self.db.execute(f"SELECT min(store_id) FROM {meta}").fetchone()
424
+ if row is not None and row[0] is not None:
425
+ return row[0]
426
+ made = uuid.uuid4().hex
427
+ if not self.db.optimistic:
428
+ self.db.execute(f"INSERT INTO {meta} VALUES (?)", [made])
429
+ return made
430
+ # Two first writers: each appends only where the table is empty (an append that read the table
431
+ # conflicts with the other's), and the least id read afterwards is the store's.
432
+ try:
433
+ self.db.execute(f"INSERT INTO {meta} SELECT ? WHERE NOT EXISTS (SELECT 1 FROM {meta})", [made])
434
+ except Exception as exc: # noqa: BLE001
435
+ if not self.db.is_conflict(exc):
436
+ raise
437
+ return self.db.execute(f"SELECT min(store_id) FROM {meta}").fetchone()[0]
438
+
439
+ def mint(self, step: str, position: int) -> str:
440
+ """A created element's IRI, from the write step's key and the element's position in it: a retried
441
+ step mints the same IRI (`fresh.mint_iri`, rule store-mint)."""
442
+ from prograph.plan.ir.fresh import mint_iri
443
+ return mint_iri(self.minted_prefix(), self.store_id, step, str(position))
444
+
445
+ def write(self, statements: Sequence[Statement], writer: str, step: Optional[str] = None) -> Optional[int]:
446
+ """Append one write step: all of `statements` under one seq (`max(seq) + 1`), in one transaction, or
447
+ nothing where `step` (the idempotency key; a new one by default) was applied already. The step's seq,
448
+ None where it was applied before."""
449
+ step = step or uuid.uuid4().hex
450
+ c, t = self.db, self.table
451
+ if c.optimistic:
452
+ return self._write_values(statements, writer, step)
453
+ with c.step_settings(not self._in_transaction()):
454
+ return self._write_rows(statements, writer, step)
455
+
456
+ def _write_rows(self, statements: Sequence[Statement], writer: str, step: str) -> Optional[int]:
457
+ """`write` in one transaction (DuckDB, PostgreSQL): the statements in one INSERT."""
458
+ c, t = self.db, self.table
459
+ own = self._begin()
460
+ try:
461
+ c.lock(f"{t}_steps")
462
+ if c.execute(f"SELECT 1 FROM {t}_steps WHERE step = ?", [step]).fetchone() is not None:
463
+ self._end(own, "COMMIT")
464
+ return None
465
+ seq = c.execute(f"SELECT coalesce(max(seq), 0) + 1 FROM {t}").fetchone()[0]
466
+ rows = [(seq, i, s.graph, s.subject, s.predicate, s.kind, s.object, s.datatype, s.lang, s.edge_id, s.op,
467
+ writer, step) for i, s in enumerate(statements)]
468
+ if rows:
469
+ c.insert_rows(t, rows)
470
+ c.execute(f"INSERT INTO {t}_steps VALUES (?, ?)", [step, seq])
471
+ c.inserted(t, seq)
472
+ self._end(own, "COMMIT")
473
+ except Exception:
474
+ self._end(own, "ROLLBACK")
475
+ self._joined = self._joined if own else step
476
+ raise
477
+ self._joined = self._joined if own else step
478
+ self._refresh()
479
+ return seq
480
+
481
+ def _write_values(self, statements: Sequence[Statement], writer: str, step: str) -> Optional[int]:
482
+ """`write` on Delta: the statements as one VALUES select, one phase of an optimistic step."""
483
+ from prograph.plan.sql.dialect import get_dialect_instance
484
+ lit = get_dialect_instance(self.dialect).literal
485
+ rows = ", ".join("(" + ", ".join(lit(v) for v in (s.graph, s.subject, s.predicate, s.kind, s.object,
486
+ s.datatype, s.lang, s.edge_id, s.op, i)) + ")"
487
+ for i, s in enumerate(statements))
488
+ names = "graph, subject, predicate, o_kind, o_text, o_datatype, o_lang, edge_id, op, _o"
489
+ selects = [f"SELECT * FROM VALUES {rows} AS _v({names})"] if statements else []
490
+ return self.write_select(selects, writer, step)[0]
491
+
492
+ def _begin(self) -> bool:
493
+ """Begin the step's transaction; False inside a transaction the caller holds open, which the step
494
+ then joins (an error leaves it for the caller to roll back)."""
495
+ if self._in_transaction():
496
+ return False
497
+ self.db.begin()
498
+ return True
499
+
500
+ def _end(self, own: bool, how: str) -> None:
501
+ if own:
502
+ self.db.commit() if how == "COMMIT" else self.db.rollback()
503
+
504
+ # The most an applied step's answer keeps (bytes of its Arrow IPC stream); a bigger one is not kept and a
505
+ # retry of the step says so (`StoreAnswerNotKeptError`).
506
+ ANSWER_CAP = 1024 * 1024 # 1 MiB of IPC bytes: about 1.4 MB of text, inlined in one statement on Delta
507
+
508
+ def _ensure(self, suffix: str) -> None:
509
+ """Make `{table}_answers` (the answer each applied step gave: its Arrow table as base64 of an IPC stream,
510
+ NULL beyond `ANSWER_CAP`; `attempt` is the step's attempt on Delta, the marker's, and '' elsewhere) or
511
+ `{table}_digests` (the digest of the query a step ran, what a retry of its key is checked against), where
512
+ the step that keeps one finds it missing."""
513
+ if suffix in self._ensured:
514
+ return
515
+ c, t, x = self.db, self.table, self.db.text
516
+ ddl = {"answers": f"(step {x} NOT NULL, attempt {x} NOT NULL, kept {x}, size BIGINT NOT NULL)",
517
+ "digests": f"(step {x} NOT NULL, attempt {x} NOT NULL, digest {x} NOT NULL)"}[suffix]
518
+ c.create(f"{t}_{suffix}", f"CREATE TABLE IF NOT EXISTS {t}_{suffix} {ddl}")
519
+ if not self._in_transaction(): # made inside the caller's transaction, it may be rolled back
520
+ self._ensured.add(suffix)
521
+
522
+ def keep_answer(self, step: str, attempt: str, table: Any) -> None:
523
+ """Keep the answer `table` of the step `step` beside its rows (rule store-step-answer), in the step's
524
+ own transaction (on Delta, before its marker, under its `attempt`: the marker makes it the step's).
525
+ An answer over `ANSWER_CAP` is recorded as not kept."""
526
+ import base64
527
+ import pyarrow as pa
528
+ sink = pa.BufferOutputStream()
529
+ with pa.ipc.new_stream(sink, table.schema) as writer:
530
+ writer.write_table(table)
531
+ raw = sink.getvalue().to_pybytes()
532
+ self._ensure("answers")
533
+ kept = base64.b64encode(raw).decode("ascii") if len(raw) <= self.ANSWER_CAP else None
534
+ self.db.execute(f"INSERT INTO {self.table}_answers VALUES (?, ?, ?, ?)", [step, attempt, kept, len(raw)])
535
+
536
+ @staticmethod
537
+ def digest(query: str, parameters: Optional[dict] = None) -> str:
538
+ """The digest of a step's query: SHA-256 of the query text with its hint block removed, as the engine
539
+ reads it (a changed space is a different query: the safest reading) and its parameters in a canonical JSON (keys sorted, each value
540
+ tagged with its type, so 1, 1.0, True and '1' differ)."""
541
+ import hashlib
542
+ import json
543
+
544
+ def tag(v: Any) -> Any:
545
+ if hasattr(v, "tolist") and not isinstance(v, (list, tuple, dict, str, bytes)):
546
+ v = v.tolist() # a numpy scalar or array: the Python value it holds
547
+ elif hasattr(v, "to_pydatetime"):
548
+ v = v.to_pydatetime() # a pandas Timestamp
549
+ if v is None:
550
+ return ["NoneType", None]
551
+ for kind in (bool, int, float, str): # a subclass (numpy's str_, float64) by its base
552
+ if isinstance(v, kind):
553
+ return [kind.__name__, repr(float(v)) if kind is float else kind(v)]
554
+ if isinstance(v, (list, tuple)):
555
+ return ["list", [tag(x) for x in v]]
556
+ if isinstance(v, dict):
557
+ return ["map", [[str(k), tag(x)] for k, x in sorted(v.items(), key=lambda kv: str(kv[0]))]]
558
+ return [type(v).__name__, repr(v)]
559
+ text = json.dumps([query, tag(dict(parameters or {}))], sort_keys=True, ensure_ascii=True)
560
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
561
+
562
+ def keep_digest(self, step: str, attempt: str, digest: str) -> None:
563
+ """Record the digest of the query of step `step`, in the step's own transaction (on Delta under its
564
+ `attempt`: the marker makes it the step's)."""
565
+ self._ensure("digests")
566
+ self.db.execute(f"INSERT INTO {self.table}_digests VALUES (?, ?, ?)", [step, attempt, digest])
567
+
568
+ def check_digest(self, step: str, digest: Optional[str]) -> None:
569
+ """Raise `StoreStepKeyReusedError` where the applied step `step` ran another query (or the same query
570
+ with other parameters) than the one with `digest`, or where it recorded no digest (applied before the
571
+ store did). None `digest`: nothing to check."""
572
+ if digest is None:
573
+ return
574
+ t = self.table
575
+ if not self.db.has_table(f"{t}_digests"):
576
+ row = None
577
+ elif self.db.optimistic:
578
+ row = self.db.execute(
579
+ f"SELECT a.digest FROM {t}_digests AS a JOIN {t} AS m ON m.step = a.step AND "
580
+ f"m.attempt = a.attempt WHERE m.op = '{COMMIT}' AND m.step = ?", [step]).fetchone()
581
+ else:
582
+ row = self.db.execute(f"SELECT digest FROM {t}_digests WHERE step = ?", [step]).fetchone()
583
+ if row is not None and row[0] == digest:
584
+ return
585
+ from prograph.utils.exceptions import StoreStepKeyReusedError
586
+ why = ("it ran another query, or this query with other parameters" if row is not None
587
+ else "it was applied before the store recorded the query of a step, so this query cannot be "
588
+ "checked to be the one it ran")
589
+ raise StoreStepKeyReusedError(
590
+ f"the step key {step!r} was applied before and {why}: nothing was written. Use a new step key for "
591
+ f"a new query, or the same query and parameters to read the answer the step kept", step)
592
+
593
+ def kept_answer(self, step: str) -> Any:
594
+ """The answer the applied step `step` gave: its Arrow table (call only for a step that answers).
595
+ Raises `StoreAnswerNotKeptError` where it was not kept (over `ANSWER_CAP`, or the step was applied
596
+ before answers were kept), never an answer read again from the store as it is now."""
597
+ import base64
598
+ import pyarrow as pa
599
+ t = self.table
600
+ if not self.db.has_table(f"{t}_answers"):
601
+ row = None
602
+ elif self.db.optimistic:
603
+ row = self.db.execute(
604
+ f"SELECT a.kept, a.size FROM {t}_answers AS a JOIN {t} AS m ON m.step = a.step AND "
605
+ f"m.attempt = a.attempt WHERE m.op = '{COMMIT}' AND m.step = ?", [step]).fetchone()
606
+ else:
607
+ row = self.db.execute(f"SELECT kept, size FROM {t}_answers WHERE step = ?", [step]).fetchone()
608
+ if row is None or row[0] is None:
609
+ from prograph.utils.exceptions import StoreAnswerNotKeptError
610
+ why = (f"it is {row[1]} bytes and the store keeps at most {self.ANSWER_CAP}" if row is not None
611
+ else "the step was applied before the store kept answers")
612
+ raise StoreAnswerNotKeptError(
613
+ f"the write step {step!r} was applied before and its answer was not kept ({why}): the write is "
614
+ f"done and visible, and nothing was written again. Read what it wrote with a query of its own",
615
+ step)
616
+ return pa.ipc.open_stream(base64.b64decode(row[0])).read_all()
617
+
618
+ def applied(self, step: str) -> bool:
619
+ """Whether the step key `step` was applied already (on Delta: a marker of that step key)."""
620
+ if self.db.optimistic:
621
+ return self.db.execute(f"SELECT 1 FROM {self.table} WHERE op = '{COMMIT}' AND step = ?",
622
+ [step]).fetchone() is not None
623
+ return self.db.execute(f"SELECT 1 FROM {self.table}_steps WHERE step = ?", [step]).fetchone() \
624
+ is not None
625
+
626
+ def write_select(self, selects: Sequence[Any], writer: str, step: str, before: Sequence[str] = (),
627
+ answer: Optional[str] = None, after: Sequence[str] = (),
628
+ digits: Sequence[Any] = (), replan: bool = False,
629
+ digest: Optional[str] = None, writers: bool = False) -> Tuple[Optional[int], Any]:
630
+ """Append one write step whose statements are the rows of `selects` (each of the columns
631
+ `STATEMENT_COLUMNS`, then `op` and `_o`, its order within the select), set-based in the engine: an
632
+ `INSERT INTO ... SELECT` per select (on DuckDB one INSERT of them all, `_write_step`), in order, all under one seq (`pos` after the ones before, so a
633
+ later one's op wins), in one transaction, or nothing where `step` was applied already (as `write`).
634
+ In the
635
+ transaction: the statements `before` first (the step's rows written to temporary tables), and after
636
+ the insert `answer`, read whole (the write's answer: where it fails, the step is rolled back), then
637
+ `after` (the temporary tables dropped) however it ends. The step's seq and the answer's Arrow table
638
+ (read as the engine reads an answer, `digits` its decimal columns'); (None, None) where the step was
639
+ applied before. On PostgreSQL the step holds the step lock (`StoreConnection.lock`) from its start. On Delta
640
+ a step that lost its race redoes the SQL it was given (`_write_optimistic`), or with `replan` raises
641
+ `StepLostRace` for the caller to plan again. With `writers`, each select carries a `_writer` column after
642
+ `op`, each statement's writer (OWL RL's rule ids); `writer` is then the step's (a Delta marker's)."""
643
+ c = self.db
644
+ if c.optimistic:
645
+ return self._write_optimistic(selects, writer, step, before, answer, after, digits, replan, digest,
646
+ writers)
647
+ with c.step_settings(not self._in_transaction()):
648
+ return self._write_step(selects, writer, step, before, answer, after, digits, digest, writers)
649
+
650
+ def _write_step(self, selects: Sequence[Any], writer: str, step: str, before: Sequence[str],
651
+ answer: Optional[str], after: Sequence[str], digits: Sequence[Any], digest: Optional[str],
652
+ writers: bool) -> Tuple[Optional[int], Any]:
653
+ """`write_select` in one transaction (DuckDB, PostgreSQL). On PostgreSQL each select is an INSERT of
654
+ the statements table in turn, a later one reading the earlier ones' rows there. On DuckDB
655
+ (`StoreConnection.one_insert`) the table gets one INSERT at the end (duckdb/duckdb#28839, `OWN_ROWS`):
656
+ each select's rows are a temporary table of their own (one CREATE ... AS, `pos` after the rows before),
657
+ the store's reads see them through `own_view` (`read_own`), redefined after each, and one INSERT
658
+ writes them all; `own_view` is then empty again and the tables dropped."""
659
+ from prograph.execution import StatementGuard, checkpoint, committing
660
+ c, t = self.db, self.table
661
+ one = c.one_insert
662
+
663
+ def run(sql: str, parameters: Any = None) -> Any:
664
+ # Each phase starts only while the query is not cancelled.
665
+ checkpoint()
666
+ return c.execute(sql, parameters) if parameters is not None else c.execute(sql)
667
+ # A cancel interrupts the running phase (`StatementGuard`) and the step rolls back; the guard
668
+ # ends before the commit, so a step committed is never reported cancelled.
669
+ guard = StatementGuard(stop=c.interrupt)
670
+ try:
671
+ own = self._begin()
672
+ except BaseException as error:
673
+ guard.stop()
674
+ if isinstance(error, Exception):
675
+ guard.check() # an interrupted begin is the cancel (or the timeout), not DuckDB's error
676
+ raise
677
+ try:
678
+ checkpoint()
679
+ c.lock(f"{t}_steps")
680
+ if run(f"SELECT 1 FROM {t}_steps WHERE step = ?", [step]).fetchone() is not None:
681
+ guard.stop()
682
+ self._end(own, "COMMIT")
683
+ return None, None
684
+ names = ", ".join(STATEMENT_COLUMNS)
685
+ columns = f"seq, pos, {names}, op, writer, step, written_at"
686
+ empty = f"SELECT {columns} FROM {t} WHERE false"
687
+ if one: # the step's own rows: none yet
688
+ run(f"CREATE OR REPLACE TEMP VIEW {self.own_view} AS {empty}")
689
+ for sql in before:
690
+ run(sql)
691
+ seq = run(f"SELECT coalesce(max(seq), 0) + 1 FROM {t}").fetchone()[0]
692
+ tag, phases, written = uuid.uuid4().hex[:12], [], 0 # DuckDB: the phases' tables, their rows
693
+ for select_sql in selects:
694
+ # A program step: ("run", sql) a statement (a temporary table of the step's rows), ("insert",
695
+ # sql) or a plain select an insert of statements.
696
+ if isinstance(select_sql, tuple):
697
+ kind, select_sql = select_sql
698
+ if kind == "run":
699
+ run(select_sql)
700
+ continue
701
+ if not one:
702
+ run(f"INSERT INTO {t} ({columns}) "
703
+ f"SELECT ?, CAST((SELECT count(*) FROM {t} WHERE seq = ?) + ROW_NUMBER() OVER (ORDER BY _o) "
704
+ f"- 1 AS INTEGER), {names}, op, {'_writer' if writers else '?'}, ?, now() "
705
+ f"FROM ({select_sql}) AS _statements",
706
+ [seq, seq, step] if writers else [seq, seq, writer, step])
707
+ continue
708
+ # The phase's rows as the statements table types them, `pos` after the step's rows so far.
709
+ phase = f"{OWN_ROWS}_{tag}_{len(phases)}"
710
+ text = ", ".join(f"CAST({n} AS {c.text}) AS {n}" for n in STATEMENT_COLUMNS)
711
+ run(f"CREATE TEMP TABLE {phase} AS SELECT CAST(? AS BIGINT) AS seq, CAST({written} + "
712
+ f"ROW_NUMBER() OVER (ORDER BY _o) - 1 AS INTEGER) AS pos, {text}, CAST(op AS {c.text}) AS op, "
713
+ f"CAST({'_writer' if writers else '?'} AS {c.text}) AS writer, CAST(? AS {c.text}) AS step, "
714
+ f"CAST(now() AS {c.stamp}) AS written_at FROM ({select_sql}) AS _statements",
715
+ [seq, step] if writers else [seq, writer, step])
716
+ phases.append(phase)
717
+ written += run(f"SELECT count(*) FROM {phase}").fetchone()[0]
718
+ held = " UNION ALL ".join(f"SELECT {columns} FROM {p}" for p in phases)
719
+ run(f"CREATE OR REPLACE TEMP VIEW {self.own_view} AS {held}")
720
+ if one:
721
+ if written: # one INSERT of the step's rows; none where it has none
722
+ run(f"INSERT INTO {t} ({columns}) " + " UNION ALL ".join(f"SELECT {columns} FROM {p}"
723
+ for p in phases))
724
+ run(f"CREATE OR REPLACE TEMP VIEW {self.own_view} AS {empty}") # they are the table's now
725
+ for p in phases:
726
+ run(f"DROP TABLE {p}")
727
+ run(f"INSERT INTO {t}_steps VALUES (?, ?)", [step, seq])
728
+ c.inserted(t, seq)
729
+ if digest is not None:
730
+ self.keep_digest(step, "", digest)
731
+ if answer is not None:
732
+ checkpoint()
733
+ answered = c.answer(answer, digits) if answer is not None else None
734
+ if answered is not None:
735
+ self.keep_answer(step, "", answered)
736
+ for sql in after:
737
+ run(sql)
738
+ guard.stop()
739
+ committing() # cancelled: rolled back; else a cancel from here on changes nothing
740
+ self._end(own, "COMMIT")
741
+ except Exception:
742
+ guard.stop()
743
+ self._end(own, "ROLLBACK")
744
+ if not own: # the caller's transaction is aborted: it rolls back, its temporary tables with it
745
+ self._joined = step
746
+ guard.check() # cancelled, or past the deadline: that, not the interrupted statement's error
747
+ raise
748
+ self._joined = self._joined if own else step
749
+ self._after_step(seq)
750
+ return seq, answered
751
+
752
+ def _write_optimistic(self, selects: Sequence[Any], writer: str, step: str, before: Sequence[str],
753
+ answer: Optional[str], after: Sequence[str], digits: Sequence[Any],
754
+ replan: bool = False, digest: Optional[str] = None,
755
+ writers: bool = False) -> Tuple[Optional[int], Any]:
756
+ """`write_select` on Delta, where every statement commits alone. Take the highest marked seq the caller has
757
+ read (`seen`: the store's `_seen_seq`, as of the plan's `sync`);
758
+ append the step's phases under a fresh `attempt` (each phase its own Delta commit, invisible until the
759
+ marker); read the answer from them; then append the marker at seq `seen + 1`, only where no marker of a
760
+ higher seq or of this step key is there (`COMMIT`). A marker that did not land (another writer's came
761
+ first, or Delta refused the append as concurrent) means the step read a store that has changed: it
762
+ redoes whole, on the store as it is now, up to `REDO_LIMIT` times, then raises
763
+ `StoreWriteConflictError` (nothing visible); with `replan` it raises `StepLostRace` at once, as the SQL it
764
+ was given may not be the SQL of a plan made on the store as it is now. Every step is thus serialised against the marked ones: seq
765
+ order is commit order. A step key already marked writes nothing: (None, None)."""
766
+ import random
767
+ import time
768
+ from prograph.execution import StatementGuard, checkpoint, committing
769
+ from prograph.plan.sql.dialect import get_dialect_instance
770
+ from prograph.utils.exceptions import StoreWriteConflictError
771
+ c, t = self.db, self.table
772
+ lit = get_dialect_instance(self.dialect).literal
773
+ names = ", ".join(STATEMENT_COLUMNS)
774
+
775
+ def run(sql: str) -> Any:
776
+ checkpoint() # each phase starts only while the query is not cancelled
777
+ return c.execute(sql)
778
+ guard = None
779
+ try:
780
+ for tries in range(self.REDO_LIMIT):
781
+ checkpoint()
782
+ if self.applied(step):
783
+ return None, None
784
+ attempt = uuid.uuid4().hex
785
+ if not replan:
786
+ self.sync() # the store as it is: the caller's SQL was not planned on it
787
+ # The store as the caller read it: a plan is made on what `sync` last read, so a marker that
788
+ # landed since (even before this attempt began) fails this step's marker and the query is
789
+ # planned again, not run on a plan that cannot see what that writer wrote.
790
+ seen = self._seen_seq
791
+ guard = StatementGuard(stop=c.interrupt)
792
+ c.scope(attempt)
793
+ won, answered = False, None
794
+ try:
795
+ for sql in before:
796
+ run(sql)
797
+ phase = 0 # a phase's `pos` starts at its index * PHASE_STRIDE: an INSERT tells no count
798
+ for select_sql in selects:
799
+ if isinstance(select_sql, tuple):
800
+ kind, select_sql = select_sql
801
+ if kind == "run":
802
+ run(select_sql)
803
+ continue
804
+ run(f"INSERT INTO {t} (seq, attempt, pos, {names}, op, writer, step, written_at) "
805
+ f"SELECT 0, {lit(attempt)}, CAST({phase * PHASE_STRIDE} + ROW_NUMBER() OVER "
806
+ f"(ORDER BY _o) - 1 AS BIGINT), {names}, op, {'_writer' if writers else lit(writer)}, "
807
+ f"{lit(step)}, now() "
808
+ f"FROM ({select_sql}) AS _statements")
809
+ phase += 1
810
+ if digest is not None:
811
+ self.keep_digest(step, attempt, digest)
812
+ if answer is not None:
813
+ checkpoint()
814
+ answered = c.answer(answer, digits)
815
+ self.keep_answer(step, attempt, answered)
816
+ guard.stop()
817
+ committing() # cancelled: nothing is marked; else a cancel from here on changes nothing
818
+ c.execute(
819
+ f"INSERT INTO {t} (seq, attempt, pos, graph, subject, predicate, o_kind, op, writer, step, "
820
+ f"written_at) SELECT {seen + 1}, {lit(attempt)}, 0, '', '', '', '', '{COMMIT}', "
821
+ f"{lit(writer)}, {lit(step)}, now() WHERE NOT EXISTS (SELECT 1 FROM {t} WHERE "
822
+ f"op = '{COMMIT}' AND (seq > {seen} OR step = {lit(step)}))")
823
+ # An INSERT tells no count: the marker is read back (its seq and op let Delta skip every
824
+ # file but the marker's).
825
+ won = c.execute(f"SELECT 1 FROM {t} WHERE op = '{COMMIT}' AND seq = {seen + 1} "
826
+ f"AND attempt = {lit(attempt)}").fetchone() is not None
827
+ except Exception as exc: # noqa: BLE001 - a lost race is a redo; anything else is the step's error
828
+ if not c.is_conflict(exc):
829
+ raise
830
+ finally:
831
+ guard.stop()
832
+ c.scope("")
833
+ if (answered is not None or digest is not None) and not won:
834
+ for kept in ("answers", "digests"): # kept for an attempt that did not mark
835
+ try:
836
+ c.execute(f"DELETE FROM {t}_{kept} WHERE attempt = {lit(attempt)}")
837
+ except Exception: # noqa: BLE001 - never read: a marker of its attempt makes one the step's
838
+ pass
839
+ for sql in after: # the step's tables, whether it won, lost or failed
840
+ try:
841
+ c.execute(sql)
842
+ except Exception: # noqa: BLE001 - dropped again by the next step's `IF EXISTS`
843
+ pass
844
+ if won:
845
+ try:
846
+ self._after_step(seen + 1)
847
+ except Exception as exc: # noqa: BLE001 - the step is committed: say so, never "nothing happened"
848
+ from prograph.utils.exceptions import StoreStepCommittedError
849
+ raise StoreStepCommittedError(
850
+ f"the write step {step!r} committed (its marker is at seq {seen + 1}) and a statement "
851
+ f"after the commit failed, so its answer could not be read: {type(exc).__name__}. The "
852
+ f"write is done and visible; call the query again with step_key={step!r} to read the "
853
+ f"answer the step kept, without writing again", step) from exc
854
+ return seen + 1, answered
855
+ if self.applied(step): # another attempt of this key marked first: its step wrote it
856
+ return None, None
857
+ if replan:
858
+ raise StepLostRace()
859
+ time.sleep(random.uniform(0, 0.05 * (tries + 1)))
860
+ raise StoreWriteConflictError(
861
+ f"write contention: the write lost its race to other writers {self.REDO_LIMIT} times (each redo "
862
+ f"reads the store again and runs the step whole); nothing was written. Call the query again, or "
863
+ f"write from fewer sessions at once")
864
+ except Exception:
865
+ if guard is not None:
866
+ guard.check() # cancelled, or past the deadline: that, not the interrupted statement's error
867
+ raise
868
+
869
+ def _after_step(self, seq: int) -> None:
870
+ """Read what the store holds again after step `seq` committed."""
871
+ t, v = self.table, self._visible()
872
+ # The step's rows alone tell what it added where it retracted nothing and wrote only about elements
873
+ # it created (their labels and types are among its rows). A retraction may take away what the store
874
+ # held, and a statement about an element made before (a SET) is joined to that element's labels and
875
+ # type, which only the store holds: then the store is read again whole.
876
+ # Two counts over the step's rows alone (one CTE of them), no LIMIT and no OR: the second an anti join
877
+ # written as a LEFT JOIN, a hash join on every engine (PostgreSQL ran an OR's NOT EXISTS once per row
878
+ # of the step). A correlated NOT EXISTS over this view fails a Databricks warehouse's optimizer
879
+ # ([INTERNAL_ERROR], SQLSTATE XX000), which the join does not.
880
+ touched = self.db.hashed(
881
+ f"WITH _step AS (SELECT * FROM {v} AS _v WHERE seq = ?) "
882
+ f"SELECT (SELECT count(*) FROM _step WHERE op = '{RETRACT}') + (SELECT count(*) FROM _step AS s "
883
+ f"LEFT JOIN (SELECT DISTINCT subject FROM _step WHERE predicate IN ('{ELEMENT}', '{REL_TYPE}')) AS x "
884
+ f"ON x.subject = s.subject WHERE s.edge_id IS NULL AND x.subject IS NULL)",
885
+ [seq]).fetchone()[0]
886
+ self._refresh(None if touched else seq)
887
+
888
+ def effective_sql(self, graphs: Optional[Iterable[str]] = None, but: Optional[Iterable[str]] = None,
889
+ about: Optional[Tuple[str, str]] = None) -> str:
890
+ """The effective statements as SQL: for each statement (graph, subject, predicate, object, edge_id)
891
+ the op of the highest (seq, pos), kept where it is an assert; only `graphs` where given, none of
892
+ `but` where given; only those of (subject, predicate) `about` where given (read before grouping)."""
893
+ t = self.table
894
+ where = ""
895
+ if graphs is not None or but is not None or about is not None:
896
+ from prograph.plan.sql.dialect import get_dialect_instance
897
+ literal = get_dialect_instance(self.dialect).literal
898
+ tests = []
899
+ if graphs is not None:
900
+ tests.append(f"graph IN ({', '.join(literal(g) for g in graphs)})")
901
+ elif but is not None:
902
+ tests.append(f"graph NOT IN ({', '.join(literal(g) for g in but)})")
903
+ if about is not None:
904
+ tests.append(f"subject = {literal(about[0])} AND predicate = {literal(about[1])}")
905
+ where = " WHERE " + " AND ".join(tests)
906
+ # One group per statement, kept where its last assert (seq, pos as one number) is after its last
907
+ # retract. The filter an IS NOT NULL, which PostgreSQL estimates as most rows (a window's `rn = 1`
908
+ # it took for one in two hundred, and every join over the view became a nested loop).
909
+ last = "seq * 2147483648 + pos" if not self.db.optimistic else \
910
+ f"(CAST(seq AS DECIMAL(38, 0)) * {1 << 50} + pos)"
911
+ source = f"{self._visible()} AS _v" if self.db.optimistic or self._own else t
912
+ return (f"SELECT graph, subject, predicate, o_kind, o_text, o_datatype, o_lang, edge_id FROM {source}{where} "
913
+ f"GROUP BY graph, subject, predicate, o_kind, o_text, o_datatype, o_lang, edge_id "
914
+ f"HAVING CASE WHEN max(CASE WHEN op = '{ASSERT}' THEN {last} END) > "
915
+ f"coalesce(max(CASE WHEN op = '{RETRACT}' THEN {last} END), -1) THEN 1 END IS NOT NULL")
916
+
917
+ def effective(self, graphs: Optional[Iterable[str]] = None) -> List[tuple]:
918
+ """The effective statements' rows, for tests and inspection."""
919
+ return self.db.execute(self.effective_sql(graphs) + " ORDER BY 1, 2, 3, 5").fetchall()
920
+
921
+ def r2rml(self, kinds: Optional[Sequence[tuple]] = None, derived_kinds: Sequence[tuple] = (),
922
+ source: Optional[str] = None, stored: bool = False) -> str:
923
+ """The generated mapping (one-plan 2.10) as R2RML Turtle (`dataset.store_mapping`): one triples map per
924
+ predicate and kind of object the store holds (IRIs; literals of each datatype; literals of each
925
+ language), each over the effective view (an `rr:sqlQuery`); with `stored`, one per `derived_kinds` of its
926
+ stored derived facts too. Regenerated after a write step: a datatype first written then gets its map.
927
+ `kinds` and `source` (a `premises` view): the maps of those kinds over that relation instead."""
928
+ from prograph.plan.ir.dataset import store_mapping
929
+ return store_mapping(self, self._kinds if kinds is None else kinds, derived_kinds,
930
+ self.effective_sql() if source is None else source,
931
+ apart=bool(derived_kinds) if source is not None else bool(self._profile.derived),
932
+ stored=stored)
933
+
934
+
935
+ @dataclass(frozen=True)
936
+ class Profile:
937
+ """What the store's statements hold, for its generated maps and its property-graph view: each
938
+ (predicate, object kind, datatype, language) of its asserted statements (`kinds`, the store's own
939
+ predicates aside) and of its stored derived facts (`derived`, `dataset.derived_graphs`); its created
940
+ nodes' classes and its relationships' types; each (class, predicate, datatype) of a created node's
941
+ literal (`node_props`, class None; `nodes` whether it holds any created node for a node of no label) and each (type, predicate, datatype) of a
942
+ relationship's (`rel_props`)."""
943
+ kinds: Tuple[tuple, ...] = ()
944
+ derived: Tuple[tuple, ...] = ()
945
+ classes: Tuple[str, ...] = ()
946
+ types: Tuple[str, ...] = ()
947
+ nodes: bool = False
948
+ node_props: frozenset = frozenset()
949
+ rel_props: frozenset = frozenset()
950
+
951
+ def with_(self, more: "Profile") -> "Profile":
952
+ """This and `more`, held together (a step that only asserts adds to what the store held)."""
953
+ return Profile(kinds=tuple(sorted(set(self.kinds) | set(more.kinds), key=_sort_key)),
954
+ derived=tuple(sorted(set(self.derived) | set(more.derived), key=_sort_key)),
955
+ classes=tuple(sorted(set(self.classes) | set(more.classes))),
956
+ types=tuple(sorted(set(self.types) | set(more.types))),
957
+ nodes=self.nodes or more.nodes, node_props=self.node_props | more.node_props, rel_props=self.rel_props | more.rel_props)
958
+
959
+
960
+ def _sort_key(row: tuple) -> tuple:
961
+ return tuple((x is not None, x or "") for x in row)
962
+
963
+
964
+ @home("store-connection", "The owned store's connection: its statements, transaction, step lock and cancel on each engine",
965
+ terms=("store connection", "step transaction", "step lock", "LOCK TABLE", "EXCLUSIVE MODE", "ADBC",
966
+ "PostgreSQL", "DuckDB", "placeholder", "read-only transaction", "interrupt", "adbc_cancel", "Delta",
967
+ "SparkStore", "DatabricksStore"))
968
+ def store_connection(connection: Any) -> "StoreConnection":
969
+ """The owned store's side of `connection`: a DuckDB connection (`DuckDBStore`) or an ADBC PostgreSQL
970
+ one (`PostgreSQLStore`, the connection a PostgreSQL engine reads on: one driver, one type mapping).
971
+ A Spark session (`SparkStore`) or a Databricks SQL connection (`DatabricksStore`) is Delta: the store's
972
+ tables are Delta tables and a step is optimistic. Anything else is refused by name."""
973
+ if isinstance(connection, StoreConnection):
974
+ return connection
975
+ module = type(connection).__module__.split(".")[0]
976
+ if module == "pyspark":
977
+ return SparkStore(connection)
978
+ if module == "databricks":
979
+ return DatabricksStore(connection)
980
+ if hasattr(connection, "adbc_get_info"):
981
+ vendor = str((connection.adbc_get_info() or {}).get("vendor_name", "")).lower()
982
+ if "postgres" in vendor:
983
+ return PostgreSQLStore(connection)
984
+ raise TypeError(f"the owned store runs on DuckDB or PostgreSQL, not on an ADBC connection to {vendor!r}")
985
+ if type(connection).__module__.split(".")[0] in ("duckdb", "_duckdb"):
986
+ return DuckDBStore(connection)
987
+ raise TypeError(f"the owned store takes a DuckDB connection, an ADBC PostgreSQL connection "
988
+ f"(adbc-driver-postgresql), a Spark session or a Databricks SQL connection, not a "
989
+ f"{type(connection).__module__}.{type(connection).__name__}")
990
+
991
+
992
+ def engine_connection(store: Any, connection: Any, frames: bool) -> Any:
993
+ """The DuckDB connection an engine over a DuckDB reads on, beside the owned store `store`: the store's own,
994
+ since a query reads the mapped tables and the store's statements in one statement. Over `frames` (pandas
995
+ frames, `connection` None) the frames are registered on the store's connection, which is returned; over a
996
+ caller's `connection` it must be the store's. A store on another connection, or not on DuckDB, raises
997
+ `StoreConnectionError`, saying what to pass."""
998
+ from prograph.utils.exceptions import StoreConnectionError
999
+ if frames and store.db.dialect != "duckdb":
1000
+ raise StoreConnectionError(
1001
+ f"an owned store beside pandas frames must be on a DuckDB connection (the frames are read on one); "
1002
+ f"this store is on {store.db.dialect}. Pass Store(duckdb.connect()) to the engine, or put the frames "
1003
+ f"in tables of that database and pass connection= and store= on the same connection.")
1004
+ if connection is not None and store.connection is not connection:
1005
+ raise StoreConnectionError(
1006
+ "store= is on another connection than connection=: a query reads the mapped tables and the store's "
1007
+ "statements in one statement, so both must be on one connection. Pass Store(connection) built on the "
1008
+ "connection the engine reads, or leave connection= out when the data is pandas frames (the engine "
1009
+ "then reads them on the store's connection).")
1010
+ return store.connection if connection is None else connection
1011
+
1012
+
1013
+ def check_frame_name(con: Any, name: str) -> None:
1014
+ """A pandas frame's source name `name`, read on the store's DuckDB connection `con` (`engine_connection`):
1015
+ its view is a TEMP view, which cannot be in a schema, so a name with a schema is refused, and a name with a
1016
+ catalog needs one that is not a database of the connection (it is attached in memory); either would
1017
+ write into the user's own database. Raises `StoreConnectionError`."""
1018
+ from prograph.utils.exceptions import StoreConnectionError
1019
+ parts = name.split(".")
1020
+ if len(parts) == 2:
1021
+ raise StoreConnectionError(
1022
+ f"the frame {name!r} is named with a schema, which would be made in the database of the "
1023
+ f"connection the owned store is on. Name the frame with no schema (a table of the mapping), or "
1024
+ f"with a catalog, schema and table (`catalog.schema.table`), whose catalog is made in memory.")
1025
+ if len(parts) == 3:
1026
+ held = con.execute("SELECT path, database_name = current_database() FROM duckdb_databases() "
1027
+ "WHERE database_name = ?", [parts[0]]).fetchall()
1028
+ if held and (held[0][0] is not None or held[0][1]):
1029
+ raise StoreConnectionError(
1030
+ f"the frame {name!r} is named in the catalog {parts[0]!r}, a database of the connection the "
1031
+ f"owned store is on: a view of the frame would be written there. Name the frame in a catalog "
1032
+ f"that is not attached to it.")
1033
+
1034
+
1035
+ class StoreConnection:
1036
+ """What the store asks of its engine's connection (`store_connection`): a statement with `?`
1037
+ parameters (`execute`, `executemany`: a result's `fetchone`, `fetchall`, `fetch_arrow_table`), the
1038
+ write's answer read as the engine's reads read one (`answer`), the step's transaction (`begin`, `commit`,
1039
+ `rollback`, `in_transaction`), the step lock (`lock`), and `interrupt` for a cancel. `dialect` names
1040
+ and quotes as the engine's SQL does; `text` and `stamp` are the DDL's types."""
1041
+ dialect = ""
1042
+ text = "VARCHAR"
1043
+ stamp = "TIMESTAMP"
1044
+ # Delta: no transaction, no lock; a step is optimistic (`Store._write_optimistic`).
1045
+ optimistic = False
1046
+ # DuckDB: a step writes the statements table with one INSERT (`Store._write_step`, `OWN_ROWS`).
1047
+ one_insert = False
1048
+ starts_with = "starts_with" # the engine's function name
1049
+
1050
+ def __init__(self, raw: Any) -> None:
1051
+ self.raw = raw
1052
+
1053
+ def step_table(self, name: str, rows: str) -> str:
1054
+ """The statement that writes a step's rows `rows` (a select) to a table of its own, `name`."""
1055
+ return f"CREATE TEMP TABLE {name} AS {rows}"
1056
+
1057
+ def drop_step_table(self, name: str) -> str:
1058
+ return f"DROP TABLE {name}"
1059
+
1060
+ def stage(self, store: str, key: str, index: int, name: str, rows: str) -> Tuple[str, str, str]:
1061
+ """How a step stages the rows `rows` of its stage `index`: (what a statement reads them as, the statement
1062
+ that writes them, the statement that removes them). Here a temporary table `name` of the session, dropped
1063
+ after the step."""
1064
+ return name, self.step_table(name, rows), self.drop_step_table(name)
1065
+
1066
+ def scope(self, attempt: str) -> None:
1067
+ """Delta: the running step's attempt, which its own reads see unmarked (empty: none)."""
1068
+
1069
+ def is_conflict(self, error: BaseException) -> bool:
1070
+ """Whether `error` is the engine refusing an append as concurrent with another writer's (Delta)."""
1071
+ return False
1072
+
1073
+ def has_table(self, table: str) -> bool:
1074
+ """Whether `table` exists (a read of a store made before a table was added says it holds nothing)."""
1075
+ return self.execute("SELECT count(*) FROM information_schema.tables WHERE table_name = ?",
1076
+ [table]).fetchone()[0] > 0
1077
+
1078
+ def has_column(self, table: str, column: str) -> None:
1079
+ """Raise where `table` lacks `column` (Delta: a store made before the commit marker)."""
1080
+
1081
+ def execute(self, sql: str, parameters: Sequence[Any] = ()) -> Any:
1082
+ raise NotImplementedError
1083
+
1084
+ def hashed(self, sql: str, parameters: Sequence[Any] = ()) -> Any:
1085
+ """`execute` for a read that joins one step's rows with themselves on the subject (what a step added,
1086
+ `Store._after_step` and `_refresh`): a hash join is right whatever the planner estimates."""
1087
+ return self.execute(sql, parameters)
1088
+
1089
+ def executemany(self, sql: str, rows: Sequence[Sequence[Any]]) -> None:
1090
+ raise NotImplementedError
1091
+
1092
+ def answer(self, sql: str, digits: Sequence[Any] = ()) -> Any:
1093
+ return self.execute(sql).fetch_arrow_table()
1094
+
1095
+ def in_transaction(self) -> bool:
1096
+ # DuckDB says no more than this: in autocommit each statement is a transaction of its own, so two
1097
+ # reads of the id differ; inside an open one they are equal. (A BEGIN there would abort it.)
1098
+ ids = [self.execute("SELECT txid_current()").fetchone()[0] for _ in range(2)]
1099
+ return ids[0] == ids[1]
1100
+
1101
+ def begin(self) -> None:
1102
+ self.execute("BEGIN TRANSACTION")
1103
+
1104
+ def commit(self) -> None:
1105
+ self.execute("COMMIT")
1106
+
1107
+ def rollback(self) -> None:
1108
+ self.execute("ROLLBACK")
1109
+
1110
+ def step_settings(self, own: bool = True) -> Any:
1111
+ """A block holding the settings a write step runs under (`own`: the step is its own transaction, not
1112
+ one the caller holds), the connection's own put back after it."""
1113
+ import contextlib
1114
+ return contextlib.nullcontext()
1115
+
1116
+ def insert_rows(self, table: str, rows: Sequence[Sequence[Any]]) -> None:
1117
+ """`rows` (seq, pos, the statement's columns, op, writer, step) appended to the statements `table`,
1118
+ written now."""
1119
+ self.executemany(f"INSERT INTO {table} VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, now())", rows)
1120
+
1121
+ def plan_json(self, sql: str) -> Optional[str]:
1122
+ """The database's plan of `sql` as JSON text, reading no row (PostgreSQL's `EXPLAIN (VERBOSE, FORMAT
1123
+ JSON)`); None where the connection gives none."""
1124
+ return None
1125
+
1126
+ def read_only(self) -> bool:
1127
+ """Whether the connection refuses writes (a read-only PostgreSQL transaction or session); False where
1128
+ that cannot be told before a write."""
1129
+ return False
1130
+
1131
+ def lock(self, table: str) -> None:
1132
+ """Hold the step lock on `table` (the steps table) until the step's transaction ends, so two
1133
+ writers never read one `max(seq)`; raises `StoreReadOnlyError` where the transaction is read-only."""
1134
+
1135
+ def create(self, table: str, ddl: str) -> None:
1136
+ """`ddl` (a CREATE TABLE IF NOT EXISTS of `table`), where `table` is missing."""
1137
+ self.execute(ddl)
1138
+
1139
+ def index(self, table: str, column: str) -> None:
1140
+ """An index on `table`'s `column`, where the engine needs one to read a step's rows."""
1141
+
1142
+ def inserted(self, table: str, seq: int) -> None:
1143
+ """After step `seq`'s inserts into `table`, in its transaction."""
1144
+
1145
+ def interrupt(self) -> None:
1146
+ """Stop the running statement (from another thread: a cancel or the deadline)."""
1147
+
1148
+
1149
+ class DuckDBStore(StoreConnection):
1150
+ """The store on a DuckDB connection: its own `?` parameters, one writer at a time (DuckDB serializes
1151
+ them), `connection.interrupt` for a cancel. A step writes the statements table with one INSERT
1152
+ (`one_insert`) under `preserve_insertion_order = false` (`step_settings`), the two ways round DuckDB's
1153
+ lost INSERT (duckdb/duckdb#28839)."""
1154
+ dialect = "duckdb"
1155
+ one_insert = True
1156
+
1157
+ def step_settings(self, own: bool = True) -> Any:
1158
+ """`preserve_insertion_order` false for the step, in this connection's session alone (a plain SET is
1159
+ DuckDB's global setting, every connection to the database's), its value (it may be the caller's) set back
1160
+ after the step however it ends. Only where the step is its own transaction (`own`): in a transaction the
1161
+ caller holds, a failed step leaves it aborted, where no SET runs, so the setting stays as it is (the
1162
+ one INSERT is the fix; the setting a second guard). Putting it back never hides the step's error."""
1163
+ import contextlib
1164
+ read = "SELECT current_setting('preserve_insertion_order')"
1165
+
1166
+ @contextlib.contextmanager
1167
+ def block():
1168
+ if not own:
1169
+ yield
1170
+ return
1171
+ held = self.raw.execute(read).fetchone()[0]
1172
+ self.raw.execute("SET SESSION preserve_insertion_order = false")
1173
+ failed = False
1174
+ try:
1175
+ yield
1176
+ except BaseException:
1177
+ failed = True
1178
+ raise
1179
+ finally:
1180
+ try:
1181
+ # The global value again, unless the session held its own before.
1182
+ self.raw.execute("RESET SESSION preserve_insertion_order")
1183
+ if self.raw.execute(read).fetchone()[0] != held:
1184
+ self.raw.execute(f"SET SESSION preserve_insertion_order = {'true' if held else 'false'}")
1185
+ except Exception: # noqa: BLE001 - the step's own error, where there is one, is the one raised
1186
+ if not failed:
1187
+ raise
1188
+ return block()
1189
+
1190
+ def insert_rows(self, table: str, rows: Sequence[Sequence[Any]]) -> None:
1191
+ """One INSERT of all `rows`, from an Arrow table registered on the connection for it."""
1192
+ import pyarrow as pa
1193
+ names = ("seq", "pos") + STATEMENT_COLUMNS + ("op", "writer", "step")
1194
+ types = [pa.int64(), pa.int32()] + [pa.string()] * (len(names) - 2)
1195
+ data = pa.table([pa.array([r[i] for r in rows], type=ty) for i, ty in enumerate(types)], names=list(names))
1196
+ view = f"{OWN_ROWS}_{uuid.uuid4().hex[:12]}"
1197
+ self.raw.register(view, data)
1198
+ try:
1199
+ self.raw.execute(f"INSERT INTO {table} ({', '.join(names)}, written_at) "
1200
+ f"SELECT {', '.join(names)}, now() FROM {view}")
1201
+ finally:
1202
+ self.raw.unregister(view)
1203
+
1204
+ def execute(self, sql: str, parameters: Sequence[Any] = ()) -> Any:
1205
+ return self.raw.execute(sql, list(parameters)) if parameters else self.raw.execute(sql)
1206
+
1207
+ def executemany(self, sql: str, rows: Sequence[Sequence[Any]]) -> None:
1208
+ self.raw.executemany(sql, rows)
1209
+
1210
+ def answer(self, sql: str, digits: Sequence[Any] = ()) -> Any:
1211
+ from prograph.graph.drivers import duckdb_arrow_table
1212
+ return duckdb_arrow_table(self.raw, sql)
1213
+
1214
+ def interrupt(self) -> None:
1215
+ self.raw.interrupt()
1216
+
1217
+
1218
+ class _Rows:
1219
+ """A PostgreSQL statement's result, read whole (the driver's Arrow), the cursor closed."""
1220
+
1221
+ def __init__(self, table: Any) -> None:
1222
+ self.table = table
1223
+
1224
+ def fetch_arrow_table(self) -> Any:
1225
+ return self.table
1226
+
1227
+ def fetchall(self) -> List[tuple]:
1228
+ if self.table is None:
1229
+ return []
1230
+ columns = [c.to_pylist() for c in self.table.columns]
1231
+ return list(zip(*columns)) if columns else []
1232
+
1233
+ def fetchone(self) -> Optional[tuple]:
1234
+ rows = self.fetchall()
1235
+ return rows[0] if rows else None
1236
+
1237
+
1238
+ class PostgreSQLStore(StoreConnection):
1239
+ """The store on an ADBC PostgreSQL connection. `?` parameters become `$n`; each statement on a
1240
+ cursor of its own, cancelled through `cursor.adbc_cancel`. On a connection in autocommit (one a URI
1241
+ opened) the step is a `BEGIN` ... `COMMIT` of its own; one not in autocommit is the caller's
1242
+ transaction, which the step joins, and after a failed statement the driver refuses every later one
1243
+ until the caller's `connection.rollback()`, so the step runs nothing more there. The step lock is
1244
+ `LOCK TABLE <steps> IN EXCLUSIVE MODE` (readers go on reading what was committed): PostgreSQL's
1245
+ single-writer rule, which DuckDB gives for free."""
1246
+ dialect = "postgresql"
1247
+ text = "TEXT"
1248
+ stamp = "TIMESTAMPTZ"
1249
+
1250
+ def __init__(self, raw: Any) -> None:
1251
+ super().__init__(raw)
1252
+ self._running: Any = None
1253
+
1254
+ def _autocommit(self) -> bool:
1255
+ try:
1256
+ return str(self.raw.adbc_connection.get_option("adbc.connection.autocommit")).lower() == "true"
1257
+ except Exception: # noqa: BLE001 - a driver that does not say: take it as the caller's transaction
1258
+ return False
1259
+
1260
+ def _run(self, sql: str, parameters: Sequence[Any] = (), many: bool = False) -> Any:
1261
+ cursor = self.raw.cursor()
1262
+ self._running = cursor
1263
+ try:
1264
+ if parameters:
1265
+ sql = _numbered(sql, len(parameters[0]) if many else len(parameters))
1266
+ if many:
1267
+ cursor.executemany(sql, [tuple(r) for r in parameters])
1268
+ return _Rows(None)
1269
+ cursor.execute(sql, tuple(parameters)) if parameters else cursor.execute(sql)
1270
+ return _Rows(cursor.fetch_arrow_table() if cursor.description else None)
1271
+ finally:
1272
+ self._running = None
1273
+ cursor.close()
1274
+
1275
+ def execute(self, sql: str, parameters: Sequence[Any] = ()) -> Any:
1276
+ return self._run(sql, parameters)
1277
+
1278
+ def plan_json(self, sql: str) -> Optional[str]:
1279
+ # ADBC reads a result through COPY (... TO STDOUT), which an EXPLAIN cannot be inside: this one
1280
+ # statement is read without it.
1281
+ # In a transaction the caller holds, none: a failed statement there leaves the transaction in error,
1282
+ # and this driver refuses every statement after it, a ROLLBACK TO SAVEPOINT included (only the
1283
+ # connection's rollback, the caller's whole transaction, clears it). No plan: the query is not sized.
1284
+ if not self._autocommit():
1285
+ return None
1286
+ cursor = self.raw.cursor()
1287
+ try:
1288
+ cursor.adbc_statement.set_options(**{"adbc.postgresql.use_copy": "false"})
1289
+ cursor.execute(f"EXPLAIN (VERBOSE, FORMAT JSON) {sql}")
1290
+ row = cursor.fetchone()
1291
+ finally:
1292
+ cursor.close()
1293
+ return None if row is None else row[0]
1294
+
1295
+ def hashed(self, sql: str, parameters: Sequence[Any] = ()) -> Any:
1296
+ """A step's rows filtered by their seq are estimated from the table's statistics, which a step made
1297
+ since the last ANALYZE is not in: about 160 rows for a step of 30,000, and PostgreSQL then joins them
1298
+ with themselves in nested loops over a spilled CTE (90 s a step once the store held about a dozen
1299
+ steps, against 0.06 s hashed). Nested loops are off for the one statement, by `SET LOCAL`, which its
1300
+ transaction's end undoes whatever happens: the session's own setting is never changed. In the
1301
+ caller's transaction the value it had is set back after the statement; on a connection in
1302
+ autocommit the statement runs in a transaction of its own."""
1303
+ if self.in_transaction():
1304
+ before = self._run("SELECT current_setting('enable_nestloop')").fetchone()[0]
1305
+ self._run("SET LOCAL enable_nestloop = off")
1306
+ try:
1307
+ return self._run(sql, parameters)
1308
+ finally:
1309
+ try:
1310
+ self._run(f"SET LOCAL enable_nestloop = {'on' if before == 'on' else 'off'}")
1311
+ except Exception: # noqa: BLE001 - an aborted transaction: its end undoes the SET LOCAL
1312
+ pass
1313
+ self._run("BEGIN")
1314
+ try:
1315
+ self._run("SET LOCAL enable_nestloop = off")
1316
+ rows = self._run(sql, parameters)
1317
+ except BaseException:
1318
+ try:
1319
+ self._run("ROLLBACK")
1320
+ except Exception: # noqa: BLE001 - the original error is the one to raise
1321
+ pass
1322
+ raise
1323
+ self._run("COMMIT")
1324
+ return rows
1325
+
1326
+ def executemany(self, sql: str, rows: Sequence[Sequence[Any]]) -> None:
1327
+ if rows:
1328
+ self._run(sql, rows, many=True)
1329
+
1330
+ def answer(self, sql: str, digits: Sequence[Any] = ()) -> Any:
1331
+ """The write's answer as a PostgreSQL engine reads any (`postgresql_graph.typed_cursor`: numerics as
1332
+ exact decimals, a term's four columns one struct)."""
1333
+ from prograph.graph.postgresql_graph import typed_cursor
1334
+ cursor = self.raw.cursor()
1335
+ self._running = cursor
1336
+ try:
1337
+ cursor, numerics = typed_cursor(self.raw, sql, digits=digits, cursor=cursor)
1338
+ return numerics.convert(cursor.fetch_arrow_table())
1339
+ finally:
1340
+ self._running = None
1341
+ cursor.close()
1342
+
1343
+ def in_transaction(self) -> bool:
1344
+ # Read from the driver, no statement: a connection out of autocommit is always in the caller's
1345
+ # transaction (the DB-API way to hold one); one in autocommit is in none (a `BEGIN` the caller sends
1346
+ # as a statement itself is not seen). No transaction id is taken: a standby assigns none.
1347
+ return not self._autocommit()
1348
+
1349
+ def begin(self) -> None:
1350
+ self.execute("BEGIN")
1351
+
1352
+ def read_only(self) -> bool:
1353
+ return self.execute("SELECT current_setting('transaction_read_only')").fetchone()[0] == "on"
1354
+
1355
+ def lock(self, table: str) -> None:
1356
+ # Read-only first: nothing before it in the step writes or takes a transaction id.
1357
+ if self.read_only():
1358
+ from prograph.utils.exceptions import StoreReadOnlyError
1359
+ raise StoreReadOnlyError(
1360
+ "the owned store's PostgreSQL connection is read-only (a read-only transaction or session, "
1361
+ "default_transaction_read_only): a write step needs a writable connection; nothing was written")
1362
+ self.execute(f"LOCK TABLE {table} IN EXCLUSIVE MODE")
1363
+
1364
+ def has_table(self, table: str) -> bool:
1365
+ return not self.execute("SELECT to_regclass(?) IS NULL", [table]).fetchone()[0]
1366
+
1367
+ def create(self, table: str, ddl: str) -> None:
1368
+ # A read-only connection reads a store made before: CREATE (even IF NOT EXISTS) is refused there.
1369
+ if self.execute("SELECT to_regclass(?) IS NULL", [table]).fetchone()[0]:
1370
+ self.execute(ddl)
1371
+
1372
+ def index(self, table: str, column: str) -> None:
1373
+ # DuckDB's zone maps skip what a monotonic seq does not hold; PostgreSQL reads a step's rows (the
1374
+ # INSERT's count of them, a step's profile) by an index, else by a scan of the whole table.
1375
+ self.create(f"{table}_{column}", f"CREATE INDEX IF NOT EXISTS {table}_{column} ON {table} ({column})")
1376
+
1377
+ def inserted(self, table: str, seq: int) -> None:
1378
+ """The table's statistics read again (ANALYZE) where the step grew it by more than a tenth of what
1379
+ PostgreSQL last counted, or it never counted: autovacuum's rule, now rather than a minute later, so
1380
+ the step's own reads and the next ones are planned on what the table holds (planned on no statistics,
1381
+ a step's self-joins run as nested loops: 86 s for 10,000 created nodes)."""
1382
+ rows = self.execute(f"SELECT count(*) FROM {table} WHERE seq = ?", [seq]).fetchone()[0]
1383
+ known = self.execute("SELECT reltuples FROM pg_class WHERE oid = to_regclass(?)", [table]).fetchone()[0]
1384
+ if known is None or known < 0 or rows > 0.1 * known:
1385
+ self.execute(f"ANALYZE {table}")
1386
+
1387
+ def interrupt(self) -> None:
1388
+ cursor = self._running
1389
+ if cursor is not None:
1390
+ cursor.adbc_cancel()
1391
+
1392
+
1393
+ class _SparkRows:
1394
+ """A Spark statement's result, collected inside the statement's session (`SparkStore.statement`)."""
1395
+
1396
+ def __init__(self, rows: List[tuple], table: Any = None) -> None:
1397
+ self.rows, self.table = rows, table
1398
+
1399
+ def fetchall(self) -> List[tuple]:
1400
+ return list(self.rows)
1401
+
1402
+ def fetchone(self) -> Optional[tuple]:
1403
+ return self.rows[0] if self.rows else None
1404
+
1405
+ def fetch_arrow_table(self) -> Any:
1406
+ return self.table
1407
+
1408
+
1409
+ class DeltaStore(StoreConnection):
1410
+ """What the two Delta connections share: the store's tables are Delta tables (`USING delta`), a statement
1411
+ commits alone, `?` parameters are written into the SQL as the dialect's literals (Spark binds neither a
1412
+ positional nor a named parameter inside the subquery of an `INSERT ... SELECT`), a step's rows go to a
1413
+ Delta table of their own (a temporary view would read the changing store again), and the running step's
1414
+ attempt is a temporary view the effective view reads."""
1415
+ optimistic = True
1416
+ starts_with = "startswith"
1417
+ text = "STRING"
1418
+ stamp = "TIMESTAMP"
1419
+
1420
+ def __init__(self, raw: Any) -> None:
1421
+ super().__init__(raw)
1422
+ self.scope("")
1423
+
1424
+ def statement(self, sql: str) -> Any:
1425
+ raise NotImplementedError
1426
+
1427
+ def execute(self, sql: str, parameters: Sequence[Any] = ()) -> Any:
1428
+ if parameters:
1429
+ from prograph.plan.sql.dialect import get_dialect_instance
1430
+ sql = _inlined(sql, parameters, get_dialect_instance(self.dialect).literal)
1431
+ return self.statement(sql)
1432
+
1433
+ def executemany(self, sql: str, rows: Sequence[Sequence[Any]]) -> None:
1434
+ for row in rows:
1435
+ self.execute(sql, row)
1436
+
1437
+ def in_transaction(self) -> bool:
1438
+ return False
1439
+
1440
+ def begin(self) -> None:
1441
+ pass
1442
+
1443
+ def commit(self) -> None:
1444
+ pass
1445
+
1446
+ def rollback(self) -> None:
1447
+ pass
1448
+
1449
+ def create(self, table: str, ddl: str) -> None:
1450
+ self.execute(f"{ddl} USING delta")
1451
+
1452
+ def has_column(self, table: str, column: str) -> None:
1453
+ if column not in self.columns(table):
1454
+ raise ValueError(f"the owned store's table {table} was made before the commit marker (it has no "
1455
+ f"{column} column) and a Delta store is not altered: write to a new table name "
1456
+ f"(Store(connection, table=...))")
1457
+
1458
+ def has_table(self, table: str) -> bool:
1459
+ try:
1460
+ self.columns(table)
1461
+ except Exception: # noqa: BLE001 - the engine's own "table not found"
1462
+ return False
1463
+ return True
1464
+
1465
+ def columns(self, table: str) -> List[str]:
1466
+ raise NotImplementedError
1467
+
1468
+ STAGE_KEEP = 86400 # seconds a staged row may stay before the first step of a connection removes it (a step killed before its cleanup)
1469
+
1470
+ @staticmethod
1471
+ def _ddl_type(text: str) -> str:
1472
+ """`text`, a type as `DESCRIBE QUERY` writes it, as DDL: a struct field's name quoted with backticks. A
1473
+ name holding a character Delta refuses in a column name (` ,;{}()=`, a tab or a newline: a map key with
1474
+ a space), one that cannot be told from the type's own syntax (`:` or `<`), and an empty struct raise
1475
+ `ValueError` naming it, before anything is retried. A NULL-only type (`void`, which a table column
1476
+ cannot hold, nested or not) is `string`: the value read back is the same null."""
1477
+ from prograph.plan.sql.dialect import get_dialect_instance
1478
+ quote = get_dialect_instance("spark").quote
1479
+
1480
+ def parse(i: int) -> Tuple[str, int]:
1481
+ j = i
1482
+ while j < len(text) and text[j] not in "<,>(":
1483
+ j += 1
1484
+ word = text[i:j]
1485
+ if j < len(text) and text[j] == "(": # decimal(10,2), varchar(3)
1486
+ k = text.index(")", j)
1487
+ return text[i:k + 1], k + 1
1488
+ if j < len(text) and text[j] == "<":
1489
+ parts, j = [], j + 1
1490
+ if word.lower() == "struct" and text[j] == ">":
1491
+ raise ValueError("an empty map")
1492
+ while True:
1493
+ if word.lower() == "struct":
1494
+ k = text.index(":", j)
1495
+ name = text[j:k]
1496
+ if any(c in name for c in " ,;{}()=\n\t<>"):
1497
+ raise ValueError(name)
1498
+ inner, j = parse(k + 1)
1499
+ parts.append(quote(name.strip("`")) + ":" + inner)
1500
+ else:
1501
+ inner, j = parse(j)
1502
+ parts.append(inner)
1503
+ if text[j] == ">":
1504
+ return f"{word}<{','.join(parts)}>", j + 1
1505
+ j += 1 # the comma
1506
+ if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*(?: [A-Za-z_][A-Za-z0-9_.]*)*", word):
1507
+ # A leaf is a type's name, words of letters, digits and '_' (timestamp_ntz, `string collate
1508
+ # UTF8_LCASE`, `interval day to second`): `b:bigint` is a field name holding a colon.
1509
+ raise ValueError(text)
1510
+ return ("string" if word.lower() in ("void", "null") else word), j
1511
+ try:
1512
+ return parse(0)[0]
1513
+ except (IndexError, ValueError) as exc:
1514
+ raise ValueError(str(exc) if isinstance(exc, ValueError) else text) from exc
1515
+
1516
+ def _stage_columns(self, table: str, slots: Dict[str, str]) -> None:
1517
+ """Make `table`'s slot columns `slots` (name -> type) exist: those it lacks added by one ALTER. Two
1518
+ writers adding at once: one's ALTER fails (the column exists, or Delta's metadata changed); read the
1519
+ columns again and add what is still missing, a bounded number of times."""
1520
+ import time
1521
+ for tries in range(8):
1522
+ have = set(self.columns(table))
1523
+ missing = {n: t for n, t in slots.items() if n not in have}
1524
+ if not missing:
1525
+ return
1526
+ try:
1527
+ self.execute(f"ALTER TABLE {table} ADD COLUMNS ({', '.join(f'{n} {t}' for n, t in missing.items())})")
1528
+ except Exception: # noqa: BLE001 - another writer's ALTER landed first: look again
1529
+ if tries == 7:
1530
+ raise
1531
+ time.sleep(0.1 * (tries + 1))
1532
+
1533
+ def stage(self, store: str, key: str, index: int, name: str, rows: str) -> Tuple[str, str, str]:
1534
+ """Delta has no temporary table, and a managed table made and dropped for every step counts against the
1535
+ schema's table quota for days (Unity Catalog keeps a dropped table about a week), so a step stages its
1536
+ rows in the one table `{store}_stage`, never in one of its own. Its columns are `_key`, `_stage`, `_at`
1537
+ and typed slots added as steps need them (`s_<hash of the exact type>_<n>`, the n-th column of that
1538
+ type in a stage's rows: a decimal(10,2) and a decimal(12,2) never share a column, and nothing is
1539
+ cast; the slots a stage does not use are null, which Delta stores for nothing). A step's rows are
1540
+ written once under its `_key` and `_stage` and read back by them (the step reads its own rows as they
1541
+ were written, whatever the store does after, and never another writer's), then deleted by `_key`.
1542
+ Delta keeps a deleted row's file until `VACUUM`; rows a killed step left are deleted by
1543
+ `clear_stale_stage` when a store opens."""
1544
+ import hashlib
1545
+ import time
1546
+ from prograph.plan.sql.dialect import get_dialect_instance
1547
+ quote = get_dialect_instance(self.dialect).quote
1548
+ literal = get_dialect_instance(self.dialect).literal
1549
+ if not getattr(self, "_stale_cleared", False): # the first step of a connection, which can write
1550
+ self._stale_cleared = True
1551
+ self.clear_stale_stage(store)
1552
+ described = [(n, t) for n, t, *_ in self.execute(f"DESCRIBE QUERY {rows}").fetchall()]
1553
+ table = f"{store}_stage"
1554
+ picks, slots, used, wanted = [], [], {}, {}
1555
+ for n, t in described:
1556
+ try:
1557
+ self._ddl_type(t)
1558
+ except ValueError as exc:
1559
+ from prograph.plan.ir.cypher_write import CypherWriteRefusedError
1560
+ raise CypherWriteRefusedError(
1561
+ f"{str(exc)!r}, a map key or field name or an empty map, cannot be held in a Delta write "
1562
+ f"step's rows (Delta refuses an empty map and a name with a space or any of , ; {{ }} ( ) = "
1563
+ f": < >): rename the key, or write the value in a query of its own. Nothing was written") from exc
1564
+ # A column holding only NULL has no type for a table: held as text (a null of any type reads as one).
1565
+ void = t.lower() in ("void", "null")
1566
+ ddl = "string" if void else self._ddl_type(t)
1567
+ tag = hashlib.sha256(ddl.encode()).hexdigest()[:8]
1568
+ used[tag] = used.get(tag, 0) + 1
1569
+ slot = f"s_{tag}_{used[tag]}"
1570
+ wanted[slot] = ddl
1571
+ slots.append(slot)
1572
+ picks.append(f"CAST({quote(n)} AS STRING)" if void else quote(n))
1573
+ for tries in range(20): # two writers making the table at once: one's create fails until the other's lands
1574
+ try:
1575
+ self.execute(f"CREATE TABLE IF NOT EXISTS {table} (_key STRING NOT NULL, _stage INT NOT NULL, "
1576
+ f"_at TIMESTAMP NOT NULL) USING delta")
1577
+ break
1578
+ except Exception: # noqa: BLE001 - made by another writer meanwhile, or a real failure after the last try
1579
+ time.sleep(0.25)
1580
+ if self.has_table(table):
1581
+ break
1582
+ if tries == 19:
1583
+ raise
1584
+ self._stage_columns(table, wanted)
1585
+ k = literal(key)
1586
+ write = (f"INSERT INTO {table} (_key, _stage, _at{''.join(', ' + c for c in slots)}) "
1587
+ f"SELECT {k}, {index}, now(){''.join(', ' + c for c in picks)} FROM ({rows}) AS _r")
1588
+ read = (f"(SELECT {', '.join(f'{c} AS {quote(n)}' for c, (n, _) in zip(slots, described))} FROM {table} "
1589
+ f"WHERE _key = {k} AND _stage = {index})")
1590
+ return read, write, f"DELETE FROM {table} WHERE _key = {k}"
1591
+
1592
+ def clear_stale_stage(self, store: str) -> None:
1593
+ """Delete the staged rows of `store` older than `STAGE_KEEP` (a step killed before its cleanup), where
1594
+ its staging table exists. Never fatal: a read-only connection or a concurrent writer leaves them."""
1595
+ try:
1596
+ if self.has_table(f"{store}_stage"):
1597
+ self.execute(f"DELETE FROM {store}_stage WHERE _at < now() - INTERVAL '{self.STAGE_KEEP}' SECOND")
1598
+ except Exception: # noqa: BLE001 - the next opening tries again
1599
+ pass
1600
+
1601
+ def scope(self, attempt: str) -> None:
1602
+ from prograph.plan.sql.dialect import get_dialect_instance
1603
+ literal = get_dialect_instance(self.dialect).literal
1604
+ self.session_statement(
1605
+ f"CREATE OR REPLACE TEMPORARY VIEW {OWN_ATTEMPT} AS SELECT {literal(attempt)} AS attempt")
1606
+
1607
+ def session_statement(self, sql: str) -> None:
1608
+ """`sql` on the connection's own session (a temporary view made there is the one statements read)."""
1609
+ self.statement(sql)
1610
+
1611
+ def is_conflict(self, error: BaseException) -> bool:
1612
+ text = f"{type(error).__name__} {error}"
1613
+ return ("Concurrent" in text or "DELTA_CONCURRENT" in text or "ConcurrentModification" in text
1614
+ or "MetadataChanged" in text or "DELTA_METADATA_CHANGED" in text)
1615
+
1616
+
1617
+ class SparkStore(DeltaStore):
1618
+ """The store on a Spark session (`spark.sql`; Delta Lake configured on it): one session per writer, as the
1619
+ own-attempt view is the session's. Each statement runs as the engine's own do (`statement_session`: a
1620
+ clone with ANSI mode on), so a step's rows are computed as a read of them is; a cancel is
1621
+ `cancelJobGroup` of the group the statements run in."""
1622
+ dialect = "spark"
1623
+
1624
+ def __init__(self, raw: Any) -> None:
1625
+ self._group = "prograph-store-" + uuid.uuid4().hex[:12]
1626
+ super().__init__(raw)
1627
+
1628
+ def statement(self, sql: str, arrow: bool = False) -> Any:
1629
+ from prograph.graph.ansi_session import statement_session
1630
+ from prograph.graph.spark_graph import _spark_arrow
1631
+ self.raw.sparkContext.setJobGroup(self._group, "prograph owned-store step", True)
1632
+ with statement_session(self.raw) as session:
1633
+ frame = session.sql(sql)
1634
+ if arrow:
1635
+ return _SparkRows([], _spark_arrow(frame))
1636
+ return _SparkRows([tuple(r) for r in frame.collect()] if frame.columns else [])
1637
+
1638
+ def answer(self, sql: str, digits: Sequence[Any] = ()) -> Any:
1639
+ return self.statement(sql, arrow=True).fetch_arrow_table()
1640
+
1641
+ def session_statement(self, sql: str) -> None:
1642
+ self.raw.sql(sql) # the session itself: each statement's clone copies its temporary views
1643
+
1644
+ def columns(self, table: str) -> List[str]:
1645
+ return [c.lower() for c in self.raw.sql(f"SELECT * FROM {table} LIMIT 0").columns]
1646
+
1647
+ def interrupt(self) -> None:
1648
+ self.raw.sparkContext.cancelJobGroup(self._group)
1649
+
1650
+
1651
+ class DatabricksStore(DeltaStore):
1652
+ """The store on a Databricks SQL warehouse connection (`databricks.sql`), a cursor per statement, cancelled
1653
+ by `cursor.cancel`. Written to the same seam as `SparkStore` and run on a warehouse (the temporary view, the
1654
+ literal inlining, the conflict errors); the cancel is unverified there. A step takes about 8 to 11 seconds
1655
+ (each statement is a round trip), so writes suit a few steps, not a stream."""
1656
+ dialect = "databricks_sql" # the engine's name for a SQL warehouse
1657
+
1658
+ def __init__(self, raw: Any) -> None:
1659
+ self._running: Any = None
1660
+ super().__init__(raw)
1661
+
1662
+ def statement(self, sql: str) -> Any:
1663
+ cursor = self.raw.cursor()
1664
+ self._running = cursor
1665
+ try:
1666
+ cursor.execute(sql)
1667
+ return _Rows(cursor.fetchall_arrow()) if cursor.description else _Rows(None)
1668
+ finally:
1669
+ self._running = None
1670
+ cursor.close()
1671
+
1672
+ def answer(self, sql: str, digits: Sequence[Any] = ()) -> Any:
1673
+ return self.statement(sql).fetch_arrow_table()
1674
+
1675
+ def columns(self, table: str) -> List[str]:
1676
+ return [c.lower() for c in self.statement(f"SELECT * FROM {table} LIMIT 0").fetch_arrow_table().column_names]
1677
+
1678
+ def interrupt(self) -> None:
1679
+ cursor = self._running
1680
+ if cursor is not None:
1681
+ cursor.cancel()
1682
+
1683
+
1684
+ def _inlined(sql: str, parameters: Sequence[Any], literal: Any) -> str:
1685
+ """`sql` with each `?` outside a quoted string written as `literal` of its parameter. Only the store's own
1686
+ SQL comes here ('' quoting, no comment, no `?` operator); one whose `?` count is not the parameters' is
1687
+ refused rather than filled wrong."""
1688
+ out, n, quote = [], 0, None
1689
+ for ch in sql:
1690
+ if quote:
1691
+ if ch == quote:
1692
+ quote = None
1693
+ elif ch in ("'", '"', "`"):
1694
+ quote = ch
1695
+ elif ch == "?":
1696
+ if n >= len(parameters):
1697
+ raise ValueError(f"the owned store's SQL has more ? than its {len(parameters)} parameters: {sql[:80]!r}")
1698
+ out.append(literal(parameters[n]))
1699
+ n += 1
1700
+ continue
1701
+ out.append(ch)
1702
+ if n != len(parameters):
1703
+ raise ValueError(f"the owned store's SQL has {n} ? outside quotes for {len(parameters)} parameters: {sql[:80]!r}")
1704
+ return "".join(out)
1705
+
1706
+
1707
+ def _numbered(sql: str, count: int) -> str:
1708
+ """`sql` with each `?` parameter outside a quoted string or name written `$1`, `$2`, ... (PostgreSQL's).
1709
+ Only the store's own SQL comes here: strings quoted with '' (an E'' string's backslash escapes too, as the
1710
+ PostgreSQL dialect writes every string), and outside them no comment, no dollar quote and no `?`
1711
+ operator. SQL it cannot read that way, or whose `?` are not the `count` parameters, is refused rather
1712
+ than numbered wrong."""
1713
+ out, n, quote, before, escaped, skip = [], 0, None, " ", False, False
1714
+ for i, ch in enumerate(sql):
1715
+ if quote:
1716
+ if skip:
1717
+ skip = False
1718
+ elif escaped and ch == "\\":
1719
+ skip = True # in an E'' string a backslash escapes the next character
1720
+ elif ch == quote:
1721
+ quote, escaped = None, False
1722
+ elif ch in ("'", '"'):
1723
+ escaped = ch == "'" and before in "eE" and (i < 2 or not (sql[i - 2].isalnum() or sql[i - 2] == "_"))
1724
+ quote = ch
1725
+ elif ch == "$" or sql.startswith("--", i) or sql.startswith("/*", i):
1726
+ raise ValueError("the owned store numbers ? parameters only in its own SQL (no comment or dollar "
1727
+ f"quote): {sql[:80]!r}")
1728
+ elif ch == "?":
1729
+ n += 1
1730
+ out.append(f"${n}")
1731
+ before = ch
1732
+ continue
1733
+ out.append(ch)
1734
+ before = ch
1735
+ if n != count:
1736
+ raise ValueError(f"the owned store's SQL has {n} ? outside quotes for {count} parameters: {sql[:80]!r}")
1737
+ return "".join(out)