prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,612 @@
|
|
|
1
|
+
"""Cypher's writes into the owned store (one-plan 2.10; decided 2026-11-12).
|
|
2
|
+
On DuckDB and PostgreSQL (the store's connection seam, `store.store_connection`).
|
|
3
|
+
|
|
4
|
+
A query that writes is one write step. The front end (`cypher_front._Translator.create`) plans each CREATE
|
|
5
|
+
over the rows before it, each created element's IRI minted per row (`fresh.mint_cell`), and records the
|
|
6
|
+
CREATE's statement templates over that plan (`Stage`): its node's `ELEMENT` statement, an `rdf:type`
|
|
7
|
+
statement per label, a statement per property that is not null; a relationship's statement from its start
|
|
8
|
+
to its end with its edge id, its type's `REL_TYPE` statement and its properties on the edge id.
|
|
9
|
+
|
|
10
|
+
`statements_sql` lowers each stage's rows once and writes a SELECT per template over them (`STATEMENT_COLUMNS`,
|
|
11
|
+
all text); `apply` appends that as one step (`Store.write_select`: one `INSERT INTO ... SELECT` in the engine, never row by
|
|
12
|
+
row in Python). A step key the store applied already appends nothing and answers with the answer it kept.
|
|
13
|
+
|
|
14
|
+
RETURN after CREATE answers from the very rows the step wrote: `apply` writes each CREATE's rows to a
|
|
15
|
+
temporary table once, in the step's transaction, and both the INSERT and the answer read it. A retry of an
|
|
16
|
+
applied step key answers with the answer the step kept (`Store.keep_answer`), never from the query's rows
|
|
17
|
+
read again. A clause that
|
|
18
|
+
reads after an updating clause (a MATCH, a MERGE) sees the query's own writes: each stage's rows are written
|
|
19
|
+
after the inserts of the stages before it, in the step's one transaction (`apply`); on DuckDB those are the
|
|
20
|
+
phases' own tables, which the store's reads see until the step's one INSERT (`Store.read_own`).
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from dataclasses import dataclass, field
|
|
26
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
27
|
+
|
|
28
|
+
from prograph.plan.ir import expr as E
|
|
29
|
+
from prograph.plan.ir import ops as O
|
|
30
|
+
from prograph.plan.ir.facts import NotBuilt
|
|
31
|
+
from prograph.rules import module_home
|
|
32
|
+
from prograph.schema import xsd
|
|
33
|
+
from prograph.schema.naming import local_name
|
|
34
|
+
from prograph.store import StepLostRace
|
|
35
|
+
from prograph.plan.ir.slot import LIST, Bound
|
|
36
|
+
from prograph.utils.exceptions import CypherExecutionError, CypherWriteRefusedError, StoreStepCommittedError
|
|
37
|
+
|
|
38
|
+
module_home("cypher-writes", "Cypher's CREATE, SET, REMOVE, DELETE and MERGE as one write step into the owned store",
|
|
39
|
+
terms=("CREATE", "SET", "REMOVE", "write step", "updating clause", "statement template",
|
|
40
|
+
"INSERT SELECT", "read after write", "RETURN after CREATE", "retract", "SET +=",
|
|
41
|
+
"replace properties", "mapped element", "DELETE", "DETACH DELETE",
|
|
42
|
+
"DeleteConnectedNode", "MERGE", "ON CREATE SET", "ON MATCH SET", "merge key", "list property",
|
|
43
|
+
"temporal property", "store write",
|
|
44
|
+
"InvalidPropertyType", "DeletedEntityAccess"))
|
|
45
|
+
|
|
46
|
+
WRITER = "cypher"
|
|
47
|
+
# A property's datatype -> whether the store holds it, and a list's elements'; a temporal value too, in its
|
|
48
|
+
# own datatype (`store.temporal_datatype`, `cypher_temporal.stored_text`).
|
|
49
|
+
STORABLE = (xsd.INTEGER, xsd.DOUBLE, xsd.BOOLEAN, xsd.STRING)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class Template:
|
|
54
|
+
"""One statement an updating clause writes per row: `subject` a column of the stage (a node's or edge's
|
|
55
|
+
IRI); `predicate` a constant IRI; the object a column (`object_column`, an IRI's or a value's) or a
|
|
56
|
+
constant text (`object_text`); `kind` iri or literal, `datatype` a literal's; `edge` the edge id's
|
|
57
|
+
column.
|
|
58
|
+
|
|
59
|
+
A retraction (`op` retract) retracts the effective statements of the subject that it matches: those of
|
|
60
|
+
one of `predicates` (of `kind`, and `object_text` where given), or with no predicates every property
|
|
61
|
+
(a literal of a predicate not the store's own). `owned`: the subject must be an element the store
|
|
62
|
+
created (SET and REMOVE; an element a mapped table defines raises, `MAPPED`)."""
|
|
63
|
+
subject: str
|
|
64
|
+
predicate: str = ""
|
|
65
|
+
kind: str = "literal"
|
|
66
|
+
object_column: Optional[str] = None
|
|
67
|
+
object_text: Optional[str] = None
|
|
68
|
+
datatype: Optional[str] = None
|
|
69
|
+
edge: Optional[str] = None
|
|
70
|
+
ends: bool = False # a relationship's statement: its subject and object are nodes that must exist
|
|
71
|
+
op: str = "assert"
|
|
72
|
+
predicates: Tuple[str, ...] = ()
|
|
73
|
+
owned: bool = False
|
|
74
|
+
# What a retraction matches of the subject (`statements_sql`): "props", its properties (or `predicates`);
|
|
75
|
+
# "node", a node's own statements (its marker, labels, properties); "rel", a relationship's (its
|
|
76
|
+
# statement from start to end, its type, its properties); "detach", a node's relationships, theirs
|
|
77
|
+
# included; "connected", none, raising where the node has a relationship (DELETE, `CONNECTED`).
|
|
78
|
+
scope: str = "props"
|
|
79
|
+
refusal: str = "" # the mark an element a mapped table defines raises (`MAPPED` where empty)
|
|
80
|
+
when: Optional[str] = None # a boolean column of the stage: the template applies to its true rows only
|
|
81
|
+
# A `refuse` template's column of the stage: it raises where that column is not null (an element written
|
|
82
|
+
# after a DELETE of the query deleted it, `cypher_front._deleted_reads`' `__gone`).
|
|
83
|
+
gone: Optional[str] = None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
# A CREATE's relationship whose end is null (an OPTIONAL MATCH that found none): the statement raises it,
|
|
87
|
+
# so the step writes nothing (Neo4j: the whole query fails).
|
|
88
|
+
MISSING_END = "ProGraph: a node of a created relationship is missing"
|
|
89
|
+
# SET or REMOVE on an element a mapped table defines: raised by the statement, so nothing is written.
|
|
90
|
+
MAPPED = "ProGraph: SET or REMOVE on an element a mapped table defines"
|
|
91
|
+
MAPPED_DELETE = "ProGraph: DELETE of an element a mapped table defines"
|
|
92
|
+
# MERGE that found no element of a label a mapped table defines: the store does not create one.
|
|
93
|
+
MAPPED_MERGE = "ProGraph: MERGE would create an element of a label a mapped table defines"
|
|
94
|
+
# MERGE of a node of several labels, one with a declared key, where no node matches the whole pattern but a
|
|
95
|
+
# node of that label holds the key (Neo4j: the uniqueness constraint fails): nothing is written. Followed by
|
|
96
|
+
# `<label>|<key, ...>|`.
|
|
97
|
+
KEY_TAKEN = "ProGraph: MERGE of a key another node holds: "
|
|
98
|
+
# A read after the query's writes of a property they changed the type of, where an element still holds the old
|
|
99
|
+
# type (`store_view._value`): a column holds one type.
|
|
100
|
+
TWO_TYPES = "ProGraph: a property held in two types after the step's writes: "
|
|
101
|
+
# MERGE of a key one of whose values is null (openCypher raises): nothing is written.
|
|
102
|
+
NULL_KEY = "ProGraph: MERGE of a null property"
|
|
103
|
+
# A property or label of an element the query deleted, read after the DELETE (openCypher's EntityNotFound,
|
|
104
|
+
# DeletedEntityAccess): the read raises, so nothing is written.
|
|
105
|
+
DELETED_ACCESS = "ProGraph: DeletedEntityAccess"
|
|
106
|
+
# A list property holding a null (Neo4j: a list property's elements are values): nothing is written.
|
|
107
|
+
NULL_IN_LIST = "ProGraph: a list property holding null"
|
|
108
|
+
# DELETE of a node that still has a relationship (openCypher's DeleteConnectedNode): nothing is written.
|
|
109
|
+
CONNECTED = "ProGraph: DeleteConnectedNode"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@dataclass
|
|
113
|
+
class Stage:
|
|
114
|
+
"""An updating clause's rows (`plan`: a CREATE's with its created elements' columns, a SET's with its
|
|
115
|
+
new values' columns, extended) and its templates, in phases: each phase one INSERT of the step, after
|
|
116
|
+
the ones before it, so a retraction sees what an earlier phase asserted (a SET after a CREATE, a second
|
|
117
|
+
SET of one key)."""
|
|
118
|
+
plan: O.Op
|
|
119
|
+
phases: List[List[Template]] = field(default_factory=list)
|
|
120
|
+
minted: List[str] = field(default_factory=list) # its created elements' columns
|
|
121
|
+
# A SET's row number column (`row`) and the order the query states its rows in (`order`: (column,
|
|
122
|
+
# descending), a WITH ... ORDER BY's keys; none: an arbitrary order): of a property's values for one
|
|
123
|
+
# element, the last row's in that order is the one kept.
|
|
124
|
+
row: Optional[str] = None
|
|
125
|
+
order: Tuple[Tuple[str, bool], ...] = ()
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def templates(self) -> List[Template]:
|
|
129
|
+
return [t for phase in self.phases for t in phase]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@dataclass
|
|
133
|
+
class WriteStep:
|
|
134
|
+
"""What a query writes: the store, the step's key, its CREATEs' rows and templates (`Stage`, merged);
|
|
135
|
+
`answers` whether it has a RETURN (else it answers no rows)."""
|
|
136
|
+
store: Any
|
|
137
|
+
step: str
|
|
138
|
+
stages: List[Stage]
|
|
139
|
+
answers: bool
|
|
140
|
+
# Columns written when the step's rows are, as SQL over the rows' columns (`{order}` the stated order's
|
|
141
|
+
# ORDER BY): a MERGE's created column (`cypher_front._created`).
|
|
142
|
+
replaced: Dict[str, str] = field(default_factory=dict)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# The engines a query's writes run on (`apply`): one transaction per step on DuckDB and PostgreSQL; on Delta
|
|
146
|
+
# (Spark, a Databricks SQL warehouse) an optimistic step with a commit marker (`Store._write_optimistic`).
|
|
147
|
+
WRITING_DIALECTS = ("duckdb", "postgresql", "spark", "databricks_sql")
|
|
148
|
+
# The most columns an engine holds in one statement's select list (PostgreSQL's target list): a write step
|
|
149
|
+
# whose rows are wider is refused before it runs.
|
|
150
|
+
COLUMN_LIMIT = {"postgresql": 1664}
|
|
151
|
+
# The Delta engines (Spark, Databricks): a statement commits alone, and the SQL is Spark's.
|
|
152
|
+
DELTA_DIALECTS = frozenset({"spark", "databricks", "databricks_sql"})
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def arbitrary_order(dialect: str) -> str:
|
|
156
|
+
"""The window ORDER BY of a numbering that wants rows in no order: none where the engine takes none; Spark
|
|
157
|
+
and Databricks require one, so the order the rows are read."""
|
|
158
|
+
return "ORDER BY monotonically_increasing_id()" if dialect in DELTA_DIALECTS else ""
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def text_cell(e: E.Call, cells: Dict[str, Any]) -> Any:
|
|
162
|
+
"""`STORE_TEXT(x)`: a statement's text for `x`, an IRI's (`lower.lexical`) or a value's canonical
|
|
163
|
+
lexical form (`lower.native_text`, rule term-text-in-sql)."""
|
|
164
|
+
from prograph.plan.ir import lower as L
|
|
165
|
+
from prograph.plan.ir.slot import Bound, value_slot
|
|
166
|
+
cell = L.expr_cell(e.args[0], cells)
|
|
167
|
+
if cell.slot.present is Bound.NEVER:
|
|
168
|
+
return L.Cell(value_slot(xsd.STRING, Bound.MAYBE), L.NULL_TEXT)
|
|
169
|
+
if cell.slot.kinds == {LIST}:
|
|
170
|
+
return L.Cell(value_slot(xsd.STRING, Bound.MAYBE), list_text(cell))
|
|
171
|
+
if _temporal(cell.slot.datatype) and len(cell.slot.datatypes) == 1:
|
|
172
|
+
return L.Cell(value_slot(xsd.STRING, Bound.MAYBE), temporal_text(cell.slot.datatype, cell.value))
|
|
173
|
+
text = L.native_text(cell) if cell.slot.native is not None and not cell.slot.terms else L.lexical(cell)
|
|
174
|
+
return L.Cell(value_slot(xsd.STRING, Bound.MAYBE), L.fn("CAST_STRING", text))
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _temporal(datatype: Optional[str]) -> bool:
|
|
178
|
+
from prograph.plan.ir.cypher_temporal import stored_kind
|
|
179
|
+
return stored_kind(datatype) is not None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def temporal_text(datatype: str, value: Any) -> Any:
|
|
183
|
+
"""A Cypher temporal's text in a statement (rule cypher-writes): a date's ISO 8601 text, any other's
|
|
184
|
+
integer parts as a JSON object in one fixed form on every engine (`cypher_temporal.stored_text`), read
|
|
185
|
+
back exactly by the store's view as a source value of those parts (`store_view._read`)."""
|
|
186
|
+
from prograph.plan.ir import lower as L
|
|
187
|
+
from prograph.plan.ir.cypher_temporal import stored_text
|
|
188
|
+
return stored_text(datatype, value, L)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def list_text(cell: Any) -> Any:
|
|
192
|
+
"""A list's text in a statement (rule cypher-writes): the JSON array of its elements (the engine's
|
|
193
|
+
own JSON writer; a list of temporals each element's `temporal_text`, a date's quoted, joined in one
|
|
194
|
+
fixed form), read back by the store's view as a list of the elements' type (`store_view._read`). A list
|
|
195
|
+
holding a null raises (`NULL_IN_LIST`): a property's list holds values."""
|
|
196
|
+
from prograph.plan.ir import lower as L
|
|
197
|
+
from prograph.plan.ir.strings import _dialect_has
|
|
198
|
+
if not _dialect_has("STORE_LIST_TEXT"):
|
|
199
|
+
raise NotBuilt("a list property written to the owned store on this engine (DuckDB and PostgreSQL hold "
|
|
200
|
+
"them; Spark and Databricks not yet)")
|
|
201
|
+
text = L.fn("STORE_LIST_TEXT", cell.value)
|
|
202
|
+
if _temporal(cell.slot.datatype):
|
|
203
|
+
from prograph.plan.ir.cypher_temporal import CYPHER_DATE
|
|
204
|
+
from prograph.plan.sql import rel_algebra as R
|
|
205
|
+
var = R.LitExpr(value="_sv", sql="_sv")
|
|
206
|
+
element = temporal_text(cell.slot.datatype, var)
|
|
207
|
+
if cell.slot.datatype == CYPHER_DATE:
|
|
208
|
+
element = L.concat([L.text('"'), element, L.text('"')])
|
|
209
|
+
text = L.concat([L.text("["), L.fn("STORE_TEXT_JOIN", L.fn("LIST_TRANSFORM", cell.value, var, element),
|
|
210
|
+
L.text(",")), L.text("]")])
|
|
211
|
+
# The slot does not say whether elements are null (its `null` is the list's): checked on every row.
|
|
212
|
+
return L.case(L.fn("STORE_LIST_HAS_NULL", cell.value),
|
|
213
|
+
L.fn("CAST_STRING", L.fn("RAISE_ERROR", L.text(NULL_IN_LIST))), text)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def deleted_read_cell(e: E.Call, cells: Dict[str, Any]) -> Any:
|
|
217
|
+
"""`DELETED_READ(gone, x)`: `x`, a property or label flag of an element read after a DELETE of the query,
|
|
218
|
+
which raises (`DELETED_ACCESS`) on each row where `gone` (its `__gone` column: the step deleted it) is not
|
|
219
|
+
null and the read is evaluated (Neo4j: EntityNotFound, DeletedEntityAccess, at run time).
|
|
220
|
+
`DELETED_NULL(gone, x)`: `x`, null where `gone` is not null (a deleted element returned whole). Each part
|
|
221
|
+
of `x` is kept where the test is null, so the cell keeps `x`'s type on every engine."""
|
|
222
|
+
import dataclasses
|
|
223
|
+
from prograph.plan.ir import lower as L
|
|
224
|
+
gone = L.expr_cell(e.args[0], cells)
|
|
225
|
+
x = L.expr_cell(e.args[1], cells)
|
|
226
|
+
if e.name == "DELETED_NULL":
|
|
227
|
+
keep = L.is_null(gone.value)
|
|
228
|
+
else:
|
|
229
|
+
raised = L.case(L.is_null(gone.value, negated=True),
|
|
230
|
+
L.fn("CAST_STRING", L.fn("RAISE_ERROR", L.text(DELETED_ACCESS))))
|
|
231
|
+
keep = L.is_null(raised)
|
|
232
|
+
|
|
233
|
+
def kept(part: Any) -> Any:
|
|
234
|
+
return None if part is None else L.case(keep, part)
|
|
235
|
+
return dataclasses.replace(x, value=kept(x.value), kind=kept(x.kind), dt=kept(x.dt), lang=kept(x.lang),
|
|
236
|
+
more=tuple(kept(m) for m in x.more))
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _no_value(element: str) -> bool:
|
|
240
|
+
"""Whether a list's element type is one no property holds: a list's, a map's, a node's, a relationship's
|
|
241
|
+
or a path's."""
|
|
242
|
+
from prograph.plan.ir.cypher_answer import element_base, path_parts
|
|
243
|
+
from prograph.plan.ir.cypher_lists import nested
|
|
244
|
+
from prograph.plan.ir.cypher_maps import is_map_type
|
|
245
|
+
return nested(element) or is_map_type(element) or element_base(element) is not None \
|
|
246
|
+
or path_parts(element) is not None
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def value_datatype(plan: O.Op, column: str, key: str) -> Optional[str]:
|
|
250
|
+
"""The datatype a property's value is stored with, from its column's slot: None for null (no
|
|
251
|
+
statement); a temporal value its own (`store.temporal_datatype`); a list of integers, floats, booleans,
|
|
252
|
+
strings or temporals of one type a list datatype (`store.list_datatype`);
|
|
253
|
+
a map, or a list of lists, maps or elements, raises openCypher's InvalidPropertyType (before anything
|
|
254
|
+
runs: nothing is written); a value the store does not hold yet is refused."""
|
|
255
|
+
from prograph.plan.ir.slot import MAP, VALUE
|
|
256
|
+
from prograph.store import list_datatype
|
|
257
|
+
slot = plan.slots[column]
|
|
258
|
+
if slot.present is Bound.NEVER:
|
|
259
|
+
return None
|
|
260
|
+
if slot.kinds == {MAP}:
|
|
261
|
+
raise CypherExecutionError(f"TypeError: InvalidPropertyType: property {key} is a map, which no property "
|
|
262
|
+
"holds (a property holds a number, string or boolean, or a list of them)")
|
|
263
|
+
from prograph.store import temporal_datatype
|
|
264
|
+
if slot.kinds == {LIST}:
|
|
265
|
+
if slot.datatype in STORABLE:
|
|
266
|
+
return list_datatype(slot.datatype)
|
|
267
|
+
if _temporal(slot.datatype) and len(slot.datatypes) == 1:
|
|
268
|
+
return list_datatype(temporal_datatype(slot.datatype))
|
|
269
|
+
if slot.datatypes and all(_no_value(dt) for dt in slot.datatypes):
|
|
270
|
+
raise CypherExecutionError(
|
|
271
|
+
f"TypeError: InvalidPropertyType: property {key} is a list of lists, maps, nodes, relationships "
|
|
272
|
+
"or paths, which no property holds (a list property holds numbers, strings or booleans)")
|
|
273
|
+
raise NotBuilt(f"writing property {key} from a list that is not of integers, floats, booleans, "
|
|
274
|
+
"strings or temporal values of one type alone (an empty list literal, or values of several "
|
|
275
|
+
"types)")
|
|
276
|
+
if slot.kinds != {VALUE} or len(slot.datatypes) != 1:
|
|
277
|
+
raise NotBuilt(f"writing property {key} from a value that is not one of integer, float, boolean, "
|
|
278
|
+
"string or a temporal value")
|
|
279
|
+
dt = next(iter(slot.datatypes))
|
|
280
|
+
if _temporal(dt):
|
|
281
|
+
return temporal_datatype(dt)
|
|
282
|
+
if dt not in STORABLE:
|
|
283
|
+
raise NotBuilt(f"writing property {key} of a value of type {local_name(dt).rsplit(':', 1)[-1]} "
|
|
284
|
+
"(the owned store holds integers, floats, booleans, strings and temporal values for now)")
|
|
285
|
+
return dt
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def merged(stages: List[Stage], stage: Stage) -> List[Stage]:
|
|
289
|
+
"""`stages` with `stage` after them, a stage whose rows the next one only extends (its plan reached
|
|
290
|
+
from the new one's through Extends alone: CREATE after CREATE) merged into it, so each set of rows is
|
|
291
|
+
lowered once."""
|
|
292
|
+
if stages:
|
|
293
|
+
x = stage.plan
|
|
294
|
+
while isinstance(x, O.Extend) and x is not stages[-1].plan:
|
|
295
|
+
x = x.input
|
|
296
|
+
if x is stages[-1].plan:
|
|
297
|
+
last = stages[-1]
|
|
298
|
+
phases = [list(p) for p in last.phases]
|
|
299
|
+
for phase in stage.phases:
|
|
300
|
+
# Phases of assertions alone are one (a CREATE after a CREATE): an INSERT each, fewer.
|
|
301
|
+
if phases and all(t.op != "retract" for t in phases[-1] + phase):
|
|
302
|
+
phases[-1] += phase
|
|
303
|
+
else:
|
|
304
|
+
phases.append(list(phase))
|
|
305
|
+
stage.phases = phases
|
|
306
|
+
stage.minted[:0] = last.minted
|
|
307
|
+
stage.row, stage.order = stage.row or last.row, stage.order or last.order
|
|
308
|
+
return stages[:-1] + [stage]
|
|
309
|
+
return stages + [stage]
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def statements_sql(stages: List[Stage], dialect: str, store: Any = None) -> List[str]:
|
|
313
|
+
"""Every stage's statements as SELECTs of `STATEMENT_COLUMNS`, `op` and `_o` (the order within a phase:
|
|
314
|
+
retractions first), all text, one per phase in order: the stage's rows with the text of each column its
|
|
315
|
+
templates read (`STORE_TEXT`), then a SELECT per template over them, UNION ALL. A row with no subject
|
|
316
|
+
or object gives no statement; a retraction is each effective statement of the store it matches."""
|
|
317
|
+
from prograph.plan.ir import lower
|
|
318
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
319
|
+
from prograph.store import ASSERT, CYPHER_GRAPH, INTERNAL, IRI, LITERAL, RETRACT, STATEMENT_COLUMNS
|
|
320
|
+
d = get_dialect_instance(dialect)
|
|
321
|
+
text_type = d.type_name("STRING")
|
|
322
|
+
null = f"CAST(NULL AS {text_type})"
|
|
323
|
+
raise_missing = f"CAST({d.render_function('RAISE_ERROR', [d.literal(MISSING_END)])} AS {text_type})"
|
|
324
|
+
def raising(mark: str) -> str:
|
|
325
|
+
return f"CAST({d.render_function('RAISE_ERROR', [d.literal(mark)])} AS {text_type})"
|
|
326
|
+
|
|
327
|
+
def raised(mark: str, where: str) -> str:
|
|
328
|
+
"""A filter that raises `mark` on each row where `where` (SQL over the row) holds: the condition
|
|
329
|
+
inside the CASE, so the raise depends on the row and no engine folds it as a constant (PostgreSQL
|
|
330
|
+
would raise while planning) or reads it before the condition; over a join it reads both sides, so
|
|
331
|
+
it is not pushed below the join to rows the join does not keep."""
|
|
332
|
+
return f"CASE WHEN {where} THEN {raising(mark)} END = ''"
|
|
333
|
+
effective = f"({store.effective_sql([CYPHER_GRAPH])})" if store is not None else ""
|
|
334
|
+
# A CTE read more than once is computed once where the engine says so (not Spark's syntax: it decides).
|
|
335
|
+
materialized = "" if dialect in DELTA_DIALECTS else "MATERIALIZED "
|
|
336
|
+
prefix = store.minted_prefix() if store is not None else ""
|
|
337
|
+
out = []
|
|
338
|
+
for index, stage in enumerate(stages):
|
|
339
|
+
texts: Dict[str, str] = {} # a column of the stage -> its text's column
|
|
340
|
+
x = stage.plan
|
|
341
|
+
for t in stage.templates:
|
|
342
|
+
for column in (t.subject, t.object_column, t.edge):
|
|
343
|
+
if column is not None and column not in texts:
|
|
344
|
+
texts[column] = f"#t{len(texts)}"
|
|
345
|
+
x = O.Extend(x, texts[column], E.Call("STORE_TEXT", (E.Col(column),), E.CYPHER))
|
|
346
|
+
x.facts # inferred one level at a time (a long run of Extends)
|
|
347
|
+
# Each row numbered (`_row`, the order its table holds): of a SET's values for one property of one
|
|
348
|
+
# element, the last row's is the one asserted (a property has one value).
|
|
349
|
+
whens = tuple(dict.fromkeys(t.when for t in stage.templates if t.when))
|
|
350
|
+
gones = tuple(dict.fromkeys(t.gone for t in stage.templates if t.gone))
|
|
351
|
+
kept = tuple(texts.values()) + ((stage.row,) if stage.row else ()) + whens + gones
|
|
352
|
+
number = d.quote(lower.sql_name(stage.row)) if stage.row else f"ROW_NUMBER() OVER ({arbitrary_order(dialect)})"
|
|
353
|
+
cte = (f"WITH _stage AS (SELECT *, {number} AS _row FROM "
|
|
354
|
+
f"({lower.to_sql(O.Project(x, kept), dialect)}) AS _rows) ")
|
|
355
|
+
|
|
356
|
+
def col(column: str, needed: bool = False, owned: Any = False) -> str:
|
|
357
|
+
c = f"_stage.{d.quote(lower.sql_name(texts[column]))}"
|
|
358
|
+
if needed: # null raises: no statement of a pattern with a missing node is written
|
|
359
|
+
return f"CASE WHEN {c} IS NULL THEN {raise_missing} ELSE {c} END"
|
|
360
|
+
if owned: # an element a mapped table defines raises: it is not the store's to change
|
|
361
|
+
mark = owned if isinstance(owned, str) else MAPPED
|
|
362
|
+
return (f"CASE WHEN {c} IS NOT NULL AND substr({c}, 1, {len(prefix)}) <> {d.literal(prefix)} "
|
|
363
|
+
f"THEN {raising(mark)} ELSE {c} END")
|
|
364
|
+
return c
|
|
365
|
+
|
|
366
|
+
def only(select: str, when: Optional[str]) -> str:
|
|
367
|
+
"""`select` over the stage's rows where the column `when` is true (all rows for none)."""
|
|
368
|
+
if not when:
|
|
369
|
+
return select
|
|
370
|
+
return select.replace("FROM _stage", f"FROM (SELECT * FROM _stage WHERE "
|
|
371
|
+
f"{d.quote(lower.sql_name(when))} IS TRUE) AS _stage")
|
|
372
|
+
for phase in stage.phases:
|
|
373
|
+
selects = []
|
|
374
|
+
# Each subject a SET or REMOVE changes is checked on every row, whether or not it matches a
|
|
375
|
+
# statement (a retraction of nothing still raises on a mapped element): a select of no row.
|
|
376
|
+
nulls = ", ".join(f"{null} AS {c}" for c in STATEMENT_COLUMNS)
|
|
377
|
+
for column, mark, when in dict.fromkeys((t.subject, t.refusal or True, t.when) for t in phase if t.owned):
|
|
378
|
+
selects.append(only(f"SELECT {nulls}, '{ASSERT}' AS op, 1 AS _o FROM _stage "
|
|
379
|
+
f"WHERE {col(column, owned=mark)} = ''", when))
|
|
380
|
+
for t in phase:
|
|
381
|
+
first = len(selects)
|
|
382
|
+
subject = col(t.subject, t.ends, (t.refusal or True) if t.owned else False)
|
|
383
|
+
columns = ("_e.graph, _e.subject, _e.predicate, _e.o_kind, _e.o_text, _e.o_datatype, _e.o_lang, "
|
|
384
|
+
f"_e.edge_id, '{RETRACT}' AS op, 0 AS _o")
|
|
385
|
+
retracted = f"SELECT DISTINCT {columns} FROM _stage JOIN _eff AS _e ON "
|
|
386
|
+
# Each match is an equality join (a hash join each; an OR of two is a nested loop).
|
|
387
|
+
if t.scope == "refuse": # raise for every row the template applies to (`refusal`)
|
|
388
|
+
applies = f"_stage.{d.quote(lower.sql_name(t.when))} IS TRUE" if t.when else \
|
|
389
|
+
f"_stage.{d.quote(lower.sql_name(t.gone))} IS NOT NULL" if t.gone else "_stage._row IS NOT NULL"
|
|
390
|
+
selects.append(only(f"SELECT {nulls}, '{ASSERT}' AS op, 1 AS _o FROM _stage "
|
|
391
|
+
f"WHERE {raised(t.refusal, applies)}", t.when))
|
|
392
|
+
continue
|
|
393
|
+
if t.scope == "connected" and t.when:
|
|
394
|
+
raise NotBuilt("a conditional DELETE")
|
|
395
|
+
if t.scope == "connected":
|
|
396
|
+
# A node DELETE leaves with a relationship (not deleted before it): raise, write nothing. The
|
|
397
|
+
# condition names a stage column that is not provably non-null (`_row >= 0`, not `IS NOT
|
|
398
|
+
# NULL`: Spark knows a row number is never null, drops that half, and then reads the
|
|
399
|
+
# condition on the view's rows alone, below the join, and raises for any relationship).
|
|
400
|
+
for end in ("subject", "o_text"):
|
|
401
|
+
selects.append(f"SELECT {nulls}, '{ASSERT}' AS op, 1 AS _o FROM _stage JOIN _eff AS _e "
|
|
402
|
+
f"ON _e.{end} = {subject} "
|
|
403
|
+
f"WHERE {raised(CONNECTED, '_e.edge_id IS NOT NULL AND _stage._row >= 0')}")
|
|
404
|
+
continue
|
|
405
|
+
if t.scope == "node":
|
|
406
|
+
selects.append(retracted + f"_e.subject = {subject} AND _e.edge_id IS NULL")
|
|
407
|
+
continue
|
|
408
|
+
if t.scope == "rel":
|
|
409
|
+
selects += [retracted + f"_e.edge_id = {subject}", retracted + f"_e.subject = {subject}"]
|
|
410
|
+
continue
|
|
411
|
+
if t.scope == "detach":
|
|
412
|
+
# The node's relationships, at either end (`_d`), and each one's own statements.
|
|
413
|
+
edges = " UNION ".join(f"SELECT _x.edge_id FROM _stage JOIN _eff AS _x ON _x.{end} = {subject} "
|
|
414
|
+
f"WHERE _x.edge_id IS NOT NULL" for end in ("subject", "o_text"))
|
|
415
|
+
for on in ("edge_id", "subject"):
|
|
416
|
+
selects.append(f"SELECT DISTINCT {columns} FROM ({edges}) AS _d JOIN _eff AS _e "
|
|
417
|
+
f"ON _e.{on} = _d.edge_id")
|
|
418
|
+
continue
|
|
419
|
+
if t.op == RETRACT:
|
|
420
|
+
if t.predicates:
|
|
421
|
+
match = [f"_e.predicate IN ({', '.join(d.literal(p) for p in t.predicates)})",
|
|
422
|
+
f"_e.o_kind = {d.literal(t.kind)}"]
|
|
423
|
+
else:
|
|
424
|
+
match = [f"_e.o_kind = '{LITERAL}'", f"NOT {store.db.starts_with}(_e.predicate, '{INTERNAL}')"]
|
|
425
|
+
if t.object_text is not None:
|
|
426
|
+
match.append(f"_e.o_text = {d.literal(t.object_text)}")
|
|
427
|
+
selects.append(only(
|
|
428
|
+
"SELECT DISTINCT _e.graph, _e.subject, _e.predicate, _e.o_kind, _e.o_text, _e.o_datatype, "
|
|
429
|
+
f"_e.o_lang, _e.edge_id, '{RETRACT}' AS op, 0 AS _o FROM _stage "
|
|
430
|
+
f"JOIN _eff AS _e ON _e.subject = {subject} "
|
|
431
|
+
f"AND {' AND '.join(match)}", t.when))
|
|
432
|
+
continue
|
|
433
|
+
obj = col(t.object_column, t.ends) if t.object_column is not None else d.literal(t.object_text)
|
|
434
|
+
where = ([f"{obj} IS NOT NULL"] if t.object_column and not t.ends else []) + \
|
|
435
|
+
([f"{subject} IS NOT NULL"] if t.owned else [])
|
|
436
|
+
select = (
|
|
437
|
+
f"SELECT {d.literal(CYPHER_GRAPH)} AS graph, {subject} AS subject, "
|
|
438
|
+
f"{d.literal(t.predicate)} AS predicate, {d.literal(t.kind)} AS o_kind, {obj} AS o_text, "
|
|
439
|
+
f"{d.literal(t.datatype) if t.datatype and t.kind != IRI else null} AS o_datatype, "
|
|
440
|
+
f"{null} AS o_lang, {col(t.edge) if t.edge else null} AS edge_id, '{ASSERT}' AS op, "
|
|
441
|
+
f"1 AS _o FROM _stage"
|
|
442
|
+
# A null value is no property; a missing end raises above, never filtered out first.
|
|
443
|
+
+ (f" WHERE {' AND '.join(where)}" if where else ""))
|
|
444
|
+
if t.owned and t.kind == LITERAL:
|
|
445
|
+
# SET of a property: one value per element, the last row's in the order the query
|
|
446
|
+
# states (rows that bind one element twice, `MATCH (h), (y) SET h.k = y.v`); where that
|
|
447
|
+
# row's value is null the property is gone.
|
|
448
|
+
select = select.replace(" AS _o FROM _stage", " AS _o, _stage._row AS _row FROM _stage", 1)
|
|
449
|
+
select = select.replace(f" WHERE {obj} IS NOT NULL AND ", " WHERE ", 1)
|
|
450
|
+
select = (f"SELECT {', '.join(STATEMENT_COLUMNS)}, op, _o FROM (SELECT *, ROW_NUMBER() OVER "
|
|
451
|
+
f"(PARTITION BY subject, predicate ORDER BY _row DESC) AS _last FROM ({select}) "
|
|
452
|
+
f"AS _a) AS _b WHERE _last = 1 AND o_text IS NOT NULL")
|
|
453
|
+
selects.append(select)
|
|
454
|
+
selects[first:] = [only(x, t.when) for x in selects[first:]]
|
|
455
|
+
if selects:
|
|
456
|
+
# The store's effective statements once per phase (after the phases before it), where a
|
|
457
|
+
# retraction reads them.
|
|
458
|
+
eff = f", _eff AS {materialized}{effective} " if any(t.op == RETRACT for t in phase) else " "
|
|
459
|
+
out.append((index, cte.rstrip() + eff + " UNION ALL ".join(selects)))
|
|
460
|
+
return out
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def apply(engine: Any, write: WriteStep, dialect: str, answer: Optional[O.Op] = None,
|
|
464
|
+
base_iri: Optional[str] = None, digest: Optional[str] = None) -> Any:
|
|
465
|
+
"""Append the query's write step to the engine's owned store, in one transaction, reading its rows once:
|
|
466
|
+
each CREATE's rows (`Stage.plan`) are written to a temporary table (`lower.MATERIALIZED`, each later
|
|
467
|
+
stage reading the earlier's), the statements inserted from them (`Store.write_select`: one `INSERT ...
|
|
468
|
+
SELECT` in the engine), and `answer` (the plan of the query's RETURN) read from them too, so it
|
|
469
|
+
answers from the very rows written, a LIMIT with ties or no ORDER BY included. An answer that fails
|
|
470
|
+
rolls the step back: nothing is written (Neo4j: the whole query fails). Everything is lowered before
|
|
471
|
+
anything runs. The answer's Arrow table, kept with the step (`Store.keep_answer`); where the store applied the
|
|
472
|
+
step's key before, nothing is written and the answer is the one the step kept (`Store.kept_answer`, or
|
|
473
|
+
`StoreAnswerNotKeptError`: never the query read again over the store as it is now). None where there is no
|
|
474
|
+
answer. With a `digest` of the query and its parameters (`Store.digest`), a key applied for another
|
|
475
|
+
query raises `StoreStepKeyReusedError`, nothing written."""
|
|
476
|
+
if dialect not in WRITING_DIALECTS:
|
|
477
|
+
raise CypherWriteRefusedError(
|
|
478
|
+
f"Cypher writes into the owned store run on DuckDB, PostgreSQL, Spark and a Databricks SQL warehouse, "
|
|
479
|
+
f"not on {dialect}: run the "
|
|
480
|
+
f"write on one of those engines, or query this engine read-only")
|
|
481
|
+
if getattr(write.store, "dialect", dialect) != dialect:
|
|
482
|
+
raise CypherWriteRefusedError(
|
|
483
|
+
f"the owned store is on a {write.store.dialect} connection and the engine on {dialect}: give the "
|
|
484
|
+
f"store the engine's own connection (Store(connection)), so the step's reads and writes are one "
|
|
485
|
+
f"transaction")
|
|
486
|
+
if write.store.applied(write.step):
|
|
487
|
+
write.store.check_digest(write.step, digest)
|
|
488
|
+
return write.store.kept_answer(write.step) if write.answers else None
|
|
489
|
+
import uuid
|
|
490
|
+
from prograph.plan.ir import lower
|
|
491
|
+
tag = uuid.uuid4().hex[:12]
|
|
492
|
+
held: Dict[int, str] = {}
|
|
493
|
+
before, after = [], []
|
|
494
|
+
done: Dict[str, str] = {} # the columns written already, by an earlier stage
|
|
495
|
+
token = lower.MATERIALIZED.set(held)
|
|
496
|
+
try:
|
|
497
|
+
for i, stage in enumerate(write.stages):
|
|
498
|
+
name = f"_prograph_step_{tag}_{i}"
|
|
499
|
+
wide = sum(len(lower.physical(c, stage.plan.slots[c])) for c in stage.plan.columns)
|
|
500
|
+
if wide > COLUMN_LIMIT.get(dialect, wide):
|
|
501
|
+
raise CypherWriteRefusedError(
|
|
502
|
+
f"this query's write carries {wide} columns through its step, and {dialect} holds at most "
|
|
503
|
+
f"{COLUMN_LIMIT[dialect]} in one statement: split it into several queries (fewer CREATE "
|
|
504
|
+
f"clauses, or fewer variables carried by WITH). Nothing was written")
|
|
505
|
+
rows = lower.physical_sql(stage.plan, dialect, base_iri)
|
|
506
|
+
# Columns written with the rows, in the order the query states (a WITH ... ORDER BY, a written
|
|
507
|
+
# UNWIND's list), nulls as Cypher orders them (last ascending, first descending); with none, in
|
|
508
|
+
# the order read: a SET's row number, a MERGE's created column (the first row of its key).
|
|
509
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
510
|
+
quote = get_dialect_instance(dialect).quote
|
|
511
|
+
keys = ", ".join(f"{quote(lower.sql_name(c))} {'DESC NULLS FIRST' if desc else 'ASC NULLS LAST'}"
|
|
512
|
+
for c, desc in stage.order)
|
|
513
|
+
order = f"ORDER BY {keys}" if keys else arbitrary_order(dialect)
|
|
514
|
+
written = {stage.row: "ROW_NUMBER() OVER ({order})"} if stage.row else {}
|
|
515
|
+
written.update({c: sql for c, sql in write.replaced.items() if c in stage.plan.columns and c not in done})
|
|
516
|
+
done.update(written)
|
|
517
|
+
if written:
|
|
518
|
+
# Every physical column, the written ones replaced (an explicit list: `SELECT * REPLACE` is
|
|
519
|
+
# DuckDB's alone).
|
|
520
|
+
made = {lower.sql_name(c): sql.format(order=order) for c, sql in written.items()}
|
|
521
|
+
names = [n for c in stage.plan.columns for n, _ in lower.physical(c, stage.plan.slots[c])] \
|
|
522
|
+
or ["__one"]
|
|
523
|
+
rows = (f"SELECT {', '.join(f'{made[n]} AS {quote(n)}' if n in made else quote(n) for n in names)} "
|
|
524
|
+
f"FROM ({rows}) AS _p")
|
|
525
|
+
read, make, drop = write.store.db.stage(write.store.table, tag, i, name, rows)
|
|
526
|
+
before.append(make)
|
|
527
|
+
if drop not in after:
|
|
528
|
+
after.append(drop)
|
|
529
|
+
held[id(stage.plan)] = read
|
|
530
|
+
select = statements_sql(write.stages, dialect, write.store)
|
|
531
|
+
# Read as the engine reads any answer (a term split into its columns where the engine's reader
|
|
532
|
+
# makes them one, its decimals' digits, on PostgreSQL an element's parts: `entry.answer_sql`).
|
|
533
|
+
from prograph.plan.ir.entry import answer_sql
|
|
534
|
+
answered = answer_sql(answer, dialect, base_iri) if answer is not None else None
|
|
535
|
+
digits = lower.ANSWER_DIGITS.get() if answer is not None else ()
|
|
536
|
+
finally:
|
|
537
|
+
lower.MATERIALIZED.reset(token)
|
|
538
|
+
# The step's program: each stage's rows written to its table after the inserts of the stages before
|
|
539
|
+
# it, so a clause that reads after a write (a MATCH, a MERGE) sees the step's own statements (one-plan
|
|
540
|
+
# 2.10, part 2: one transaction, one seq, several inserts with reads between them).
|
|
541
|
+
program: List[Tuple[str, str]] = []
|
|
542
|
+
for index, create in enumerate(before):
|
|
543
|
+
program.append(("run", create))
|
|
544
|
+
program += [("insert", sql) for i, sql in select if i == index]
|
|
545
|
+
try:
|
|
546
|
+
seq, table = write.store.write_select(program, WRITER, write.step, answer=answered, after=after,
|
|
547
|
+
digits=digits, replan=True, digest=digest)
|
|
548
|
+
# Applied by another attempt of the key meanwhile: the answer it kept, never a re-read one.
|
|
549
|
+
if seq is None:
|
|
550
|
+
write.store.check_digest(write.step, digest)
|
|
551
|
+
return table if seq is not None or not write.answers else write.store.kept_answer(write.step)
|
|
552
|
+
except (StepLostRace, StoreStepCommittedError):
|
|
553
|
+
raise # a lost race: the query is planned again on the store as the winner left it (`entry._lowered`)
|
|
554
|
+
except Exception as exc:
|
|
555
|
+
if TWO_TYPES in str(exc):
|
|
556
|
+
key = str(exc)[str(exc).index(TWO_TYPES) + len(TWO_TYPES):].split("'")[0].split('"')[0].split()[0]
|
|
557
|
+
raise NotBuilt(f"a read of property {key} after this query's writes changed its type, where some "
|
|
558
|
+
f"element still holds the old type (a column holds one type): give every element "
|
|
559
|
+
f"one type, or read in a query of its own") from exc
|
|
560
|
+
if KEY_TAKEN in str(exc):
|
|
561
|
+
label, key = str(exc)[str(exc).index(KEY_TAKEN) + len(KEY_TAKEN):].split("|")[:2]
|
|
562
|
+
raise CypherExecutionError(
|
|
563
|
+
f"ConstraintValidationFailed: MERGE found no node of the whole pattern, and a node of {label} "
|
|
564
|
+
f"already holds its declared key ({key.replace(',', ', ')}) with those values; nothing was "
|
|
565
|
+
f"written. MERGE the node by its key alone and add the other labels with SET") from exc
|
|
566
|
+
if NULL_KEY in str(exc):
|
|
567
|
+
raise CypherExecutionError(
|
|
568
|
+
"MERGE of a null property: a key's value is null in a row (openCypher's MergeReadOwnWrites "
|
|
569
|
+
"error); nothing was written. Filter out the rows where it is null") from exc
|
|
570
|
+
if dialect == "postgresql" and "row is too big" in str(exc):
|
|
571
|
+
# A step's rows written to its temporary table are one heap row each: PostgreSQL holds at most
|
|
572
|
+
# about 8 kB of them (a wide CREATE of many elements' values), as the select list's limit above.
|
|
573
|
+
raise CypherWriteRefusedError(
|
|
574
|
+
"this query's write carries rows wider than PostgreSQL holds in one row (about 8 kB through "
|
|
575
|
+
"its step): split it into several queries (fewer CREATE clauses, or fewer variables carried by "
|
|
576
|
+
"WITH). Nothing was written") from exc
|
|
577
|
+
if DELETED_ACCESS in str(exc):
|
|
578
|
+
raise CypherExecutionError(
|
|
579
|
+
"EntityNotFound: DeletedEntityAccess: an element this query deleted is read (a property or label) "
|
|
580
|
+
"or written after its DELETE; nothing was written. Read or write it before the DELETE") from exc
|
|
581
|
+
if NULL_IN_LIST in str(exc):
|
|
582
|
+
raise CypherExecutionError(
|
|
583
|
+
"TypeError: InvalidPropertyType: a list property holds a null element in a row; a list property "
|
|
584
|
+
"holds values only. Nothing was written. Filter the nulls out first ([x IN list WHERE x IS NOT "
|
|
585
|
+
"NULL])") from exc
|
|
586
|
+
if MAPPED_MERGE in str(exc):
|
|
587
|
+
raise CypherWriteRefusedError(
|
|
588
|
+
"MERGE found no element, and its label is one a mapped table defines: the store does not create "
|
|
589
|
+
"elements of a mapped label; add the row to the table, or MERGE a label the store holds. Nothing "
|
|
590
|
+
"was written") from exc
|
|
591
|
+
if MAPPED_DELETE in str(exc):
|
|
592
|
+
raise CypherWriteRefusedError(
|
|
593
|
+
"DELETE removes only elements a query created: an element a mapped table defines stays until "
|
|
594
|
+
"hiding mapped elements is opted into, which a later version adds. Nothing was written") from exc
|
|
595
|
+
if CONNECTED in str(exc):
|
|
596
|
+
raise CypherExecutionError(
|
|
597
|
+
"DeleteConnectedNode: a node to delete still has relationships; nothing was written. Delete "
|
|
598
|
+
"them too, or use DETACH DELETE") from exc
|
|
599
|
+
if MAPPED in str(exc):
|
|
600
|
+
raise CypherWriteRefusedError(
|
|
601
|
+
"SET and REMOVE change only elements a query created: a property or label of an element a "
|
|
602
|
+
"mapped table defines is read-only until the mapping declares it overridable, which a later "
|
|
603
|
+
"version adds. Nothing was written") from exc
|
|
604
|
+
if MISSING_END in str(exc):
|
|
605
|
+
raise CypherExecutionError(
|
|
606
|
+
"a node of a created relationship is missing (null: an OPTIONAL MATCH found none); nothing "
|
|
607
|
+
"was written. Match the node first, or filter out the rows where it is null") from exc
|
|
608
|
+
from prograph.plan.ir.entry import _typed
|
|
609
|
+
typed = _typed(exc, "; ".join(sql for _, sql in select))
|
|
610
|
+
if typed is exc:
|
|
611
|
+
raise
|
|
612
|
+
raise typed from exc
|