prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,644 @@
|
|
|
1
|
+
"""PostgreSQL Graph Backend.
|
|
2
|
+
|
|
3
|
+
Reads and writes PostgreSQL through Apache Arrow ADBC
|
|
4
|
+
(``adbc-driver-postgresql``), on one connection: a result is the Arrow the
|
|
5
|
+
driver returns, and a write is visible to the next read.
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
|
|
9
|
+
# A URI: ProGraph opens the connection, in autocommit
|
|
10
|
+
graph = PostgreSQLGraph(schema, "postgresql://user:pass@host/db")
|
|
11
|
+
|
|
12
|
+
# An ADBC connection: committing stays the caller's
|
|
13
|
+
import adbc_driver_postgresql.dbapi
|
|
14
|
+
conn = adbc_driver_postgresql.dbapi.connect("postgresql://user:pass@host/db")
|
|
15
|
+
graph = PostgreSQLGraph(schema, conn)
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
21
|
+
|
|
22
|
+
from prograph.rules import home
|
|
23
|
+
|
|
24
|
+
from .drivers import connection_refused, require_driver, scrub_credentials, secrets_of
|
|
25
|
+
from .sql_graph import SQLGraph
|
|
26
|
+
|
|
27
|
+
PACKAGE = "adbc-driver-postgresql"
|
|
28
|
+
_TYPE_NAME = b"ADBC:postgresql:typname"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def open_connection(target: Any) -> Any:
|
|
32
|
+
"""The ADBC connection for ``target``: a URI, which ProGraph opens in
|
|
33
|
+
autocommit (the caller holds nothing to commit on), or an ADBC
|
|
34
|
+
connection, returned as it is. Anything else is refused: psycopg2 and
|
|
35
|
+
SQLAlchemy connections return rows, not Arrow."""
|
|
36
|
+
dbapi = require_driver("adbc_driver_postgresql.dbapi", PACKAGE, "postgresql")
|
|
37
|
+
if isinstance(target, str):
|
|
38
|
+
try:
|
|
39
|
+
return dbapi.connect(target, autocommit=True)
|
|
40
|
+
except Exception as error: # noqa: BLE001 - the driver's own error types
|
|
41
|
+
raise connection_refused("postgresql", error, secrets_of(target)) from None
|
|
42
|
+
if not hasattr(target, "adbc_get_info"):
|
|
43
|
+
raise TypeError(
|
|
44
|
+
f"The postgresql backend takes a URI ('postgresql://user:password@host/db') or an "
|
|
45
|
+
f"ADBC connection ({PACKAGE}), not a {type(target).__module__}."
|
|
46
|
+
f"{type(target).__name__}: its result is the Arrow the ADBC driver returns.")
|
|
47
|
+
return target
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# PostgreSQL types the driver has no Arrow type for, and the type each is
|
|
51
|
+
# read as. The cast is made in the SQL: `uuid` would
|
|
52
|
+
# otherwise arrive as sixteen opaque bytes. `numeric` is read as its text and
|
|
53
|
+
# made a decimal by Arrow (`Numerics.convert`).
|
|
54
|
+
READ_AS = {b"uuid": "text"}
|
|
55
|
+
NUMERIC = b"numeric"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _typname(field) -> Optional[bytes]:
|
|
59
|
+
return (field.metadata or {}).get(_TYPE_NAME)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _cast(field) -> str:
|
|
63
|
+
"""The cast that reads a column as a type Arrow has, if the column, or
|
|
64
|
+
the element of an array column, is one of READ_AS."""
|
|
65
|
+
import pyarrow as pa
|
|
66
|
+
target = READ_AS.get(_typname(field))
|
|
67
|
+
if target:
|
|
68
|
+
return f"::{target}"
|
|
69
|
+
if pa.types.is_list(field.type) or pa.types.is_large_list(field.type):
|
|
70
|
+
inner = _cast(field.type.value_field)
|
|
71
|
+
return inner and inner + "[]"
|
|
72
|
+
return ""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _is_numeric_list(field) -> bool:
|
|
76
|
+
import pyarrow as pa
|
|
77
|
+
return ((pa.types.is_list(field.type) or pa.types.is_large_list(field.type))
|
|
78
|
+
and _typname(field.type.value_field) == NUMERIC)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@home("postgresql-numeric-result", "A PostgreSQL numeric result column, as an exact decimal",
|
|
82
|
+
terms=("numeric", "NUMERIC", "decimal128", "ADBC", "PostgreSQL", "Lakebase", "exact decimal", "D7"))
|
|
83
|
+
def numeric_digits(known: Optional[Tuple[int, int]]) -> Tuple[Tuple[int, int], bool]:
|
|
84
|
+
"""The Arrow decimal128(p, s) a `numeric` result column is read as, and whether its text is
|
|
85
|
+
rounded to s places first. The ADBC driver gives a numeric as text with no precision or scale,
|
|
86
|
+
so the digits come from the plan (`lower.answer_digits`: a SUM's NUMERIC(38, 0), a quotient's
|
|
87
|
+
DECIMAL(38, 18), a column's recorded NUMERIC(p, s)); held to D7's 38 digits and 18 places
|
|
88
|
+
(sparql-choices.md D7), more places rounded half away from zero, as ROUND does. A column of no
|
|
89
|
+
known digits (an unconstrained NUMERIC, a statement ProGraph did not plan) is DECIMAL(38, 18),
|
|
90
|
+
rounded past 18 places. A value whose whole part does not fit is an error naming the column
|
|
91
|
+
(`Numerics.convert`), never a float."""
|
|
92
|
+
from prograph.schema.xsd import QUOTIENT_PRECISION, QUOTIENT_SCALE
|
|
93
|
+
if known is None or not 0 <= known[1] <= known[0] <= QUOTIENT_PRECISION:
|
|
94
|
+
# No digits, or none a decimal128 holds (PostgreSQL 15's negative scale, a scale past
|
|
95
|
+
# the precision): D7's DECIMAL(38, 18).
|
|
96
|
+
return (QUOTIENT_PRECISION, QUOTIENT_SCALE), True
|
|
97
|
+
p, s = known
|
|
98
|
+
held = min(s, QUOTIENT_SCALE)
|
|
99
|
+
return (max(p, 1), held), held != s
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _text(column: str, kind, places: Optional[int]) -> str:
|
|
103
|
+
"""`column`'s text where its value fits `kind` (rounded to `places` first, if given), else a
|
|
104
|
+
text that is no number, which the Arrow cast refuses (`Numerics.convert`): a whole part too
|
|
105
|
+
long, a NaN or an infinity, or (not rounded) more places than the plan said. Arrow's own cast
|
|
106
|
+
wraps a text of more than 38 digits silently, so the bound is checked here."""
|
|
107
|
+
value = f"ROUND({column}, {places})" if places is not None else column
|
|
108
|
+
exact = "" if places is not None else f" AND {column} = ROUND({column}, {kind.scale})"
|
|
109
|
+
return (f"CASE WHEN {column} IS NULL THEN NULL WHEN ABS({value}) < 1e{kind.precision - kind.scale}{exact} "
|
|
110
|
+
f"THEN ({value})::text ELSE 'not a DECIMAL({kind.precision},{kind.scale})' END")
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# -- an element's struct from its flat columns -----------------------------------------------------------
|
|
114
|
+
#
|
|
115
|
+
# The statement writes a node, relationship, path, map, or a list of them as flat typed columns
|
|
116
|
+
# (`cypher_answer.Shape`: the layout, one definition for the writer and this reader); `assemble` makes the Arrow
|
|
117
|
+
# value DuckDB's struct is, column-wise: `StructArray.from_arrays` over the parts, a `MapArray` for a
|
|
118
|
+
# node's properties built from the per-key columns (the keys of non-null values only, a null property being
|
|
119
|
+
# a key the element lacks), a `ListArray` over offsets for a list. No cell is read in Python, and no jsonb.
|
|
120
|
+
|
|
121
|
+
def _listed(array, name: str = "l"):
|
|
122
|
+
"""A list column (the driver's `list<item: T>`) as `list<name: T>`, its offsets and nulls as they are."""
|
|
123
|
+
import pyarrow as pa
|
|
124
|
+
kind = pa.list_(pa.field(name, array.type.value_type))
|
|
125
|
+
return pa.ListArray.from_arrays(array.offsets, array.values, type=kind, mask=array.is_null())
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _offsets(counts):
|
|
129
|
+
"""The int32 offsets of lists of `counts` (a null count is 0)."""
|
|
130
|
+
import pyarrow as pa
|
|
131
|
+
import pyarrow.compute as pc
|
|
132
|
+
sums = pc.cumulative_sum(pc.cast(pc.fill_null(counts, 0), pa.int64()))
|
|
133
|
+
made = pa.concat_arrays([pa.array([0], type=pa.int32()), pc.cast(sums, pa.int32())])
|
|
134
|
+
# Offsets with no null, said so: a list's `from_arrays` refuses offsets whose null count is not yet known
|
|
135
|
+
# beside a validity mask.
|
|
136
|
+
return pa.Array.from_buffers(pa.int32(), len(made), [None, made.buffers()[1]], null_count=0, offset=made.offset)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _common(arrays):
|
|
140
|
+
"""The Arrow type one map holds all its values as: the widest integer or float of the keys' columns, the
|
|
141
|
+
decimal both fit (as the engines unify them), else the columns' own."""
|
|
142
|
+
import pyarrow as pa
|
|
143
|
+
types = [a.type for a in arrays]
|
|
144
|
+
if all(pa.types.is_integer(t) for t in types):
|
|
145
|
+
return pa.int64()
|
|
146
|
+
if all(pa.types.is_floating(t) for t in types):
|
|
147
|
+
return pa.float64() if any(t.bit_width == 64 for t in types) else pa.float32()
|
|
148
|
+
if all(pa.types.is_decimal(t) for t in types):
|
|
149
|
+
scale = max(t.scale for t in types)
|
|
150
|
+
return pa.decimal128(min(max(t.precision - t.scale for t in types) + scale, 38), scale)
|
|
151
|
+
return types[0]
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _properties(shape, arrays, length: int, mask=None):
|
|
155
|
+
"""A node's or relationship's `properties` as `map<string, V>` of the keys it has (`cypher_answer.map_form`'s
|
|
156
|
+
form: V the one type of every key, else the dense union of integer, float, string and boolean, and the
|
|
157
|
+
exact decimal or a list where there is one), `arrays` the per-key columns of `shape.fields`, `mask` the rows
|
|
158
|
+
that hold no map (a null element's)."""
|
|
159
|
+
import pyarrow as pa
|
|
160
|
+
import pyarrow.compute as pc
|
|
161
|
+
from prograph.plan.ir import cypher_mixed as M
|
|
162
|
+
from prograph.plan.ir.cypher_answer import split_decimal
|
|
163
|
+
from prograph.plan.ir.cypher_lists import nested
|
|
164
|
+
if not shape.fields:
|
|
165
|
+
# No key the pattern reads: DuckDB's empty map, which its engine types map<int32, int32>.
|
|
166
|
+
return pa.MapArray.from_arrays(_offsets(pa.repeat(pa.scalar(0, pa.int32()), length)),
|
|
167
|
+
pa.array([], type=pa.int32()), pa.array([], type=pa.int32()), mask=mask)
|
|
168
|
+
keys = [k for k, _ in shape.fields]
|
|
169
|
+
valid = [pc.is_valid(a) for a in arrays]
|
|
170
|
+
found = [pc.indices_nonzero(v) for v in valid]
|
|
171
|
+
rows = pa.concat_arrays(found)
|
|
172
|
+
which = pa.concat_arrays([pa.repeat(pa.scalar(i, pa.int64()), len(f)) for i, f in enumerate(found)])
|
|
173
|
+
# Row-major, the keys in order within a row: the entries sorted by (row, key).
|
|
174
|
+
order = pc.sort_indices(pc.add(pc.multiply(rows, len(keys)), which))
|
|
175
|
+
present = [pc.filter(a, v) for a, v in zip(arrays, valid)]
|
|
176
|
+
total = pc.cast(valid[0], pa.int64())
|
|
177
|
+
for v in valid[1:]:
|
|
178
|
+
total = pc.add(total, pc.cast(v, pa.int64()))
|
|
179
|
+
offsets = _offsets(total)
|
|
180
|
+
names = pc.take(pa.array(keys), pc.take(which, order))
|
|
181
|
+
if shape.held.startswith("v"):
|
|
182
|
+
kind = _common(present)
|
|
183
|
+
items = pc.take(pa.concat_arrays([pc.cast(a, kind) for a in present]), order)
|
|
184
|
+
if nested(split_decimal(shape.fields[0][1])[0]):
|
|
185
|
+
items = _listed(items)
|
|
186
|
+
return pa.MapArray.from_arrays(offsets, names, items, mask=mask)
|
|
187
|
+
# Values of several types: each entry a tagged struct, made the dense union (`cypher_mixed._union`), as
|
|
188
|
+
# DuckDB's answer is.
|
|
189
|
+
tags = []
|
|
190
|
+
for _, t in shape.fields:
|
|
191
|
+
base = split_decimal(t)[0]
|
|
192
|
+
tags.append("m" if M.is_mixed(base) else "l" if nested(base) else "d" if base == xsd_decimal() else
|
|
193
|
+
M.family(base) or ("s" if base in _temporal() else None))
|
|
194
|
+
# A property of several types over its maps (tag `m`): its tagged text, each row's kind its own.
|
|
195
|
+
mixed = {i: M.from_text(a) for i, (a, tag) in enumerate(zip(present, tags)) if tag == "m"}
|
|
196
|
+
pieces = {M.INT: ("i", pa.int64()), M.FLOAT: ("f", pa.float64()), M.STR: ("s", pa.string()),
|
|
197
|
+
M.BOOL: ("b", pa.bool_())}
|
|
198
|
+
decimals = [a for a, tag in zip(present, tags) if tag == "d"]
|
|
199
|
+
lists = [a for a, tag in zip(present, tags) if tag == "l"]
|
|
200
|
+
if decimals:
|
|
201
|
+
pieces[M.DEC] = ("d", _common(decimals))
|
|
202
|
+
if lists:
|
|
203
|
+
pieces[M.LIST_FIELD] = ("l", _listed(lists[0]).type)
|
|
204
|
+
fields = {M.KIND: pc.take(pa.concat_arrays([mixed[i].field(M.KIND) if i in mixed else
|
|
205
|
+
pa.array([tag] * len(a), type=pa.string())
|
|
206
|
+
for i, (a, tag) in enumerate(zip(present, tags))]), order)}
|
|
207
|
+
for name, (tag, kind) in pieces.items():
|
|
208
|
+
fields[name] = pc.take(pa.concat_arrays([
|
|
209
|
+
mixed[i].field(name) if i in mixed and name in M.FIELDS else
|
|
210
|
+
(_listed(a) if tag == "l" else pc.cast(a, kind)) if t == tag else pa.nulls(len(a), kind)
|
|
211
|
+
for i, (a, t) in enumerate(zip(present, tags))]), order)
|
|
212
|
+
names_in_order = [M.KIND, M.INT, M.FLOAT, M.STR, M.BOOL] + [n for n in (M.DEC, M.LIST_FIELD) if n in pieces]
|
|
213
|
+
tagged = pa.StructArray.from_arrays([fields[n] for n in names_in_order], names=names_in_order)
|
|
214
|
+
return pa.MapArray.from_arrays(offsets, names, M._union(tagged), mask=mask)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def xsd_decimal() -> str:
|
|
218
|
+
from prograph.schema import xsd
|
|
219
|
+
return xsd.DECIMAL
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _temporal():
|
|
223
|
+
from prograph.plan.ir.cypher_temporal import TEMPORAL
|
|
224
|
+
return TEMPORAL
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _map(fields, arrays, length: int, top: bool):
|
|
228
|
+
"""(the struct of a map's `fields`, the arrays left) from its columns, each already flattened to this level's
|
|
229
|
+
length: a field's own column, a null field int32 nulls (as DuckDB types it), a nested map its marker and its
|
|
230
|
+
own fields' columns, a list its counts then its items (a plain list's flat array, a list of maps' fields one
|
|
231
|
+
level down, of as many instances as the counts sum to), an empty map DuckDB's empty map; a decimal field's digits
|
|
232
|
+
are its type's (the reader's `Numerics` cast it already)."""
|
|
233
|
+
import pyarrow as pa
|
|
234
|
+
import pyarrow.compute as pc
|
|
235
|
+
from prograph.plan.ir.cypher_lists import element_of, nested
|
|
236
|
+
from prograph.plan.ir.cypher_maps import is_map_type, type_fields
|
|
237
|
+
names, made = [], []
|
|
238
|
+
for k, t in fields:
|
|
239
|
+
names.append(k)
|
|
240
|
+
if t is None:
|
|
241
|
+
arrays.pop(0)
|
|
242
|
+
made.append(pa.nulls(length, pa.int32()))
|
|
243
|
+
elif is_map_type(t):
|
|
244
|
+
arrays.pop(0)
|
|
245
|
+
sub = tuple(sorted(type_fields(t).items()))
|
|
246
|
+
value, arrays = _map(sub, arrays, length, top=False)
|
|
247
|
+
made.append(value)
|
|
248
|
+
elif nested(t):
|
|
249
|
+
counts = arrays.pop(0)
|
|
250
|
+
inner = element_of(t)
|
|
251
|
+
offsets = _offsets(counts)
|
|
252
|
+
if inner is not None and is_map_type(inner):
|
|
253
|
+
total = int(pc.sum(pc.fill_null(counts, 0)).as_py() or 0)
|
|
254
|
+
items, arrays = _map(tuple(sorted(type_fields(inner).items())), arrays, total, top=False)
|
|
255
|
+
else:
|
|
256
|
+
items = arrays.pop(0)
|
|
257
|
+
made.append(pa.ListArray.from_arrays(offsets, items, mask=pc.is_null(counts)))
|
|
258
|
+
else:
|
|
259
|
+
made.append(arrays.pop(0))
|
|
260
|
+
if not fields:
|
|
261
|
+
if top:
|
|
262
|
+
kind = pa.struct([])
|
|
263
|
+
return pc.fill_null(pa.nulls(length, kind), pa.scalar({}, kind)), arrays
|
|
264
|
+
return pa.MapArray.from_arrays(_offsets(pa.repeat(pa.scalar(0, pa.int32()), length)),
|
|
265
|
+
pa.array([], type=pa.int32()), pa.array([], type=pa.int32())), arrays
|
|
266
|
+
return pa.StructArray.from_arrays(made, names=names), arrays
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _elements(shape, elements):
|
|
270
|
+
"""The struct array of the elements of a list shape from its parts already flattened to one array each
|
|
271
|
+
(`assemble`): a node's id, labels (cut out of the flat labels by the counts) and properties, a relationship's."""
|
|
272
|
+
import pyarrow as pa
|
|
273
|
+
import pyarrow.compute as pc
|
|
274
|
+
from prograph.plan.ir.cypher_answer import NODE_FIELDS, REL_FIELDS
|
|
275
|
+
if shape.kind == "node":
|
|
276
|
+
labels = pa.ListArray.from_arrays(_offsets(elements[2]), elements[1],
|
|
277
|
+
type=pa.list_(pa.field("l", elements[1].type)))
|
|
278
|
+
parts, fields, rest = [elements[0], labels], NODE_FIELDS, elements[3:]
|
|
279
|
+
else:
|
|
280
|
+
parts, fields, rest = list(elements[:4]), REL_FIELDS, elements[4:]
|
|
281
|
+
parts.append(_properties(shape, rest, len(elements[0])))
|
|
282
|
+
return pa.StructArray.from_arrays(parts, names=list(fields), mask=pc.is_null(elements[0]))
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def assemble(shape, arrays):
|
|
286
|
+
"""The Arrow value of `shape` (`cypher_answer.Shape`) made of its columns `arrays` (`shape.parts(False)`, in
|
|
287
|
+
order): a struct of id, labels and properties for a node, of id, type, start, end and properties for a
|
|
288
|
+
relationship, of the two lists for a path, a map's fields for a map literal; a list of them where the shape is
|
|
289
|
+
one. A null id (an element no row matched) is a null element, a null node-id list a null list."""
|
|
290
|
+
import pyarrow as pa
|
|
291
|
+
import pyarrow.compute as pc
|
|
292
|
+
from prograph.plan.ir.cypher_answer import NODE_FIELDS, PATH_FIELDS, REL_FIELDS
|
|
293
|
+
if shape.kind == "path" and shape.many:
|
|
294
|
+
# A list of paths: each path's node count and relationship count (arrays per row), then the elements of
|
|
295
|
+
# every path of the row in order; the elements are made once, the paths cut out of them by the counts.
|
|
296
|
+
nodes, rels = shape.inner
|
|
297
|
+
cut = 1 + len(nodes.parts(False))
|
|
298
|
+
n_counts, node_cols = pc.list_flatten(arrays[0]), [pc.list_flatten(a) for a in arrays[1:cut]]
|
|
299
|
+
r_counts, rel_cols = pc.list_flatten(arrays[cut]), [pc.list_flatten(a) for a in arrays[cut + 1:]]
|
|
300
|
+
node_list = pa.ListArray.from_arrays(_offsets(n_counts), _elements(nodes, node_cols))
|
|
301
|
+
rel_list = pa.ListArray.from_arrays(_offsets(r_counts), _elements(rels, rel_cols))
|
|
302
|
+
paths = pa.StructArray.from_arrays([node_list, rel_list], names=list(PATH_FIELDS))
|
|
303
|
+
return pa.ListArray.from_arrays(_offsets(pc.list_value_length(arrays[0])), paths, mask=pc.is_null(arrays[0]))
|
|
304
|
+
if shape.kind == "path":
|
|
305
|
+
cut = len(shape.inner[0].parts(False))
|
|
306
|
+
nodes, rels = assemble(shape.inner[0], arrays[:cut]), assemble(shape.inner[1], arrays[cut:])
|
|
307
|
+
return pa.StructArray.from_arrays([nodes, rels], names=list(PATH_FIELDS), mask=pc.is_null(arrays[0]))
|
|
308
|
+
if shape.kind == "map" and not shape.many:
|
|
309
|
+
# The first column is a marker (a constant): the rows' count, for a map of no key. A column inside a list is an
|
|
310
|
+
# array per row of every instance, flattened to its level's length.
|
|
311
|
+
parts = shape.parts(False)
|
|
312
|
+
flat = [pc.list_flatten(a) if p.depth > 0 else a for p, a in zip(parts, arrays)]
|
|
313
|
+
made, _ = _map(shape.fields, flat[1:], len(arrays[0]), top=True)
|
|
314
|
+
return made
|
|
315
|
+
if shape.kind == "mixed": # a value of several types: its tagged text, made the tagged struct
|
|
316
|
+
from prograph.plan.ir.cypher_mixed import from_text
|
|
317
|
+
return from_text(arrays[0])
|
|
318
|
+
if shape.kind == "empty":
|
|
319
|
+
# A path that is only its start: no relationship of any type, DuckDB's empty list of int32.
|
|
320
|
+
return pa.ListArray.from_arrays(_offsets(pa.repeat(pa.scalar(0, pa.int32()), len(arrays[0]))),
|
|
321
|
+
pa.array([], type=pa.int32()), type=pa.list_(pa.field("l", pa.int32())))
|
|
322
|
+
if shape.kind == "props" and not shape.many:
|
|
323
|
+
return _properties(shape, arrays[1:], len(arrays[0]), mask=pc.is_null(arrays[0]))
|
|
324
|
+
if shape.many:
|
|
325
|
+
ids = arrays[0]
|
|
326
|
+
elements = [pc.list_flatten(a) for a in arrays]
|
|
327
|
+
if shape.kind == "map": # every column an array per row: flattened once to its level's length
|
|
328
|
+
made, _ = _map(shape.fields, list(elements[1:]), len(elements[0]), top=False)
|
|
329
|
+
return pa.ListArray.from_arrays(_offsets(pc.list_value_length(ids)), made, mask=pc.is_null(ids))
|
|
330
|
+
if shape.kind == "props":
|
|
331
|
+
maps = _properties(shape, elements[1:], len(elements[0]))
|
|
332
|
+
return pa.ListArray.from_arrays(_offsets(pc.list_value_length(ids)), maps, mask=pc.is_null(ids))
|
|
333
|
+
struct = _elements(shape, elements)
|
|
334
|
+
return pa.ListArray.from_arrays(_offsets(pc.list_value_length(ids)), struct, mask=pc.is_null(ids))
|
|
335
|
+
if shape.kind == "node":
|
|
336
|
+
parts, fields, rest = [arrays[0], _listed(arrays[1])], NODE_FIELDS, arrays[2:]
|
|
337
|
+
else:
|
|
338
|
+
parts, fields, rest = list(arrays[:4]), REL_FIELDS, arrays[4:]
|
|
339
|
+
parts.append(_properties(shape, rest, len(arrays[0])))
|
|
340
|
+
return pa.StructArray.from_arrays(parts, names=list(fields), mask=pc.is_null(arrays[0]))
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
class Numerics:
|
|
344
|
+
"""The numeric columns of one PostgreSQL result: each read as text in the SQL (`typed_sql`) and
|
|
345
|
+
made decimal128(p, s) by one Arrow cast per column (`numeric_digits`), no Python per row. And its
|
|
346
|
+
term columns: the four columns the plan wrote for a term (`lower.TERM_FIELDS`, marked
|
|
347
|
+
`lower.TERM_COLUMNS` in `digits`) made one struct column by Arrow, as DuckDB answers it."""
|
|
348
|
+
|
|
349
|
+
def __init__(self, schema, digits: Sequence[Any] = (), exact: bool = True):
|
|
350
|
+
"""`exact` False: ProGraph's own readers of the database (`read_table`), which read a numeric
|
|
351
|
+
as a double, so a NaN or an infinity in a table they scan is a value, not an error. `digits`
|
|
352
|
+
is per answer column; a term column is four columns of the result, so its entry is spread."""
|
|
353
|
+
import pyarrow as pa
|
|
354
|
+
from prograph.plan.ir.lower import TERM_COLUMNS, TERM_FIELDS
|
|
355
|
+
from prograph.plan.ir.cypher_answer import ElementColumns
|
|
356
|
+
self.terms = [] # the position of each term's first of four columns
|
|
357
|
+
self.elements = {} # the position of an element's first column -> its shape (`cypher_answer.Shape`)
|
|
358
|
+
spread: List[Any] = []
|
|
359
|
+
for entry in digits:
|
|
360
|
+
if entry == TERM_COLUMNS:
|
|
361
|
+
self.terms.append(len(spread))
|
|
362
|
+
spread += [None] * len(TERM_FIELDS)
|
|
363
|
+
elif isinstance(entry, ElementColumns):
|
|
364
|
+
self.elements[len(spread)] = entry.shape
|
|
365
|
+
spread += list(entry.digits)
|
|
366
|
+
else:
|
|
367
|
+
spread.append(entry)
|
|
368
|
+
digits = spread
|
|
369
|
+
self.columns = [] # (position, name, Arrow type, rounded to places or None)
|
|
370
|
+
self.doubles = {} # position -> the float8 cast, where not exact
|
|
371
|
+
self.unknown = set() # positions of no known digits (an unconstrained NUMERIC, or one computed from it)
|
|
372
|
+
for i, field in enumerate(schema):
|
|
373
|
+
if not exact and (_typname(field) == NUMERIC or _is_numeric_list(field)):
|
|
374
|
+
self.doubles[i] = "::float8[]" if _is_numeric_list(field) else "::float8"
|
|
375
|
+
elif _typname(field) == NUMERIC:
|
|
376
|
+
known = digits[i] if i < len(digits) else None
|
|
377
|
+
(p, s), rounded = numeric_digits(known)
|
|
378
|
+
self.columns.append((i, field.name, pa.decimal128(p, s), s if rounded else None))
|
|
379
|
+
if known is None:
|
|
380
|
+
self.unknown.add(i)
|
|
381
|
+
elif _is_numeric_list(field):
|
|
382
|
+
# An element's digits: the plan's where it records them (an element's property), else D7's.
|
|
383
|
+
(p, s), rounded = numeric_digits(digits[i] if i < len(digits) else None)
|
|
384
|
+
self.columns.append((i, field.name, pa.list_(pa.decimal128(p, s)), s if rounded else None))
|
|
385
|
+
|
|
386
|
+
def select(self, i: int, column: str) -> Optional[str]:
|
|
387
|
+
"""The select-list expression reading column `column` (position `i`) as text, if numeric."""
|
|
388
|
+
if i in self.doubles:
|
|
389
|
+
return column + self.doubles[i]
|
|
390
|
+
for j, _, kind, places in self.columns:
|
|
391
|
+
if j != i:
|
|
392
|
+
continue
|
|
393
|
+
if hasattr(kind, "value_type"):
|
|
394
|
+
return (f"CASE WHEN {column} IS NULL THEN NULL ELSE ARRAY(SELECT {_text('x', kind.value_type, places)} "
|
|
395
|
+
f"FROM unnest({column}) WITH ORDINALITY AS u(x, n) ORDER BY n) END")
|
|
396
|
+
return _text(column, kind, places)
|
|
397
|
+
return None
|
|
398
|
+
|
|
399
|
+
def groups(self):
|
|
400
|
+
"""{position of a group's first column: (its width, how it is made one column)}: a term's four
|
|
401
|
+
columns (`lower.TERM_FIELDS`), an element's parts (`cypher_answer.Shape`)."""
|
|
402
|
+
from prograph.plan.ir.lower import TERM_FIELDS
|
|
403
|
+
out = {i: (len(TERM_FIELDS), None) for i in self.terms}
|
|
404
|
+
for i, shape in self.elements.items():
|
|
405
|
+
out[i] = (len(shape.parts(False)), shape)
|
|
406
|
+
return out
|
|
407
|
+
|
|
408
|
+
def schema(self, schema):
|
|
409
|
+
"""The result's schema: its numerics as decimals, each term's four columns and each element's parts
|
|
410
|
+
one column (named for the first, without its suffix)."""
|
|
411
|
+
import pyarrow as pa
|
|
412
|
+
from prograph.plan.ir.lower import TERM_FIELDS, term_struct_type
|
|
413
|
+
for i, _, kind, _ in self.columns:
|
|
414
|
+
schema = schema.set(i, schema.field(i).with_type(kind))
|
|
415
|
+
groups = self.groups()
|
|
416
|
+
if not groups:
|
|
417
|
+
return schema
|
|
418
|
+
fields, i = [], 0
|
|
419
|
+
while i < len(schema):
|
|
420
|
+
if i in groups:
|
|
421
|
+
width, shape = groups[i]
|
|
422
|
+
name = schema.field(i).name
|
|
423
|
+
if shape is None:
|
|
424
|
+
fields.append(pa.field(name[:-len(f"__{TERM_FIELDS[0]}")], term_struct_type()))
|
|
425
|
+
else:
|
|
426
|
+
empty = [pa.array([], type=schema.field(j).type) for j in range(i, i + width)]
|
|
427
|
+
fields.append(pa.field(name[:-len("__e0")], assemble(shape, empty).type))
|
|
428
|
+
i += width
|
|
429
|
+
else:
|
|
430
|
+
fields.append(schema.field(i))
|
|
431
|
+
i += 1
|
|
432
|
+
return pa.schema(fields)
|
|
433
|
+
|
|
434
|
+
def convert(self, data):
|
|
435
|
+
"""`data` (a Table or a RecordBatch) with its numeric columns as decimals."""
|
|
436
|
+
import pyarrow as pa
|
|
437
|
+
import pyarrow.compute as pc
|
|
438
|
+
if not self.columns and not self.terms and not self.elements:
|
|
439
|
+
return data
|
|
440
|
+
if isinstance(data, pa.Table) and (self.terms or self.elements):
|
|
441
|
+
# Struct arrays are built from arrays, not chunked ones: batch by batch, the table's own batches.
|
|
442
|
+
return pa.Table.from_batches([self.convert(b) for b in data.to_batches()],
|
|
443
|
+
schema=self.schema(data.schema))
|
|
444
|
+
arrays = list(data.columns)
|
|
445
|
+
for i, name, kind, _ in self.columns:
|
|
446
|
+
try:
|
|
447
|
+
arrays[i] = pc.cast(arrays[i], kind)
|
|
448
|
+
except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as error:
|
|
449
|
+
from prograph.utils.exceptions import SQLRuntimeError
|
|
450
|
+
raise SQLRuntimeError(
|
|
451
|
+
f"The column {name!r} holds a number that does not fit {kind}, or no number (a NaN "
|
|
452
|
+
f"or an infinity): ProGraph's decimals are exact to 38 digits, 18 of them after "
|
|
453
|
+
f"the point ({error})" + (
|
|
454
|
+
". Its precision and scale are not known (an unconstrained NUMERIC, or a value computed "
|
|
455
|
+
"from one): for a column's exact type, map a table, or a view created in the database, "
|
|
456
|
+
"whose column declares NUMERIC(p, s)" if i in self.unknown else ""),
|
|
457
|
+
reason="numeric_overflow") from None
|
|
458
|
+
schema = self.schema(data.schema)
|
|
459
|
+
groups = self.groups()
|
|
460
|
+
if groups:
|
|
461
|
+
from prograph.plan.ir.lower import TERM_FIELDS, term_struct_type
|
|
462
|
+
made, i = [], 0
|
|
463
|
+
while i < len(arrays):
|
|
464
|
+
if i in groups:
|
|
465
|
+
width, shape = groups[i]
|
|
466
|
+
parts = arrays[i:i + width]
|
|
467
|
+
if shape is not None:
|
|
468
|
+
made.append(assemble(shape, parts))
|
|
469
|
+
else:
|
|
470
|
+
# an unbound term is a null struct: its `type` is null (a bound term has one)
|
|
471
|
+
made.append(pa.StructArray.from_arrays(parts, fields=list(term_struct_type()),
|
|
472
|
+
mask=parts[TERM_FIELDS.index("type")].is_null()))
|
|
473
|
+
i += width
|
|
474
|
+
else:
|
|
475
|
+
made.append(arrays[i])
|
|
476
|
+
i += 1
|
|
477
|
+
arrays = made
|
|
478
|
+
if isinstance(data, pa.RecordBatch):
|
|
479
|
+
return pa.RecordBatch.from_arrays(arrays, schema=schema)
|
|
480
|
+
return pa.Table.from_arrays(arrays, schema=schema)
|
|
481
|
+
|
|
482
|
+
def reader(self, reader, close=None):
|
|
483
|
+
"""`reader`'s batches converted, under one schema; `close` called when it is done."""
|
|
484
|
+
import pyarrow as pa
|
|
485
|
+
|
|
486
|
+
def batches():
|
|
487
|
+
try:
|
|
488
|
+
for batch in reader:
|
|
489
|
+
yield self.convert(batch)
|
|
490
|
+
finally:
|
|
491
|
+
if close is not None:
|
|
492
|
+
close()
|
|
493
|
+
|
|
494
|
+
return pa.RecordBatchReader.from_batches(self.schema(reader.schema), batches())
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def typed_sql(sql: str, schema, quote, numerics: Optional[Numerics] = None) -> str:
|
|
498
|
+
"""``sql`` with its ``uuid`` columns read as ``text`` and its ``numeric`` columns as their text (``Numerics``), or ``sql``
|
|
499
|
+
itself when its result has neither.
|
|
500
|
+
|
|
501
|
+
The columns are renamed by position and given their own names back, so
|
|
502
|
+
duplicate and quoted names survive. The query sits on lines of its own
|
|
503
|
+
with no trailing semicolon, so a closing ``--`` comment cannot swallow
|
|
504
|
+
the parenthesis. PostgreSQL keeps the inner query's row order through
|
|
505
|
+
this plain projection; SQL does not promise it.
|
|
506
|
+
"""
|
|
507
|
+
numerics = numerics or Numerics(schema)
|
|
508
|
+
reads = [numerics.select(i, f"c{i}") or f"c{i}{_cast(field)}" for i, field in enumerate(schema)]
|
|
509
|
+
if all(r == f"c{i}" for i, r in enumerate(reads)):
|
|
510
|
+
return sql
|
|
511
|
+
columns = ", ".join(f"c{i}" for i in range(len(reads)))
|
|
512
|
+
select = ", ".join(f"{r} AS {quote(field.name)}" for r, field in zip(reads, schema))
|
|
513
|
+
return f"SELECT {select} FROM (\n{sql.rstrip().rstrip(';').rstrip()}\n) AS q({columns})"
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def typed_cursor(connection: Any, sql: str, parameters: Sequence[Any] = (), digits: Sequence[Any] = (),
|
|
517
|
+
exact: bool = True, cursor: Any = None):
|
|
518
|
+
"""(an open cursor on an ADBC connection that has executed ``sql`` with
|
|
519
|
+
its result read as types Arrow has (``typed_sql``), the ``Numerics`` that
|
|
520
|
+
make its numeric columns decimals). The driver is asked for the result's
|
|
521
|
+
schema first, which fetches no row and runs no write. ``digits``: each
|
|
522
|
+
answer column's DECIMAL(p, s) where the plan knows it (``numeric_digits``). ``cursor``: the
|
|
523
|
+
connection's cursor to run it on (a new one by default); closed where it fails."""
|
|
524
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
525
|
+
cursor = cursor if cursor is not None else connection.cursor()
|
|
526
|
+
try:
|
|
527
|
+
schema = cursor.adbc_execute_schema(sql)
|
|
528
|
+
numerics = Numerics(schema, digits, exact)
|
|
529
|
+
typed = typed_sql(sql, schema, get_dialect_instance('postgresql').quote, numerics)
|
|
530
|
+
cursor.execute(typed, parameters) if parameters else cursor.execute(typed)
|
|
531
|
+
except Exception:
|
|
532
|
+
cursor.close()
|
|
533
|
+
raise
|
|
534
|
+
return cursor, numerics
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def read_table(connection: Any, sql: str, parameters: Sequence[Any] = ()):
|
|
538
|
+
"""The result of ``sql`` as one Arrow table of types Arrow has, for
|
|
539
|
+
ProGraph's own readers of the database (``drivers.fetch_rows``): a
|
|
540
|
+
numeric is a double there, so a NaN in a scanned table is no error
|
|
541
|
+
(an answer's numerics are exact, ``Numerics``); None for a statement
|
|
542
|
+
with no result."""
|
|
543
|
+
cursor, numerics = typed_cursor(connection, sql, parameters, exact=False)
|
|
544
|
+
try:
|
|
545
|
+
if not cursor.description:
|
|
546
|
+
return None
|
|
547
|
+
return numerics.convert(cursor.fetch_arrow_table())
|
|
548
|
+
finally:
|
|
549
|
+
cursor.close()
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def _close(connection: Any) -> None:
|
|
553
|
+
try:
|
|
554
|
+
connection.close()
|
|
555
|
+
except Exception: # noqa: BLE001 - already closed by the caller
|
|
556
|
+
pass
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
class PostgreSQLGraph(SQLGraph):
|
|
560
|
+
"""PostgreSQL-backed graph, on one ADBC connection."""
|
|
561
|
+
|
|
562
|
+
def __init__(
|
|
563
|
+
self,
|
|
564
|
+
schema: Any,
|
|
565
|
+
connection: Any,
|
|
566
|
+
table_stats: Optional[Dict[str, int]] = None,
|
|
567
|
+
):
|
|
568
|
+
opened_here = isinstance(connection, str)
|
|
569
|
+
self._secrets = secrets_of(connection) if opened_here else []
|
|
570
|
+
super().__init__(schema, open_connection(connection), dialect='postgresql',
|
|
571
|
+
table_stats=table_stats)
|
|
572
|
+
if opened_here:
|
|
573
|
+
self._turn_jit_off()
|
|
574
|
+
# The session is ProGraph's, so ProGraph closes it: when the
|
|
575
|
+
# graph is dropped, or at exit before the driver is torn down
|
|
576
|
+
# (an ADBC connection left open then fails in its destructor).
|
|
577
|
+
import weakref
|
|
578
|
+
weakref.finalize(self, _close, self._connection)
|
|
579
|
+
|
|
580
|
+
def _turn_jit_off(self) -> None:
|
|
581
|
+
"""`SET jit = off` on the session ProGraph opened (never on a connection the caller gave). One
|
|
582
|
+
generated statement is large (a saturated class is a union of many branches), so its estimated cost
|
|
583
|
+
passes PostgreSQL's JIT thresholds while it returns few rows: 8044 functions compiled for 100 s on a
|
|
584
|
+
statement that runs for a second without. Measured on TPC-H SF1 (PostgreSQL 16): JIT gains 6% over the 17
|
|
585
|
+
queries that finish both ways; q1 and q11 are faster with it (34 -> 21 s, about 150 -> 90 s), on SQL whose
|
|
586
|
+
per-property scans of one table are slow either way (the reference SQL takes under a second). A server that refuses the setting (a managed service that has not
|
|
587
|
+
got it) leaves the session as it is."""
|
|
588
|
+
try:
|
|
589
|
+
cursor = self._connection.cursor()
|
|
590
|
+
try:
|
|
591
|
+
cursor.execute("SET jit = off")
|
|
592
|
+
finally:
|
|
593
|
+
cursor.close()
|
|
594
|
+
except Exception: # noqa: BLE001 - the setting is an optimisation, not required
|
|
595
|
+
pass
|
|
596
|
+
|
|
597
|
+
# The plan's answer digits reach the reader (`entry._with_digits`): numerics arrive as text.
|
|
598
|
+
reads_answer_digits = True
|
|
599
|
+
|
|
600
|
+
def _run(self, sql: str, digits: Sequence[Any] = ()):
|
|
601
|
+
"""(an open cursor that has executed ``sql``, read as types Arrow
|
|
602
|
+
have, its ``Numerics``, and the statement's ``StatementGuard``, which
|
|
603
|
+
a cancel stops it through: ``cursor.adbc_cancel()``)."""
|
|
604
|
+
from prograph.execution import StatementGuard, checkpoint
|
|
605
|
+
checkpoint()
|
|
606
|
+
cursor = self._connection.cursor()
|
|
607
|
+
guard = StatementGuard(stop=cursor.adbc_cancel)
|
|
608
|
+
try:
|
|
609
|
+
return typed_cursor(self._connection, sql, digits=digits, cursor=cursor) + (guard,)
|
|
610
|
+
except Exception as error:
|
|
611
|
+
guard.stop()
|
|
612
|
+
guard.check()
|
|
613
|
+
message = str(error)
|
|
614
|
+
if scrub_credentials(message, self._secrets) != message:
|
|
615
|
+
from prograph.utils.exceptions import SQLRuntimeError
|
|
616
|
+
raise SQLRuntimeError(
|
|
617
|
+
f"SQL execution failed: {scrub_credentials(message, self._secrets)}",
|
|
618
|
+
reason="execution_error", sql=sql) from None
|
|
619
|
+
raise
|
|
620
|
+
|
|
621
|
+
def _execute_sql_arrow(self, sql: str, digits: Sequence[Any] = ()):
|
|
622
|
+
"""Execute SQL and return the driver's Arrow table, its numerics as
|
|
623
|
+
decimals (``Numerics``; ``digits`` the plan's, ``numeric_digits``)."""
|
|
624
|
+
cursor, numerics, guard = self._run(sql, digits)
|
|
625
|
+
with guard:
|
|
626
|
+
try:
|
|
627
|
+
return numerics.convert(cursor.fetch_arrow_table())
|
|
628
|
+
finally:
|
|
629
|
+
cursor.close()
|
|
630
|
+
|
|
631
|
+
def _execute_sql(self, sql: str) -> List[Dict[str, Any]]:
|
|
632
|
+
"""Execute SQL against PostgreSQL; rows as dicts."""
|
|
633
|
+
return self._execute_sql_arrow(sql).to_pylist(maps_as_pydicts="strict")
|
|
634
|
+
|
|
635
|
+
def _execute_sql_batches(self, sql: str, batch_size: int, digits: Sequence[Any] = ()):
|
|
636
|
+
"""Stream the driver's result as Arrow record batches, its numerics
|
|
637
|
+
as decimals under one schema (``Numerics.reader``). The driver
|
|
638
|
+
sizes its batches in bytes, so ``batch_size`` rows is not what a
|
|
639
|
+
batch holds (the contract is a stream, not a size). The cursor
|
|
640
|
+
stays open while the reader is read."""
|
|
641
|
+
cursor, numerics, guard = self._run(sql, digits)
|
|
642
|
+
with guard.opening(close=cursor.close):
|
|
643
|
+
reader = numerics.reader(cursor.fetch_record_batch())
|
|
644
|
+
return guard.reader(reader, close=cursor.close)
|