prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
"""Oracle Graph Backend.
|
|
2
|
+
|
|
3
|
+
Reads and writes Oracle through ``oracledb`` (thin mode), on the connection
|
|
4
|
+
the caller gives: a result is the Arrow the driver builds, with
|
|
5
|
+
each column's Arrow type requested from its declared Oracle type.
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
|
|
9
|
+
import oracledb
|
|
10
|
+
conn = oracledb.connect(user="prograph", password="...",
|
|
11
|
+
dsn="localhost:1521/XEPDB1")
|
|
12
|
+
graph = OracleGraph(schema, conn)
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
18
|
+
|
|
19
|
+
from .drivers import require_driver
|
|
20
|
+
from .sql_graph import SQLGraph
|
|
21
|
+
from prograph.rules import home
|
|
22
|
+
|
|
23
|
+
PACKAGE = "oracledb"
|
|
24
|
+
# Below 26.0.1 a bare NULL column makes the Arrow fetch lose its rows, or
|
|
25
|
+
# crash the process.
|
|
26
|
+
MINIMUM_VERSION = (26, 0, 1)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _version(text: str) -> Tuple[int, ...]:
|
|
30
|
+
"""A release as a comparable tuple: ``26.0.1`` is (26, 0, 1, 1). A
|
|
31
|
+
pre-release (``26.0.1rc1``, ``26.0.1.dev0``) sorts before its release, so
|
|
32
|
+
``26.0.0rc1`` and ``26.0.1rc1`` are both below the floor."""
|
|
33
|
+
import re
|
|
34
|
+
match = re.match(r"(\d+)(?:\.(\d+))?(?:\.(\d+))?(.*)$", text.strip())
|
|
35
|
+
if not match:
|
|
36
|
+
return (0,)
|
|
37
|
+
major, minor, micro, rest = match.groups()
|
|
38
|
+
return (int(major), int(minor or 0), int(micro or 0), 0 if rest else 1)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def require_oracledb() -> Any:
|
|
42
|
+
"""The ``oracledb`` module, refused like a missing driver if it is older
|
|
43
|
+
than the floor."""
|
|
44
|
+
module = require_driver("oracledb", PACKAGE, "oracle")
|
|
45
|
+
if _version(module.__version__) < MINIMUM_VERSION + (1,):
|
|
46
|
+
from prograph.utils.exceptions import MissingDriverError
|
|
47
|
+
floor = ".".join(map(str, MINIMUM_VERSION))
|
|
48
|
+
raise MissingDriverError(
|
|
49
|
+
f"The oracle backend needs {PACKAGE} {floor} or later (pip install "
|
|
50
|
+
f"'{PACKAGE}>={floor}'); {module.__version__} loses rows of a result with a "
|
|
51
|
+
f"bare NULL column.", package=PACKAGE, backend="oracle")
|
|
52
|
+
return module
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@home("oracle-integer-number", "Oracle integer NUMBER",
|
|
56
|
+
terms=("Oracle", "NUMBER", "scale", "integer column"))
|
|
57
|
+
def number_is_integer(scale: Optional[int]) -> bool:
|
|
58
|
+
"""Whether an Oracle ``NUMBER`` of this declared scale holds integers only:
|
|
59
|
+
a known scale of 0 or below (``NUMBER(9)``, ``INTEGER``, ``NUMBER(5,-2)``).
|
|
60
|
+
An unknown scale (a plain ``NUMBER``, an expression: none, or -127 from a
|
|
61
|
+
cursor's description) may hold fractions. The one home of this rule
|
|
62
|
+
(docs/architecture/rules.md): result types and column types both ask it."""
|
|
63
|
+
return scale is not None and scale != -127 and scale <= 0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def number_digits(precision: Optional[int], scale: Optional[int]) -> Optional[Tuple[int, int]]:
|
|
67
|
+
"""The exact decimal an Oracle ``NUMBER`` of this declared precision and scale holds, as
|
|
68
|
+
(precision, scale), where the scale is above 0 and both fit a 38-digit decimal (``NUMBER(20,2)``;
|
|
69
|
+
``NUMBER(3,5)`` holds 0.00123, five digits); None for an integer (`number_is_integer`) or an
|
|
70
|
+
unknown precision or scale (a plain ``NUMBER``, ``FLOAT``: a floating decimal). Result types and
|
|
71
|
+
column types both ask it, as they ask `number_is_integer`."""
|
|
72
|
+
if scale is None or scale == -127 or scale <= 0 or not precision or precision > 38 or scale > 38:
|
|
73
|
+
return None
|
|
74
|
+
return (max(precision, scale), scale)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def returns_rows(sql: str) -> bool:
|
|
78
|
+
"""Whether ``sql`` can have result columns: its first keyword, after
|
|
79
|
+
comments and parentheses, is SELECT or WITH. Oracle runs DDL when it is
|
|
80
|
+
parsed, so only such a statement is parsed for its types; anything else
|
|
81
|
+
runs once, on a cursor."""
|
|
82
|
+
import re
|
|
83
|
+
text = re.sub(r"--[^\n]*|/\*.*?\*/", " ", sql, flags=re.S).lstrip(" \t\r\n(")
|
|
84
|
+
return re.match(r"(?i)(select|with)\b", text) is not None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _arrow_type(column) -> Optional[Any]:
|
|
88
|
+
"""The Arrow type to request for a result column, from its declared
|
|
89
|
+
Oracle type, precision and scale, or None for a
|
|
90
|
+
type the table has no row for. ``int64`` is requested for a declared scale of 0 or below, never for a
|
|
91
|
+
number of unknown scale, which the driver would truncate; a NUMBER of known precision and a scale
|
|
92
|
+
above 0 is the exact ``decimal128`` of its digits (`number_digits`); one of unknown precision or
|
|
93
|
+
scale (a plain NUMBER) is ``float64``, which keeps about 15 significant digits."""
|
|
94
|
+
import pyarrow as pa
|
|
95
|
+
name = column.type_code.name
|
|
96
|
+
if name == "DB_TYPE_NUMBER":
|
|
97
|
+
if number_is_integer(column.scale):
|
|
98
|
+
return pa.int64()
|
|
99
|
+
digits = number_digits(column.precision, column.scale)
|
|
100
|
+
return pa.decimal128(*digits) if digits else pa.float64()
|
|
101
|
+
if name in ("DB_TYPE_BINARY_DOUBLE",):
|
|
102
|
+
return pa.float64()
|
|
103
|
+
if name in ("DB_TYPE_BINARY_FLOAT",):
|
|
104
|
+
return pa.float32()
|
|
105
|
+
if name in ("DB_TYPE_VARCHAR", "DB_TYPE_NVARCHAR", "DB_TYPE_CHAR", "DB_TYPE_NCHAR",
|
|
106
|
+
"DB_TYPE_CLOB", "DB_TYPE_NCLOB"):
|
|
107
|
+
return pa.string()
|
|
108
|
+
if name == "DB_TYPE_DATE":
|
|
109
|
+
return pa.timestamp("s")
|
|
110
|
+
if name == "DB_TYPE_TIMESTAMP" or name in _ZONED:
|
|
111
|
+
# The driver keeps microseconds only, even asked for nanoseconds:
|
|
112
|
+
# more fractional digits would be cut, so such a column is not
|
|
113
|
+
# requested.
|
|
114
|
+
if (column.scale or 0) > 6:
|
|
115
|
+
return None
|
|
116
|
+
return pa.timestamp("us", tz="UTC" if name in _ZONED else None)
|
|
117
|
+
if name in ("DB_TYPE_RAW", "DB_TYPE_BLOB"):
|
|
118
|
+
return pa.binary()
|
|
119
|
+
# Not listed: LONG and LONG RAW (deprecated), BOOLEAN (Oracle 23ai, not
|
|
120
|
+
# the 21c the tests run on), INTERVAL and the object types. Each is
|
|
121
|
+
# refused by name until a row and its test are added.
|
|
122
|
+
return None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _description(connection: Any, sql: str):
|
|
126
|
+
"""The result columns of ``sql`` as Oracle declares them: parsed, not run."""
|
|
127
|
+
cursor = connection.cursor()
|
|
128
|
+
try:
|
|
129
|
+
cursor.parse(sql)
|
|
130
|
+
return cursor.description or []
|
|
131
|
+
finally:
|
|
132
|
+
cursor.close()
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
_ZONED = ("DB_TYPE_TIMESTAMP_TZ", "DB_TYPE_TIMESTAMP_LTZ")
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _in_utc(sql: str, description) -> str:
|
|
139
|
+
"""``sql`` with each zoned timestamp column converted to UTC
|
|
140
|
+
(``SYS_EXTRACT_UTC``), or ``sql`` itself if it has none.
|
|
141
|
+
|
|
142
|
+
The query is selected from as an inline view, each column by the exact
|
|
143
|
+
name the parse reported, so a query with its own ``WITH`` can be
|
|
144
|
+
wrapped (Oracle refuses a ``WITH`` inside a ``WITH`` column list,
|
|
145
|
+
ORA-32034). Where two columns share a name, a view cannot tell them
|
|
146
|
+
apart, so they are renamed by position in a ``WITH`` column list
|
|
147
|
+
instead, which a query that has its own ``WITH`` cannot take."""
|
|
148
|
+
zoned = [i for i, column in enumerate(description) if column.type_code.name in _ZONED]
|
|
149
|
+
if not zoned:
|
|
150
|
+
return sql
|
|
151
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
152
|
+
quote = get_dialect_instance('oracle').quote_exact
|
|
153
|
+
names = [column.name for column in description]
|
|
154
|
+
|
|
155
|
+
def value(i, ref):
|
|
156
|
+
return f"SYS_EXTRACT_UTC({ref})" if i in zoned else ref
|
|
157
|
+
|
|
158
|
+
if len(set(names)) == len(names):
|
|
159
|
+
select = ", ".join(f'{value(i, f"q.{quote(name)}")} AS {quote(name)}' for i, name in enumerate(names))
|
|
160
|
+
return f"SELECT {select} FROM (\n{sql}\n) q"
|
|
161
|
+
positions = ", ".join(f"c{i}" for i in range(len(names)))
|
|
162
|
+
select = ", ".join(f"{value(i, f'c{i}')} AS {quote(name)}" for i, name in enumerate(names))
|
|
163
|
+
return f"WITH q ({positions}) AS (\n{sql}\n) SELECT {select} FROM q"
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _terminated(sql: str) -> str:
|
|
167
|
+
# Oracle takes one statement with no terminator; a caller's SQL may end
|
|
168
|
+
# with a semicolon.
|
|
169
|
+
return sql.rstrip().rstrip(';').rstrip()
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def utc_statement(connection: Any, sql: str) -> str:
|
|
173
|
+
"""``sql`` as ProGraph's own readers of the database run it on Oracle
|
|
174
|
+
(``drivers.fetch_rows``): a zoned timestamp read as its UTC instant, as a
|
|
175
|
+
query's result is. The driver gives a reader the
|
|
176
|
+
naive time in the value's own zone otherwise."""
|
|
177
|
+
sql = _terminated(sql)
|
|
178
|
+
if not returns_rows(sql):
|
|
179
|
+
return sql
|
|
180
|
+
return _in_utc(sql, _description(connection, sql))
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def typed_statement(connection: Any, sql: str, digits: Sequence[Any] = ()):
|
|
184
|
+
"""(statement, requested Arrow schema) for ``sql`` on Oracle. ``digits``: each answer column's
|
|
185
|
+
DECIMAL(p, s) where the plan knows it (`lower.ANSWER_DIGITS`); an integer-scaled column (an integer by
|
|
186
|
+
`number_is_integer`) of known digits is the exact ``decimal128(p, 0)`` (a SUM of integers), not int64.
|
|
187
|
+
|
|
188
|
+
The statement is parsed, not run, for each result column's declared
|
|
189
|
+
type; a zoned timestamp is converted to UTC in the SQL (``_in_utc``). A
|
|
190
|
+
column of a type with no row in the table is refused by name."""
|
|
191
|
+
import pyarrow as pa
|
|
192
|
+
from prograph.utils.exceptions import SQLRuntimeError
|
|
193
|
+
sql = _terminated(sql)
|
|
194
|
+
if not returns_rows(sql):
|
|
195
|
+
return sql, pa.schema([])
|
|
196
|
+
description = _description(connection, sql)
|
|
197
|
+
fields = []
|
|
198
|
+
known = list(digits) if len(digits) == len(description) else [None] * len(description)
|
|
199
|
+
for column, entry in zip(description, known):
|
|
200
|
+
arrow = _arrow_type(column)
|
|
201
|
+
if pa.types.is_int64(arrow or pa.null()) and _is_digits(entry) and entry[1] == 0:
|
|
202
|
+
arrow = pa.decimal128(entry[0], 0)
|
|
203
|
+
if arrow is None:
|
|
204
|
+
if "TIMESTAMP" in column.type_code.name and (column.scale or 0) > 6:
|
|
205
|
+
message = (f"Result column {column.name!r} is an Oracle TIMESTAMP with {column.scale} "
|
|
206
|
+
f"fractional digits: the driver keeps 6 and would cut the rest. "
|
|
207
|
+
f"Cast it to TIMESTAMP(6) in the query to accept that.")
|
|
208
|
+
else:
|
|
209
|
+
message = (f"Result column {column.name!r} is Oracle {column.type_code.name}, which has "
|
|
210
|
+
f"no Arrow type in the Oracle type table; cast it in the query.")
|
|
211
|
+
raise SQLRuntimeError(message, reason="unsupported_result_type", sql=sql)
|
|
212
|
+
fields.append(pa.field(column.name, arrow))
|
|
213
|
+
return _in_utc(sql, description), pa.schema(fields)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _is_digits(entry: Any) -> bool:
|
|
217
|
+
"""Whether ``entry`` is a (precision, scale) pair of a decimal128 (an answer column's digits)."""
|
|
218
|
+
return (isinstance(entry, tuple) and len(entry) == 2 and all(isinstance(n, int) for n in entry)
|
|
219
|
+
and 1 <= entry[0] <= 38 and 0 <= entry[1] <= entry[0])
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _refuse_other_connections(connection: Any) -> None:
|
|
223
|
+
if not hasattr(connection, "fetch_df_all"):
|
|
224
|
+
raise TypeError(
|
|
225
|
+
f"The oracle backend takes an oracledb connection, not a "
|
|
226
|
+
f"{type(connection).__module__}.{type(connection).__name__}: its result is the "
|
|
227
|
+
f"Arrow the driver builds.")
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
class OracleGraph(SQLGraph):
|
|
231
|
+
"""Oracle-backed graph, on the caller's ``oracledb`` connection."""
|
|
232
|
+
|
|
233
|
+
def __init__(
|
|
234
|
+
self,
|
|
235
|
+
schema: Any,
|
|
236
|
+
connection: Any,
|
|
237
|
+
table_stats: Optional[Dict[str, int]] = None,
|
|
238
|
+
):
|
|
239
|
+
require_oracledb()
|
|
240
|
+
_refuse_other_connections(connection)
|
|
241
|
+
super().__init__(schema, connection, dialect='oracle', table_stats=table_stats)
|
|
242
|
+
|
|
243
|
+
# The plan's answer digits reach the reader (`entry._with_digits`): a scale of 0 is decimal128 where known.
|
|
244
|
+
reads_answer_digits = True
|
|
245
|
+
|
|
246
|
+
def _execute_sql_arrow(self, sql: str, digits: Sequence[Any] = ()):
|
|
247
|
+
"""Execute SQL and return the Arrow table the driver builds, each
|
|
248
|
+
column of its requested type (``digits``: the plan's, `typed_statement`). A statement with no result (a write)
|
|
249
|
+
runs on a cursor and returns an empty table; committing stays the
|
|
250
|
+
caller's."""
|
|
251
|
+
import pyarrow as pa
|
|
252
|
+
from prograph.execution import StatementGuard
|
|
253
|
+
# A cancel breaks the connection's statement (`connection.cancel()`).
|
|
254
|
+
with StatementGuard(stop=self._connection.cancel):
|
|
255
|
+
statement, schema = typed_statement(self._connection, sql, digits)
|
|
256
|
+
if not len(schema):
|
|
257
|
+
cursor = self._connection.cursor()
|
|
258
|
+
try:
|
|
259
|
+
cursor.execute(statement)
|
|
260
|
+
finally:
|
|
261
|
+
cursor.close()
|
|
262
|
+
return pa.table({})
|
|
263
|
+
return pa.table(self._connection.fetch_df_all(statement=statement, requested_schema=schema))
|
|
264
|
+
|
|
265
|
+
def _execute_sql(self, sql: str) -> List[Dict[str, Any]]:
|
|
266
|
+
"""Execute SQL against Oracle; rows as dicts."""
|
|
267
|
+
return self._execute_sql_arrow(sql).to_pylist()
|
|
268
|
+
|
|
269
|
+
def _execute_sql_batches(self, sql: str, batch_size: int, digits: Sequence[Any] = ()):
|
|
270
|
+
"""Stream the driver's result as Arrow record batches of
|
|
271
|
+
``batch_size`` rows."""
|
|
272
|
+
import pyarrow as pa
|
|
273
|
+
from prograph.execution import StatementGuard
|
|
274
|
+
# A cancel breaks the connection's statement until the reader ends.
|
|
275
|
+
guard = StatementGuard(stop=self._connection.cancel)
|
|
276
|
+
with guard.opening():
|
|
277
|
+
statement, schema = typed_statement(self._connection, sql, digits)
|
|
278
|
+
frames = self._connection.fetch_df_batches(
|
|
279
|
+
statement=statement, size=batch_size, requested_schema=schema)
|
|
280
|
+
|
|
281
|
+
def batches():
|
|
282
|
+
for frame in frames:
|
|
283
|
+
yield from pa.table(frame).to_batches()
|
|
284
|
+
|
|
285
|
+
return guard.reader(pa.RecordBatchReader.from_batches(schema, batches()))
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
"""PandasGraph - A virtual graph backed by Pandas DataFrames.
|
|
2
|
+
|
|
3
|
+
This holds the DataFrames and the embedded DuckDB that runs the generated SQL over them. Read-only: its mutators refuse (prograph/read_only.py).
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import logging
|
|
7
|
+
import uuid
|
|
8
|
+
import weakref
|
|
9
|
+
from typing import Any, Dict, List, Optional
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
try:
|
|
14
|
+
import numpy as np
|
|
15
|
+
import pandas as pd
|
|
16
|
+
PANDAS_AVAILABLE = True
|
|
17
|
+
except ImportError:
|
|
18
|
+
PANDAS_AVAILABLE = False
|
|
19
|
+
np = None
|
|
20
|
+
pd = None
|
|
21
|
+
|
|
22
|
+
from prograph.schema.mapping import GraphSchema
|
|
23
|
+
from prograph.read_only import GraphRefusesWrites
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _nullable_dtype(dtype):
|
|
27
|
+
"""The nullable type a column of `dtype` takes to hold a missing value
|
|
28
|
+
and keep its type, or None if it holds one already. An integer column
|
|
29
|
+
given a missing value became floats (30 read as 30.0) and DuckDB's view
|
|
30
|
+
over the frame refused it; it becomes Int64, a boolean one
|
|
31
|
+
boolean. The one home of this rule: appends and cell writes both ask."""
|
|
32
|
+
if pd.api.types.is_integer_dtype(dtype) and not pd.api.types.is_extension_array_dtype(dtype):
|
|
33
|
+
return "Int64"
|
|
34
|
+
if pd.api.types.is_bool_dtype(dtype) and not pd.api.types.is_extension_array_dtype(dtype):
|
|
35
|
+
return "boolean"
|
|
36
|
+
return None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _set_cells(df, mask, column: str, value: Any) -> None:
|
|
40
|
+
"""df.loc[mask, column] = value, the column keeping its type: a missing
|
|
41
|
+
value makes it nullable first (_nullable_dtype), and a value that does not
|
|
42
|
+
fit widens it (a nullable column refuses a string, which plain pandas
|
|
43
|
+
columns turned into objects). The cells `set_properties` writes on a
|
|
44
|
+
mutable mapping's frame (prograph.writes)."""
|
|
45
|
+
from prograph.schema.types import is_missing
|
|
46
|
+
if column in df.columns and is_missing(value):
|
|
47
|
+
nullable = _nullable_dtype(df[column].dtype)
|
|
48
|
+
if nullable is not None:
|
|
49
|
+
df[column] = df[column].astype(nullable)
|
|
50
|
+
try:
|
|
51
|
+
df.loc[mask, column] = value
|
|
52
|
+
except (TypeError, ValueError):
|
|
53
|
+
df[column] = df[column].astype(object)
|
|
54
|
+
df.loc[mask, column] = value
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _q(name: str) -> str:
|
|
58
|
+
"""A DuckDB identifier, quoted (docs/architecture/rules.md)."""
|
|
59
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
60
|
+
return get_dialect_instance("duckdb").quote(name)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def frames_connection():
|
|
64
|
+
"""An embedded DuckDB connection for reading pandas frames, which types
|
|
65
|
+
an object column (Python Decimals, strings) from every row: DuckDB
|
|
66
|
+
types one from a sample of its first 1000 rows by default, and its scan
|
|
67
|
+
then rounds the rows past the sample to that size, so
|
|
68
|
+
123456789012.123456 after a thousand 1.50 was read as DECIMAL(19,2),
|
|
69
|
+
123456789012.12. The setting is the database's, so
|
|
70
|
+
every cursor of the connection reads frames the same way."""
|
|
71
|
+
import duckdb
|
|
72
|
+
con = duckdb.connect()
|
|
73
|
+
con.execute(f"SET GLOBAL pandas_analyze_sample = {2 ** 63 - 1}")
|
|
74
|
+
return con
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def register_frame(con, name: str, internal: str, frame, shared: bool = False) -> None:
|
|
78
|
+
"""Make `frame` readable on DuckDB connection `con` under its source
|
|
79
|
+
name `name`. A qualified name (`catalog.schema.table`) is a view in an
|
|
80
|
+
attached in-memory catalog, because DuckDB reads the dots as a path.
|
|
81
|
+
|
|
82
|
+
`shared` is a connection that is someone else's (an owned store's, rule `store-connection`): the view is a
|
|
83
|
+
TEMP view, which lives in the connection's own temporary catalog (nothing is written to the user's
|
|
84
|
+
database file, a view or table of the user's of that name is not replaced, and it is read first where a
|
|
85
|
+
query names it), and a qualified name is refused where it would need a schema in the user's database: a
|
|
86
|
+
three-part name's catalog must be one this call makes (in memory)."""
|
|
87
|
+
con.register(internal, frame)
|
|
88
|
+
parts = name.split(".")
|
|
89
|
+
if shared:
|
|
90
|
+
from prograph.store import check_frame_name
|
|
91
|
+
check_frame_name(con, name)
|
|
92
|
+
if len(parts) == 3:
|
|
93
|
+
con.execute(f"ATTACH IF NOT EXISTS ':memory:' AS {_q(parts[0])}")
|
|
94
|
+
con.execute(f"CREATE SCHEMA IF NOT EXISTS {_q(parts[0])}.{_q(parts[1])}")
|
|
95
|
+
elif len(parts) == 2:
|
|
96
|
+
con.execute(f"CREATE SCHEMA IF NOT EXISTS {_q(parts[0])}")
|
|
97
|
+
quoted = ".".join(_q(p) for p in parts)
|
|
98
|
+
temp = "TEMP " if shared and len(parts) == 1 else ""
|
|
99
|
+
con.execute(f"CREATE OR REPLACE {temp}VIEW {quoted} AS SELECT * FROM {_q(internal)}")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _frame_state(frame) -> tuple:
|
|
103
|
+
"""What tells the embedded DuckDB's view of `frame` is out of date,
|
|
104
|
+
beside the frame being another object: its columns and their types, its
|
|
105
|
+
length, and the arrays that hold its data (_column_data; None where
|
|
106
|
+
they cannot be told, which is always out of date)."""
|
|
107
|
+
return (tuple(zip(map(str, frame.columns), map(str, frame.dtypes))), len(frame),
|
|
108
|
+
_column_data(frame))
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# Extension arrays whose data is these attributes, each an array a write
|
|
112
|
+
# either changes in place or replaces (a pyarrow array is immutable, so a
|
|
113
|
+
# write replaces `_pa_array`; a new category replaces the dtype).
|
|
114
|
+
_ARRAY_PARTS = ("_ndarray", "_data", "_mask", "_codes", "_pa_array", "dtype")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _column_data(frame) -> Optional[tuple]:
|
|
118
|
+
"""The arrays that hold `frame`'s data now, as objects, or None where
|
|
119
|
+
one cannot be told.
|
|
120
|
+
|
|
121
|
+
DuckDB reads a registered frame through the arrays it held when it was
|
|
122
|
+
registered. A write in place into one of them is read by the next
|
|
123
|
+
statement; a write that gives a column new arrays is not: an assigned
|
|
124
|
+
column (`df["age"] = df["age"] + 1`), `drop` and `insert`, and with
|
|
125
|
+
copy-on-write (pandas 3's default) every write, `df.loc`, `df.at` and
|
|
126
|
+
the engine's own SET included, since the registration holds the old
|
|
127
|
+
arrays and pandas copies them first. So the
|
|
128
|
+
frame is registered again when any of these is another object. They
|
|
129
|
+
are pandas' block arrays (`frame._mgr.arrays`, an internal of pandas 2
|
|
130
|
+
and 3) and, for an extension array, the parts named in _ARRAY_PARTS.
|
|
131
|
+
The objects are kept, held with the state, so none of them can be freed
|
|
132
|
+
and another take its id while it is compared. An array of another kind,
|
|
133
|
+
or a pandas without `_mgr.arrays`, cannot be told: None."""
|
|
134
|
+
import numpy
|
|
135
|
+
arrays = getattr(getattr(frame, "_mgr", None), "arrays", None)
|
|
136
|
+
if arrays is None:
|
|
137
|
+
return None
|
|
138
|
+
held = []
|
|
139
|
+
for array in arrays:
|
|
140
|
+
held.append(array)
|
|
141
|
+
if isinstance(array, numpy.ndarray):
|
|
142
|
+
continue
|
|
143
|
+
parts = [getattr(array, name) for name in _ARRAY_PARTS if hasattr(array, name)]
|
|
144
|
+
if len(parts) < 2: # its dtype alone: data this does not know
|
|
145
|
+
return None
|
|
146
|
+
held.extend(parts)
|
|
147
|
+
return tuple(held)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _same_state(held: tuple, state: tuple) -> bool:
|
|
151
|
+
"""Whether two _frame_state are of the same data: the same columns,
|
|
152
|
+
types and length, and the very same arrays."""
|
|
153
|
+
if held[2] is None or state[2] is None:
|
|
154
|
+
return False
|
|
155
|
+
return (held[:2] == state[:2] and len(held[2]) == len(state[2])
|
|
156
|
+
and all(a is b for a, b in zip(held[2], state[2])))
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class PandasGraph(GraphRefusesWrites):
|
|
160
|
+
"""A virtual graph backed by Pandas DataFrames.
|
|
161
|
+
|
|
162
|
+
Runs the generated SQL in an embedded DuckDB over the DataFrames.
|
|
163
|
+
Read-only over the caller's frames: its mutators refuse.
|
|
164
|
+
|
|
165
|
+
Example:
|
|
166
|
+
>>> schema = GraphSchema()
|
|
167
|
+
>>> schema.add_node_mapping(NodeMapping(
|
|
168
|
+
... source_table='movies',
|
|
169
|
+
... label='Movie',
|
|
170
|
+
... id_column='movie_id',
|
|
171
|
+
... property_columns={'title': 'title'},
|
|
172
|
+
... ))
|
|
173
|
+
>>> graph = PandasGraph(schema, {'movies': movies_df})
|
|
174
|
+
>>> graph.enable_local_sql()
|
|
175
|
+
"""
|
|
176
|
+
|
|
177
|
+
def __init__(
|
|
178
|
+
self,
|
|
179
|
+
schema: GraphSchema,
|
|
180
|
+
dataframes: Dict[str, 'pd.DataFrame']
|
|
181
|
+
):
|
|
182
|
+
"""Create a PandasGraph.
|
|
183
|
+
|
|
184
|
+
Args:
|
|
185
|
+
schema: GraphSchema defining the mapping
|
|
186
|
+
dataframes: Dictionary mapping table names to DataFrames
|
|
187
|
+
"""
|
|
188
|
+
if not PANDAS_AVAILABLE:
|
|
189
|
+
raise ImportError("pandas is required for the Pandas backend")
|
|
190
|
+
|
|
191
|
+
self.schema = schema
|
|
192
|
+
self.dataframes = dataframes
|
|
193
|
+
# An embedded DuckDB over these same frames, when enabled. See
|
|
194
|
+
# enable_local_sql.
|
|
195
|
+
self._local_sql = None
|
|
196
|
+
# True on a connection that is the caller's (an owned store's): see `register_frame`.
|
|
197
|
+
self._shared = False
|
|
198
|
+
# Each graph's registered names are its own: two graphs on one connection never read one another's frames.
|
|
199
|
+
self._frame_prefix = f"__prograph_frame_{uuid.uuid4().hex[:12]}_"
|
|
200
|
+
self._local_views: Dict[str, str] = {}
|
|
201
|
+
# By frame name: (a weak reference to the frame registered, then
|
|
202
|
+
# _frame_state of it then, which holds its arrays).
|
|
203
|
+
self._local_view_sources: Dict[str, tuple] = {}
|
|
204
|
+
|
|
205
|
+
# ------------------------------------------------------------------
|
|
206
|
+
# Local SQL: DuckDB over the frames, with this graph as the fallback
|
|
207
|
+
# ------------------------------------------------------------------
|
|
208
|
+
|
|
209
|
+
def enable_local_sql(self, connection=None) -> None:
|
|
210
|
+
"""Run generated SQL in an embedded DuckDB over these frames: on `connection`, a DuckDB connection the
|
|
211
|
+
frames are registered on (an owned store's own, so a query reads the frames and the store's statements
|
|
212
|
+
on one connection; `store.engine_connection`), else a new one (`frames_connection`).
|
|
213
|
+
|
|
214
|
+
This is what `ProGraph(backend='duckdb', dataframes=...)` builds:
|
|
215
|
+
every query runs as SQL, through DuckDB, over the frames.
|
|
216
|
+
"""
|
|
217
|
+
self._shared = connection is not None
|
|
218
|
+
if connection is None:
|
|
219
|
+
self._local_sql = frames_connection()
|
|
220
|
+
# A timestamp's parts (year(), hour()) are read in the session time
|
|
221
|
+
# zone, UTC; the machine's zone made
|
|
222
|
+
# 2011-02-01T01:02:03Z day 31 west of Greenwich.
|
|
223
|
+
self._local_sql.execute("SET TimeZone = 'UTC'")
|
|
224
|
+
else:
|
|
225
|
+
# The caller's connection: its session time zone stays the caller's, and the frames' typing is the
|
|
226
|
+
# database's setting, as `frames_connection` sets it.
|
|
227
|
+
connection.execute(f"SET GLOBAL pandas_analyze_sample = {2 ** 63 - 1}")
|
|
228
|
+
self._local_sql = connection
|
|
229
|
+
self._local_views = {}
|
|
230
|
+
self._local_view_sources = {}
|
|
231
|
+
|
|
232
|
+
def sql_dialect(self) -> Optional[str]:
|
|
233
|
+
"""The dialect this graph executes SQL in, or None if it has none."""
|
|
234
|
+
return "duckdb" if self._local_sql is not None else None
|
|
235
|
+
|
|
236
|
+
def execute_local_sql_arrow(self, sql: str):
|
|
237
|
+
"""execute_local_sql, returning DuckDB's Arrow table as is. A cancel
|
|
238
|
+
interrupts the statement (`StatementGuard`)."""
|
|
239
|
+
from prograph.execution import StatementGuard
|
|
240
|
+
from prograph.graph.drivers import duckdb_arrow_table
|
|
241
|
+
with StatementGuard(stop=self._local_sql.interrupt):
|
|
242
|
+
return duckdb_arrow_table(self._prepare_local_sql(), sql)
|
|
243
|
+
|
|
244
|
+
def execute_local_sql_batches(self, sql: str, batch_size: int):
|
|
245
|
+
"""execute_local_sql as a stream of Arrow record batches.
|
|
246
|
+
|
|
247
|
+
The reader is read after execute() returns, outside the engine's
|
|
248
|
+
lock, so it runs on a cursor of its own: another query on the
|
|
249
|
+
shared connection would otherwise end its result. The cursor
|
|
250
|
+
registers the same frames (registrations are per connection; the
|
|
251
|
+
views over them are shared)."""
|
|
252
|
+
from prograph.execution import StatementGuard, checkpoint
|
|
253
|
+
from prograph.graph.drivers import duckdb_arrow_reader
|
|
254
|
+
checkpoint()
|
|
255
|
+
self._prepare_local_sql() # the views and their frame names exist
|
|
256
|
+
cursor = self._local_sql.cursor()
|
|
257
|
+
for name, frame in self.dataframes.items():
|
|
258
|
+
cursor.register(self._local_views[name], frame)
|
|
259
|
+
self._views_on(cursor)
|
|
260
|
+
# A cancel interrupts the cursor's statement until the reader ends.
|
|
261
|
+
guard = StatementGuard(stop=cursor.interrupt)
|
|
262
|
+
with guard.opening(close=cursor.close):
|
|
263
|
+
reader = duckdb_arrow_reader(cursor, sql, batch_size)
|
|
264
|
+
return guard.reader(reader, close=cursor.close)
|
|
265
|
+
|
|
266
|
+
def _views_on(self, cursor) -> None:
|
|
267
|
+
"""On a connection that is the caller's (an owned store's) the frames' views are TEMP views, each
|
|
268
|
+
connection's own: a cursor makes its own over the frames it registered (`register_frame`)."""
|
|
269
|
+
if self._shared:
|
|
270
|
+
for name, frame in self.dataframes.items():
|
|
271
|
+
if "." not in name:
|
|
272
|
+
register_frame(cursor, name, self._local_views[name], frame, shared=True)
|
|
273
|
+
|
|
274
|
+
def _run_local_sql(self, sql: str, parameters: Optional[Dict[str, Any]] = None):
|
|
275
|
+
"""Register the frames that changed since the last statement, then
|
|
276
|
+
run `sql`; the open result. `parameters` are bound by DuckDB
|
|
277
|
+
(`$name`)."""
|
|
278
|
+
con = self._prepare_local_sql()
|
|
279
|
+
return con.execute(sql, parameters) if parameters else con.execute(sql)
|
|
280
|
+
|
|
281
|
+
def _prepare_local_sql(self):
|
|
282
|
+
"""Register the frames that changed since the last statement; the
|
|
283
|
+
connection a statement then runs on.
|
|
284
|
+
|
|
285
|
+
Each frame is a view under its source name over the frame registered
|
|
286
|
+
as `__prograph_frame_N`. DuckDB reads a registered frame when a
|
|
287
|
+
statement runs, through the arrays it was given, so a cell written into one of them,
|
|
288
|
+
by the engine or by the frame's owner, is read with no new
|
|
289
|
+
registration. A frame is registered again when it is another object
|
|
290
|
+
(a mutation replaces it in `self.dataframes`: concatenation, boolean
|
|
291
|
+
filters), has other columns, column types or length, or holds its
|
|
292
|
+
data in other arrays (_frame_state).
|
|
293
|
+
Registering every frame before every statement cost three analyses
|
|
294
|
+
of each object column (Python Decimals, strings) every time, which
|
|
295
|
+
DuckDB types by reading every value (frames_connection): 6.9 s of a
|
|
296
|
+
10 s query at 1M rows."""
|
|
297
|
+
con = self._local_sql
|
|
298
|
+
for name, frame in self.dataframes.items():
|
|
299
|
+
if len(frame.columns) == 0:
|
|
300
|
+
# A frame with no columns is no table, and DuckDB refuses to
|
|
301
|
+
# register one. Registering every frame before every
|
|
302
|
+
# statement, one such frame anywhere among them failed
|
|
303
|
+
# every statement, whatever it read; in `auto` mode that
|
|
304
|
+
# was a silent fall back to Python for the whole engine.
|
|
305
|
+
# A query that does read it gets DuckDB's own "no such
|
|
306
|
+
# table".
|
|
307
|
+
continue
|
|
308
|
+
state = _frame_state(frame)
|
|
309
|
+
held = self._local_view_sources.get(name)
|
|
310
|
+
if held is not None and held[0]() is frame and _same_state(held[1:], state):
|
|
311
|
+
if self._shared and "." not in name:
|
|
312
|
+
# Another graph on this connection may have made its own view of this name since: make
|
|
313
|
+
# this graph's again (a statement of the catalog alone, no frame is read).
|
|
314
|
+
quoted = _q(name)
|
|
315
|
+
con.execute(f"CREATE OR REPLACE TEMP VIEW {quoted} AS SELECT * FROM {_q(self._local_views[name])}")
|
|
316
|
+
continue
|
|
317
|
+
internal = self._local_views.get(name)
|
|
318
|
+
if internal is None:
|
|
319
|
+
internal = f"{self._frame_prefix}{len(self._local_views)}"
|
|
320
|
+
self._local_views[name] = internal
|
|
321
|
+
register_frame(con, name, internal, frame, shared=self._shared)
|
|
322
|
+
else:
|
|
323
|
+
con.register(internal, frame)
|
|
324
|
+
# A view fixes its column types when created; a frame of
|
|
325
|
+
# other columns or types needs the view made again.
|
|
326
|
+
# On a connection that is the caller's another graph may have made its view of this name since
|
|
327
|
+
# (`register_frame`): this graph's is made again each time.
|
|
328
|
+
if held is None or held[1] != state[0] or (self._shared and "." not in name):
|
|
329
|
+
quoted = ".".join(_q(p) for p in name.split("."))
|
|
330
|
+
con.execute(f"CREATE OR REPLACE {'TEMP ' if self._shared and '.' not in name else ''}VIEW {quoted} "
|
|
331
|
+
f"AS SELECT * FROM {_q(internal)}")
|
|
332
|
+
self._local_view_sources[name] = (weakref.ref(frame),) + state
|
|
333
|
+
return con
|
|
334
|
+
|
|
335
|
+
def _invalidate_caches(self, changes: 'SchemaChangeSet') -> None:
|
|
336
|
+
"""Drop the frames of tables a schema change removed."""
|
|
337
|
+
if not changes.has_changes:
|
|
338
|
+
return
|
|
339
|
+
for table in changes.removed_tables:
|
|
340
|
+
self.dataframes.pop(table, None)
|
|
341
|
+
|
|
342
|
+
def __repr__(self) -> str:
|
|
343
|
+
return f"PandasGraph(tables={sorted(self.dataframes)})"
|