prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,285 @@
1
+ """Oracle Graph Backend.
2
+
3
+ Reads and writes Oracle through ``oracledb`` (thin mode), on the connection
4
+ the caller gives: a result is the Arrow the driver builds, with
5
+ each column's Arrow type requested from its declared Oracle type.
6
+
7
+ Usage:
8
+
9
+ import oracledb
10
+ conn = oracledb.connect(user="prograph", password="...",
11
+ dsn="localhost:1521/XEPDB1")
12
+ graph = OracleGraph(schema, conn)
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from typing import Any, Dict, List, Optional, Sequence, Tuple
18
+
19
+ from .drivers import require_driver
20
+ from .sql_graph import SQLGraph
21
+ from prograph.rules import home
22
+
23
+ PACKAGE = "oracledb"
24
+ # Below 26.0.1 a bare NULL column makes the Arrow fetch lose its rows, or
25
+ # crash the process.
26
+ MINIMUM_VERSION = (26, 0, 1)
27
+
28
+
29
+ def _version(text: str) -> Tuple[int, ...]:
30
+ """A release as a comparable tuple: ``26.0.1`` is (26, 0, 1, 1). A
31
+ pre-release (``26.0.1rc1``, ``26.0.1.dev0``) sorts before its release, so
32
+ ``26.0.0rc1`` and ``26.0.1rc1`` are both below the floor."""
33
+ import re
34
+ match = re.match(r"(\d+)(?:\.(\d+))?(?:\.(\d+))?(.*)$", text.strip())
35
+ if not match:
36
+ return (0,)
37
+ major, minor, micro, rest = match.groups()
38
+ return (int(major), int(minor or 0), int(micro or 0), 0 if rest else 1)
39
+
40
+
41
+ def require_oracledb() -> Any:
42
+ """The ``oracledb`` module, refused like a missing driver if it is older
43
+ than the floor."""
44
+ module = require_driver("oracledb", PACKAGE, "oracle")
45
+ if _version(module.__version__) < MINIMUM_VERSION + (1,):
46
+ from prograph.utils.exceptions import MissingDriverError
47
+ floor = ".".join(map(str, MINIMUM_VERSION))
48
+ raise MissingDriverError(
49
+ f"The oracle backend needs {PACKAGE} {floor} or later (pip install "
50
+ f"'{PACKAGE}>={floor}'); {module.__version__} loses rows of a result with a "
51
+ f"bare NULL column.", package=PACKAGE, backend="oracle")
52
+ return module
53
+
54
+
55
+ @home("oracle-integer-number", "Oracle integer NUMBER",
56
+ terms=("Oracle", "NUMBER", "scale", "integer column"))
57
+ def number_is_integer(scale: Optional[int]) -> bool:
58
+ """Whether an Oracle ``NUMBER`` of this declared scale holds integers only:
59
+ a known scale of 0 or below (``NUMBER(9)``, ``INTEGER``, ``NUMBER(5,-2)``).
60
+ An unknown scale (a plain ``NUMBER``, an expression: none, or -127 from a
61
+ cursor's description) may hold fractions. The one home of this rule
62
+ (docs/architecture/rules.md): result types and column types both ask it."""
63
+ return scale is not None and scale != -127 and scale <= 0
64
+
65
+
66
+ def number_digits(precision: Optional[int], scale: Optional[int]) -> Optional[Tuple[int, int]]:
67
+ """The exact decimal an Oracle ``NUMBER`` of this declared precision and scale holds, as
68
+ (precision, scale), where the scale is above 0 and both fit a 38-digit decimal (``NUMBER(20,2)``;
69
+ ``NUMBER(3,5)`` holds 0.00123, five digits); None for an integer (`number_is_integer`) or an
70
+ unknown precision or scale (a plain ``NUMBER``, ``FLOAT``: a floating decimal). Result types and
71
+ column types both ask it, as they ask `number_is_integer`."""
72
+ if scale is None or scale == -127 or scale <= 0 or not precision or precision > 38 or scale > 38:
73
+ return None
74
+ return (max(precision, scale), scale)
75
+
76
+
77
+ def returns_rows(sql: str) -> bool:
78
+ """Whether ``sql`` can have result columns: its first keyword, after
79
+ comments and parentheses, is SELECT or WITH. Oracle runs DDL when it is
80
+ parsed, so only such a statement is parsed for its types; anything else
81
+ runs once, on a cursor."""
82
+ import re
83
+ text = re.sub(r"--[^\n]*|/\*.*?\*/", " ", sql, flags=re.S).lstrip(" \t\r\n(")
84
+ return re.match(r"(?i)(select|with)\b", text) is not None
85
+
86
+
87
+ def _arrow_type(column) -> Optional[Any]:
88
+ """The Arrow type to request for a result column, from its declared
89
+ Oracle type, precision and scale, or None for a
90
+ type the table has no row for. ``int64`` is requested for a declared scale of 0 or below, never for a
91
+ number of unknown scale, which the driver would truncate; a NUMBER of known precision and a scale
92
+ above 0 is the exact ``decimal128`` of its digits (`number_digits`); one of unknown precision or
93
+ scale (a plain NUMBER) is ``float64``, which keeps about 15 significant digits."""
94
+ import pyarrow as pa
95
+ name = column.type_code.name
96
+ if name == "DB_TYPE_NUMBER":
97
+ if number_is_integer(column.scale):
98
+ return pa.int64()
99
+ digits = number_digits(column.precision, column.scale)
100
+ return pa.decimal128(*digits) if digits else pa.float64()
101
+ if name in ("DB_TYPE_BINARY_DOUBLE",):
102
+ return pa.float64()
103
+ if name in ("DB_TYPE_BINARY_FLOAT",):
104
+ return pa.float32()
105
+ if name in ("DB_TYPE_VARCHAR", "DB_TYPE_NVARCHAR", "DB_TYPE_CHAR", "DB_TYPE_NCHAR",
106
+ "DB_TYPE_CLOB", "DB_TYPE_NCLOB"):
107
+ return pa.string()
108
+ if name == "DB_TYPE_DATE":
109
+ return pa.timestamp("s")
110
+ if name == "DB_TYPE_TIMESTAMP" or name in _ZONED:
111
+ # The driver keeps microseconds only, even asked for nanoseconds:
112
+ # more fractional digits would be cut, so such a column is not
113
+ # requested.
114
+ if (column.scale or 0) > 6:
115
+ return None
116
+ return pa.timestamp("us", tz="UTC" if name in _ZONED else None)
117
+ if name in ("DB_TYPE_RAW", "DB_TYPE_BLOB"):
118
+ return pa.binary()
119
+ # Not listed: LONG and LONG RAW (deprecated), BOOLEAN (Oracle 23ai, not
120
+ # the 21c the tests run on), INTERVAL and the object types. Each is
121
+ # refused by name until a row and its test are added.
122
+ return None
123
+
124
+
125
+ def _description(connection: Any, sql: str):
126
+ """The result columns of ``sql`` as Oracle declares them: parsed, not run."""
127
+ cursor = connection.cursor()
128
+ try:
129
+ cursor.parse(sql)
130
+ return cursor.description or []
131
+ finally:
132
+ cursor.close()
133
+
134
+
135
+ _ZONED = ("DB_TYPE_TIMESTAMP_TZ", "DB_TYPE_TIMESTAMP_LTZ")
136
+
137
+
138
+ def _in_utc(sql: str, description) -> str:
139
+ """``sql`` with each zoned timestamp column converted to UTC
140
+ (``SYS_EXTRACT_UTC``), or ``sql`` itself if it has none.
141
+
142
+ The query is selected from as an inline view, each column by the exact
143
+ name the parse reported, so a query with its own ``WITH`` can be
144
+ wrapped (Oracle refuses a ``WITH`` inside a ``WITH`` column list,
145
+ ORA-32034). Where two columns share a name, a view cannot tell them
146
+ apart, so they are renamed by position in a ``WITH`` column list
147
+ instead, which a query that has its own ``WITH`` cannot take."""
148
+ zoned = [i for i, column in enumerate(description) if column.type_code.name in _ZONED]
149
+ if not zoned:
150
+ return sql
151
+ from prograph.plan.sql.dialect import get_dialect_instance
152
+ quote = get_dialect_instance('oracle').quote_exact
153
+ names = [column.name for column in description]
154
+
155
+ def value(i, ref):
156
+ return f"SYS_EXTRACT_UTC({ref})" if i in zoned else ref
157
+
158
+ if len(set(names)) == len(names):
159
+ select = ", ".join(f'{value(i, f"q.{quote(name)}")} AS {quote(name)}' for i, name in enumerate(names))
160
+ return f"SELECT {select} FROM (\n{sql}\n) q"
161
+ positions = ", ".join(f"c{i}" for i in range(len(names)))
162
+ select = ", ".join(f"{value(i, f'c{i}')} AS {quote(name)}" for i, name in enumerate(names))
163
+ return f"WITH q ({positions}) AS (\n{sql}\n) SELECT {select} FROM q"
164
+
165
+
166
+ def _terminated(sql: str) -> str:
167
+ # Oracle takes one statement with no terminator; a caller's SQL may end
168
+ # with a semicolon.
169
+ return sql.rstrip().rstrip(';').rstrip()
170
+
171
+
172
+ def utc_statement(connection: Any, sql: str) -> str:
173
+ """``sql`` as ProGraph's own readers of the database run it on Oracle
174
+ (``drivers.fetch_rows``): a zoned timestamp read as its UTC instant, as a
175
+ query's result is. The driver gives a reader the
176
+ naive time in the value's own zone otherwise."""
177
+ sql = _terminated(sql)
178
+ if not returns_rows(sql):
179
+ return sql
180
+ return _in_utc(sql, _description(connection, sql))
181
+
182
+
183
+ def typed_statement(connection: Any, sql: str, digits: Sequence[Any] = ()):
184
+ """(statement, requested Arrow schema) for ``sql`` on Oracle. ``digits``: each answer column's
185
+ DECIMAL(p, s) where the plan knows it (`lower.ANSWER_DIGITS`); an integer-scaled column (an integer by
186
+ `number_is_integer`) of known digits is the exact ``decimal128(p, 0)`` (a SUM of integers), not int64.
187
+
188
+ The statement is parsed, not run, for each result column's declared
189
+ type; a zoned timestamp is converted to UTC in the SQL (``_in_utc``). A
190
+ column of a type with no row in the table is refused by name."""
191
+ import pyarrow as pa
192
+ from prograph.utils.exceptions import SQLRuntimeError
193
+ sql = _terminated(sql)
194
+ if not returns_rows(sql):
195
+ return sql, pa.schema([])
196
+ description = _description(connection, sql)
197
+ fields = []
198
+ known = list(digits) if len(digits) == len(description) else [None] * len(description)
199
+ for column, entry in zip(description, known):
200
+ arrow = _arrow_type(column)
201
+ if pa.types.is_int64(arrow or pa.null()) and _is_digits(entry) and entry[1] == 0:
202
+ arrow = pa.decimal128(entry[0], 0)
203
+ if arrow is None:
204
+ if "TIMESTAMP" in column.type_code.name and (column.scale or 0) > 6:
205
+ message = (f"Result column {column.name!r} is an Oracle TIMESTAMP with {column.scale} "
206
+ f"fractional digits: the driver keeps 6 and would cut the rest. "
207
+ f"Cast it to TIMESTAMP(6) in the query to accept that.")
208
+ else:
209
+ message = (f"Result column {column.name!r} is Oracle {column.type_code.name}, which has "
210
+ f"no Arrow type in the Oracle type table; cast it in the query.")
211
+ raise SQLRuntimeError(message, reason="unsupported_result_type", sql=sql)
212
+ fields.append(pa.field(column.name, arrow))
213
+ return _in_utc(sql, description), pa.schema(fields)
214
+
215
+
216
+ def _is_digits(entry: Any) -> bool:
217
+ """Whether ``entry`` is a (precision, scale) pair of a decimal128 (an answer column's digits)."""
218
+ return (isinstance(entry, tuple) and len(entry) == 2 and all(isinstance(n, int) for n in entry)
219
+ and 1 <= entry[0] <= 38 and 0 <= entry[1] <= entry[0])
220
+
221
+
222
+ def _refuse_other_connections(connection: Any) -> None:
223
+ if not hasattr(connection, "fetch_df_all"):
224
+ raise TypeError(
225
+ f"The oracle backend takes an oracledb connection, not a "
226
+ f"{type(connection).__module__}.{type(connection).__name__}: its result is the "
227
+ f"Arrow the driver builds.")
228
+
229
+
230
+ class OracleGraph(SQLGraph):
231
+ """Oracle-backed graph, on the caller's ``oracledb`` connection."""
232
+
233
+ def __init__(
234
+ self,
235
+ schema: Any,
236
+ connection: Any,
237
+ table_stats: Optional[Dict[str, int]] = None,
238
+ ):
239
+ require_oracledb()
240
+ _refuse_other_connections(connection)
241
+ super().__init__(schema, connection, dialect='oracle', table_stats=table_stats)
242
+
243
+ # The plan's answer digits reach the reader (`entry._with_digits`): a scale of 0 is decimal128 where known.
244
+ reads_answer_digits = True
245
+
246
+ def _execute_sql_arrow(self, sql: str, digits: Sequence[Any] = ()):
247
+ """Execute SQL and return the Arrow table the driver builds, each
248
+ column of its requested type (``digits``: the plan's, `typed_statement`). A statement with no result (a write)
249
+ runs on a cursor and returns an empty table; committing stays the
250
+ caller's."""
251
+ import pyarrow as pa
252
+ from prograph.execution import StatementGuard
253
+ # A cancel breaks the connection's statement (`connection.cancel()`).
254
+ with StatementGuard(stop=self._connection.cancel):
255
+ statement, schema = typed_statement(self._connection, sql, digits)
256
+ if not len(schema):
257
+ cursor = self._connection.cursor()
258
+ try:
259
+ cursor.execute(statement)
260
+ finally:
261
+ cursor.close()
262
+ return pa.table({})
263
+ return pa.table(self._connection.fetch_df_all(statement=statement, requested_schema=schema))
264
+
265
+ def _execute_sql(self, sql: str) -> List[Dict[str, Any]]:
266
+ """Execute SQL against Oracle; rows as dicts."""
267
+ return self._execute_sql_arrow(sql).to_pylist()
268
+
269
+ def _execute_sql_batches(self, sql: str, batch_size: int, digits: Sequence[Any] = ()):
270
+ """Stream the driver's result as Arrow record batches of
271
+ ``batch_size`` rows."""
272
+ import pyarrow as pa
273
+ from prograph.execution import StatementGuard
274
+ # A cancel breaks the connection's statement until the reader ends.
275
+ guard = StatementGuard(stop=self._connection.cancel)
276
+ with guard.opening():
277
+ statement, schema = typed_statement(self._connection, sql, digits)
278
+ frames = self._connection.fetch_df_batches(
279
+ statement=statement, size=batch_size, requested_schema=schema)
280
+
281
+ def batches():
282
+ for frame in frames:
283
+ yield from pa.table(frame).to_batches()
284
+
285
+ return guard.reader(pa.RecordBatchReader.from_batches(schema, batches()))
@@ -0,0 +1,343 @@
1
+ """PandasGraph - A virtual graph backed by Pandas DataFrames.
2
+
3
+ This holds the DataFrames and the embedded DuckDB that runs the generated SQL over them. Read-only: its mutators refuse (prograph/read_only.py).
4
+ """
5
+
6
+ import logging
7
+ import uuid
8
+ import weakref
9
+ from typing import Any, Dict, List, Optional
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+ try:
14
+ import numpy as np
15
+ import pandas as pd
16
+ PANDAS_AVAILABLE = True
17
+ except ImportError:
18
+ PANDAS_AVAILABLE = False
19
+ np = None
20
+ pd = None
21
+
22
+ from prograph.schema.mapping import GraphSchema
23
+ from prograph.read_only import GraphRefusesWrites
24
+
25
+
26
+ def _nullable_dtype(dtype):
27
+ """The nullable type a column of `dtype` takes to hold a missing value
28
+ and keep its type, or None if it holds one already. An integer column
29
+ given a missing value became floats (30 read as 30.0) and DuckDB's view
30
+ over the frame refused it; it becomes Int64, a boolean one
31
+ boolean. The one home of this rule: appends and cell writes both ask."""
32
+ if pd.api.types.is_integer_dtype(dtype) and not pd.api.types.is_extension_array_dtype(dtype):
33
+ return "Int64"
34
+ if pd.api.types.is_bool_dtype(dtype) and not pd.api.types.is_extension_array_dtype(dtype):
35
+ return "boolean"
36
+ return None
37
+
38
+
39
+ def _set_cells(df, mask, column: str, value: Any) -> None:
40
+ """df.loc[mask, column] = value, the column keeping its type: a missing
41
+ value makes it nullable first (_nullable_dtype), and a value that does not
42
+ fit widens it (a nullable column refuses a string, which plain pandas
43
+ columns turned into objects). The cells `set_properties` writes on a
44
+ mutable mapping's frame (prograph.writes)."""
45
+ from prograph.schema.types import is_missing
46
+ if column in df.columns and is_missing(value):
47
+ nullable = _nullable_dtype(df[column].dtype)
48
+ if nullable is not None:
49
+ df[column] = df[column].astype(nullable)
50
+ try:
51
+ df.loc[mask, column] = value
52
+ except (TypeError, ValueError):
53
+ df[column] = df[column].astype(object)
54
+ df.loc[mask, column] = value
55
+
56
+
57
+ def _q(name: str) -> str:
58
+ """A DuckDB identifier, quoted (docs/architecture/rules.md)."""
59
+ from prograph.plan.sql.dialect import get_dialect_instance
60
+ return get_dialect_instance("duckdb").quote(name)
61
+
62
+
63
+ def frames_connection():
64
+ """An embedded DuckDB connection for reading pandas frames, which types
65
+ an object column (Python Decimals, strings) from every row: DuckDB
66
+ types one from a sample of its first 1000 rows by default, and its scan
67
+ then rounds the rows past the sample to that size, so
68
+ 123456789012.123456 after a thousand 1.50 was read as DECIMAL(19,2),
69
+ 123456789012.12. The setting is the database's, so
70
+ every cursor of the connection reads frames the same way."""
71
+ import duckdb
72
+ con = duckdb.connect()
73
+ con.execute(f"SET GLOBAL pandas_analyze_sample = {2 ** 63 - 1}")
74
+ return con
75
+
76
+
77
+ def register_frame(con, name: str, internal: str, frame, shared: bool = False) -> None:
78
+ """Make `frame` readable on DuckDB connection `con` under its source
79
+ name `name`. A qualified name (`catalog.schema.table`) is a view in an
80
+ attached in-memory catalog, because DuckDB reads the dots as a path.
81
+
82
+ `shared` is a connection that is someone else's (an owned store's, rule `store-connection`): the view is a
83
+ TEMP view, which lives in the connection's own temporary catalog (nothing is written to the user's
84
+ database file, a view or table of the user's of that name is not replaced, and it is read first where a
85
+ query names it), and a qualified name is refused where it would need a schema in the user's database: a
86
+ three-part name's catalog must be one this call makes (in memory)."""
87
+ con.register(internal, frame)
88
+ parts = name.split(".")
89
+ if shared:
90
+ from prograph.store import check_frame_name
91
+ check_frame_name(con, name)
92
+ if len(parts) == 3:
93
+ con.execute(f"ATTACH IF NOT EXISTS ':memory:' AS {_q(parts[0])}")
94
+ con.execute(f"CREATE SCHEMA IF NOT EXISTS {_q(parts[0])}.{_q(parts[1])}")
95
+ elif len(parts) == 2:
96
+ con.execute(f"CREATE SCHEMA IF NOT EXISTS {_q(parts[0])}")
97
+ quoted = ".".join(_q(p) for p in parts)
98
+ temp = "TEMP " if shared and len(parts) == 1 else ""
99
+ con.execute(f"CREATE OR REPLACE {temp}VIEW {quoted} AS SELECT * FROM {_q(internal)}")
100
+
101
+
102
+ def _frame_state(frame) -> tuple:
103
+ """What tells the embedded DuckDB's view of `frame` is out of date,
104
+ beside the frame being another object: its columns and their types, its
105
+ length, and the arrays that hold its data (_column_data; None where
106
+ they cannot be told, which is always out of date)."""
107
+ return (tuple(zip(map(str, frame.columns), map(str, frame.dtypes))), len(frame),
108
+ _column_data(frame))
109
+
110
+
111
+ # Extension arrays whose data is these attributes, each an array a write
112
+ # either changes in place or replaces (a pyarrow array is immutable, so a
113
+ # write replaces `_pa_array`; a new category replaces the dtype).
114
+ _ARRAY_PARTS = ("_ndarray", "_data", "_mask", "_codes", "_pa_array", "dtype")
115
+
116
+
117
+ def _column_data(frame) -> Optional[tuple]:
118
+ """The arrays that hold `frame`'s data now, as objects, or None where
119
+ one cannot be told.
120
+
121
+ DuckDB reads a registered frame through the arrays it held when it was
122
+ registered. A write in place into one of them is read by the next
123
+ statement; a write that gives a column new arrays is not: an assigned
124
+ column (`df["age"] = df["age"] + 1`), `drop` and `insert`, and with
125
+ copy-on-write (pandas 3's default) every write, `df.loc`, `df.at` and
126
+ the engine's own SET included, since the registration holds the old
127
+ arrays and pandas copies them first. So the
128
+ frame is registered again when any of these is another object. They
129
+ are pandas' block arrays (`frame._mgr.arrays`, an internal of pandas 2
130
+ and 3) and, for an extension array, the parts named in _ARRAY_PARTS.
131
+ The objects are kept, held with the state, so none of them can be freed
132
+ and another take its id while it is compared. An array of another kind,
133
+ or a pandas without `_mgr.arrays`, cannot be told: None."""
134
+ import numpy
135
+ arrays = getattr(getattr(frame, "_mgr", None), "arrays", None)
136
+ if arrays is None:
137
+ return None
138
+ held = []
139
+ for array in arrays:
140
+ held.append(array)
141
+ if isinstance(array, numpy.ndarray):
142
+ continue
143
+ parts = [getattr(array, name) for name in _ARRAY_PARTS if hasattr(array, name)]
144
+ if len(parts) < 2: # its dtype alone: data this does not know
145
+ return None
146
+ held.extend(parts)
147
+ return tuple(held)
148
+
149
+
150
+ def _same_state(held: tuple, state: tuple) -> bool:
151
+ """Whether two _frame_state are of the same data: the same columns,
152
+ types and length, and the very same arrays."""
153
+ if held[2] is None or state[2] is None:
154
+ return False
155
+ return (held[:2] == state[:2] and len(held[2]) == len(state[2])
156
+ and all(a is b for a, b in zip(held[2], state[2])))
157
+
158
+
159
+ class PandasGraph(GraphRefusesWrites):
160
+ """A virtual graph backed by Pandas DataFrames.
161
+
162
+ Runs the generated SQL in an embedded DuckDB over the DataFrames.
163
+ Read-only over the caller's frames: its mutators refuse.
164
+
165
+ Example:
166
+ >>> schema = GraphSchema()
167
+ >>> schema.add_node_mapping(NodeMapping(
168
+ ... source_table='movies',
169
+ ... label='Movie',
170
+ ... id_column='movie_id',
171
+ ... property_columns={'title': 'title'},
172
+ ... ))
173
+ >>> graph = PandasGraph(schema, {'movies': movies_df})
174
+ >>> graph.enable_local_sql()
175
+ """
176
+
177
+ def __init__(
178
+ self,
179
+ schema: GraphSchema,
180
+ dataframes: Dict[str, 'pd.DataFrame']
181
+ ):
182
+ """Create a PandasGraph.
183
+
184
+ Args:
185
+ schema: GraphSchema defining the mapping
186
+ dataframes: Dictionary mapping table names to DataFrames
187
+ """
188
+ if not PANDAS_AVAILABLE:
189
+ raise ImportError("pandas is required for the Pandas backend")
190
+
191
+ self.schema = schema
192
+ self.dataframes = dataframes
193
+ # An embedded DuckDB over these same frames, when enabled. See
194
+ # enable_local_sql.
195
+ self._local_sql = None
196
+ # True on a connection that is the caller's (an owned store's): see `register_frame`.
197
+ self._shared = False
198
+ # Each graph's registered names are its own: two graphs on one connection never read one another's frames.
199
+ self._frame_prefix = f"__prograph_frame_{uuid.uuid4().hex[:12]}_"
200
+ self._local_views: Dict[str, str] = {}
201
+ # By frame name: (a weak reference to the frame registered, then
202
+ # _frame_state of it then, which holds its arrays).
203
+ self._local_view_sources: Dict[str, tuple] = {}
204
+
205
+ # ------------------------------------------------------------------
206
+ # Local SQL: DuckDB over the frames, with this graph as the fallback
207
+ # ------------------------------------------------------------------
208
+
209
+ def enable_local_sql(self, connection=None) -> None:
210
+ """Run generated SQL in an embedded DuckDB over these frames: on `connection`, a DuckDB connection the
211
+ frames are registered on (an owned store's own, so a query reads the frames and the store's statements
212
+ on one connection; `store.engine_connection`), else a new one (`frames_connection`).
213
+
214
+ This is what `ProGraph(backend='duckdb', dataframes=...)` builds:
215
+ every query runs as SQL, through DuckDB, over the frames.
216
+ """
217
+ self._shared = connection is not None
218
+ if connection is None:
219
+ self._local_sql = frames_connection()
220
+ # A timestamp's parts (year(), hour()) are read in the session time
221
+ # zone, UTC; the machine's zone made
222
+ # 2011-02-01T01:02:03Z day 31 west of Greenwich.
223
+ self._local_sql.execute("SET TimeZone = 'UTC'")
224
+ else:
225
+ # The caller's connection: its session time zone stays the caller's, and the frames' typing is the
226
+ # database's setting, as `frames_connection` sets it.
227
+ connection.execute(f"SET GLOBAL pandas_analyze_sample = {2 ** 63 - 1}")
228
+ self._local_sql = connection
229
+ self._local_views = {}
230
+ self._local_view_sources = {}
231
+
232
+ def sql_dialect(self) -> Optional[str]:
233
+ """The dialect this graph executes SQL in, or None if it has none."""
234
+ return "duckdb" if self._local_sql is not None else None
235
+
236
+ def execute_local_sql_arrow(self, sql: str):
237
+ """execute_local_sql, returning DuckDB's Arrow table as is. A cancel
238
+ interrupts the statement (`StatementGuard`)."""
239
+ from prograph.execution import StatementGuard
240
+ from prograph.graph.drivers import duckdb_arrow_table
241
+ with StatementGuard(stop=self._local_sql.interrupt):
242
+ return duckdb_arrow_table(self._prepare_local_sql(), sql)
243
+
244
+ def execute_local_sql_batches(self, sql: str, batch_size: int):
245
+ """execute_local_sql as a stream of Arrow record batches.
246
+
247
+ The reader is read after execute() returns, outside the engine's
248
+ lock, so it runs on a cursor of its own: another query on the
249
+ shared connection would otherwise end its result. The cursor
250
+ registers the same frames (registrations are per connection; the
251
+ views over them are shared)."""
252
+ from prograph.execution import StatementGuard, checkpoint
253
+ from prograph.graph.drivers import duckdb_arrow_reader
254
+ checkpoint()
255
+ self._prepare_local_sql() # the views and their frame names exist
256
+ cursor = self._local_sql.cursor()
257
+ for name, frame in self.dataframes.items():
258
+ cursor.register(self._local_views[name], frame)
259
+ self._views_on(cursor)
260
+ # A cancel interrupts the cursor's statement until the reader ends.
261
+ guard = StatementGuard(stop=cursor.interrupt)
262
+ with guard.opening(close=cursor.close):
263
+ reader = duckdb_arrow_reader(cursor, sql, batch_size)
264
+ return guard.reader(reader, close=cursor.close)
265
+
266
+ def _views_on(self, cursor) -> None:
267
+ """On a connection that is the caller's (an owned store's) the frames' views are TEMP views, each
268
+ connection's own: a cursor makes its own over the frames it registered (`register_frame`)."""
269
+ if self._shared:
270
+ for name, frame in self.dataframes.items():
271
+ if "." not in name:
272
+ register_frame(cursor, name, self._local_views[name], frame, shared=True)
273
+
274
+ def _run_local_sql(self, sql: str, parameters: Optional[Dict[str, Any]] = None):
275
+ """Register the frames that changed since the last statement, then
276
+ run `sql`; the open result. `parameters` are bound by DuckDB
277
+ (`$name`)."""
278
+ con = self._prepare_local_sql()
279
+ return con.execute(sql, parameters) if parameters else con.execute(sql)
280
+
281
+ def _prepare_local_sql(self):
282
+ """Register the frames that changed since the last statement; the
283
+ connection a statement then runs on.
284
+
285
+ Each frame is a view under its source name over the frame registered
286
+ as `__prograph_frame_N`. DuckDB reads a registered frame when a
287
+ statement runs, through the arrays it was given, so a cell written into one of them,
288
+ by the engine or by the frame's owner, is read with no new
289
+ registration. A frame is registered again when it is another object
290
+ (a mutation replaces it in `self.dataframes`: concatenation, boolean
291
+ filters), has other columns, column types or length, or holds its
292
+ data in other arrays (_frame_state).
293
+ Registering every frame before every statement cost three analyses
294
+ of each object column (Python Decimals, strings) every time, which
295
+ DuckDB types by reading every value (frames_connection): 6.9 s of a
296
+ 10 s query at 1M rows."""
297
+ con = self._local_sql
298
+ for name, frame in self.dataframes.items():
299
+ if len(frame.columns) == 0:
300
+ # A frame with no columns is no table, and DuckDB refuses to
301
+ # register one. Registering every frame before every
302
+ # statement, one such frame anywhere among them failed
303
+ # every statement, whatever it read; in `auto` mode that
304
+ # was a silent fall back to Python for the whole engine.
305
+ # A query that does read it gets DuckDB's own "no such
306
+ # table".
307
+ continue
308
+ state = _frame_state(frame)
309
+ held = self._local_view_sources.get(name)
310
+ if held is not None and held[0]() is frame and _same_state(held[1:], state):
311
+ if self._shared and "." not in name:
312
+ # Another graph on this connection may have made its own view of this name since: make
313
+ # this graph's again (a statement of the catalog alone, no frame is read).
314
+ quoted = _q(name)
315
+ con.execute(f"CREATE OR REPLACE TEMP VIEW {quoted} AS SELECT * FROM {_q(self._local_views[name])}")
316
+ continue
317
+ internal = self._local_views.get(name)
318
+ if internal is None:
319
+ internal = f"{self._frame_prefix}{len(self._local_views)}"
320
+ self._local_views[name] = internal
321
+ register_frame(con, name, internal, frame, shared=self._shared)
322
+ else:
323
+ con.register(internal, frame)
324
+ # A view fixes its column types when created; a frame of
325
+ # other columns or types needs the view made again.
326
+ # On a connection that is the caller's another graph may have made its view of this name since
327
+ # (`register_frame`): this graph's is made again each time.
328
+ if held is None or held[1] != state[0] or (self._shared and "." not in name):
329
+ quoted = ".".join(_q(p) for p in name.split("."))
330
+ con.execute(f"CREATE OR REPLACE {'TEMP ' if self._shared and '.' not in name else ''}VIEW {quoted} "
331
+ f"AS SELECT * FROM {_q(internal)}")
332
+ self._local_view_sources[name] = (weakref.ref(frame),) + state
333
+ return con
334
+
335
+ def _invalidate_caches(self, changes: 'SchemaChangeSet') -> None:
336
+ """Drop the frames of tables a schema change removed."""
337
+ if not changes.has_changes:
338
+ return
339
+ for table in changes.removed_tables:
340
+ self.dataframes.pop(table, None)
341
+
342
+ def __repr__(self) -> str:
343
+ return f"PandasGraph(tables={sorted(self.dataframes)})"