prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,306 @@
1
+ """Upload a ProGraph ``GenieSpace`` to Databricks via the Genie API.
2
+
3
+ The :class:`GenieSpace` dataclass produced by
4
+ :mod:`prograph.genie.space_emitter` carries the same information as
5
+ Databricks' native Genie Space JSON. This module is the bridge:
6
+
7
+ - :func:`to_databricks_serialized_space` converts a ``GenieSpace`` to
8
+ the JSON shape Databricks' ``WorkspaceClient.genie.create_space()``
9
+ accepts as the ``serialized_space`` argument.
10
+ - :func:`upload_space` calls the Databricks SDK and creates / updates
11
+ a real Space in your workspace.
12
+
13
+ Databricks Genie's ``serialized_space`` schema (v2), inferred from the
14
+ ``genie.get_space(..., include_serialized_space=True)`` output of a
15
+ known-good Space:
16
+
17
+ .. code-block:: json
18
+
19
+ {
20
+ "version": 2,
21
+ "config": {
22
+ "sample_questions": [
23
+ {"id": "<32-hex-id>", "question": ["...?"]}
24
+ ]
25
+ },
26
+ "data_sources": {
27
+ "tables": [{"identifier": "catalog.schema.table"}]
28
+ },
29
+ "instructions": {
30
+ "text_instructions": [
31
+ {"id": "<id>", "content": ["line 1\\n", "line 2\\n", ...]}
32
+ ],
33
+ "example_question_sqls": [
34
+ {
35
+ "id": "<id>",
36
+ "question": ["..."],
37
+ "answer": [
38
+ {"format": "SQL", "content": ["SQL line 1\\n", ...]}
39
+ ]
40
+ }
41
+ ]
42
+ },
43
+ "benchmarks": {
44
+ "questions": [...]
45
+ }
46
+ }
47
+
48
+ Notes on the format:
49
+
50
+ - ``content`` and ``question`` are arrays of strings (one per line).
51
+ Databricks renders them concatenated; we always emit a single
52
+ element for simplicity.
53
+ - IDs are 32-char hex (UUID without dashes). The SDK accepts any
54
+ unique string, but Databricks-generated IDs follow this convention.
55
+ - The ``benchmarks`` key carries SQL exemplars Databricks uses for
56
+ internal evaluation. We omit it; ProGraph's sample queries already
57
+ serve the few-shot role.
58
+ """
59
+
60
+ from __future__ import annotations
61
+
62
+ import json
63
+ import uuid
64
+ from typing import TYPE_CHECKING, Any, Optional
65
+
66
+ if TYPE_CHECKING:
67
+ from prograph.genie.formats import GenieSpace
68
+
69
+
70
+ SERIALIZED_SPACE_VERSION = 2
71
+
72
+
73
+ def to_databricks_serialized_space(space: "GenieSpace") -> str:
74
+ """Convert a ProGraph :class:`GenieSpace` to Databricks' v2 JSON.
75
+
76
+ Returns a JSON string ready to pass as the ``serialized_space``
77
+ argument to ``WorkspaceClient.genie.create_space()``.
78
+
79
+ Fields:
80
+
81
+ - ``data_sources.tables`` ← ``GenieSpace.tables`` (one entry per
82
+ fully-qualified table name).
83
+ - ``instructions.text_instructions`` ← the rendered instructions
84
+ body, split into lines for Databricks' line-array shape.
85
+ - ``config.sample_questions`` ← the *question text* from
86
+ ``GenieSpace.sample_queries`` (Databricks uses these as
87
+ "explore" prompts in the UI).
88
+ - ``instructions.example_question_sqls`` ← the *full question +
89
+ SQL pairs* from ``GenieSpace.sample_queries`` (these are the
90
+ few-shot examples Genie's reasoning model sees).
91
+
92
+ ``GenieSpace.mcp_servers`` is not used here. Databricks' Genie
93
+ Space format has no slot for attaching external MCP servers, and
94
+ per the managed-MCP documentation Genie Space is positioned as an
95
+ MCP *server* not *client*, the convergence path for outbound-MCP
96
+ is the Agent
97
+ Bricks Supervisor Agent, not the Space itself. If the field is
98
+ set when this function runs, the caller is asking for something
99
+ Databricks doesn't support, and :func:`upload_space` raises.
100
+ """
101
+ # Databricks' import API requires id-sorted lists in
102
+ # config.sample_questions, instructions.example_question_sqls, and
103
+ # instructions.text_instructions. Generate IDs once, then sort.
104
+ sample_question_entries = sorted(
105
+ (
106
+ {"id": _new_id(), "question": [q.question]}
107
+ for q in space.sample_queries
108
+ ),
109
+ key=lambda e: e["id"],
110
+ )
111
+ example_sql_entries = sorted(
112
+ (
113
+ {
114
+ "id": _new_id(),
115
+ "question": [q.question],
116
+ "sql": _split_lines_for_databricks(q.sql),
117
+ }
118
+ for q in space.sample_queries
119
+ ),
120
+ key=lambda e: e["id"],
121
+ )
122
+
123
+ payload: dict[str, Any] = {
124
+ "version": SERIALIZED_SPACE_VERSION,
125
+ "config": {
126
+ "sample_questions": sample_question_entries,
127
+ },
128
+ "data_sources": {
129
+ # Genie's API requires tables to be sorted by identifier.
130
+ "tables": [
131
+ {"identifier": t} for t in sorted(set(space.tables))
132
+ ],
133
+ },
134
+ "instructions": {
135
+ "text_instructions": _build_text_instructions(space),
136
+ "example_question_sqls": example_sql_entries,
137
+ },
138
+ }
139
+ return json.dumps(payload, indent=2)
140
+
141
+
142
+ def upload_space(
143
+ space: "GenieSpace",
144
+ warehouse_id: str,
145
+ *,
146
+ workspace_client: Optional[Any] = None,
147
+ host: Optional[str] = None,
148
+ token: Optional[str] = None,
149
+ parent_path: Optional[str] = None,
150
+ update_if_exists: bool = False,
151
+ existing_space_id: Optional[str] = None,
152
+ ) -> dict[str, Any]:
153
+ """Create (or update) a Genie Space on Databricks from a ProGraph ``GenieSpace``.
154
+
155
+ Args:
156
+ space: The :class:`GenieSpace` to upload.
157
+ warehouse_id: The SQL Warehouse Genie should run queries
158
+ against (e.g. ``"06bb2559f22f28f4"``).
159
+ workspace_client: Optional pre-constructed
160
+ ``databricks.sdk.WorkspaceClient``. When ``None``, one is
161
+ built from ``host`` + ``token``, falling back to the SDK's
162
+ environment-based auth (``DATABRICKS_HOST`` /
163
+ ``DATABRICKS_TOKEN``).
164
+ host / token: Explicit credentials for constructing a fresh
165
+ ``WorkspaceClient``. Ignored if ``workspace_client`` is
166
+ provided.
167
+ parent_path: Workspace path under which to create the Space
168
+ (e.g. ``"/Users/me@company.com"``). ``None`` lets
169
+ Databricks pick the default.
170
+ update_if_exists: When ``True`` and ``existing_space_id`` is
171
+ also provided, call ``update_space`` instead of
172
+ ``create_space``. Useful for re-running the same
173
+ generation pipeline and overwriting an old Space in
174
+ place.
175
+ existing_space_id: Space ID to update (only meaningful with
176
+ ``update_if_exists=True``).
177
+
178
+ Returns:
179
+ Dict with the created/updated Space's id, title, warehouse_id,
180
+ and the serialized payload size in bytes.
181
+
182
+ Raises:
183
+ ValueError: If ``space.mcp_servers`` is non-empty. Databricks
184
+ Genie Space has no inbound-MCP attachment path (per the
185
+ managed-MCP documentation),
186
+ external-MCP orchestration goes through the Agent Bricks
187
+ Supervisor Agent, not the Space itself.
188
+ """
189
+ if space.mcp_servers:
190
+ raise ValueError(
191
+ "GenieSpace.mcp_servers is set but Databricks Genie Space "
192
+ "does not support attaching external MCP servers (neither "
193
+ "via API nor UI). For the outbound-MCP pattern, build an "
194
+ "Agent Bricks Supervisor Agent that orchestrates the Genie "
195
+ "Space + your MCP servers. See docs/guides/genie.md."
196
+ )
197
+
198
+ w = workspace_client or _build_workspace_client(host=host, token=token)
199
+
200
+ serialized = to_databricks_serialized_space(space)
201
+
202
+ if update_if_exists and existing_space_id:
203
+ result = w.genie.update_space(
204
+ space_id=existing_space_id,
205
+ title=space.name,
206
+ description=space.description or None,
207
+ serialized_space=serialized,
208
+ warehouse_id=warehouse_id,
209
+ )
210
+ action = "updated"
211
+ else:
212
+ result = w.genie.create_space(
213
+ warehouse_id=warehouse_id,
214
+ serialized_space=serialized,
215
+ title=space.name,
216
+ description=space.description or None,
217
+ parent_path=parent_path,
218
+ )
219
+ action = "created"
220
+
221
+ return {
222
+ "action": action,
223
+ "space_id": result.space_id,
224
+ "title": result.title,
225
+ "warehouse_id": result.warehouse_id,
226
+ "parent_path": getattr(result, "parent_path", None),
227
+ "serialized_bytes": len(serialized),
228
+ }
229
+
230
+
231
+ # ─── Helpers ─────────────────────────────────────────────────────────
232
+
233
+
234
+ def _new_id() -> str:
235
+ """Generate a 32-char hex id in the style Databricks uses."""
236
+ return uuid.uuid4().hex
237
+
238
+
239
+ def _split_lines_for_databricks(text: str) -> list[str]:
240
+ """Split a multi-line string into Databricks' "line array" form.
241
+
242
+ Databricks stores SQL bodies and instruction text as arrays of
243
+ lines, each line keeping its trailing ``\\n`` (except possibly the
244
+ last). Mirror that shape so the strings render correctly in the
245
+ Genie UI.
246
+ """
247
+ if not text:
248
+ return [""]
249
+ lines = text.split("\n")
250
+ # Re-attach a trailing \n to every line except the last; if the
251
+ # original ended with a newline, the last line will be empty and
252
+ # we just drop it (Databricks doesn't include the trailing empty).
253
+ out: list[str] = []
254
+ for i, line in enumerate(lines):
255
+ if i < len(lines) - 1:
256
+ out.append(line + "\n")
257
+ else:
258
+ if line: # don't emit a trailing empty
259
+ out.append(line)
260
+ return out or [""]
261
+
262
+
263
+ def _build_text_instructions(space: "GenieSpace") -> list[dict[str, Any]]:
264
+ """Build the ``text_instructions`` list from the rendered body.
265
+
266
+ Today: one big entry containing the full rendered markdown.
267
+ Databricks renders it as a single instruction block.
268
+
269
+ Future option: split per-section so users can selectively edit /
270
+ enable individual sections in the Genie UI. Skipped for now,
271
+ the single-block approach roundtrips cleanly.
272
+ """
273
+ if not space.instructions.rendered:
274
+ return []
275
+ return [
276
+ {
277
+ "id": _new_id(),
278
+ "content": _split_lines_for_databricks(space.instructions.rendered),
279
+ }
280
+ ]
281
+
282
+
283
+ def _build_workspace_client(
284
+ *, host: Optional[str], token: Optional[str]
285
+ ) -> Any:
286
+ """Build a :class:`WorkspaceClient` from explicit creds or env."""
287
+ try:
288
+ from databricks.sdk import WorkspaceClient
289
+ except ImportError as e: # pragma: no cover
290
+ raise ImportError(
291
+ "databricks-sdk is required for upload_space. "
292
+ "Install with: pip install databricks-sdk"
293
+ ) from e
294
+
295
+ if host and token:
296
+ return WorkspaceClient(host=host, token=token)
297
+ # Falls back to env vars (DATABRICKS_HOST, DATABRICKS_TOKEN) or
298
+ # ~/.databrickscfg.
299
+ return WorkspaceClient()
300
+
301
+
302
+ __all__ = [
303
+ "to_databricks_serialized_space",
304
+ "upload_space",
305
+ "SERIALIZED_SPACE_VERSION",
306
+ ]
@@ -0,0 +1,393 @@
1
+ """Library API for deploying a Genie Space to Databricks.
2
+
3
+ Single entry point :func:`deploy_space` that creates (or updates) a
4
+ Genie Space on Databricks from a ProGraph schema. Designed for use
5
+ from any Python context: scripts, notebooks (Databricks workspace or
6
+ local Jupyter), Databricks Apps, Lakeflow jobs.
7
+
8
+ The CLI command ``prograph upload-genie-space`` is a thin wrapper
9
+ around this function. It adds argparse + print statements and that's
10
+ all. Anything callable from the CLI is callable from Python with the
11
+ same arguments.
12
+
13
+ Three input modes for the schema source, mix-and-match as convenient:
14
+
15
+ 1. **File paths**, ``config_path=...``, ``ontology_path=...``.
16
+ Same as the CLI; works in notebooks that load YAML from a
17
+ workspace path or DBFS.
18
+
19
+ 2. **Raw strings**, ``config_yaml=...``, ``ontology_ttl=...``.
20
+ Pass YAML / Turtle as a string when you've loaded it some other
21
+ way (e.g. from a Databricks Volume via ``dbutils.fs.head``, from a
22
+ web fetch, from a templating system).
23
+
24
+ 3. **In-memory objects**, ``schema=...``, ``ontology=...``.
25
+ Pass a constructed :class:`GraphSchema` and (optional)
26
+ :class:`OntologyLayer` directly. Useful when the schema is built
27
+ programmatically.
28
+
29
+ Auth follows the same precedence:
30
+
31
+ - ``workspace_client=``, already-constructed
32
+ :class:`databricks.sdk.WorkspaceClient`. Notebooks running on
33
+ Databricks use this with no arguments: the runtime auto-authenticates.
34
+ - ``host=`` + ``token=``, explicit credentials when not on Databricks.
35
+ - (fallback) Environment variables: ``DATABRICKS_HOST`` /
36
+ ``DATABRICKS_TOKEN``, or ``~/.databrickscfg``.
37
+
38
+ Mixing ``workspace_client`` with ``host``/``token`` raises, the
39
+ intent is ambiguous and silent precedence is harder to debug than a
40
+ clear error.
41
+ """
42
+
43
+ from __future__ import annotations
44
+
45
+ from pathlib import Path
46
+ from typing import TYPE_CHECKING, Any, List, Optional, Union
47
+
48
+ if TYPE_CHECKING:
49
+ from prograph.genie.formats import GenieSpace
50
+ from prograph.ontology.layer import OntologyLayer
51
+ from prograph.plan.sql.dialect import DialectConfig
52
+ from prograph.schema import GraphSchema
53
+
54
+
55
+ def deploy_space(
56
+ *,
57
+ # ── Schema source (exactly one mode) ──
58
+ config_path: Optional[Union[str, Path]] = None,
59
+ config_yaml: Optional[str] = None,
60
+ schema: Optional["GraphSchema"] = None,
61
+
62
+ # ── Ontology source (optional, at most one mode) ──
63
+ ontology_path: Optional[Union[str, Path]] = None,
64
+ ontology_ttl: Optional[str] = None,
65
+ ontology: Optional["OntologyLayer"] = None,
66
+
67
+ # ── Target (required) ──
68
+ warehouse_id: str,
69
+
70
+ # ── View placement ──
71
+ target_catalog: Optional[str] = None,
72
+ target_schema: str = "prograph_views",
73
+ dialect: Optional["DialectConfig"] = None,
74
+ dialect_name: Optional[str] = None,
75
+ include_labels: Optional[List[str]] = None,
76
+ exclude_labels: Optional[List[str]] = None,
77
+
78
+ # ── Space metadata ──
79
+ space_name: Optional[str] = None,
80
+ space_description: str = "",
81
+ summary_instructions: bool = False,
82
+ max_sample_queries: int = 25,
83
+
84
+ # ── Apply generated views to the warehouse in one shot ──
85
+ apply_views: bool = False,
86
+
87
+ # ── Databricks auth ──
88
+ workspace_client: Optional[Any] = None,
89
+ host: Optional[str] = None,
90
+ token: Optional[str] = None,
91
+
92
+ # ── Create-vs-update ──
93
+ parent_path: Optional[str] = None,
94
+ update_if_exists: bool = False,
95
+ existing_space_id: Optional[str] = None,
96
+
97
+ # ── Source provenance (lands in space.metadata) ──
98
+ source_label: Optional[str] = None,
99
+ ) -> dict:
100
+ """Deploy a Genie Space to Databricks from any input mode.
101
+
102
+ See module docstring for the three input modes (paths / strings /
103
+ objects) and the auth options. Returns the same result dict the
104
+ underlying :func:`prograph.genie.upload_space` produces, plus a
105
+ few aggregate counts surfaced for caller convenience.
106
+
107
+ Args:
108
+ warehouse_id: Databricks SQL Warehouse ID for the Space.
109
+ config_path / config_yaml / schema: Exactly one. The schema source.
110
+ ontology_path / ontology_ttl / ontology: At most one (optional).
111
+ target_catalog / target_schema: Where generated views live in
112
+ the warehouse. Same fields as the CLI.
113
+ dialect / dialect_name: SQL dialect. ``dialect`` accepts a
114
+ :class:`DialectConfig` directly; ``dialect_name`` accepts
115
+ the string used by the CLI (``"databricks_sql"`` etc.).
116
+ include_labels / exclude_labels: NodeMapping label filters.
117
+ space_name / space_description / summary_instructions /
118
+ max_sample_queries: Pass through to the
119
+ :class:`GenieSpace` assembly.
120
+ apply_views: When ``True``, opens a databricks-sql-connector
121
+ session against ``warehouse_id`` and runs
122
+ ``CREATE OR REPLACE VIEW`` for every generated view before
123
+ the Space is created. Closes the deploy → views → query
124
+ loop in one call. Requires ``databricks-sql-connector`` to
125
+ be installed; auth re-uses ``host``/``token`` (or env
126
+ vars). Defaults to ``False`` for backwards compatibility.
127
+ workspace_client: Pre-built :class:`WorkspaceClient`. When
128
+ running in a Databricks notebook, build it with no args
129
+ (the runtime auto-authenticates) and pass it here.
130
+ host / token: Explicit credentials for non-Databricks
131
+ environments. Mutually exclusive with ``workspace_client``.
132
+ parent_path: Workspace path under which to create the Space.
133
+ update_if_exists / existing_space_id: Update an existing
134
+ Space in place instead of creating a new one.
135
+ source_label: Optional string to record in
136
+ ``space.metadata["source_config"]`` for provenance. The
137
+ CLI sets it to the absolute config-file path; callers
138
+ using in-memory inputs can pass anything meaningful.
139
+
140
+ Returns:
141
+ Result dict from :func:`upload_space` with extra fields:
142
+ ``table_count``, ``view_count``, ``sample_query_count``.
143
+
144
+ Raises:
145
+ ValueError: If more than one schema-source / ontology-source
146
+ mode is provided, if zero schema sources are provided,
147
+ if ``workspace_client`` is given alongside ``host`` /
148
+ ``token``, or if ``update_if_exists`` is set without
149
+ ``existing_space_id``.
150
+ """
151
+ _reject_auth_conflict(workspace_client, host, token)
152
+ _reject_update_without_id(update_if_exists, existing_space_id)
153
+
154
+ loaded_schema = _resolve_schema(config_path, config_yaml, schema)
155
+ loaded_ontology = _resolve_ontology(ontology_path, ontology_ttl, ontology)
156
+ resolved_dialect = _resolve_dialect(dialect, dialect_name)
157
+
158
+ from prograph.genie.space_emitter import emit_genie_space
159
+ from prograph.genie.databricks_uploader import upload_space
160
+
161
+ space = emit_genie_space(
162
+ schema=loaded_schema,
163
+ ontology=loaded_ontology,
164
+ name=space_name,
165
+ description=space_description,
166
+ target_catalog=target_catalog,
167
+ target_schema=target_schema,
168
+ dialect=resolved_dialect,
169
+ include_labels=include_labels,
170
+ exclude_labels=exclude_labels,
171
+ summary_instructions=summary_instructions,
172
+ max_sample_queries=max_sample_queries,
173
+ source_config=source_label,
174
+ )
175
+
176
+ applied_views: List[str] = []
177
+ if apply_views:
178
+ applied_views = _apply_views(
179
+ views=space.views,
180
+ warehouse_id=warehouse_id,
181
+ target_catalog=target_catalog,
182
+ target_schema=target_schema,
183
+ dialect=resolved_dialect,
184
+ host=host,
185
+ token=token,
186
+ )
187
+
188
+ result = upload_space(
189
+ space=space,
190
+ warehouse_id=warehouse_id,
191
+ workspace_client=workspace_client,
192
+ host=host,
193
+ token=token,
194
+ parent_path=parent_path,
195
+ update_if_exists=update_if_exists,
196
+ existing_space_id=existing_space_id,
197
+ )
198
+ result["table_count"] = len(space.tables)
199
+ result["view_count"] = len(space.views)
200
+ result["sample_query_count"] = len(space.sample_queries)
201
+ result["applied_views"] = applied_views
202
+ return result
203
+
204
+
205
+ # ─── Input resolution ────────────────────────────────────────────────
206
+
207
+
208
+ def _resolve_schema(
209
+ config_path: Optional[Union[str, Path]],
210
+ config_yaml: Optional[str],
211
+ schema: Optional["GraphSchema"],
212
+ ) -> "GraphSchema":
213
+ """Load a GraphSchema from exactly one of the three input modes.
214
+
215
+ Raises ``ValueError`` if zero or more than one source is provided.
216
+ For the path / string modes, resolves
217
+ ``sources.<key>.table`` references the same way the CLI does, so
218
+ NodeMapping.source_table ends up as the fully-qualified table
219
+ name in the generated view SQL.
220
+ """
221
+ provided = sum(
222
+ x is not None for x in (config_path, config_yaml, schema)
223
+ )
224
+ if provided == 0:
225
+ raise ValueError(
226
+ "deploy_space: provide exactly one schema source, "
227
+ "config_path, config_yaml, or schema"
228
+ )
229
+ if provided > 1:
230
+ raise ValueError(
231
+ "deploy_space: provide exactly one schema source, "
232
+ "got multiple of (config_path, config_yaml, schema)"
233
+ )
234
+
235
+ if schema is not None:
236
+ return schema
237
+
238
+ from prograph.config import load_config, load_config_from_string, validate_config
239
+
240
+ if config_path is not None:
241
+ config = load_config(Path(config_path))
242
+ else:
243
+ config = load_config_from_string(config_yaml)
244
+ validate_config(config) # surface warnings on stderr left to caller
245
+
246
+ return config.to_schema(by_table=True)
247
+
248
+
249
+ def _resolve_ontology(
250
+ ontology_path: Optional[Union[str, Path]],
251
+ ontology_ttl: Optional[str],
252
+ ontology: Optional["OntologyLayer"],
253
+ ) -> Optional["OntologyLayer"]:
254
+ """Load an OntologyLayer from at most one of the three modes."""
255
+ provided = sum(
256
+ x is not None for x in (ontology_path, ontology_ttl, ontology)
257
+ )
258
+ if provided == 0:
259
+ return None
260
+ if provided > 1:
261
+ raise ValueError(
262
+ "deploy_space: provide at most one ontology source, "
263
+ "got multiple of (ontology_path, ontology_ttl, ontology)"
264
+ )
265
+
266
+ if ontology is not None:
267
+ return ontology
268
+
269
+ from prograph.ontology import OntologyLayer
270
+
271
+ if ontology_path is not None:
272
+ return OntologyLayer.from_owl(str(ontology_path))
273
+ return OntologyLayer.from_turtle(ontology_ttl)
274
+
275
+
276
+ def _resolve_dialect(
277
+ dialect: Optional["DialectConfig"],
278
+ dialect_name: Optional[str],
279
+ ) -> Optional["DialectConfig"]:
280
+ """At most one of ``dialect`` or ``dialect_name``. ``None`` lets the
281
+ view generator pick its default."""
282
+ if dialect is not None and dialect_name is not None:
283
+ raise ValueError(
284
+ "deploy_space: pass either dialect (a DialectConfig instance) "
285
+ "or dialect_name (a string), not both"
286
+ )
287
+ if dialect is not None:
288
+ return dialect
289
+ if dialect_name is None:
290
+ return None
291
+
292
+ from prograph.plan.sql import dialect as dialect_module
293
+
294
+ mapping = {
295
+ "databricks_sql": dialect_module.SPARK_DIALECT,
296
+ "spark": dialect_module.SPARK_DIALECT,
297
+ "postgresql": dialect_module.POSTGRESQL_DIALECT,
298
+ }
299
+ sqlserver = getattr(dialect_module, "SQLSERVER_DIALECT", None)
300
+ if sqlserver is not None:
301
+ mapping["sqlserver"] = sqlserver
302
+ if dialect_name not in mapping:
303
+ raise ValueError(
304
+ f"Unknown dialect_name {dialect_name!r}. "
305
+ f"Known: {', '.join(sorted(mapping))}"
306
+ )
307
+ return mapping[dialect_name]
308
+
309
+
310
+ def _reject_auth_conflict(
311
+ workspace_client: Optional[Any],
312
+ host: Optional[str],
313
+ token: Optional[str],
314
+ ) -> None:
315
+ if workspace_client is not None and (host is not None or token is not None):
316
+ raise ValueError(
317
+ "deploy_space: pass either workspace_client (already-built "
318
+ "WorkspaceClient) or host+token (explicit credentials), not both. "
319
+ "When running inside a Databricks notebook, prefer "
320
+ "workspace_client=WorkspaceClient(), the runtime auto-"
321
+ "authenticates."
322
+ )
323
+
324
+
325
+ def _reject_update_without_id(
326
+ update_if_exists: bool,
327
+ existing_space_id: Optional[str],
328
+ ) -> None:
329
+ if update_if_exists and not existing_space_id:
330
+ raise ValueError(
331
+ "deploy_space: update_if_exists=True requires "
332
+ "existing_space_id to identify which Space to update"
333
+ )
334
+
335
+
336
+ def _apply_views(
337
+ *,
338
+ views,
339
+ warehouse_id: str,
340
+ target_catalog: Optional[str],
341
+ target_schema: str,
342
+ dialect,
343
+ host: Optional[str],
344
+ token: Optional[str],
345
+ ) -> List[str]:
346
+ """Open a databricks-sql session to the warehouse and apply each view.
347
+
348
+ Returns the list of fully-qualified view names that landed. Raises a
349
+ clear error if ``databricks-sql-connector`` isn't installed or if
350
+ credentials can't be resolved.
351
+ """
352
+ import os
353
+
354
+ try:
355
+ import databricks.sql as dbsql
356
+ except ImportError as e:
357
+ raise ImportError(
358
+ "apply_views=True requires databricks-sql-connector. "
359
+ "Install with: pip install 'databricks-sql-connector>=3.0'"
360
+ ) from e
361
+
362
+ server_hostname = host or os.environ.get("DATABRICKS_HOST") or ""
363
+ server_hostname = server_hostname.replace("https://", "").rstrip("/")
364
+ access_token = token or os.environ.get("DATABRICKS_TOKEN")
365
+ if not server_hostname or not access_token:
366
+ raise ValueError(
367
+ "apply_views=True needs host+token (or DATABRICKS_HOST + "
368
+ "DATABRICKS_TOKEN env vars) to open a SQL session against "
369
+ "the warehouse."
370
+ )
371
+
372
+ from prograph.genie.view_writer import write_views
373
+
374
+ conn = dbsql.connect(
375
+ server_hostname=server_hostname,
376
+ http_path=f"/sql/1.0/warehouses/{warehouse_id}",
377
+ access_token=access_token,
378
+ )
379
+ try:
380
+ applied = write_views(
381
+ views=views,
382
+ connection=conn,
383
+ target_catalog=target_catalog,
384
+ target_schema=target_schema,
385
+ dialect=dialect,
386
+ )
387
+ finally:
388
+ conn.close()
389
+
390
+ return [fq for _, fq in applied]
391
+
392
+
393
+ __all__ = ["deploy_space"]