prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,434 @@
1
+ # Databricks notebook source
2
+ # MAGIC %md
3
+ # MAGIC # ProGraph: Genie Q&A end-to-end demo
4
+ # MAGIC
5
+ # MAGIC Loads the canonical turbofan TTL into Delta tables on the current
6
+ # MAGIC workspace, calls `deploy_space()` to create a Genie Space wired to
7
+ # MAGIC ProGraph-generated views, and asks three questions through the
8
+ # MAGIC Genie Conversations API. The Space + tables are left in place so
9
+ # MAGIC you can open the Space afterwards and inspect the conversation
10
+ # MAGIC history.
11
+ # MAGIC
12
+ # MAGIC Mirror of `tests/integration/genie/test_w9_genie_qa.py` but as a
13
+ # MAGIC notebook so you can re-run it in any workspace and watch the
14
+ # MAGIC pipeline execute inline.
15
+ # MAGIC
16
+ # MAGIC **What it exercises**
17
+ # MAGIC - Shared-table NodeMappings with `filter_predicate` (10 tables → 51 labels; 2 more are queries)
18
+ # MAGIC - Abstract parent NodeMappings (Module / Component / Sensor / …)
19
+ # MAGIC - Subclass-closure views from the ontology's class hierarchy
20
+ # MAGIC - `deploy_space()` library API
21
+ # MAGIC - Genie Conversations API (`start_conversation_and_wait`, `create_message_and_wait`)
22
+
23
+ # COMMAND ----------
24
+ # MAGIC %md ## 1. Install ProGraph
25
+
26
+ # COMMAND ----------
27
+ # MAGIC %pip install --quiet 'prograph[genie]>=0.5.0' rdflib pyyaml
28
+ # MAGIC dbutils.library.restartPython()
29
+
30
+ # COMMAND ----------
31
+ # MAGIC %md ## 2. Parameters
32
+ # MAGIC
33
+ # MAGIC Set the SQL Warehouse ID, target catalog, and a suffix that scopes
34
+ # MAGIC this run's schemas. Multiple runs with different suffixes can
35
+ # MAGIC coexist.
36
+
37
+ # COMMAND ----------
38
+ dbutils.widgets.text("warehouse_id", "", "SQL Warehouse ID")
39
+ # ``catalog`` is a SINGLE-PART name (e.g. ``workspace``, ``main``, ``bg``).
40
+ # Don't pass ``catalog.schema`` here, the notebook builds sub-schemas
41
+ # underneath this catalog itself.
42
+ dbutils.widgets.text("catalog", "workspace", "Target catalog (single-part)")
43
+ dbutils.widgets.text("suffix", "demo", "Run suffix")
44
+
45
+ WAREHOUSE_ID = dbutils.widgets.get("warehouse_id")
46
+ CATALOG = dbutils.widgets.get("catalog")
47
+ SUFFIX = dbutils.widgets.get("suffix")
48
+
49
+ if not WAREHOUSE_ID:
50
+ raise ValueError("Set the warehouse_id widget before running.")
51
+ if "." in CATALOG:
52
+ # Fail fast; Databricks would otherwise raise a less-helpful
53
+ # AnalysisException once the notebook tried CREATE SCHEMA <catalog>.<sub>.
54
+ raise ValueError(
55
+ f"catalog widget must be a single-part name (got {CATALOG!r}). "
56
+ f"The notebook builds prograph_turbofan_data_<suffix> and "
57
+ f"prograph_turbofan_views_<suffix> underneath this catalog."
58
+ )
59
+
60
+ DATA_SCHEMA = f"prograph_turbofan_data_{SUFFIX}"
61
+ VIEWS_SCHEMA = f"prograph_turbofan_views_{SUFFIX}"
62
+
63
+ print(f"Warehouse: {WAREHOUSE_ID}")
64
+ print(f"Data schema: {CATALOG}.{DATA_SCHEMA}")
65
+ print(f"Views schema: {CATALOG}.{VIEWS_SCHEMA}")
66
+
67
+ # COMMAND ----------
68
+ # MAGIC %md ## 3. Load + consolidate the turbofan TTL
69
+ # MAGIC
70
+ # MAGIC Reads the canonical `engine-instances.ttl` shipped with the
71
+ # MAGIC `prograph` package, classifies its triples into per-label pandas
72
+ # MAGIC DataFrames, then consolidates them into the 10 shared node
73
+ # MAGIC tables + 17 relationship tables the turbofan schema expects.
74
+ # MAGIC
75
+ # MAGIC The consolidation map is inlined below so this notebook is
76
+ # MAGIC self-contained, no test-only modules required.
77
+
78
+ # COMMAND ----------
79
+ import re
80
+ import pandas as pd
81
+ from prograph.schema.rows import iter_rows
82
+ from prograph.schema.types import is_missing
83
+ from prograph.cli.ttl_loader import load_ttl
84
+ from prograph.cli._iri_utils import extract_local_name
85
+
86
+ # --- Consolidation constants (mirrors tests/integration/genie/_turbofan_seed.py) ---
87
+
88
+ SCHEMA_LABEL_MAP = {
89
+ "LowPressureCompressor": "LPC",
90
+ "HighPressureCompressor": "HPC",
91
+ "LowPressureTurbine": "LPT",
92
+ "HighPressureTurbine": "HPT",
93
+ }
94
+
95
+ _MODULES = ["FanModule", "LPC", "HPC", "CombustionChamber", "HPT", "LPT",
96
+ "ExhaustNozzle", "AccessoryGearbox", "OilSystem", "FuelSystem"]
97
+ _COMPONENTS = ["Blade", "Disk", "Shaft", "Bearing", "Seal", "Combustor",
98
+ "FuelNozzle", "OilPump", "OilFilter", "HeatExchanger", "FADEC", "Vane"]
99
+ _SENSORS = ["VibrationSensor", "TemperatureSensor", "PressureSensor",
100
+ "SpeedSensor", "OilDebrisSensor", "FlowSensor"]
101
+ _DEGRADATIONS = ["Erosion", "Corrosion", "FatigueCracking", "ThermalDegradation",
102
+ "ForeignObjectDamage", "WearAbrasion", "SealLeakage", "CarbonBuildup"]
103
+ _MAINTENANCES = ["BoroscopeInspection", "OilAnalysis", "BlendRepair",
104
+ "Replacement", "ScheduledOverhaul"]
105
+
106
+ TABLE_FOR_LABEL = {
107
+ "Engine": "engines",
108
+ **{lbl: "modules" for lbl in _MODULES},
109
+ **{lbl: "components" for lbl in _COMPONENTS},
110
+ **{lbl: "sensors" for lbl in _SENSORS},
111
+ "SensorReading": "sensor_readings",
112
+ **{lbl: "degradation_modes" for lbl in _DEGRADATIONS},
113
+ **{lbl: "maintenance_actions" for lbl in _MAINTENANCES},
114
+ "Fleet": "fleets",
115
+ "Aircraft": "aircraft",
116
+ "OperatingEnvironment": "operating_environments",
117
+ }
118
+ TYPED_TABLES = {"modules", "components", "sensors", "degradation_modes",
119
+ "maintenance_actions"}
120
+
121
+ REL_TYPE_TO_TABLE = {
122
+ "IS_PART_OF": "is_part_of", "DOWNSTREAM_OF": "downstream_of",
123
+ "CONNECTED_TO": "connected_to", "DRIVES_SHAFT": "drives_shaft",
124
+ "SUPPLIES_FLUID_TO": "supplies_fluid_to", "MONITORS": "monitors",
125
+ "INSTALLED_ON": "installed_on", "HAS_READING": "has_reading",
126
+ "EXHIBITS_DEGRADATION": "exhibits_degradation",
127
+ "SUSCEPTIBLE_TO": "susceptible_to", "INDICATED_BY": "indicated_by",
128
+ "PROPAGATES_TO": "propagates_to", "PERFORMED_ON": "performed_on",
129
+ "ADDRESSES_DEGRADATION": "addresses_degradation",
130
+ "INSTALLED_ON_AIRCRAFT": "installed_on_aircraft",
131
+ "BELONGS_TO_FLEET": "belongs_to_fleet", "OPERATES_IN": "operates_in",
132
+ }
133
+
134
+ _CAMEL_RE = re.compile(r"(?<!^)(?=[A-Z])")
135
+ def _camel_to_snake(name): return _CAMEL_RE.sub("_", name).lower()
136
+
137
+
138
+ def consolidate(per_label):
139
+ """Per-label frames → shared-table frames with `type` discriminators."""
140
+ nodes = {t: [] for t in set(TABLE_FOR_LABEL.values())}
141
+ rels = {t: [] for t in REL_TYPE_TO_TABLE.values()}
142
+ for raw_label, df in per_label.items():
143
+ if "_start_uri" in df.columns:
144
+ rt = REL_TYPE_TO_TABLE.get(raw_label)
145
+ if rt is None: continue
146
+ for row in iter_rows(df):
147
+ rels[rt].append({
148
+ "start_id": extract_local_name(str(row["_start_uri"])),
149
+ "end_id": extract_local_name(str(row["_end_uri"])),
150
+ })
151
+ continue
152
+ if "_uri" not in df.columns: continue
153
+ schema_label = SCHEMA_LABEL_MAP.get(raw_label, raw_label)
154
+ tbl = TABLE_FOR_LABEL.get(schema_label)
155
+ if tbl is None: continue
156
+ typed = tbl in TYPED_TABLES
157
+ for row in iter_rows(df):
158
+ r = {"id": extract_local_name(str(row["_uri"]))}
159
+ for col, val in row.items():
160
+ if col == "_uri": continue
161
+ r[_camel_to_snake(col)] = None if is_missing(val) else val
162
+ if typed: r["type"] = schema_label
163
+ nodes[tbl].append(r)
164
+ return (
165
+ {t: pd.DataFrame(rows) for t, rows in nodes.items()},
166
+ {t: pd.DataFrame(rows) for t, rows in rels.items()},
167
+ )
168
+
169
+
170
+ # --- Run the consolidation ------------------------------------------------
171
+
172
+ import importlib.resources as ir
173
+ ttl_path = ir.files("prograph.demos.turbofan") / "data" / "engine-instances.ttl"
174
+ with ir.as_file(ttl_path) as p:
175
+ per_label = load_ttl([str(p)])
176
+
177
+ node_frames, rel_frames = consolidate(per_label)
178
+
179
+ print("Node tables:")
180
+ for t, df in node_frames.items():
181
+ print(f" {t:24s} {len(df):4d} rows")
182
+ print("\nRelationship tables:")
183
+ for t, df in rel_frames.items():
184
+ print(f" {t:24s} {len(df):4d} rows")
185
+
186
+ # COMMAND ----------
187
+ # MAGIC %md ## 4. Write Delta tables
188
+ # MAGIC
189
+ # MAGIC `spark.createDataFrame(...).write.saveAsTable(...)` lands each
190
+ # MAGIC frame as a managed Delta table, no Volume / `COPY INTO` needed
191
+ # MAGIC since we're inside the Databricks runtime.
192
+
193
+ # COMMAND ----------
194
+ spark.sql(f"CREATE SCHEMA IF NOT EXISTS `{CATALOG}`.`{DATA_SCHEMA}`")
195
+
196
+ table_fqs = {}
197
+ skipped_empty = []
198
+ for table, df in {**node_frames, **rel_frames}.items():
199
+ if df.empty:
200
+ # Skip, guessing a DDL for an empty frame risks shape drift,
201
+ # and the corresponding labels won't be in include_labels anyway.
202
+ skipped_empty.append(table)
203
+ continue
204
+ # Coerce all-None object columns to string so Spark infers a real type.
205
+ coerced = df.copy()
206
+ for col in coerced.columns:
207
+ if coerced[col].isna().all() and coerced[col].dtype == "object":
208
+ coerced[col] = coerced[col].astype("string")
209
+ sdf = spark.createDataFrame(coerced)
210
+ sdf.write.mode("overwrite").saveAsTable(f"{CATALOG}.{DATA_SCHEMA}.{table}")
211
+ table_fqs[table] = f"{CATALOG}.{DATA_SCHEMA}.{table}"
212
+
213
+ print(f"Wrote {len(table_fqs)} tables to {CATALOG}.{DATA_SCHEMA}")
214
+ if skipped_empty:
215
+ print(f"Skipped (empty in TTL data): {', '.join(skipped_empty)}")
216
+
217
+ # COMMAND ----------
218
+ # MAGIC %md ## 5. Generate views + apply them to the warehouse
219
+ # MAGIC
220
+ # MAGIC View generation must receive the same `include_labels` we'll pass
221
+ # MAGIC to `deploy_space`: subclass-closure views are filter-sensitive,
222
+ # MAGIC so a mismatch between the applied set and the Space-registered
223
+ # MAGIC set makes Genie reject the Space.
224
+
225
+ # COMMAND ----------
226
+ import yaml
227
+ from prograph.config import load_config_from_string
228
+ from prograph.ontology import OntologyLayer
229
+ from prograph.genie import generate_views
230
+
231
+ # Read the shipped graph.yaml + ontology TTL.
232
+ yaml_text = (
233
+ ir.files("prograph.demos.turbofan") / "graph.yaml"
234
+ ).read_text(encoding="utf-8")
235
+ ontology_text = (
236
+ ir.files("prograph.demos.turbofan") / "ontology" / "engine-maintenance.ttl"
237
+ ).read_text(encoding="utf-8")
238
+
239
+ # Rewrite sources[*].table to point at the per-run FQ names.
240
+ yaml_doc = yaml.safe_load(yaml_text)
241
+ for key in list((yaml_doc.get("sources") or {}).keys()):
242
+ if key in table_fqs:
243
+ yaml_doc["sources"][key]["table"] = table_fqs[key]
244
+ rewritten_yaml = yaml.safe_dump(yaml_doc)
245
+
246
+ config = load_config_from_string(rewritten_yaml)
247
+ schema = config.to_schema(by_table=True)
248
+
249
+ ontology = OntologyLayer.from_turtle(ontology_text)
250
+
251
+ # Genie Spaces cap at 30 tables, restrict to a focused subset.
252
+ # Includes the abstract Component mapping, the parent of the subclass-closure views.
253
+ INCLUDED_LABELS = [
254
+ "Engine",
255
+ "Component",
256
+ "FanModule", "LPC", "HPC",
257
+ "Blade", "Disk", "FADEC", "Combustor",
258
+ ]
259
+
260
+ views = generate_views(schema, ontology=ontology, include_labels=INCLUDED_LABELS)
261
+ print(f"{len(views)} views generated:")
262
+ for v in views:
263
+ print(f" {v.name} ({v.source.value})")
264
+
265
+ # Apply them to the warehouse.
266
+ spark.sql(f"CREATE SCHEMA IF NOT EXISTS `{CATALOG}`.`{VIEWS_SCHEMA}`")
267
+ for v in views:
268
+ fq = f"`{CATALOG}`.`{VIEWS_SCHEMA}`.`{v.name}`"
269
+ spark.sql(f"CREATE OR REPLACE VIEW {fq} AS\n{v.sql}")
270
+ print(f"\nApplied to {CATALOG}.{VIEWS_SCHEMA}")
271
+
272
+ # COMMAND ----------
273
+ # MAGIC %md ## 6. Deploy the Genie Space
274
+
275
+ # COMMAND ----------
276
+ from prograph.genie import deploy_space
277
+ from databricks.sdk import WorkspaceClient
278
+
279
+ wc = WorkspaceClient() # auto-auth from notebook runtime
280
+ result = deploy_space(
281
+ config_yaml=rewritten_yaml,
282
+ ontology_ttl=ontology_text,
283
+ warehouse_id=WAREHOUSE_ID,
284
+ target_catalog=CATALOG,
285
+ target_schema=VIEWS_SCHEMA,
286
+ space_name=f"ProGraph Turbofan Q&A, {SUFFIX}",
287
+ space_description="Generated from prograph/demos/turbofan via the Genie notebook.",
288
+ include_labels=INCLUDED_LABELS,
289
+ workspace_client=wc,
290
+ )
291
+ space_id = result["space_id"]
292
+ print(f"Space ID: {space_id}")
293
+ print(f"Action: {result['action']}")
294
+ print(f"View count: {result['view_count']}")
295
+ print(f"Sample count: {result['sample_query_count']}")
296
+ print()
297
+ displayHTML(
298
+ f'<a href="/genie/rooms/{space_id}" target="_blank" '
299
+ f'style="display:inline-block;padding:8px 16px;background:#0f62fe;'
300
+ f'color:white;text-decoration:none;border-radius:4px;">'
301
+ f'Open in Genie →</a>'
302
+ )
303
+
304
+ # COMMAND ----------
305
+ # MAGIC %md ## 7. Ask three questions through the Conversations API
306
+ # MAGIC
307
+ # MAGIC `start_conversation_and_wait` / `create_message_and_wait` block
308
+ # MAGIC until Genie's response is ready (typically 20-90s per turn). For
309
+ # MAGIC each turn we display the question, the SQL Genie wrote, and the
310
+ # MAGIC row count of the result.
311
+
312
+ # COMMAND ----------
313
+ import time
314
+
315
+ def _attachment_sql(msg):
316
+ for att in msg.attachments or []:
317
+ if att.query and att.query.query:
318
+ return att.query.query
319
+ return ""
320
+
321
+ def _attachment_rowcount(space_id, conversation_id, msg):
322
+ for att in msg.attachments or []:
323
+ if not att.query:
324
+ continue
325
+ try:
326
+ resp = wc.genie.get_message_query_result_by_attachment(
327
+ space_id=space_id, conversation_id=conversation_id,
328
+ message_id=msg.id, attachment_id=att.attachment_id,
329
+ )
330
+ except Exception:
331
+ resp = wc.genie.execute_message_attachment_query(
332
+ space_id=space_id, conversation_id=conversation_id,
333
+ message_id=msg.id, attachment_id=att.attachment_id,
334
+ )
335
+ if resp and resp.statement_response and resp.statement_response.result:
336
+ data = getattr(resp.statement_response.result, "data_array", None)
337
+ return len(data) if data is not None else None
338
+ return None
339
+
340
+
341
+ turns: list[dict] = [] # captured for the JSON summary at the end
342
+
343
+ def _show_turn(label, question, msg):
344
+ sql = _attachment_sql(msg)
345
+ rows = _attachment_rowcount(space_id, msg.conversation_id, msg)
346
+ text = msg.attachments[0].text.content if (
347
+ msg.attachments and msg.attachments[0].text
348
+ ) else "(no text response)"
349
+ turns.append({
350
+ "label": label, "question": question,
351
+ "generated_sql": sql or None, "row_count": rows,
352
+ "answer_text": text,
353
+ })
354
+ displayHTML(f"""
355
+ <div style="font-family:system-ui;border:1px solid #ddd;padding:12px;
356
+ border-radius:8px;margin:8px 0;">
357
+ <div style="font-weight:600;font-size:14px;color:#525252;">{label}</div>
358
+ <div style="font-size:16px;margin:4px 0 12px 0;">{question}</div>
359
+ <div style="font-size:13px;color:#525252;">Answer:</div>
360
+ <div style="margin:4px 0 12px 0;white-space:pre-wrap;">{text}</div>
361
+ <div style="font-size:13px;color:#525252;">Generated SQL:</div>
362
+ <pre style="background:#f4f4f4;padding:8px;border-radius:4px;
363
+ font-size:12px;overflow-x:auto;">{sql or '(no SQL)'}</pre>
364
+ <div style="font-size:13px;color:#525252;">Result rows: {rows}</div>
365
+ </div>
366
+ """)
367
+
368
+
369
+ time.sleep(5) # brief grace period for Space readiness
370
+
371
+ q1 = "How many engines are in the dataset?"
372
+ msg1 = wc.genie.start_conversation_and_wait(space_id=space_id, content=q1)
373
+ _show_turn("Turn 1", q1, msg1)
374
+
375
+ q2 = "How many components have a wear_index of at least 0.7?"
376
+ msg2 = wc.genie.create_message_and_wait(
377
+ space_id=space_id, conversation_id=msg1.conversation_id, content=q2,
378
+ )
379
+ _show_turn("Turn 2", q2, msg2)
380
+
381
+ q3 = "List the blade ids and their wear_index values."
382
+ msg3 = wc.genie.create_message_and_wait(
383
+ space_id=space_id, conversation_id=msg1.conversation_id, content=q3,
384
+ )
385
+ _show_turn("Turn 3", q3, msg3)
386
+
387
+ # COMMAND ----------
388
+ # MAGIC %md ## 8. Summary
389
+
390
+ # COMMAND ----------
391
+ host = spark.conf.get("spark.databricks.workspaceUrl", "<workspace>")
392
+ displayHTML(f"""
393
+ <h3>Run complete</h3>
394
+ <ul>
395
+ <li><b>Space:</b>
396
+ <a href="/genie/rooms/{space_id}" target="_blank">{space_id}</a></li>
397
+ <li><b>Data schema:</b> <code>{CATALOG}.{DATA_SCHEMA}</code>
398
+ ({len(table_fqs)} tables)</li>
399
+ <li><b>Views schema:</b> <code>{CATALOG}.{VIEWS_SCHEMA}</code>
400
+ ({len(views)} views)</li>
401
+ <li><b>Conversation:</b> open the Space link above to see the
402
+ three turns and ask follow-ups</li>
403
+ </ul>
404
+ <p>To tear down, run the cell below. Otherwise the Space + schemas
405
+ stay in place.</p>
406
+ """)
407
+
408
+ # Programmatic summary, picked up by callers running this notebook as
409
+ # a job. Use ``dbutils.notebook.run(...)`` and parse the returned
410
+ # string as JSON.
411
+ import json
412
+ _summary = {
413
+ "space_id": space_id,
414
+ "space_url": f"https://{host}/genie/rooms/{space_id}",
415
+ "data_schema": f"{CATALOG}.{DATA_SCHEMA}",
416
+ "views_schema": f"{CATALOG}.{VIEWS_SCHEMA}",
417
+ "table_count": len(table_fqs),
418
+ "view_count": len(views),
419
+ "turns": turns,
420
+ }
421
+ dbutils.notebook.exit(json.dumps(_summary, indent=2))
422
+
423
+ # COMMAND ----------
424
+ # MAGIC %md ## 9. (Optional) Tear down
425
+ # MAGIC
426
+ # MAGIC Drops the Space + the data and views schemas. Skip this cell if
427
+ # MAGIC you want to keep exploring the Space.
428
+
429
+ # COMMAND ----------
430
+ # Uncomment to clean up:
431
+ # wc.genie.trash_space(space_id)
432
+ # spark.sql(f"DROP SCHEMA IF EXISTS `{CATALOG}`.`{VIEWS_SCHEMA}` CASCADE")
433
+ # spark.sql(f"DROP SCHEMA IF EXISTS `{CATALOG}`.`{DATA_SCHEMA}` CASCADE")
434
+ # print("Cleaned up.")