prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
# Databricks notebook source
|
|
2
|
+
# MAGIC %md
|
|
3
|
+
# MAGIC # ProGraph: Genie Q&A end-to-end demo
|
|
4
|
+
# MAGIC
|
|
5
|
+
# MAGIC Loads the canonical turbofan TTL into Delta tables on the current
|
|
6
|
+
# MAGIC workspace, calls `deploy_space()` to create a Genie Space wired to
|
|
7
|
+
# MAGIC ProGraph-generated views, and asks three questions through the
|
|
8
|
+
# MAGIC Genie Conversations API. The Space + tables are left in place so
|
|
9
|
+
# MAGIC you can open the Space afterwards and inspect the conversation
|
|
10
|
+
# MAGIC history.
|
|
11
|
+
# MAGIC
|
|
12
|
+
# MAGIC Mirror of `tests/integration/genie/test_w9_genie_qa.py` but as a
|
|
13
|
+
# MAGIC notebook so you can re-run it in any workspace and watch the
|
|
14
|
+
# MAGIC pipeline execute inline.
|
|
15
|
+
# MAGIC
|
|
16
|
+
# MAGIC **What it exercises**
|
|
17
|
+
# MAGIC - Shared-table NodeMappings with `filter_predicate` (10 tables → 51 labels; 2 more are queries)
|
|
18
|
+
# MAGIC - Abstract parent NodeMappings (Module / Component / Sensor / …)
|
|
19
|
+
# MAGIC - Subclass-closure views from the ontology's class hierarchy
|
|
20
|
+
# MAGIC - `deploy_space()` library API
|
|
21
|
+
# MAGIC - Genie Conversations API (`start_conversation_and_wait`, `create_message_and_wait`)
|
|
22
|
+
|
|
23
|
+
# COMMAND ----------
|
|
24
|
+
# MAGIC %md ## 1. Install ProGraph
|
|
25
|
+
|
|
26
|
+
# COMMAND ----------
|
|
27
|
+
# MAGIC %pip install --quiet 'prograph[genie]>=0.5.0' rdflib pyyaml
|
|
28
|
+
# MAGIC dbutils.library.restartPython()
|
|
29
|
+
|
|
30
|
+
# COMMAND ----------
|
|
31
|
+
# MAGIC %md ## 2. Parameters
|
|
32
|
+
# MAGIC
|
|
33
|
+
# MAGIC Set the SQL Warehouse ID, target catalog, and a suffix that scopes
|
|
34
|
+
# MAGIC this run's schemas. Multiple runs with different suffixes can
|
|
35
|
+
# MAGIC coexist.
|
|
36
|
+
|
|
37
|
+
# COMMAND ----------
|
|
38
|
+
dbutils.widgets.text("warehouse_id", "", "SQL Warehouse ID")
|
|
39
|
+
# ``catalog`` is a SINGLE-PART name (e.g. ``workspace``, ``main``, ``bg``).
|
|
40
|
+
# Don't pass ``catalog.schema`` here, the notebook builds sub-schemas
|
|
41
|
+
# underneath this catalog itself.
|
|
42
|
+
dbutils.widgets.text("catalog", "workspace", "Target catalog (single-part)")
|
|
43
|
+
dbutils.widgets.text("suffix", "demo", "Run suffix")
|
|
44
|
+
|
|
45
|
+
WAREHOUSE_ID = dbutils.widgets.get("warehouse_id")
|
|
46
|
+
CATALOG = dbutils.widgets.get("catalog")
|
|
47
|
+
SUFFIX = dbutils.widgets.get("suffix")
|
|
48
|
+
|
|
49
|
+
if not WAREHOUSE_ID:
|
|
50
|
+
raise ValueError("Set the warehouse_id widget before running.")
|
|
51
|
+
if "." in CATALOG:
|
|
52
|
+
# Fail fast; Databricks would otherwise raise a less-helpful
|
|
53
|
+
# AnalysisException once the notebook tried CREATE SCHEMA <catalog>.<sub>.
|
|
54
|
+
raise ValueError(
|
|
55
|
+
f"catalog widget must be a single-part name (got {CATALOG!r}). "
|
|
56
|
+
f"The notebook builds prograph_turbofan_data_<suffix> and "
|
|
57
|
+
f"prograph_turbofan_views_<suffix> underneath this catalog."
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
DATA_SCHEMA = f"prograph_turbofan_data_{SUFFIX}"
|
|
61
|
+
VIEWS_SCHEMA = f"prograph_turbofan_views_{SUFFIX}"
|
|
62
|
+
|
|
63
|
+
print(f"Warehouse: {WAREHOUSE_ID}")
|
|
64
|
+
print(f"Data schema: {CATALOG}.{DATA_SCHEMA}")
|
|
65
|
+
print(f"Views schema: {CATALOG}.{VIEWS_SCHEMA}")
|
|
66
|
+
|
|
67
|
+
# COMMAND ----------
|
|
68
|
+
# MAGIC %md ## 3. Load + consolidate the turbofan TTL
|
|
69
|
+
# MAGIC
|
|
70
|
+
# MAGIC Reads the canonical `engine-instances.ttl` shipped with the
|
|
71
|
+
# MAGIC `prograph` package, classifies its triples into per-label pandas
|
|
72
|
+
# MAGIC DataFrames, then consolidates them into the 10 shared node
|
|
73
|
+
# MAGIC tables + 17 relationship tables the turbofan schema expects.
|
|
74
|
+
# MAGIC
|
|
75
|
+
# MAGIC The consolidation map is inlined below so this notebook is
|
|
76
|
+
# MAGIC self-contained, no test-only modules required.
|
|
77
|
+
|
|
78
|
+
# COMMAND ----------
|
|
79
|
+
import re
|
|
80
|
+
import pandas as pd
|
|
81
|
+
from prograph.schema.rows import iter_rows
|
|
82
|
+
from prograph.schema.types import is_missing
|
|
83
|
+
from prograph.cli.ttl_loader import load_ttl
|
|
84
|
+
from prograph.cli._iri_utils import extract_local_name
|
|
85
|
+
|
|
86
|
+
# --- Consolidation constants (mirrors tests/integration/genie/_turbofan_seed.py) ---
|
|
87
|
+
|
|
88
|
+
SCHEMA_LABEL_MAP = {
|
|
89
|
+
"LowPressureCompressor": "LPC",
|
|
90
|
+
"HighPressureCompressor": "HPC",
|
|
91
|
+
"LowPressureTurbine": "LPT",
|
|
92
|
+
"HighPressureTurbine": "HPT",
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
_MODULES = ["FanModule", "LPC", "HPC", "CombustionChamber", "HPT", "LPT",
|
|
96
|
+
"ExhaustNozzle", "AccessoryGearbox", "OilSystem", "FuelSystem"]
|
|
97
|
+
_COMPONENTS = ["Blade", "Disk", "Shaft", "Bearing", "Seal", "Combustor",
|
|
98
|
+
"FuelNozzle", "OilPump", "OilFilter", "HeatExchanger", "FADEC", "Vane"]
|
|
99
|
+
_SENSORS = ["VibrationSensor", "TemperatureSensor", "PressureSensor",
|
|
100
|
+
"SpeedSensor", "OilDebrisSensor", "FlowSensor"]
|
|
101
|
+
_DEGRADATIONS = ["Erosion", "Corrosion", "FatigueCracking", "ThermalDegradation",
|
|
102
|
+
"ForeignObjectDamage", "WearAbrasion", "SealLeakage", "CarbonBuildup"]
|
|
103
|
+
_MAINTENANCES = ["BoroscopeInspection", "OilAnalysis", "BlendRepair",
|
|
104
|
+
"Replacement", "ScheduledOverhaul"]
|
|
105
|
+
|
|
106
|
+
TABLE_FOR_LABEL = {
|
|
107
|
+
"Engine": "engines",
|
|
108
|
+
**{lbl: "modules" for lbl in _MODULES},
|
|
109
|
+
**{lbl: "components" for lbl in _COMPONENTS},
|
|
110
|
+
**{lbl: "sensors" for lbl in _SENSORS},
|
|
111
|
+
"SensorReading": "sensor_readings",
|
|
112
|
+
**{lbl: "degradation_modes" for lbl in _DEGRADATIONS},
|
|
113
|
+
**{lbl: "maintenance_actions" for lbl in _MAINTENANCES},
|
|
114
|
+
"Fleet": "fleets",
|
|
115
|
+
"Aircraft": "aircraft",
|
|
116
|
+
"OperatingEnvironment": "operating_environments",
|
|
117
|
+
}
|
|
118
|
+
TYPED_TABLES = {"modules", "components", "sensors", "degradation_modes",
|
|
119
|
+
"maintenance_actions"}
|
|
120
|
+
|
|
121
|
+
REL_TYPE_TO_TABLE = {
|
|
122
|
+
"IS_PART_OF": "is_part_of", "DOWNSTREAM_OF": "downstream_of",
|
|
123
|
+
"CONNECTED_TO": "connected_to", "DRIVES_SHAFT": "drives_shaft",
|
|
124
|
+
"SUPPLIES_FLUID_TO": "supplies_fluid_to", "MONITORS": "monitors",
|
|
125
|
+
"INSTALLED_ON": "installed_on", "HAS_READING": "has_reading",
|
|
126
|
+
"EXHIBITS_DEGRADATION": "exhibits_degradation",
|
|
127
|
+
"SUSCEPTIBLE_TO": "susceptible_to", "INDICATED_BY": "indicated_by",
|
|
128
|
+
"PROPAGATES_TO": "propagates_to", "PERFORMED_ON": "performed_on",
|
|
129
|
+
"ADDRESSES_DEGRADATION": "addresses_degradation",
|
|
130
|
+
"INSTALLED_ON_AIRCRAFT": "installed_on_aircraft",
|
|
131
|
+
"BELONGS_TO_FLEET": "belongs_to_fleet", "OPERATES_IN": "operates_in",
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
_CAMEL_RE = re.compile(r"(?<!^)(?=[A-Z])")
|
|
135
|
+
def _camel_to_snake(name): return _CAMEL_RE.sub("_", name).lower()
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def consolidate(per_label):
|
|
139
|
+
"""Per-label frames → shared-table frames with `type` discriminators."""
|
|
140
|
+
nodes = {t: [] for t in set(TABLE_FOR_LABEL.values())}
|
|
141
|
+
rels = {t: [] for t in REL_TYPE_TO_TABLE.values()}
|
|
142
|
+
for raw_label, df in per_label.items():
|
|
143
|
+
if "_start_uri" in df.columns:
|
|
144
|
+
rt = REL_TYPE_TO_TABLE.get(raw_label)
|
|
145
|
+
if rt is None: continue
|
|
146
|
+
for row in iter_rows(df):
|
|
147
|
+
rels[rt].append({
|
|
148
|
+
"start_id": extract_local_name(str(row["_start_uri"])),
|
|
149
|
+
"end_id": extract_local_name(str(row["_end_uri"])),
|
|
150
|
+
})
|
|
151
|
+
continue
|
|
152
|
+
if "_uri" not in df.columns: continue
|
|
153
|
+
schema_label = SCHEMA_LABEL_MAP.get(raw_label, raw_label)
|
|
154
|
+
tbl = TABLE_FOR_LABEL.get(schema_label)
|
|
155
|
+
if tbl is None: continue
|
|
156
|
+
typed = tbl in TYPED_TABLES
|
|
157
|
+
for row in iter_rows(df):
|
|
158
|
+
r = {"id": extract_local_name(str(row["_uri"]))}
|
|
159
|
+
for col, val in row.items():
|
|
160
|
+
if col == "_uri": continue
|
|
161
|
+
r[_camel_to_snake(col)] = None if is_missing(val) else val
|
|
162
|
+
if typed: r["type"] = schema_label
|
|
163
|
+
nodes[tbl].append(r)
|
|
164
|
+
return (
|
|
165
|
+
{t: pd.DataFrame(rows) for t, rows in nodes.items()},
|
|
166
|
+
{t: pd.DataFrame(rows) for t, rows in rels.items()},
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# --- Run the consolidation ------------------------------------------------
|
|
171
|
+
|
|
172
|
+
import importlib.resources as ir
|
|
173
|
+
ttl_path = ir.files("prograph.demos.turbofan") / "data" / "engine-instances.ttl"
|
|
174
|
+
with ir.as_file(ttl_path) as p:
|
|
175
|
+
per_label = load_ttl([str(p)])
|
|
176
|
+
|
|
177
|
+
node_frames, rel_frames = consolidate(per_label)
|
|
178
|
+
|
|
179
|
+
print("Node tables:")
|
|
180
|
+
for t, df in node_frames.items():
|
|
181
|
+
print(f" {t:24s} {len(df):4d} rows")
|
|
182
|
+
print("\nRelationship tables:")
|
|
183
|
+
for t, df in rel_frames.items():
|
|
184
|
+
print(f" {t:24s} {len(df):4d} rows")
|
|
185
|
+
|
|
186
|
+
# COMMAND ----------
|
|
187
|
+
# MAGIC %md ## 4. Write Delta tables
|
|
188
|
+
# MAGIC
|
|
189
|
+
# MAGIC `spark.createDataFrame(...).write.saveAsTable(...)` lands each
|
|
190
|
+
# MAGIC frame as a managed Delta table, no Volume / `COPY INTO` needed
|
|
191
|
+
# MAGIC since we're inside the Databricks runtime.
|
|
192
|
+
|
|
193
|
+
# COMMAND ----------
|
|
194
|
+
spark.sql(f"CREATE SCHEMA IF NOT EXISTS `{CATALOG}`.`{DATA_SCHEMA}`")
|
|
195
|
+
|
|
196
|
+
table_fqs = {}
|
|
197
|
+
skipped_empty = []
|
|
198
|
+
for table, df in {**node_frames, **rel_frames}.items():
|
|
199
|
+
if df.empty:
|
|
200
|
+
# Skip, guessing a DDL for an empty frame risks shape drift,
|
|
201
|
+
# and the corresponding labels won't be in include_labels anyway.
|
|
202
|
+
skipped_empty.append(table)
|
|
203
|
+
continue
|
|
204
|
+
# Coerce all-None object columns to string so Spark infers a real type.
|
|
205
|
+
coerced = df.copy()
|
|
206
|
+
for col in coerced.columns:
|
|
207
|
+
if coerced[col].isna().all() and coerced[col].dtype == "object":
|
|
208
|
+
coerced[col] = coerced[col].astype("string")
|
|
209
|
+
sdf = spark.createDataFrame(coerced)
|
|
210
|
+
sdf.write.mode("overwrite").saveAsTable(f"{CATALOG}.{DATA_SCHEMA}.{table}")
|
|
211
|
+
table_fqs[table] = f"{CATALOG}.{DATA_SCHEMA}.{table}"
|
|
212
|
+
|
|
213
|
+
print(f"Wrote {len(table_fqs)} tables to {CATALOG}.{DATA_SCHEMA}")
|
|
214
|
+
if skipped_empty:
|
|
215
|
+
print(f"Skipped (empty in TTL data): {', '.join(skipped_empty)}")
|
|
216
|
+
|
|
217
|
+
# COMMAND ----------
|
|
218
|
+
# MAGIC %md ## 5. Generate views + apply them to the warehouse
|
|
219
|
+
# MAGIC
|
|
220
|
+
# MAGIC View generation must receive the same `include_labels` we'll pass
|
|
221
|
+
# MAGIC to `deploy_space`: subclass-closure views are filter-sensitive,
|
|
222
|
+
# MAGIC so a mismatch between the applied set and the Space-registered
|
|
223
|
+
# MAGIC set makes Genie reject the Space.
|
|
224
|
+
|
|
225
|
+
# COMMAND ----------
|
|
226
|
+
import yaml
|
|
227
|
+
from prograph.config import load_config_from_string
|
|
228
|
+
from prograph.ontology import OntologyLayer
|
|
229
|
+
from prograph.genie import generate_views
|
|
230
|
+
|
|
231
|
+
# Read the shipped graph.yaml + ontology TTL.
|
|
232
|
+
yaml_text = (
|
|
233
|
+
ir.files("prograph.demos.turbofan") / "graph.yaml"
|
|
234
|
+
).read_text(encoding="utf-8")
|
|
235
|
+
ontology_text = (
|
|
236
|
+
ir.files("prograph.demos.turbofan") / "ontology" / "engine-maintenance.ttl"
|
|
237
|
+
).read_text(encoding="utf-8")
|
|
238
|
+
|
|
239
|
+
# Rewrite sources[*].table to point at the per-run FQ names.
|
|
240
|
+
yaml_doc = yaml.safe_load(yaml_text)
|
|
241
|
+
for key in list((yaml_doc.get("sources") or {}).keys()):
|
|
242
|
+
if key in table_fqs:
|
|
243
|
+
yaml_doc["sources"][key]["table"] = table_fqs[key]
|
|
244
|
+
rewritten_yaml = yaml.safe_dump(yaml_doc)
|
|
245
|
+
|
|
246
|
+
config = load_config_from_string(rewritten_yaml)
|
|
247
|
+
schema = config.to_schema(by_table=True)
|
|
248
|
+
|
|
249
|
+
ontology = OntologyLayer.from_turtle(ontology_text)
|
|
250
|
+
|
|
251
|
+
# Genie Spaces cap at 30 tables, restrict to a focused subset.
|
|
252
|
+
# Includes the abstract Component mapping, the parent of the subclass-closure views.
|
|
253
|
+
INCLUDED_LABELS = [
|
|
254
|
+
"Engine",
|
|
255
|
+
"Component",
|
|
256
|
+
"FanModule", "LPC", "HPC",
|
|
257
|
+
"Blade", "Disk", "FADEC", "Combustor",
|
|
258
|
+
]
|
|
259
|
+
|
|
260
|
+
views = generate_views(schema, ontology=ontology, include_labels=INCLUDED_LABELS)
|
|
261
|
+
print(f"{len(views)} views generated:")
|
|
262
|
+
for v in views:
|
|
263
|
+
print(f" {v.name} ({v.source.value})")
|
|
264
|
+
|
|
265
|
+
# Apply them to the warehouse.
|
|
266
|
+
spark.sql(f"CREATE SCHEMA IF NOT EXISTS `{CATALOG}`.`{VIEWS_SCHEMA}`")
|
|
267
|
+
for v in views:
|
|
268
|
+
fq = f"`{CATALOG}`.`{VIEWS_SCHEMA}`.`{v.name}`"
|
|
269
|
+
spark.sql(f"CREATE OR REPLACE VIEW {fq} AS\n{v.sql}")
|
|
270
|
+
print(f"\nApplied to {CATALOG}.{VIEWS_SCHEMA}")
|
|
271
|
+
|
|
272
|
+
# COMMAND ----------
|
|
273
|
+
# MAGIC %md ## 6. Deploy the Genie Space
|
|
274
|
+
|
|
275
|
+
# COMMAND ----------
|
|
276
|
+
from prograph.genie import deploy_space
|
|
277
|
+
from databricks.sdk import WorkspaceClient
|
|
278
|
+
|
|
279
|
+
wc = WorkspaceClient() # auto-auth from notebook runtime
|
|
280
|
+
result = deploy_space(
|
|
281
|
+
config_yaml=rewritten_yaml,
|
|
282
|
+
ontology_ttl=ontology_text,
|
|
283
|
+
warehouse_id=WAREHOUSE_ID,
|
|
284
|
+
target_catalog=CATALOG,
|
|
285
|
+
target_schema=VIEWS_SCHEMA,
|
|
286
|
+
space_name=f"ProGraph Turbofan Q&A, {SUFFIX}",
|
|
287
|
+
space_description="Generated from prograph/demos/turbofan via the Genie notebook.",
|
|
288
|
+
include_labels=INCLUDED_LABELS,
|
|
289
|
+
workspace_client=wc,
|
|
290
|
+
)
|
|
291
|
+
space_id = result["space_id"]
|
|
292
|
+
print(f"Space ID: {space_id}")
|
|
293
|
+
print(f"Action: {result['action']}")
|
|
294
|
+
print(f"View count: {result['view_count']}")
|
|
295
|
+
print(f"Sample count: {result['sample_query_count']}")
|
|
296
|
+
print()
|
|
297
|
+
displayHTML(
|
|
298
|
+
f'<a href="/genie/rooms/{space_id}" target="_blank" '
|
|
299
|
+
f'style="display:inline-block;padding:8px 16px;background:#0f62fe;'
|
|
300
|
+
f'color:white;text-decoration:none;border-radius:4px;">'
|
|
301
|
+
f'Open in Genie →</a>'
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
# COMMAND ----------
|
|
305
|
+
# MAGIC %md ## 7. Ask three questions through the Conversations API
|
|
306
|
+
# MAGIC
|
|
307
|
+
# MAGIC `start_conversation_and_wait` / `create_message_and_wait` block
|
|
308
|
+
# MAGIC until Genie's response is ready (typically 20-90s per turn). For
|
|
309
|
+
# MAGIC each turn we display the question, the SQL Genie wrote, and the
|
|
310
|
+
# MAGIC row count of the result.
|
|
311
|
+
|
|
312
|
+
# COMMAND ----------
|
|
313
|
+
import time
|
|
314
|
+
|
|
315
|
+
def _attachment_sql(msg):
|
|
316
|
+
for att in msg.attachments or []:
|
|
317
|
+
if att.query and att.query.query:
|
|
318
|
+
return att.query.query
|
|
319
|
+
return ""
|
|
320
|
+
|
|
321
|
+
def _attachment_rowcount(space_id, conversation_id, msg):
|
|
322
|
+
for att in msg.attachments or []:
|
|
323
|
+
if not att.query:
|
|
324
|
+
continue
|
|
325
|
+
try:
|
|
326
|
+
resp = wc.genie.get_message_query_result_by_attachment(
|
|
327
|
+
space_id=space_id, conversation_id=conversation_id,
|
|
328
|
+
message_id=msg.id, attachment_id=att.attachment_id,
|
|
329
|
+
)
|
|
330
|
+
except Exception:
|
|
331
|
+
resp = wc.genie.execute_message_attachment_query(
|
|
332
|
+
space_id=space_id, conversation_id=conversation_id,
|
|
333
|
+
message_id=msg.id, attachment_id=att.attachment_id,
|
|
334
|
+
)
|
|
335
|
+
if resp and resp.statement_response and resp.statement_response.result:
|
|
336
|
+
data = getattr(resp.statement_response.result, "data_array", None)
|
|
337
|
+
return len(data) if data is not None else None
|
|
338
|
+
return None
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
turns: list[dict] = [] # captured for the JSON summary at the end
|
|
342
|
+
|
|
343
|
+
def _show_turn(label, question, msg):
|
|
344
|
+
sql = _attachment_sql(msg)
|
|
345
|
+
rows = _attachment_rowcount(space_id, msg.conversation_id, msg)
|
|
346
|
+
text = msg.attachments[0].text.content if (
|
|
347
|
+
msg.attachments and msg.attachments[0].text
|
|
348
|
+
) else "(no text response)"
|
|
349
|
+
turns.append({
|
|
350
|
+
"label": label, "question": question,
|
|
351
|
+
"generated_sql": sql or None, "row_count": rows,
|
|
352
|
+
"answer_text": text,
|
|
353
|
+
})
|
|
354
|
+
displayHTML(f"""
|
|
355
|
+
<div style="font-family:system-ui;border:1px solid #ddd;padding:12px;
|
|
356
|
+
border-radius:8px;margin:8px 0;">
|
|
357
|
+
<div style="font-weight:600;font-size:14px;color:#525252;">{label}</div>
|
|
358
|
+
<div style="font-size:16px;margin:4px 0 12px 0;">{question}</div>
|
|
359
|
+
<div style="font-size:13px;color:#525252;">Answer:</div>
|
|
360
|
+
<div style="margin:4px 0 12px 0;white-space:pre-wrap;">{text}</div>
|
|
361
|
+
<div style="font-size:13px;color:#525252;">Generated SQL:</div>
|
|
362
|
+
<pre style="background:#f4f4f4;padding:8px;border-radius:4px;
|
|
363
|
+
font-size:12px;overflow-x:auto;">{sql or '(no SQL)'}</pre>
|
|
364
|
+
<div style="font-size:13px;color:#525252;">Result rows: {rows}</div>
|
|
365
|
+
</div>
|
|
366
|
+
""")
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
time.sleep(5) # brief grace period for Space readiness
|
|
370
|
+
|
|
371
|
+
q1 = "How many engines are in the dataset?"
|
|
372
|
+
msg1 = wc.genie.start_conversation_and_wait(space_id=space_id, content=q1)
|
|
373
|
+
_show_turn("Turn 1", q1, msg1)
|
|
374
|
+
|
|
375
|
+
q2 = "How many components have a wear_index of at least 0.7?"
|
|
376
|
+
msg2 = wc.genie.create_message_and_wait(
|
|
377
|
+
space_id=space_id, conversation_id=msg1.conversation_id, content=q2,
|
|
378
|
+
)
|
|
379
|
+
_show_turn("Turn 2", q2, msg2)
|
|
380
|
+
|
|
381
|
+
q3 = "List the blade ids and their wear_index values."
|
|
382
|
+
msg3 = wc.genie.create_message_and_wait(
|
|
383
|
+
space_id=space_id, conversation_id=msg1.conversation_id, content=q3,
|
|
384
|
+
)
|
|
385
|
+
_show_turn("Turn 3", q3, msg3)
|
|
386
|
+
|
|
387
|
+
# COMMAND ----------
|
|
388
|
+
# MAGIC %md ## 8. Summary
|
|
389
|
+
|
|
390
|
+
# COMMAND ----------
|
|
391
|
+
host = spark.conf.get("spark.databricks.workspaceUrl", "<workspace>")
|
|
392
|
+
displayHTML(f"""
|
|
393
|
+
<h3>Run complete</h3>
|
|
394
|
+
<ul>
|
|
395
|
+
<li><b>Space:</b>
|
|
396
|
+
<a href="/genie/rooms/{space_id}" target="_blank">{space_id}</a></li>
|
|
397
|
+
<li><b>Data schema:</b> <code>{CATALOG}.{DATA_SCHEMA}</code>
|
|
398
|
+
({len(table_fqs)} tables)</li>
|
|
399
|
+
<li><b>Views schema:</b> <code>{CATALOG}.{VIEWS_SCHEMA}</code>
|
|
400
|
+
({len(views)} views)</li>
|
|
401
|
+
<li><b>Conversation:</b> open the Space link above to see the
|
|
402
|
+
three turns and ask follow-ups</li>
|
|
403
|
+
</ul>
|
|
404
|
+
<p>To tear down, run the cell below. Otherwise the Space + schemas
|
|
405
|
+
stay in place.</p>
|
|
406
|
+
""")
|
|
407
|
+
|
|
408
|
+
# Programmatic summary, picked up by callers running this notebook as
|
|
409
|
+
# a job. Use ``dbutils.notebook.run(...)`` and parse the returned
|
|
410
|
+
# string as JSON.
|
|
411
|
+
import json
|
|
412
|
+
_summary = {
|
|
413
|
+
"space_id": space_id,
|
|
414
|
+
"space_url": f"https://{host}/genie/rooms/{space_id}",
|
|
415
|
+
"data_schema": f"{CATALOG}.{DATA_SCHEMA}",
|
|
416
|
+
"views_schema": f"{CATALOG}.{VIEWS_SCHEMA}",
|
|
417
|
+
"table_count": len(table_fqs),
|
|
418
|
+
"view_count": len(views),
|
|
419
|
+
"turns": turns,
|
|
420
|
+
}
|
|
421
|
+
dbutils.notebook.exit(json.dumps(_summary, indent=2))
|
|
422
|
+
|
|
423
|
+
# COMMAND ----------
|
|
424
|
+
# MAGIC %md ## 9. (Optional) Tear down
|
|
425
|
+
# MAGIC
|
|
426
|
+
# MAGIC Drops the Space + the data and views schemas. Skip this cell if
|
|
427
|
+
# MAGIC you want to keep exploring the Space.
|
|
428
|
+
|
|
429
|
+
# COMMAND ----------
|
|
430
|
+
# Uncomment to clean up:
|
|
431
|
+
# wc.genie.trash_space(space_id)
|
|
432
|
+
# spark.sql(f"DROP SCHEMA IF EXISTS `{CATALOG}`.`{VIEWS_SCHEMA}` CASCADE")
|
|
433
|
+
# spark.sql(f"DROP SCHEMA IF EXISTS `{CATALOG}`.`{DATA_SCHEMA}` CASCADE")
|
|
434
|
+
# print("Cleaned up.")
|