prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
"""Generate natural-language instructions for a Genie Space from schema state.
|
|
2
|
+
|
|
3
|
+
A Genie Space accepts an instructions string that the underlying LLM
|
|
4
|
+
sees as context for every question. ProGraph has rich structured
|
|
5
|
+
metadata about the data model (schema, ontology, shapes, generated
|
|
6
|
+
views); the instruction generator translates that into prose Genie's
|
|
7
|
+
reasoning model can use.
|
|
8
|
+
|
|
9
|
+
Output shape: :class:`prograph.genie.InstructionsDoc`,
|
|
10
|
+
a list of :class:`InstructionsSection` plus a rendered markdown body.
|
|
11
|
+
The Space emitter (W5) consumes ``doc.rendered`` for the Space config;
|
|
12
|
+
the sections are kept around for round-trip and selective editing.
|
|
13
|
+
|
|
14
|
+
Section catalog (Decision 6 from the spike):
|
|
15
|
+
|
|
16
|
+
- order=10 Overview
|
|
17
|
+
- order=20 Entities (one paragraph per NodeMapping label)
|
|
18
|
+
- order=30 Relationships (paragraph per RelationshipMapping)
|
|
19
|
+
- order=40 Subclass / type hierarchies (from ontology)
|
|
20
|
+
- order=50 OWL-defined classes (the RL+ restrictions)
|
|
21
|
+
- order=60 SHACL constraints (skipped in W4. See view generator notes)
|
|
22
|
+
- order=70 Metrics (Phase 5)
|
|
23
|
+
- order=80 Generated views (their names + purpose, so Genie picks them)
|
|
24
|
+
- order=90 Glossary
|
|
25
|
+
|
|
26
|
+
Token budget: the assembler caps the rendered body at a configurable
|
|
27
|
+
character budget (default 16000 chars, roughly 4000 tokens). Sections
|
|
28
|
+
that would push past the budget get trimmed; the overview + glossary
|
|
29
|
+
get priority because they're load-bearing for Genie's reasoning.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import re
|
|
35
|
+
from typing import TYPE_CHECKING, List, Optional
|
|
36
|
+
|
|
37
|
+
from prograph.genie.formats import InstructionsDoc, InstructionsSection
|
|
38
|
+
|
|
39
|
+
if TYPE_CHECKING:
|
|
40
|
+
from prograph.genie.formats import ViewDefinition
|
|
41
|
+
from prograph.ontology.layer import OntologyLayer
|
|
42
|
+
from prograph.schema import GraphSchema
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
# Approximate character → token ratio. Genie's tokens are about 4 chars
|
|
46
|
+
# on average. We use chars as the budget unit because they're cheap to
|
|
47
|
+
# compute deterministically; the constant is tuned for the typical
|
|
48
|
+
# English-prose-with-identifiers content this generator emits.
|
|
49
|
+
DEFAULT_CHAR_BUDGET = 16000
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# ─── Public API ──────────────────────────────────────────────────────
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def generate_instructions(
|
|
56
|
+
schema: "GraphSchema",
|
|
57
|
+
ontology: Optional["OntologyLayer"] = None,
|
|
58
|
+
views: Optional[List["ViewDefinition"]] = None,
|
|
59
|
+
char_budget: int = DEFAULT_CHAR_BUDGET,
|
|
60
|
+
summary_only: bool = False,
|
|
61
|
+
) -> InstructionsDoc:
|
|
62
|
+
"""Build a Genie-ready instructions doc from schema state.
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
schema: The :class:`GraphSchema` to describe.
|
|
66
|
+
ontology: Optional :class:`OntologyLayer`. When provided, the
|
|
67
|
+
doc gains hierarchy and defined-class sections.
|
|
68
|
+
views: Optional list of generated :class:`ViewDefinition`.
|
|
69
|
+
When provided, the doc gains a "Generated Views" section
|
|
70
|
+
telling Genie which views exist and what each is for.
|
|
71
|
+
char_budget: Max characters for the rendered body. Sections
|
|
72
|
+
that would push past this get trimmed in priority order
|
|
73
|
+
(entities + glossary kept first; relationships next; OWL
|
|
74
|
+
and views dropped last). Defaults to ``DEFAULT_CHAR_BUDGET``
|
|
75
|
+
(~4000 Genie tokens).
|
|
76
|
+
summary_only: When ``True``, emit just the overview + a
|
|
77
|
+
condensed label/relationship list, no per-entity prose.
|
|
78
|
+
Useful for token-constrained Genie Spaces or as a starting
|
|
79
|
+
template the user edits.
|
|
80
|
+
|
|
81
|
+
Returns:
|
|
82
|
+
A :class:`InstructionsDoc` with ``sections`` populated and
|
|
83
|
+
``rendered`` ready for the Genie Space config.
|
|
84
|
+
"""
|
|
85
|
+
sections: List[InstructionsSection] = []
|
|
86
|
+
|
|
87
|
+
sections.append(_section_overview(schema, ontology, views, summary_only))
|
|
88
|
+
|
|
89
|
+
if not summary_only:
|
|
90
|
+
sections.append(_section_entities(schema))
|
|
91
|
+
if schema.relationship_mappings:
|
|
92
|
+
sections.append(_section_relationships(schema))
|
|
93
|
+
if ontology is not None:
|
|
94
|
+
hierarchy = _section_hierarchy(schema, ontology)
|
|
95
|
+
if hierarchy is not None:
|
|
96
|
+
sections.append(hierarchy)
|
|
97
|
+
defined = _section_defined_classes(schema, ontology)
|
|
98
|
+
if defined is not None:
|
|
99
|
+
sections.append(defined)
|
|
100
|
+
if views:
|
|
101
|
+
sections.append(_section_generated_views(views))
|
|
102
|
+
sections.append(_section_glossary(schema, ontology))
|
|
103
|
+
|
|
104
|
+
rendered = _assemble(sections, char_budget=char_budget)
|
|
105
|
+
return InstructionsDoc(sections=sections, rendered=rendered)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# ─── Section builders ────────────────────────────────────────────────
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _section_overview(
|
|
112
|
+
schema: "GraphSchema",
|
|
113
|
+
ontology: Optional["OntologyLayer"],
|
|
114
|
+
views: Optional[List["ViewDefinition"]],
|
|
115
|
+
summary_only: bool,
|
|
116
|
+
) -> InstructionsSection:
|
|
117
|
+
name = getattr(schema, "name", None) or "graph"
|
|
118
|
+
label_count = len(schema.node_mappings)
|
|
119
|
+
rel_count = len(schema.relationship_mappings)
|
|
120
|
+
|
|
121
|
+
lines: List[str] = []
|
|
122
|
+
lines.append(
|
|
123
|
+
f"This Space queries a graph-shaped dataset called **{name}** that "
|
|
124
|
+
f"models {label_count} entity type(s) connected by "
|
|
125
|
+
f"{rel_count} relationship type(s)."
|
|
126
|
+
)
|
|
127
|
+
if ontology is not None:
|
|
128
|
+
lines.append(
|
|
129
|
+
"An OWL ontology defines class hierarchies and (where "
|
|
130
|
+
"applicable) defined-class restrictions that classify "
|
|
131
|
+
"entities automatically."
|
|
132
|
+
)
|
|
133
|
+
if views:
|
|
134
|
+
kinds = sorted({v.source.value for v in views})
|
|
135
|
+
lines.append(
|
|
136
|
+
f"ProGraph generates {len(views)} SQL view(s) ("
|
|
137
|
+
f"{', '.join(kinds)}) that expose the semantic projections "
|
|
138
|
+
"as standard tables; prefer those when the question maps "
|
|
139
|
+
"to a defined class or subclass closure."
|
|
140
|
+
)
|
|
141
|
+
if summary_only:
|
|
142
|
+
labels = ", ".join(m.label for m in schema.node_mappings if m.label is not None)
|
|
143
|
+
lines.append(f"Entities: {labels}.")
|
|
144
|
+
|
|
145
|
+
return InstructionsSection(
|
|
146
|
+
heading="Overview",
|
|
147
|
+
body="\n\n".join(lines),
|
|
148
|
+
order=10,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _section_entities(schema: "GraphSchema") -> InstructionsSection:
|
|
153
|
+
paragraphs: List[str] = []
|
|
154
|
+
for mapping in schema.node_mappings:
|
|
155
|
+
if mapping.label is None:
|
|
156
|
+
continue # an unlabelled node mapping: no label to name
|
|
157
|
+
props = list(mapping.property_columns.keys())
|
|
158
|
+
if props:
|
|
159
|
+
prop_text = (
|
|
160
|
+
f"Properties: {', '.join('`' + p + '`' for p in props)}."
|
|
161
|
+
)
|
|
162
|
+
else:
|
|
163
|
+
prop_text = "No properties beyond the entity id."
|
|
164
|
+
id_cols = (
|
|
165
|
+
mapping.id_columns[0]
|
|
166
|
+
if len(mapping.id_columns) == 1
|
|
167
|
+
else "(" + ", ".join(mapping.id_columns) + ")"
|
|
168
|
+
)
|
|
169
|
+
filter_text = ""
|
|
170
|
+
if mapping.filter_predicate:
|
|
171
|
+
filt = ", ".join(
|
|
172
|
+
f"{k}={v!r}" for k, v in mapping.filter_predicate.items()
|
|
173
|
+
)
|
|
174
|
+
filter_text = (
|
|
175
|
+
f" Filtered subset of `{mapping.source_table}` "
|
|
176
|
+
f"where {filt}."
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
paragraphs.append(
|
|
180
|
+
f"**{mapping.label}**, rows in `{mapping.source_table}` "
|
|
181
|
+
f"identified by `{id_cols}`.{filter_text} {prop_text}"
|
|
182
|
+
)
|
|
183
|
+
return InstructionsSection(
|
|
184
|
+
heading="Entities",
|
|
185
|
+
body="\n\n".join(paragraphs),
|
|
186
|
+
order=20,
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _section_relationships(schema: "GraphSchema") -> InstructionsSection:
|
|
191
|
+
paragraphs: List[str] = []
|
|
192
|
+
for rel in schema.relationship_mappings:
|
|
193
|
+
if rel.start_node_label is None or rel.end_node_label is None:
|
|
194
|
+
continue # an end given by an identity: no one label to name
|
|
195
|
+
paragraphs.append(
|
|
196
|
+
f"**{rel.rel_type}**, {rel.start_node_label} → {rel.end_node_label}. "
|
|
197
|
+
f"Stored in `{rel.source_table}`."
|
|
198
|
+
)
|
|
199
|
+
return InstructionsSection(
|
|
200
|
+
heading="Relationships",
|
|
201
|
+
body="\n\n".join(paragraphs),
|
|
202
|
+
order=30,
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _section_hierarchy(
|
|
207
|
+
schema: "GraphSchema",
|
|
208
|
+
ontology: "OntologyLayer",
|
|
209
|
+
) -> Optional[InstructionsSection]:
|
|
210
|
+
"""Describe rdfs:subClassOf chains among mapped labels."""
|
|
211
|
+
pairs: List[str] = []
|
|
212
|
+
mapped = {m.label for m in schema.node_mappings if m.label is not None}
|
|
213
|
+
for label in sorted(mapped):
|
|
214
|
+
try:
|
|
215
|
+
supers = ontology.get_superclasses(label, include_self=False)
|
|
216
|
+
except Exception:
|
|
217
|
+
continue
|
|
218
|
+
meaningful = [s for s in supers if s != label]
|
|
219
|
+
if not meaningful:
|
|
220
|
+
continue
|
|
221
|
+
chain = " → ".join(sorted(meaningful))
|
|
222
|
+
pairs.append(f"`{label}` is a subclass of {chain}.")
|
|
223
|
+
if not pairs:
|
|
224
|
+
return None
|
|
225
|
+
return InstructionsSection(
|
|
226
|
+
heading="Type Hierarchies",
|
|
227
|
+
body="\n\n".join(pairs),
|
|
228
|
+
order=40,
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _section_defined_classes(
|
|
233
|
+
schema: "GraphSchema",
|
|
234
|
+
ontology: "OntologyLayer",
|
|
235
|
+
) -> Optional[InstructionsSection]:
|
|
236
|
+
from prograph.owl.defined_classes import (
|
|
237
|
+
DataRangeConstraint,
|
|
238
|
+
InversePropertyConstraint,
|
|
239
|
+
MinCardinalityConstraint,
|
|
240
|
+
parse_defined_classes,
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
defined = parse_defined_classes(ontology)
|
|
244
|
+
if not defined:
|
|
245
|
+
return None
|
|
246
|
+
|
|
247
|
+
paragraphs: List[str] = []
|
|
248
|
+
for dc in defined:
|
|
249
|
+
if isinstance(dc.constraint, DataRangeConstraint):
|
|
250
|
+
c = dc.constraint
|
|
251
|
+
bounds: List[str] = []
|
|
252
|
+
if c.min_inclusive is not None:
|
|
253
|
+
bounds.append(f"`{c.property_name}` ≥ {c.min_inclusive}")
|
|
254
|
+
if c.max_inclusive is not None:
|
|
255
|
+
bounds.append(f"`{c.property_name}` ≤ {c.max_inclusive}")
|
|
256
|
+
if c.min_exclusive is not None:
|
|
257
|
+
bounds.append(f"`{c.property_name}` > {c.min_exclusive}")
|
|
258
|
+
if c.max_exclusive is not None:
|
|
259
|
+
bounds.append(f"`{c.property_name}` < {c.max_exclusive}")
|
|
260
|
+
if not bounds:
|
|
261
|
+
continue
|
|
262
|
+
paragraphs.append(
|
|
263
|
+
f"**{dc.name}**, a `{dc.base_class}` where "
|
|
264
|
+
f"{' and '.join(bounds)}. "
|
|
265
|
+
f"To find them, query `prograph_views.{_to_snake(dc.name)}`."
|
|
266
|
+
)
|
|
267
|
+
elif isinstance(dc.constraint, MinCardinalityConstraint):
|
|
268
|
+
c = dc.constraint
|
|
269
|
+
paragraphs.append(
|
|
270
|
+
f"**{dc.name}**, a `{dc.base_class}` with at least "
|
|
271
|
+
f"{c.min_count} outgoing `{c.property_name}` edges. "
|
|
272
|
+
f"Query `prograph_views.{_to_snake(dc.name)}`."
|
|
273
|
+
)
|
|
274
|
+
elif isinstance(dc.constraint, InversePropertyConstraint):
|
|
275
|
+
c = dc.constraint
|
|
276
|
+
paragraphs.append(
|
|
277
|
+
f"**{dc.name}**, a `{dc.base_class}` with at least one "
|
|
278
|
+
f"incoming `{c.property_name}` edge from a `{c.target_class}`. "
|
|
279
|
+
f"Query `prograph_views.{_to_snake(dc.name)}`."
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
if not paragraphs:
|
|
283
|
+
return None
|
|
284
|
+
return InstructionsSection(
|
|
285
|
+
heading="Defined Classes (OWL-Entailed)",
|
|
286
|
+
body="\n\n".join(paragraphs),
|
|
287
|
+
order=50,
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _section_generated_views(views: List["ViewDefinition"]) -> InstructionsSection:
|
|
292
|
+
"""Tell Genie which views to prefer for which intent."""
|
|
293
|
+
lines: List[str] = []
|
|
294
|
+
for v in views:
|
|
295
|
+
cols = (
|
|
296
|
+
f" Columns: {', '.join('`' + c + '`' for c in v.columns)}."
|
|
297
|
+
if v.columns
|
|
298
|
+
else ""
|
|
299
|
+
)
|
|
300
|
+
lines.append(f"- `prograph_views.{v.name}`, {v.description}{cols}")
|
|
301
|
+
return InstructionsSection(
|
|
302
|
+
heading="Generated Views",
|
|
303
|
+
body="\n".join(lines),
|
|
304
|
+
order=80,
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _section_glossary(
|
|
309
|
+
schema: "GraphSchema",
|
|
310
|
+
ontology: Optional["OntologyLayer"],
|
|
311
|
+
) -> InstructionsSection:
|
|
312
|
+
"""One-line per term reference."""
|
|
313
|
+
lines: List[str] = []
|
|
314
|
+
for mapping in schema.node_mappings:
|
|
315
|
+
if mapping.label is None:
|
|
316
|
+
continue # an unlabelled node mapping: no label to name
|
|
317
|
+
lines.append(
|
|
318
|
+
f"- **{mapping.label}**: an entity stored in "
|
|
319
|
+
f"`{mapping.source_table}`."
|
|
320
|
+
)
|
|
321
|
+
for rel in schema.relationship_mappings:
|
|
322
|
+
if rel.start_node_label is None or rel.end_node_label is None:
|
|
323
|
+
continue
|
|
324
|
+
lines.append(
|
|
325
|
+
f"- **{rel.rel_type}**: a {rel.start_node_label}-to-"
|
|
326
|
+
f"{rel.end_node_label} relationship."
|
|
327
|
+
)
|
|
328
|
+
return InstructionsSection(
|
|
329
|
+
heading="Glossary",
|
|
330
|
+
body="\n".join(lines),
|
|
331
|
+
order=90,
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
# ─── Assembly + trimming ─────────────────────────────────────────────
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _assemble(
|
|
339
|
+
sections: List[InstructionsSection],
|
|
340
|
+
char_budget: int,
|
|
341
|
+
) -> str:
|
|
342
|
+
"""Concatenate sections in ``order``-ascending sequence, capped at the budget.
|
|
343
|
+
|
|
344
|
+
Trim strategy: append sections in priority order (low ``order``
|
|
345
|
+
first). When adding a section would exceed the budget, drop it
|
|
346
|
+
and continue with subsequent sections that are smaller. This
|
|
347
|
+
favors structurally-important sections (overview, entities) over
|
|
348
|
+
optional ones (views, glossary).
|
|
349
|
+
"""
|
|
350
|
+
ordered = sorted(sections, key=lambda s: s.order)
|
|
351
|
+
parts: List[str] = []
|
|
352
|
+
used = 0
|
|
353
|
+
for section in ordered:
|
|
354
|
+
rendered = f"## {section.heading}\n\n{section.body}".strip()
|
|
355
|
+
# Separator cost: 0 for the first section, len("\n\n") otherwise.
|
|
356
|
+
cost = len(rendered) + (2 if parts else 0)
|
|
357
|
+
if used + cost > char_budget:
|
|
358
|
+
# Doesn't fit. Skip and try the next (smaller) section.
|
|
359
|
+
continue
|
|
360
|
+
parts.append(rendered)
|
|
361
|
+
used += cost
|
|
362
|
+
return "\n\n".join(parts)
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
# ─── Helpers ─────────────────────────────────────────────────────────
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _to_snake(label: str) -> str:
|
|
369
|
+
"""PascalCase / camelCase → snake_case, mirroring the view generator."""
|
|
370
|
+
s = re.sub(r"(?<=[a-z0-9])([A-Z])", r"_\1", label)
|
|
371
|
+
s = re.sub(r"(?<=[A-Z])([A-Z][a-z])", r"_\1", s)
|
|
372
|
+
return s.lower()
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
__all__ = ["generate_instructions", "DEFAULT_CHAR_BUDGET"]
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
"""Generate sample NL→SQL query pairs for a Genie Space.
|
|
2
|
+
|
|
3
|
+
Genie consumes sample queries as few-shot examples in its reasoning.
|
|
4
|
+
Good samples teach Genie:
|
|
5
|
+
|
|
6
|
+
- Which generated view answers each kind of question (so it uses
|
|
7
|
+
``prograph_views.high_wear_components`` instead of rolling its own
|
|
8
|
+
filter over the base table).
|
|
9
|
+
- Which patterns are graph-shaped vs. plain SQL.
|
|
10
|
+
- Which joins follow naturally from the schema relationships.
|
|
11
|
+
|
|
12
|
+
This generator produces 5–30 :class:`SampleQuery` instances per
|
|
13
|
+
schema, deterministically. Output is added to the Genie Space config
|
|
14
|
+
by the emitter (W5).
|
|
15
|
+
|
|
16
|
+
Generation strategy: template per (source kind, structural pattern).
|
|
17
|
+
Each template carries an NL question template and a SQL skeleton
|
|
18
|
+
referenced by structural elements (label, view name, property,
|
|
19
|
+
relationship). Templates are kept conservative, questions that
|
|
20
|
+
sound natural for any schema in the relevant pattern, queries that
|
|
21
|
+
exercise the generated views and use clear, readable SQL.
|
|
22
|
+
|
|
23
|
+
Token-friendly: every sample stays under ~200 tokens of SQL. The
|
|
24
|
+
emitter caps the total count if the Space's prompt budget is tight.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import re
|
|
30
|
+
from typing import TYPE_CHECKING, List, Optional
|
|
31
|
+
|
|
32
|
+
from prograph.genie.formats import SampleQuery, ViewSource
|
|
33
|
+
|
|
34
|
+
if TYPE_CHECKING:
|
|
35
|
+
from prograph.genie.formats import ViewDefinition
|
|
36
|
+
from prograph.ontology.layer import OntologyLayer
|
|
37
|
+
from prograph.schema import GraphSchema
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
DEFAULT_MAX_QUERIES = 25
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def generate_sample_queries(
|
|
44
|
+
schema: "GraphSchema",
|
|
45
|
+
ontology: Optional["OntologyLayer"] = None,
|
|
46
|
+
views: Optional[List["ViewDefinition"]] = None,
|
|
47
|
+
max_queries: int = DEFAULT_MAX_QUERIES,
|
|
48
|
+
view_namespace: str = "prograph_views",
|
|
49
|
+
) -> List[SampleQuery]:
|
|
50
|
+
"""Build NL→SQL samples Genie can use as few-shot examples.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
schema: The :class:`GraphSchema`.
|
|
54
|
+
ontology: Optional :class:`OntologyLayer` (used for defined-
|
|
55
|
+
class samples).
|
|
56
|
+
views: Optional list of generated views. When provided, samples
|
|
57
|
+
preferentially reference them so Genie learns to use them.
|
|
58
|
+
max_queries: Cap on the total number of samples. Patterns are
|
|
59
|
+
generated in priority order (most-useful first); excess
|
|
60
|
+
samples are dropped. Default 25.
|
|
61
|
+
view_namespace: Schema/catalog prefix for view references in
|
|
62
|
+
the generated SQL. Matches the Space's
|
|
63
|
+
``target_schema``.
|
|
64
|
+
|
|
65
|
+
Returns:
|
|
66
|
+
List of :class:`SampleQuery`. Order is stable across runs.
|
|
67
|
+
"""
|
|
68
|
+
if max_queries <= 0:
|
|
69
|
+
return []
|
|
70
|
+
|
|
71
|
+
samples: List[SampleQuery] = []
|
|
72
|
+
views = views or []
|
|
73
|
+
views_by_source: dict = {}
|
|
74
|
+
for v in views:
|
|
75
|
+
views_by_source.setdefault(v.source, []).append(v)
|
|
76
|
+
|
|
77
|
+
# Look up source tables by node label so we can fall back when no
|
|
78
|
+
# view is registered for an endpoint.
|
|
79
|
+
table_by_label = {m.label: m.source_table for m in schema.node_mappings if m.label is not None}
|
|
80
|
+
|
|
81
|
+
namespace_quoted = _quote_qualified(view_namespace)
|
|
82
|
+
|
|
83
|
+
# ── 1. One "count" sample per entity label ──
|
|
84
|
+
for mapping in schema.node_mappings:
|
|
85
|
+
if mapping.label is None:
|
|
86
|
+
continue # an unlabelled node mapping: no label to name
|
|
87
|
+
view = _view_for_label(views_by_source, mapping.label, ViewSource.SCHEMA_LABEL)
|
|
88
|
+
table = (
|
|
89
|
+
f"{namespace_quoted}.`{view.name}`"
|
|
90
|
+
if view is not None
|
|
91
|
+
else _quote_qualified(mapping.source_table)
|
|
92
|
+
)
|
|
93
|
+
samples.append(SampleQuery(
|
|
94
|
+
question=f"How many {_human(mapping.label)}s are there?",
|
|
95
|
+
sql=f"SELECT COUNT(*) AS total FROM {table}",
|
|
96
|
+
description=f"Count of {mapping.label} entities.",
|
|
97
|
+
tags=["count", mapping.label.lower()],
|
|
98
|
+
))
|
|
99
|
+
if len(samples) >= max_queries:
|
|
100
|
+
return samples
|
|
101
|
+
|
|
102
|
+
# ── 2. One "list" sample per entity with at least one property ──
|
|
103
|
+
for mapping in schema.node_mappings:
|
|
104
|
+
if not mapping.property_columns or mapping.label is None:
|
|
105
|
+
continue
|
|
106
|
+
view = _view_for_label(views_by_source, mapping.label, ViewSource.SCHEMA_LABEL)
|
|
107
|
+
table = (
|
|
108
|
+
f"{namespace_quoted}.`{view.name}`"
|
|
109
|
+
if view is not None
|
|
110
|
+
else _quote_qualified(mapping.source_table)
|
|
111
|
+
)
|
|
112
|
+
first_prop = next(iter(mapping.property_columns))
|
|
113
|
+
samples.append(SampleQuery(
|
|
114
|
+
question=f"List the first 10 {_human(mapping.label)}s with their {first_prop}.",
|
|
115
|
+
sql=(
|
|
116
|
+
f"SELECT `id`, `{first_prop}` FROM {table} "
|
|
117
|
+
f"ORDER BY `{first_prop}` LIMIT 10"
|
|
118
|
+
),
|
|
119
|
+
description=f"Sample {mapping.label} rows projecting {first_prop}.",
|
|
120
|
+
tags=["list", "limit", mapping.label.lower()],
|
|
121
|
+
))
|
|
122
|
+
if len(samples) >= max_queries:
|
|
123
|
+
return samples
|
|
124
|
+
|
|
125
|
+
# ── 3. One per OWL-defined class (the high-leverage ones) ──
|
|
126
|
+
for v in views_by_source.get(ViewSource.OWL_DEFINED_CLASS, []):
|
|
127
|
+
samples.append(SampleQuery(
|
|
128
|
+
question=_question_for_defined_class(v),
|
|
129
|
+
sql=f"SELECT `id` FROM {namespace_quoted}.`{v.name}` LIMIT 50",
|
|
130
|
+
description=v.description,
|
|
131
|
+
tags=["defined-class", _to_snake(v.source_label or v.name)],
|
|
132
|
+
))
|
|
133
|
+
if len(samples) >= max_queries:
|
|
134
|
+
return samples
|
|
135
|
+
|
|
136
|
+
# ── 4. One per subclass-closure view ──
|
|
137
|
+
for v in views_by_source.get(ViewSource.SUBCLASS_CLOSURE, []):
|
|
138
|
+
parent = v.source_label or v.name
|
|
139
|
+
samples.append(SampleQuery(
|
|
140
|
+
question=f"Show me all {_human(parent)}s across every subclass.",
|
|
141
|
+
sql=(
|
|
142
|
+
f"SELECT `id`, `__subclass_label` "
|
|
143
|
+
f"FROM {namespace_quoted}.`{v.name}` "
|
|
144
|
+
f"ORDER BY `__subclass_label`, `id` LIMIT 50"
|
|
145
|
+
),
|
|
146
|
+
description=v.description,
|
|
147
|
+
tags=["closure", _to_snake(parent)],
|
|
148
|
+
))
|
|
149
|
+
if len(samples) >= max_queries:
|
|
150
|
+
return samples
|
|
151
|
+
|
|
152
|
+
# ── 5. One "traversal" sample per relationship (with views if available) ──
|
|
153
|
+
for rel in schema.relationship_mappings:
|
|
154
|
+
if rel.start_node_label is None or rel.end_node_label is None:
|
|
155
|
+
continue # an end given by an identity: no one label to name
|
|
156
|
+
start_view = _view_for_label(
|
|
157
|
+
views_by_source, rel.start_node_label, ViewSource.SCHEMA_LABEL
|
|
158
|
+
)
|
|
159
|
+
end_view = _view_for_label(
|
|
160
|
+
views_by_source, rel.end_node_label, ViewSource.SCHEMA_LABEL
|
|
161
|
+
)
|
|
162
|
+
start_table = (
|
|
163
|
+
f"{namespace_quoted}.`{start_view.name}`"
|
|
164
|
+
if start_view is not None
|
|
165
|
+
else _quote_qualified(table_by_label.get(rel.start_node_label, ""))
|
|
166
|
+
)
|
|
167
|
+
end_table = (
|
|
168
|
+
f"{namespace_quoted}.`{end_view.name}`"
|
|
169
|
+
if end_view is not None
|
|
170
|
+
else _quote_qualified(table_by_label.get(rel.end_node_label, ""))
|
|
171
|
+
)
|
|
172
|
+
rel_table = _quote_qualified(rel.source_table)
|
|
173
|
+
|
|
174
|
+
start_id = (
|
|
175
|
+
rel.start_id_columns[0] if len(rel.start_id_columns) == 1 else "src"
|
|
176
|
+
)
|
|
177
|
+
end_id = rel.end_id_columns[0] if len(rel.end_id_columns) == 1 else "dst"
|
|
178
|
+
|
|
179
|
+
samples.append(SampleQuery(
|
|
180
|
+
question=(
|
|
181
|
+
f"Which {_human(rel.end_node_label)}s are connected to "
|
|
182
|
+
f"a {_human(rel.start_node_label)} via {rel.rel_type}?"
|
|
183
|
+
),
|
|
184
|
+
sql=(
|
|
185
|
+
f"SELECT DISTINCT e.`id` "
|
|
186
|
+
f"FROM {rel_table} r "
|
|
187
|
+
f"JOIN {end_table} e ON e.`id` = r.`{end_id}` LIMIT 50"
|
|
188
|
+
),
|
|
189
|
+
description=(
|
|
190
|
+
f"Traversal pattern over the {rel.rel_type} relationship."
|
|
191
|
+
),
|
|
192
|
+
tags=["traversal", rel.rel_type.lower()],
|
|
193
|
+
))
|
|
194
|
+
if len(samples) >= max_queries:
|
|
195
|
+
return samples
|
|
196
|
+
|
|
197
|
+
return samples
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
# ─── Helpers ─────────────────────────────────────────────────────────
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _view_for_label(
|
|
204
|
+
views_by_source: dict, label: str, source: ViewSource
|
|
205
|
+
) -> Optional["ViewDefinition"]:
|
|
206
|
+
for v in views_by_source.get(source, []):
|
|
207
|
+
if v.source_label == label:
|
|
208
|
+
return v
|
|
209
|
+
return None
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _quote_qualified(name: str) -> str:
|
|
213
|
+
"""A possibly-dotted name quoted part by part for Databricks SQL, the
|
|
214
|
+
dialect Genie runs (Dialect.quote_qualified)."""
|
|
215
|
+
if not name:
|
|
216
|
+
return ""
|
|
217
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
218
|
+
return get_dialect_instance("databricks_sql").quote_qualified(name)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _to_snake(text: str) -> str:
|
|
222
|
+
"""PascalCase / camelCase → snake_case."""
|
|
223
|
+
s = re.sub(r"(?<=[a-z0-9])([A-Z])", r"_\1", text)
|
|
224
|
+
s = re.sub(r"(?<=[A-Z])([A-Z][a-z])", r"_\1", s)
|
|
225
|
+
return s.lower()
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _human(label: str) -> str:
|
|
229
|
+
"""Convert a PascalCase label to a lowercase natural phrase.
|
|
230
|
+
|
|
231
|
+
``HighWearComponent`` → ``high wear component``.
|
|
232
|
+
``Sensor`` → ``sensor``.
|
|
233
|
+
"""
|
|
234
|
+
return _to_snake(label).replace("_", " ")
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _question_for_defined_class(view: "ViewDefinition") -> str:
|
|
238
|
+
"""Phrase a question that naturally maps to this defined class.
|
|
239
|
+
|
|
240
|
+
The view's ``description`` already explains what the class means;
|
|
241
|
+
just turn it into a question the user might actually ask.
|
|
242
|
+
"""
|
|
243
|
+
label = view.source_label or view.name
|
|
244
|
+
human = _human(label)
|
|
245
|
+
# Hand-craft phrasing for the three known restriction patterns.
|
|
246
|
+
desc = view.description.lower()
|
|
247
|
+
if "wearindex" in desc or "data range" in desc:
|
|
248
|
+
return f"Which {human.replace(' high wear', '')}s are at high wear risk?"
|
|
249
|
+
if "outgoing" in desc and "edge" in desc:
|
|
250
|
+
return f"Which entities qualify as {human}?"
|
|
251
|
+
if "incoming" in desc and "edge" in desc:
|
|
252
|
+
return f"Which entities qualify as {human}?"
|
|
253
|
+
return f"List all {human}s."
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
__all__ = ["generate_sample_queries", "DEFAULT_MAX_QUERIES"]
|