prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
"""Upload a ProGraph ``GenieSpace`` to Databricks via the Genie API.
|
|
2
|
+
|
|
3
|
+
The :class:`GenieSpace` dataclass produced by
|
|
4
|
+
:mod:`prograph.genie.space_emitter` carries the same information as
|
|
5
|
+
Databricks' native Genie Space JSON. This module is the bridge:
|
|
6
|
+
|
|
7
|
+
- :func:`to_databricks_serialized_space` converts a ``GenieSpace`` to
|
|
8
|
+
the JSON shape Databricks' ``WorkspaceClient.genie.create_space()``
|
|
9
|
+
accepts as the ``serialized_space`` argument.
|
|
10
|
+
- :func:`upload_space` calls the Databricks SDK and creates / updates
|
|
11
|
+
a real Space in your workspace.
|
|
12
|
+
|
|
13
|
+
Databricks Genie's ``serialized_space`` schema (v2), inferred from the
|
|
14
|
+
``genie.get_space(..., include_serialized_space=True)`` output of a
|
|
15
|
+
known-good Space:
|
|
16
|
+
|
|
17
|
+
.. code-block:: json
|
|
18
|
+
|
|
19
|
+
{
|
|
20
|
+
"version": 2,
|
|
21
|
+
"config": {
|
|
22
|
+
"sample_questions": [
|
|
23
|
+
{"id": "<32-hex-id>", "question": ["...?"]}
|
|
24
|
+
]
|
|
25
|
+
},
|
|
26
|
+
"data_sources": {
|
|
27
|
+
"tables": [{"identifier": "catalog.schema.table"}]
|
|
28
|
+
},
|
|
29
|
+
"instructions": {
|
|
30
|
+
"text_instructions": [
|
|
31
|
+
{"id": "<id>", "content": ["line 1\\n", "line 2\\n", ...]}
|
|
32
|
+
],
|
|
33
|
+
"example_question_sqls": [
|
|
34
|
+
{
|
|
35
|
+
"id": "<id>",
|
|
36
|
+
"question": ["..."],
|
|
37
|
+
"answer": [
|
|
38
|
+
{"format": "SQL", "content": ["SQL line 1\\n", ...]}
|
|
39
|
+
]
|
|
40
|
+
}
|
|
41
|
+
]
|
|
42
|
+
},
|
|
43
|
+
"benchmarks": {
|
|
44
|
+
"questions": [...]
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
Notes on the format:
|
|
49
|
+
|
|
50
|
+
- ``content`` and ``question`` are arrays of strings (one per line).
|
|
51
|
+
Databricks renders them concatenated; we always emit a single
|
|
52
|
+
element for simplicity.
|
|
53
|
+
- IDs are 32-char hex (UUID without dashes). The SDK accepts any
|
|
54
|
+
unique string, but Databricks-generated IDs follow this convention.
|
|
55
|
+
- The ``benchmarks`` key carries SQL exemplars Databricks uses for
|
|
56
|
+
internal evaluation. We omit it; ProGraph's sample queries already
|
|
57
|
+
serve the few-shot role.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
from __future__ import annotations
|
|
61
|
+
|
|
62
|
+
import json
|
|
63
|
+
import uuid
|
|
64
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
65
|
+
|
|
66
|
+
if TYPE_CHECKING:
|
|
67
|
+
from prograph.genie.formats import GenieSpace
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
SERIALIZED_SPACE_VERSION = 2
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def to_databricks_serialized_space(space: "GenieSpace") -> str:
|
|
74
|
+
"""Convert a ProGraph :class:`GenieSpace` to Databricks' v2 JSON.
|
|
75
|
+
|
|
76
|
+
Returns a JSON string ready to pass as the ``serialized_space``
|
|
77
|
+
argument to ``WorkspaceClient.genie.create_space()``.
|
|
78
|
+
|
|
79
|
+
Fields:
|
|
80
|
+
|
|
81
|
+
- ``data_sources.tables`` ← ``GenieSpace.tables`` (one entry per
|
|
82
|
+
fully-qualified table name).
|
|
83
|
+
- ``instructions.text_instructions`` ← the rendered instructions
|
|
84
|
+
body, split into lines for Databricks' line-array shape.
|
|
85
|
+
- ``config.sample_questions`` ← the *question text* from
|
|
86
|
+
``GenieSpace.sample_queries`` (Databricks uses these as
|
|
87
|
+
"explore" prompts in the UI).
|
|
88
|
+
- ``instructions.example_question_sqls`` ← the *full question +
|
|
89
|
+
SQL pairs* from ``GenieSpace.sample_queries`` (these are the
|
|
90
|
+
few-shot examples Genie's reasoning model sees).
|
|
91
|
+
|
|
92
|
+
``GenieSpace.mcp_servers`` is not used here. Databricks' Genie
|
|
93
|
+
Space format has no slot for attaching external MCP servers, and
|
|
94
|
+
per the managed-MCP documentation Genie Space is positioned as an
|
|
95
|
+
MCP *server* not *client*, the convergence path for outbound-MCP
|
|
96
|
+
is the Agent
|
|
97
|
+
Bricks Supervisor Agent, not the Space itself. If the field is
|
|
98
|
+
set when this function runs, the caller is asking for something
|
|
99
|
+
Databricks doesn't support, and :func:`upload_space` raises.
|
|
100
|
+
"""
|
|
101
|
+
# Databricks' import API requires id-sorted lists in
|
|
102
|
+
# config.sample_questions, instructions.example_question_sqls, and
|
|
103
|
+
# instructions.text_instructions. Generate IDs once, then sort.
|
|
104
|
+
sample_question_entries = sorted(
|
|
105
|
+
(
|
|
106
|
+
{"id": _new_id(), "question": [q.question]}
|
|
107
|
+
for q in space.sample_queries
|
|
108
|
+
),
|
|
109
|
+
key=lambda e: e["id"],
|
|
110
|
+
)
|
|
111
|
+
example_sql_entries = sorted(
|
|
112
|
+
(
|
|
113
|
+
{
|
|
114
|
+
"id": _new_id(),
|
|
115
|
+
"question": [q.question],
|
|
116
|
+
"sql": _split_lines_for_databricks(q.sql),
|
|
117
|
+
}
|
|
118
|
+
for q in space.sample_queries
|
|
119
|
+
),
|
|
120
|
+
key=lambda e: e["id"],
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
payload: dict[str, Any] = {
|
|
124
|
+
"version": SERIALIZED_SPACE_VERSION,
|
|
125
|
+
"config": {
|
|
126
|
+
"sample_questions": sample_question_entries,
|
|
127
|
+
},
|
|
128
|
+
"data_sources": {
|
|
129
|
+
# Genie's API requires tables to be sorted by identifier.
|
|
130
|
+
"tables": [
|
|
131
|
+
{"identifier": t} for t in sorted(set(space.tables))
|
|
132
|
+
],
|
|
133
|
+
},
|
|
134
|
+
"instructions": {
|
|
135
|
+
"text_instructions": _build_text_instructions(space),
|
|
136
|
+
"example_question_sqls": example_sql_entries,
|
|
137
|
+
},
|
|
138
|
+
}
|
|
139
|
+
return json.dumps(payload, indent=2)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def upload_space(
|
|
143
|
+
space: "GenieSpace",
|
|
144
|
+
warehouse_id: str,
|
|
145
|
+
*,
|
|
146
|
+
workspace_client: Optional[Any] = None,
|
|
147
|
+
host: Optional[str] = None,
|
|
148
|
+
token: Optional[str] = None,
|
|
149
|
+
parent_path: Optional[str] = None,
|
|
150
|
+
update_if_exists: bool = False,
|
|
151
|
+
existing_space_id: Optional[str] = None,
|
|
152
|
+
) -> dict[str, Any]:
|
|
153
|
+
"""Create (or update) a Genie Space on Databricks from a ProGraph ``GenieSpace``.
|
|
154
|
+
|
|
155
|
+
Args:
|
|
156
|
+
space: The :class:`GenieSpace` to upload.
|
|
157
|
+
warehouse_id: The SQL Warehouse Genie should run queries
|
|
158
|
+
against (e.g. ``"06bb2559f22f28f4"``).
|
|
159
|
+
workspace_client: Optional pre-constructed
|
|
160
|
+
``databricks.sdk.WorkspaceClient``. When ``None``, one is
|
|
161
|
+
built from ``host`` + ``token``, falling back to the SDK's
|
|
162
|
+
environment-based auth (``DATABRICKS_HOST`` /
|
|
163
|
+
``DATABRICKS_TOKEN``).
|
|
164
|
+
host / token: Explicit credentials for constructing a fresh
|
|
165
|
+
``WorkspaceClient``. Ignored if ``workspace_client`` is
|
|
166
|
+
provided.
|
|
167
|
+
parent_path: Workspace path under which to create the Space
|
|
168
|
+
(e.g. ``"/Users/me@company.com"``). ``None`` lets
|
|
169
|
+
Databricks pick the default.
|
|
170
|
+
update_if_exists: When ``True`` and ``existing_space_id`` is
|
|
171
|
+
also provided, call ``update_space`` instead of
|
|
172
|
+
``create_space``. Useful for re-running the same
|
|
173
|
+
generation pipeline and overwriting an old Space in
|
|
174
|
+
place.
|
|
175
|
+
existing_space_id: Space ID to update (only meaningful with
|
|
176
|
+
``update_if_exists=True``).
|
|
177
|
+
|
|
178
|
+
Returns:
|
|
179
|
+
Dict with the created/updated Space's id, title, warehouse_id,
|
|
180
|
+
and the serialized payload size in bytes.
|
|
181
|
+
|
|
182
|
+
Raises:
|
|
183
|
+
ValueError: If ``space.mcp_servers`` is non-empty. Databricks
|
|
184
|
+
Genie Space has no inbound-MCP attachment path (per the
|
|
185
|
+
managed-MCP documentation),
|
|
186
|
+
external-MCP orchestration goes through the Agent Bricks
|
|
187
|
+
Supervisor Agent, not the Space itself.
|
|
188
|
+
"""
|
|
189
|
+
if space.mcp_servers:
|
|
190
|
+
raise ValueError(
|
|
191
|
+
"GenieSpace.mcp_servers is set but Databricks Genie Space "
|
|
192
|
+
"does not support attaching external MCP servers (neither "
|
|
193
|
+
"via API nor UI). For the outbound-MCP pattern, build an "
|
|
194
|
+
"Agent Bricks Supervisor Agent that orchestrates the Genie "
|
|
195
|
+
"Space + your MCP servers. See docs/guides/genie.md."
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
w = workspace_client or _build_workspace_client(host=host, token=token)
|
|
199
|
+
|
|
200
|
+
serialized = to_databricks_serialized_space(space)
|
|
201
|
+
|
|
202
|
+
if update_if_exists and existing_space_id:
|
|
203
|
+
result = w.genie.update_space(
|
|
204
|
+
space_id=existing_space_id,
|
|
205
|
+
title=space.name,
|
|
206
|
+
description=space.description or None,
|
|
207
|
+
serialized_space=serialized,
|
|
208
|
+
warehouse_id=warehouse_id,
|
|
209
|
+
)
|
|
210
|
+
action = "updated"
|
|
211
|
+
else:
|
|
212
|
+
result = w.genie.create_space(
|
|
213
|
+
warehouse_id=warehouse_id,
|
|
214
|
+
serialized_space=serialized,
|
|
215
|
+
title=space.name,
|
|
216
|
+
description=space.description or None,
|
|
217
|
+
parent_path=parent_path,
|
|
218
|
+
)
|
|
219
|
+
action = "created"
|
|
220
|
+
|
|
221
|
+
return {
|
|
222
|
+
"action": action,
|
|
223
|
+
"space_id": result.space_id,
|
|
224
|
+
"title": result.title,
|
|
225
|
+
"warehouse_id": result.warehouse_id,
|
|
226
|
+
"parent_path": getattr(result, "parent_path", None),
|
|
227
|
+
"serialized_bytes": len(serialized),
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
# ─── Helpers ─────────────────────────────────────────────────────────
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _new_id() -> str:
|
|
235
|
+
"""Generate a 32-char hex id in the style Databricks uses."""
|
|
236
|
+
return uuid.uuid4().hex
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _split_lines_for_databricks(text: str) -> list[str]:
|
|
240
|
+
"""Split a multi-line string into Databricks' "line array" form.
|
|
241
|
+
|
|
242
|
+
Databricks stores SQL bodies and instruction text as arrays of
|
|
243
|
+
lines, each line keeping its trailing ``\\n`` (except possibly the
|
|
244
|
+
last). Mirror that shape so the strings render correctly in the
|
|
245
|
+
Genie UI.
|
|
246
|
+
"""
|
|
247
|
+
if not text:
|
|
248
|
+
return [""]
|
|
249
|
+
lines = text.split("\n")
|
|
250
|
+
# Re-attach a trailing \n to every line except the last; if the
|
|
251
|
+
# original ended with a newline, the last line will be empty and
|
|
252
|
+
# we just drop it (Databricks doesn't include the trailing empty).
|
|
253
|
+
out: list[str] = []
|
|
254
|
+
for i, line in enumerate(lines):
|
|
255
|
+
if i < len(lines) - 1:
|
|
256
|
+
out.append(line + "\n")
|
|
257
|
+
else:
|
|
258
|
+
if line: # don't emit a trailing empty
|
|
259
|
+
out.append(line)
|
|
260
|
+
return out or [""]
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _build_text_instructions(space: "GenieSpace") -> list[dict[str, Any]]:
|
|
264
|
+
"""Build the ``text_instructions`` list from the rendered body.
|
|
265
|
+
|
|
266
|
+
Today: one big entry containing the full rendered markdown.
|
|
267
|
+
Databricks renders it as a single instruction block.
|
|
268
|
+
|
|
269
|
+
Future option: split per-section so users can selectively edit /
|
|
270
|
+
enable individual sections in the Genie UI. Skipped for now,
|
|
271
|
+
the single-block approach roundtrips cleanly.
|
|
272
|
+
"""
|
|
273
|
+
if not space.instructions.rendered:
|
|
274
|
+
return []
|
|
275
|
+
return [
|
|
276
|
+
{
|
|
277
|
+
"id": _new_id(),
|
|
278
|
+
"content": _split_lines_for_databricks(space.instructions.rendered),
|
|
279
|
+
}
|
|
280
|
+
]
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _build_workspace_client(
|
|
284
|
+
*, host: Optional[str], token: Optional[str]
|
|
285
|
+
) -> Any:
|
|
286
|
+
"""Build a :class:`WorkspaceClient` from explicit creds or env."""
|
|
287
|
+
try:
|
|
288
|
+
from databricks.sdk import WorkspaceClient
|
|
289
|
+
except ImportError as e: # pragma: no cover
|
|
290
|
+
raise ImportError(
|
|
291
|
+
"databricks-sdk is required for upload_space. "
|
|
292
|
+
"Install with: pip install databricks-sdk"
|
|
293
|
+
) from e
|
|
294
|
+
|
|
295
|
+
if host and token:
|
|
296
|
+
return WorkspaceClient(host=host, token=token)
|
|
297
|
+
# Falls back to env vars (DATABRICKS_HOST, DATABRICKS_TOKEN) or
|
|
298
|
+
# ~/.databrickscfg.
|
|
299
|
+
return WorkspaceClient()
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
__all__ = [
|
|
303
|
+
"to_databricks_serialized_space",
|
|
304
|
+
"upload_space",
|
|
305
|
+
"SERIALIZED_SPACE_VERSION",
|
|
306
|
+
]
|
prograph/genie/deploy.py
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
"""Library API for deploying a Genie Space to Databricks.
|
|
2
|
+
|
|
3
|
+
Single entry point :func:`deploy_space` that creates (or updates) a
|
|
4
|
+
Genie Space on Databricks from a ProGraph schema. Designed for use
|
|
5
|
+
from any Python context: scripts, notebooks (Databricks workspace or
|
|
6
|
+
local Jupyter), Databricks Apps, Lakeflow jobs.
|
|
7
|
+
|
|
8
|
+
The CLI command ``prograph upload-genie-space`` is a thin wrapper
|
|
9
|
+
around this function. It adds argparse + print statements and that's
|
|
10
|
+
all. Anything callable from the CLI is callable from Python with the
|
|
11
|
+
same arguments.
|
|
12
|
+
|
|
13
|
+
Three input modes for the schema source, mix-and-match as convenient:
|
|
14
|
+
|
|
15
|
+
1. **File paths**, ``config_path=...``, ``ontology_path=...``.
|
|
16
|
+
Same as the CLI; works in notebooks that load YAML from a
|
|
17
|
+
workspace path or DBFS.
|
|
18
|
+
|
|
19
|
+
2. **Raw strings**, ``config_yaml=...``, ``ontology_ttl=...``.
|
|
20
|
+
Pass YAML / Turtle as a string when you've loaded it some other
|
|
21
|
+
way (e.g. from a Databricks Volume via ``dbutils.fs.head``, from a
|
|
22
|
+
web fetch, from a templating system).
|
|
23
|
+
|
|
24
|
+
3. **In-memory objects**, ``schema=...``, ``ontology=...``.
|
|
25
|
+
Pass a constructed :class:`GraphSchema` and (optional)
|
|
26
|
+
:class:`OntologyLayer` directly. Useful when the schema is built
|
|
27
|
+
programmatically.
|
|
28
|
+
|
|
29
|
+
Auth follows the same precedence:
|
|
30
|
+
|
|
31
|
+
- ``workspace_client=``, already-constructed
|
|
32
|
+
:class:`databricks.sdk.WorkspaceClient`. Notebooks running on
|
|
33
|
+
Databricks use this with no arguments: the runtime auto-authenticates.
|
|
34
|
+
- ``host=`` + ``token=``, explicit credentials when not on Databricks.
|
|
35
|
+
- (fallback) Environment variables: ``DATABRICKS_HOST`` /
|
|
36
|
+
``DATABRICKS_TOKEN``, or ``~/.databrickscfg``.
|
|
37
|
+
|
|
38
|
+
Mixing ``workspace_client`` with ``host``/``token`` raises, the
|
|
39
|
+
intent is ambiguous and silent precedence is harder to debug than a
|
|
40
|
+
clear error.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
from __future__ import annotations
|
|
44
|
+
|
|
45
|
+
from pathlib import Path
|
|
46
|
+
from typing import TYPE_CHECKING, Any, List, Optional, Union
|
|
47
|
+
|
|
48
|
+
if TYPE_CHECKING:
|
|
49
|
+
from prograph.genie.formats import GenieSpace
|
|
50
|
+
from prograph.ontology.layer import OntologyLayer
|
|
51
|
+
from prograph.plan.sql.dialect import DialectConfig
|
|
52
|
+
from prograph.schema import GraphSchema
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def deploy_space(
|
|
56
|
+
*,
|
|
57
|
+
# ── Schema source (exactly one mode) ──
|
|
58
|
+
config_path: Optional[Union[str, Path]] = None,
|
|
59
|
+
config_yaml: Optional[str] = None,
|
|
60
|
+
schema: Optional["GraphSchema"] = None,
|
|
61
|
+
|
|
62
|
+
# ── Ontology source (optional, at most one mode) ──
|
|
63
|
+
ontology_path: Optional[Union[str, Path]] = None,
|
|
64
|
+
ontology_ttl: Optional[str] = None,
|
|
65
|
+
ontology: Optional["OntologyLayer"] = None,
|
|
66
|
+
|
|
67
|
+
# ── Target (required) ──
|
|
68
|
+
warehouse_id: str,
|
|
69
|
+
|
|
70
|
+
# ── View placement ──
|
|
71
|
+
target_catalog: Optional[str] = None,
|
|
72
|
+
target_schema: str = "prograph_views",
|
|
73
|
+
dialect: Optional["DialectConfig"] = None,
|
|
74
|
+
dialect_name: Optional[str] = None,
|
|
75
|
+
include_labels: Optional[List[str]] = None,
|
|
76
|
+
exclude_labels: Optional[List[str]] = None,
|
|
77
|
+
|
|
78
|
+
# ── Space metadata ──
|
|
79
|
+
space_name: Optional[str] = None,
|
|
80
|
+
space_description: str = "",
|
|
81
|
+
summary_instructions: bool = False,
|
|
82
|
+
max_sample_queries: int = 25,
|
|
83
|
+
|
|
84
|
+
# ── Apply generated views to the warehouse in one shot ──
|
|
85
|
+
apply_views: bool = False,
|
|
86
|
+
|
|
87
|
+
# ── Databricks auth ──
|
|
88
|
+
workspace_client: Optional[Any] = None,
|
|
89
|
+
host: Optional[str] = None,
|
|
90
|
+
token: Optional[str] = None,
|
|
91
|
+
|
|
92
|
+
# ── Create-vs-update ──
|
|
93
|
+
parent_path: Optional[str] = None,
|
|
94
|
+
update_if_exists: bool = False,
|
|
95
|
+
existing_space_id: Optional[str] = None,
|
|
96
|
+
|
|
97
|
+
# ── Source provenance (lands in space.metadata) ──
|
|
98
|
+
source_label: Optional[str] = None,
|
|
99
|
+
) -> dict:
|
|
100
|
+
"""Deploy a Genie Space to Databricks from any input mode.
|
|
101
|
+
|
|
102
|
+
See module docstring for the three input modes (paths / strings /
|
|
103
|
+
objects) and the auth options. Returns the same result dict the
|
|
104
|
+
underlying :func:`prograph.genie.upload_space` produces, plus a
|
|
105
|
+
few aggregate counts surfaced for caller convenience.
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
warehouse_id: Databricks SQL Warehouse ID for the Space.
|
|
109
|
+
config_path / config_yaml / schema: Exactly one. The schema source.
|
|
110
|
+
ontology_path / ontology_ttl / ontology: At most one (optional).
|
|
111
|
+
target_catalog / target_schema: Where generated views live in
|
|
112
|
+
the warehouse. Same fields as the CLI.
|
|
113
|
+
dialect / dialect_name: SQL dialect. ``dialect`` accepts a
|
|
114
|
+
:class:`DialectConfig` directly; ``dialect_name`` accepts
|
|
115
|
+
the string used by the CLI (``"databricks_sql"`` etc.).
|
|
116
|
+
include_labels / exclude_labels: NodeMapping label filters.
|
|
117
|
+
space_name / space_description / summary_instructions /
|
|
118
|
+
max_sample_queries: Pass through to the
|
|
119
|
+
:class:`GenieSpace` assembly.
|
|
120
|
+
apply_views: When ``True``, opens a databricks-sql-connector
|
|
121
|
+
session against ``warehouse_id`` and runs
|
|
122
|
+
``CREATE OR REPLACE VIEW`` for every generated view before
|
|
123
|
+
the Space is created. Closes the deploy → views → query
|
|
124
|
+
loop in one call. Requires ``databricks-sql-connector`` to
|
|
125
|
+
be installed; auth re-uses ``host``/``token`` (or env
|
|
126
|
+
vars). Defaults to ``False`` for backwards compatibility.
|
|
127
|
+
workspace_client: Pre-built :class:`WorkspaceClient`. When
|
|
128
|
+
running in a Databricks notebook, build it with no args
|
|
129
|
+
(the runtime auto-authenticates) and pass it here.
|
|
130
|
+
host / token: Explicit credentials for non-Databricks
|
|
131
|
+
environments. Mutually exclusive with ``workspace_client``.
|
|
132
|
+
parent_path: Workspace path under which to create the Space.
|
|
133
|
+
update_if_exists / existing_space_id: Update an existing
|
|
134
|
+
Space in place instead of creating a new one.
|
|
135
|
+
source_label: Optional string to record in
|
|
136
|
+
``space.metadata["source_config"]`` for provenance. The
|
|
137
|
+
CLI sets it to the absolute config-file path; callers
|
|
138
|
+
using in-memory inputs can pass anything meaningful.
|
|
139
|
+
|
|
140
|
+
Returns:
|
|
141
|
+
Result dict from :func:`upload_space` with extra fields:
|
|
142
|
+
``table_count``, ``view_count``, ``sample_query_count``.
|
|
143
|
+
|
|
144
|
+
Raises:
|
|
145
|
+
ValueError: If more than one schema-source / ontology-source
|
|
146
|
+
mode is provided, if zero schema sources are provided,
|
|
147
|
+
if ``workspace_client`` is given alongside ``host`` /
|
|
148
|
+
``token``, or if ``update_if_exists`` is set without
|
|
149
|
+
``existing_space_id``.
|
|
150
|
+
"""
|
|
151
|
+
_reject_auth_conflict(workspace_client, host, token)
|
|
152
|
+
_reject_update_without_id(update_if_exists, existing_space_id)
|
|
153
|
+
|
|
154
|
+
loaded_schema = _resolve_schema(config_path, config_yaml, schema)
|
|
155
|
+
loaded_ontology = _resolve_ontology(ontology_path, ontology_ttl, ontology)
|
|
156
|
+
resolved_dialect = _resolve_dialect(dialect, dialect_name)
|
|
157
|
+
|
|
158
|
+
from prograph.genie.space_emitter import emit_genie_space
|
|
159
|
+
from prograph.genie.databricks_uploader import upload_space
|
|
160
|
+
|
|
161
|
+
space = emit_genie_space(
|
|
162
|
+
schema=loaded_schema,
|
|
163
|
+
ontology=loaded_ontology,
|
|
164
|
+
name=space_name,
|
|
165
|
+
description=space_description,
|
|
166
|
+
target_catalog=target_catalog,
|
|
167
|
+
target_schema=target_schema,
|
|
168
|
+
dialect=resolved_dialect,
|
|
169
|
+
include_labels=include_labels,
|
|
170
|
+
exclude_labels=exclude_labels,
|
|
171
|
+
summary_instructions=summary_instructions,
|
|
172
|
+
max_sample_queries=max_sample_queries,
|
|
173
|
+
source_config=source_label,
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
applied_views: List[str] = []
|
|
177
|
+
if apply_views:
|
|
178
|
+
applied_views = _apply_views(
|
|
179
|
+
views=space.views,
|
|
180
|
+
warehouse_id=warehouse_id,
|
|
181
|
+
target_catalog=target_catalog,
|
|
182
|
+
target_schema=target_schema,
|
|
183
|
+
dialect=resolved_dialect,
|
|
184
|
+
host=host,
|
|
185
|
+
token=token,
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
result = upload_space(
|
|
189
|
+
space=space,
|
|
190
|
+
warehouse_id=warehouse_id,
|
|
191
|
+
workspace_client=workspace_client,
|
|
192
|
+
host=host,
|
|
193
|
+
token=token,
|
|
194
|
+
parent_path=parent_path,
|
|
195
|
+
update_if_exists=update_if_exists,
|
|
196
|
+
existing_space_id=existing_space_id,
|
|
197
|
+
)
|
|
198
|
+
result["table_count"] = len(space.tables)
|
|
199
|
+
result["view_count"] = len(space.views)
|
|
200
|
+
result["sample_query_count"] = len(space.sample_queries)
|
|
201
|
+
result["applied_views"] = applied_views
|
|
202
|
+
return result
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
# ─── Input resolution ────────────────────────────────────────────────
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _resolve_schema(
|
|
209
|
+
config_path: Optional[Union[str, Path]],
|
|
210
|
+
config_yaml: Optional[str],
|
|
211
|
+
schema: Optional["GraphSchema"],
|
|
212
|
+
) -> "GraphSchema":
|
|
213
|
+
"""Load a GraphSchema from exactly one of the three input modes.
|
|
214
|
+
|
|
215
|
+
Raises ``ValueError`` if zero or more than one source is provided.
|
|
216
|
+
For the path / string modes, resolves
|
|
217
|
+
``sources.<key>.table`` references the same way the CLI does, so
|
|
218
|
+
NodeMapping.source_table ends up as the fully-qualified table
|
|
219
|
+
name in the generated view SQL.
|
|
220
|
+
"""
|
|
221
|
+
provided = sum(
|
|
222
|
+
x is not None for x in (config_path, config_yaml, schema)
|
|
223
|
+
)
|
|
224
|
+
if provided == 0:
|
|
225
|
+
raise ValueError(
|
|
226
|
+
"deploy_space: provide exactly one schema source, "
|
|
227
|
+
"config_path, config_yaml, or schema"
|
|
228
|
+
)
|
|
229
|
+
if provided > 1:
|
|
230
|
+
raise ValueError(
|
|
231
|
+
"deploy_space: provide exactly one schema source, "
|
|
232
|
+
"got multiple of (config_path, config_yaml, schema)"
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
if schema is not None:
|
|
236
|
+
return schema
|
|
237
|
+
|
|
238
|
+
from prograph.config import load_config, load_config_from_string, validate_config
|
|
239
|
+
|
|
240
|
+
if config_path is not None:
|
|
241
|
+
config = load_config(Path(config_path))
|
|
242
|
+
else:
|
|
243
|
+
config = load_config_from_string(config_yaml)
|
|
244
|
+
validate_config(config) # surface warnings on stderr left to caller
|
|
245
|
+
|
|
246
|
+
return config.to_schema(by_table=True)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _resolve_ontology(
|
|
250
|
+
ontology_path: Optional[Union[str, Path]],
|
|
251
|
+
ontology_ttl: Optional[str],
|
|
252
|
+
ontology: Optional["OntologyLayer"],
|
|
253
|
+
) -> Optional["OntologyLayer"]:
|
|
254
|
+
"""Load an OntologyLayer from at most one of the three modes."""
|
|
255
|
+
provided = sum(
|
|
256
|
+
x is not None for x in (ontology_path, ontology_ttl, ontology)
|
|
257
|
+
)
|
|
258
|
+
if provided == 0:
|
|
259
|
+
return None
|
|
260
|
+
if provided > 1:
|
|
261
|
+
raise ValueError(
|
|
262
|
+
"deploy_space: provide at most one ontology source, "
|
|
263
|
+
"got multiple of (ontology_path, ontology_ttl, ontology)"
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
if ontology is not None:
|
|
267
|
+
return ontology
|
|
268
|
+
|
|
269
|
+
from prograph.ontology import OntologyLayer
|
|
270
|
+
|
|
271
|
+
if ontology_path is not None:
|
|
272
|
+
return OntologyLayer.from_owl(str(ontology_path))
|
|
273
|
+
return OntologyLayer.from_turtle(ontology_ttl)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _resolve_dialect(
|
|
277
|
+
dialect: Optional["DialectConfig"],
|
|
278
|
+
dialect_name: Optional[str],
|
|
279
|
+
) -> Optional["DialectConfig"]:
|
|
280
|
+
"""At most one of ``dialect`` or ``dialect_name``. ``None`` lets the
|
|
281
|
+
view generator pick its default."""
|
|
282
|
+
if dialect is not None and dialect_name is not None:
|
|
283
|
+
raise ValueError(
|
|
284
|
+
"deploy_space: pass either dialect (a DialectConfig instance) "
|
|
285
|
+
"or dialect_name (a string), not both"
|
|
286
|
+
)
|
|
287
|
+
if dialect is not None:
|
|
288
|
+
return dialect
|
|
289
|
+
if dialect_name is None:
|
|
290
|
+
return None
|
|
291
|
+
|
|
292
|
+
from prograph.plan.sql import dialect as dialect_module
|
|
293
|
+
|
|
294
|
+
mapping = {
|
|
295
|
+
"databricks_sql": dialect_module.SPARK_DIALECT,
|
|
296
|
+
"spark": dialect_module.SPARK_DIALECT,
|
|
297
|
+
"postgresql": dialect_module.POSTGRESQL_DIALECT,
|
|
298
|
+
}
|
|
299
|
+
sqlserver = getattr(dialect_module, "SQLSERVER_DIALECT", None)
|
|
300
|
+
if sqlserver is not None:
|
|
301
|
+
mapping["sqlserver"] = sqlserver
|
|
302
|
+
if dialect_name not in mapping:
|
|
303
|
+
raise ValueError(
|
|
304
|
+
f"Unknown dialect_name {dialect_name!r}. "
|
|
305
|
+
f"Known: {', '.join(sorted(mapping))}"
|
|
306
|
+
)
|
|
307
|
+
return mapping[dialect_name]
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _reject_auth_conflict(
|
|
311
|
+
workspace_client: Optional[Any],
|
|
312
|
+
host: Optional[str],
|
|
313
|
+
token: Optional[str],
|
|
314
|
+
) -> None:
|
|
315
|
+
if workspace_client is not None and (host is not None or token is not None):
|
|
316
|
+
raise ValueError(
|
|
317
|
+
"deploy_space: pass either workspace_client (already-built "
|
|
318
|
+
"WorkspaceClient) or host+token (explicit credentials), not both. "
|
|
319
|
+
"When running inside a Databricks notebook, prefer "
|
|
320
|
+
"workspace_client=WorkspaceClient(), the runtime auto-"
|
|
321
|
+
"authenticates."
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _reject_update_without_id(
|
|
326
|
+
update_if_exists: bool,
|
|
327
|
+
existing_space_id: Optional[str],
|
|
328
|
+
) -> None:
|
|
329
|
+
if update_if_exists and not existing_space_id:
|
|
330
|
+
raise ValueError(
|
|
331
|
+
"deploy_space: update_if_exists=True requires "
|
|
332
|
+
"existing_space_id to identify which Space to update"
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _apply_views(
|
|
337
|
+
*,
|
|
338
|
+
views,
|
|
339
|
+
warehouse_id: str,
|
|
340
|
+
target_catalog: Optional[str],
|
|
341
|
+
target_schema: str,
|
|
342
|
+
dialect,
|
|
343
|
+
host: Optional[str],
|
|
344
|
+
token: Optional[str],
|
|
345
|
+
) -> List[str]:
|
|
346
|
+
"""Open a databricks-sql session to the warehouse and apply each view.
|
|
347
|
+
|
|
348
|
+
Returns the list of fully-qualified view names that landed. Raises a
|
|
349
|
+
clear error if ``databricks-sql-connector`` isn't installed or if
|
|
350
|
+
credentials can't be resolved.
|
|
351
|
+
"""
|
|
352
|
+
import os
|
|
353
|
+
|
|
354
|
+
try:
|
|
355
|
+
import databricks.sql as dbsql
|
|
356
|
+
except ImportError as e:
|
|
357
|
+
raise ImportError(
|
|
358
|
+
"apply_views=True requires databricks-sql-connector. "
|
|
359
|
+
"Install with: pip install 'databricks-sql-connector>=3.0'"
|
|
360
|
+
) from e
|
|
361
|
+
|
|
362
|
+
server_hostname = host or os.environ.get("DATABRICKS_HOST") or ""
|
|
363
|
+
server_hostname = server_hostname.replace("https://", "").rstrip("/")
|
|
364
|
+
access_token = token or os.environ.get("DATABRICKS_TOKEN")
|
|
365
|
+
if not server_hostname or not access_token:
|
|
366
|
+
raise ValueError(
|
|
367
|
+
"apply_views=True needs host+token (or DATABRICKS_HOST + "
|
|
368
|
+
"DATABRICKS_TOKEN env vars) to open a SQL session against "
|
|
369
|
+
"the warehouse."
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
from prograph.genie.view_writer import write_views
|
|
373
|
+
|
|
374
|
+
conn = dbsql.connect(
|
|
375
|
+
server_hostname=server_hostname,
|
|
376
|
+
http_path=f"/sql/1.0/warehouses/{warehouse_id}",
|
|
377
|
+
access_token=access_token,
|
|
378
|
+
)
|
|
379
|
+
try:
|
|
380
|
+
applied = write_views(
|
|
381
|
+
views=views,
|
|
382
|
+
connection=conn,
|
|
383
|
+
target_catalog=target_catalog,
|
|
384
|
+
target_schema=target_schema,
|
|
385
|
+
dialect=dialect,
|
|
386
|
+
)
|
|
387
|
+
finally:
|
|
388
|
+
conn.close()
|
|
389
|
+
|
|
390
|
+
return [fq for _, fq in applied]
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
__all__ = ["deploy_space"]
|