prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""``prograph generate-views``, emit a ``.sql`` file of Genie-consumable views.
|
|
2
|
+
|
|
3
|
+
Wraps :mod:`prograph.genie.view_generator` and :mod:`prograph.genie.view_writer`
|
|
4
|
+
for command-line use. The user gives a schema YAML (and optionally an
|
|
5
|
+
ontology TTL); we write a ``CREATE OR REPLACE VIEW`` script to disk.
|
|
6
|
+
|
|
7
|
+
The script runs unchanged against the target warehouse via
|
|
8
|
+
``databricks-sql-cli``, ``psql -f``, etc. No connection is required at
|
|
9
|
+
generate-time; running the SQL is the user's choice and their tool.
|
|
10
|
+
|
|
11
|
+
For the Python equivalent (which can also execute the views directly
|
|
12
|
+
through the engine's own connection), see :meth:`ProGraph.write_views`.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import sys
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, Optional
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def generate_views_command(
|
|
23
|
+
config_path: str,
|
|
24
|
+
ontology_path: Optional[str] = None,
|
|
25
|
+
output_path: str = "views.sql",
|
|
26
|
+
dialect_name: str = "databricks_sql",
|
|
27
|
+
target_catalog: Optional[str] = None,
|
|
28
|
+
target_schema: str = "prograph_views",
|
|
29
|
+
include_labels: Optional[list[str]] = None,
|
|
30
|
+
exclude_labels: Optional[list[str]] = None,
|
|
31
|
+
) -> dict[str, Any]:
|
|
32
|
+
"""Execute the ``generate-views`` command.
|
|
33
|
+
|
|
34
|
+
Decoupled from ``argparse`` so it's testable directly.
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
config_path: Path to the ProGraph YAML schema config.
|
|
38
|
+
ontology_path: Optional path to an OWL/TTL ontology file. When
|
|
39
|
+
present, subclass-closure and OWL-defined-class views are
|
|
40
|
+
emitted alongside the schema-label views.
|
|
41
|
+
output_path: Where to write the generated ``.sql`` script.
|
|
42
|
+
dialect_name: ``"databricks_sql"`` | ``"postgresql"`` | ``"spark"``.
|
|
43
|
+
Drives identifier quoting.
|
|
44
|
+
target_catalog: Catalog prefix for three-part names. Required
|
|
45
|
+
for Databricks SQL when running views against Unity
|
|
46
|
+
Catalog tables. Omit for Postgres / two-part backends.
|
|
47
|
+
target_schema: Where the views live. Conventional default is
|
|
48
|
+
``prograph_views``.
|
|
49
|
+
include_labels / exclude_labels: Optional whitelists /
|
|
50
|
+
blacklists by NodeMapping label.
|
|
51
|
+
|
|
52
|
+
Returns:
|
|
53
|
+
Dict with summary stats: counts per view source kind, output
|
|
54
|
+
path, dialect. Used by the CLI dispatcher to print a summary.
|
|
55
|
+
"""
|
|
56
|
+
from prograph.config import load_config, validate_config
|
|
57
|
+
from prograph.genie import generate_views, render_views_script
|
|
58
|
+
from prograph.plan.sql import dialect as dialect_module
|
|
59
|
+
|
|
60
|
+
config = load_config(Path(config_path))
|
|
61
|
+
warnings = validate_config(config)
|
|
62
|
+
if warnings:
|
|
63
|
+
# `validate_config` returns warning strings, not errors. Surface them
|
|
64
|
+
# but don't fail the command; the user can decide what to do.
|
|
65
|
+
for w in warnings:
|
|
66
|
+
print(f"warning: {w}", file=sys.stderr)
|
|
67
|
+
|
|
68
|
+
schema = config.to_schema(by_table=True)
|
|
69
|
+
ontology = _load_ontology(ontology_path)
|
|
70
|
+
dialect = _resolve_dialect(dialect_name, dialect_module)
|
|
71
|
+
|
|
72
|
+
views = generate_views(
|
|
73
|
+
schema=schema,
|
|
74
|
+
ontology=ontology,
|
|
75
|
+
dialect=dialect,
|
|
76
|
+
include_labels=include_labels,
|
|
77
|
+
exclude_labels=exclude_labels,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
script = render_views_script(
|
|
81
|
+
views,
|
|
82
|
+
target_catalog=target_catalog,
|
|
83
|
+
target_schema=target_schema,
|
|
84
|
+
dialect=dialect,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
output = Path(output_path)
|
|
88
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
89
|
+
output.write_text(script, encoding="utf-8")
|
|
90
|
+
|
|
91
|
+
counts: dict[str, int] = {}
|
|
92
|
+
for v in views:
|
|
93
|
+
counts[v.source.value] = counts.get(v.source.value, 0) + 1
|
|
94
|
+
|
|
95
|
+
return {
|
|
96
|
+
"output": str(output.resolve()),
|
|
97
|
+
"view_count": len(views),
|
|
98
|
+
"counts_by_source": counts,
|
|
99
|
+
"dialect": dialect_name,
|
|
100
|
+
"target_namespace": (
|
|
101
|
+
f"{target_catalog}.{target_schema}" if target_catalog else target_schema
|
|
102
|
+
),
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _load_ontology(ontology_path: Optional[str]):
|
|
107
|
+
"""Load an OWL/TTL ontology from disk if a path was provided.
|
|
108
|
+
|
|
109
|
+
The YAML config doesn't auto-bind an ontology today, users pass
|
|
110
|
+
it to ``ProGraph(ontology=...)`` explicitly. Mirror that here:
|
|
111
|
+
only load when ``--ontology`` is given. When absent, the view
|
|
112
|
+
generator emits schema-label views only (no subclass closures or
|
|
113
|
+
OWL-defined-class views).
|
|
114
|
+
"""
|
|
115
|
+
if ontology_path is None:
|
|
116
|
+
return None
|
|
117
|
+
from prograph.ontology import OntologyLayer
|
|
118
|
+
return OntologyLayer.from_owl(ontology_path)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _resolve_dialect(name: str, dialect_module: Any):
|
|
122
|
+
"""Map a CLI dialect name to a :class:`DialectConfig` instance."""
|
|
123
|
+
mapping = {
|
|
124
|
+
"databricks_sql": dialect_module.SPARK_DIALECT,
|
|
125
|
+
"spark": dialect_module.SPARK_DIALECT,
|
|
126
|
+
"postgresql": dialect_module.POSTGRESQL_DIALECT,
|
|
127
|
+
}
|
|
128
|
+
# SQL Server's dialect was added in a remote host's Phase 4a work; include if present.
|
|
129
|
+
sqlserver = getattr(dialect_module, "SQLSERVER_DIALECT", None)
|
|
130
|
+
if sqlserver is not None:
|
|
131
|
+
mapping["sqlserver"] = sqlserver
|
|
132
|
+
if name not in mapping:
|
|
133
|
+
raise ValueError(
|
|
134
|
+
f"Unknown dialect {name!r}. "
|
|
135
|
+
f"Known: {', '.join(sorted(mapping))}"
|
|
136
|
+
)
|
|
137
|
+
return mapping[name]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
__all__ = ["generate_views_command"]
|
prograph/cli/main.py
ADDED
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
"""ProGraph CLI entry point.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
prograph load-ttl FILE [FILE ...] [--schema YAML] [--output DIR] [--format csv|parquet]
|
|
5
|
+
prograph infer-schema FILE [FILE ...] [--output DIR] [--dialect postgresql|spark]
|
|
6
|
+
[--catalog NAME] [--db-schema NAME]
|
|
7
|
+
[--strategy shared-tables|one-table-per-class]
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import sys
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _cmd_load_ttl(args: argparse.Namespace) -> None:
|
|
15
|
+
"""Execute the load-ttl command."""
|
|
16
|
+
from prograph.cli.ttl_loader import load_ttl, write_frames
|
|
17
|
+
|
|
18
|
+
frames = load_ttl(
|
|
19
|
+
files=args.files,
|
|
20
|
+
schema_path=args.schema,
|
|
21
|
+
include_ontology=args.include_ontology,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
write_frames(frames, args.output, fmt=args.format)
|
|
25
|
+
|
|
26
|
+
total_rows = sum(len(df) for df in frames.values())
|
|
27
|
+
node_frames = {k: v for k, v in frames.items() if '_start_uri' not in v.columns}
|
|
28
|
+
rel_frames = {k: v for k, v in frames.items() if '_start_uri' in v.columns}
|
|
29
|
+
|
|
30
|
+
print(f"Loaded {total_rows} rows from {len(args.files)} file(s)")
|
|
31
|
+
print(f" Node labels ({len(node_frames)}): {', '.join(sorted(node_frames.keys()))}")
|
|
32
|
+
print(f" Relationship types ({len(rel_frames)}): {', '.join(sorted(rel_frames.keys()))}")
|
|
33
|
+
print(f" Output: {args.output}/ ({args.format})")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _cmd_generate_views(args: argparse.Namespace) -> None:
|
|
37
|
+
"""Execute the generate-views command."""
|
|
38
|
+
from prograph.cli.generate_views import generate_views_command
|
|
39
|
+
|
|
40
|
+
result = generate_views_command(
|
|
41
|
+
config_path=args.config,
|
|
42
|
+
ontology_path=args.ontology,
|
|
43
|
+
output_path=args.output,
|
|
44
|
+
dialect_name=args.dialect,
|
|
45
|
+
target_catalog=args.target_catalog,
|
|
46
|
+
target_schema=args.target_schema,
|
|
47
|
+
include_labels=args.include_label or None,
|
|
48
|
+
exclude_labels=args.exclude_label or None,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
print(f"Wrote {result['view_count']} views to {result['output']}")
|
|
52
|
+
if result["counts_by_source"]:
|
|
53
|
+
print("By source:")
|
|
54
|
+
for source, count in sorted(result["counts_by_source"].items()):
|
|
55
|
+
print(f" {source:<20} {count}")
|
|
56
|
+
print(f"Target namespace: {result['target_namespace']}")
|
|
57
|
+
print(f"Dialect: {result['dialect']}")
|
|
58
|
+
print()
|
|
59
|
+
print("To apply the views, run the generated .sql against your warehouse:")
|
|
60
|
+
if args.dialect in ("databricks_sql", "spark"):
|
|
61
|
+
print(f" databricks-sql-cli --file {result['output']}")
|
|
62
|
+
elif args.dialect == "postgresql":
|
|
63
|
+
print(f" psql -f {result['output']}")
|
|
64
|
+
else:
|
|
65
|
+
print(f" <your SQL client> --file {result['output']}")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _cmd_upload_genie_space(args: argparse.Namespace) -> None:
|
|
69
|
+
"""Execute the upload-genie-space command."""
|
|
70
|
+
from prograph.cli.upload_genie_space import upload_genie_space_command
|
|
71
|
+
|
|
72
|
+
result = upload_genie_space_command(
|
|
73
|
+
config_path=args.config,
|
|
74
|
+
warehouse_id=args.warehouse_id,
|
|
75
|
+
ontology_path=args.ontology,
|
|
76
|
+
space_name=args.name,
|
|
77
|
+
space_description=args.description,
|
|
78
|
+
target_catalog=args.target_catalog,
|
|
79
|
+
target_schema=args.target_schema,
|
|
80
|
+
dialect_name=args.dialect,
|
|
81
|
+
include_labels=args.include_label or None,
|
|
82
|
+
exclude_labels=args.exclude_label or None,
|
|
83
|
+
summary_instructions=args.summary_instructions,
|
|
84
|
+
max_sample_queries=args.max_sample_queries,
|
|
85
|
+
host=args.host,
|
|
86
|
+
token=args.token,
|
|
87
|
+
parent_path=args.parent_path,
|
|
88
|
+
update_if_exists=args.update_if_exists,
|
|
89
|
+
existing_space_id=args.existing_space_id,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
print(f"Space {result['action']}: {result['title']}")
|
|
93
|
+
print(f" space_id: {result['space_id']}")
|
|
94
|
+
print(f" warehouse_id: {result['warehouse_id']}")
|
|
95
|
+
if result.get("parent_path"):
|
|
96
|
+
print(f" parent_path: {result['parent_path']}")
|
|
97
|
+
print(f" serialized: {result['serialized_bytes']} bytes")
|
|
98
|
+
print(f" tables: {result['table_count']}")
|
|
99
|
+
print(f" views: {result['view_count']}")
|
|
100
|
+
print(f" sample queries: {result['sample_query_count']}")
|
|
101
|
+
print()
|
|
102
|
+
print("Open the Space in your Databricks workspace to interact with it:")
|
|
103
|
+
print(f" https://<your-workspace>/spaces/{result['space_id']}")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _cmd_emit_genie_space(args: argparse.Namespace) -> None:
|
|
107
|
+
"""Execute the emit-genie-space command."""
|
|
108
|
+
from prograph.cli.emit_genie_space import emit_genie_space_command
|
|
109
|
+
|
|
110
|
+
result = emit_genie_space_command(
|
|
111
|
+
config_path=args.config,
|
|
112
|
+
ontology_path=args.ontology,
|
|
113
|
+
output_path=args.output,
|
|
114
|
+
space_name=args.name,
|
|
115
|
+
space_description=args.description,
|
|
116
|
+
target_catalog=args.target_catalog,
|
|
117
|
+
target_schema=args.target_schema,
|
|
118
|
+
dialect_name=args.dialect,
|
|
119
|
+
include_labels=args.include_label or None,
|
|
120
|
+
exclude_labels=args.exclude_label or None,
|
|
121
|
+
summary_instructions=args.summary_instructions,
|
|
122
|
+
max_sample_queries=args.max_sample_queries,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
print(f"Wrote Genie Space YAML → {result['output']}")
|
|
126
|
+
print(f" Space name: {result['space_name']}")
|
|
127
|
+
print(f" Tables: {result['table_count']} (user + generated)")
|
|
128
|
+
print(f" Views generated: {result['view_count']}")
|
|
129
|
+
print(f" Sample queries: {result['sample_query_count']}")
|
|
130
|
+
print(f" Instructions: {result['instruction_chars']} chars")
|
|
131
|
+
print(f" Target namespace: {result['target_namespace']}")
|
|
132
|
+
print()
|
|
133
|
+
print("Next steps:")
|
|
134
|
+
print(" 1. Apply the views (if not already): prograph generate-views ...")
|
|
135
|
+
print(" 2. Import the YAML into Databricks Genie:")
|
|
136
|
+
print(" - Manually: paste fields into a new Genie Space")
|
|
137
|
+
print(" - Programmatically: feed to the Databricks Genie REST API")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _cmd_infer_schema(args: argparse.Namespace) -> None:
|
|
141
|
+
"""Execute the infer-schema command."""
|
|
142
|
+
from prograph.cli.schema_inferrer import infer_schema, write_inference_result
|
|
143
|
+
|
|
144
|
+
result = infer_schema(
|
|
145
|
+
files=args.files,
|
|
146
|
+
dialect=args.dialect,
|
|
147
|
+
strategy=args.strategy,
|
|
148
|
+
catalog=args.catalog,
|
|
149
|
+
db_schema=args.db_schema,
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
write_inference_result(result, args.output)
|
|
153
|
+
|
|
154
|
+
print(result.summary)
|
|
155
|
+
print(f"\nOutput written to: {args.output}/")
|
|
156
|
+
print(f" graph.yaml: ProGraph YAML schema config")
|
|
157
|
+
print(f" create_tables.sql: SQL DDL ({args.dialect})")
|
|
158
|
+
print(f" summary.txt: Human-readable summary")
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def main(argv=None) -> None:
|
|
162
|
+
"""Main CLI entry point."""
|
|
163
|
+
parser = argparse.ArgumentParser(
|
|
164
|
+
prog='prograph',
|
|
165
|
+
description='ProGraph CLI: data-prep commands for TTL loading and schema inference, and a SPARQL endpoint',
|
|
166
|
+
)
|
|
167
|
+
subparsers = parser.add_subparsers(dest='command')
|
|
168
|
+
|
|
169
|
+
# --- load-ttl ---
|
|
170
|
+
load_parser = subparsers.add_parser(
|
|
171
|
+
'load-ttl',
|
|
172
|
+
help='Parse TTL files and build per-label/rel-type DataFrames',
|
|
173
|
+
)
|
|
174
|
+
load_parser.add_argument(
|
|
175
|
+
'files', nargs='+', help='TTL file(s) to parse',
|
|
176
|
+
)
|
|
177
|
+
load_parser.add_argument(
|
|
178
|
+
'--schema', default=None,
|
|
179
|
+
help='Optional ProGraph YAML config for column name mapping',
|
|
180
|
+
)
|
|
181
|
+
load_parser.add_argument(
|
|
182
|
+
'--output', '-o', default='./output',
|
|
183
|
+
help='Output directory (default: ./output)',
|
|
184
|
+
)
|
|
185
|
+
load_parser.add_argument(
|
|
186
|
+
'--format', choices=['csv', 'parquet'], default='csv',
|
|
187
|
+
help='Output format (default: csv)',
|
|
188
|
+
)
|
|
189
|
+
load_parser.add_argument(
|
|
190
|
+
'--include-ontology', action='store_true', default=False,
|
|
191
|
+
help='Include OWL/RDFS schema triples (default: skip)',
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
# --- infer-schema ---
|
|
195
|
+
infer_parser = subparsers.add_parser(
|
|
196
|
+
'infer-schema',
|
|
197
|
+
help='Infer ProGraph YAML schema + SQL DDL from TTL ontology',
|
|
198
|
+
)
|
|
199
|
+
infer_parser.add_argument(
|
|
200
|
+
'files', nargs='+', help='TTL file(s) to analyze (ontology + optional instance data)',
|
|
201
|
+
)
|
|
202
|
+
infer_parser.add_argument(
|
|
203
|
+
'--output', '-o', default='.',
|
|
204
|
+
help='Output directory (default: .)',
|
|
205
|
+
)
|
|
206
|
+
infer_parser.add_argument(
|
|
207
|
+
'--dialect', choices=['postgresql', 'spark'], default='postgresql',
|
|
208
|
+
help='SQL dialect for DDL generation (default: postgresql)',
|
|
209
|
+
)
|
|
210
|
+
infer_parser.add_argument(
|
|
211
|
+
'--catalog', default=None,
|
|
212
|
+
help='Catalog name for qualified table names (e.g., my_catalog)',
|
|
213
|
+
)
|
|
214
|
+
infer_parser.add_argument(
|
|
215
|
+
'--db-schema', default=None,
|
|
216
|
+
help='Schema name for qualified table names (e.g., my_schema)',
|
|
217
|
+
)
|
|
218
|
+
infer_parser.add_argument(
|
|
219
|
+
'--strategy', choices=['shared-tables', 'one-table-per-class'],
|
|
220
|
+
default='shared-tables',
|
|
221
|
+
help='Table grouping strategy (default: shared-tables)',
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
# --- generate-views ---
|
|
225
|
+
gv_parser = subparsers.add_parser(
|
|
226
|
+
'generate-views',
|
|
227
|
+
help='Emit a .sql file of Genie-consumable SQL views from a ProGraph schema',
|
|
228
|
+
)
|
|
229
|
+
gv_parser.add_argument(
|
|
230
|
+
'--config', '-c', required=True,
|
|
231
|
+
help='Path to the ProGraph YAML schema config',
|
|
232
|
+
)
|
|
233
|
+
gv_parser.add_argument(
|
|
234
|
+
'--ontology', default=None,
|
|
235
|
+
help='Optional path to OWL/TTL ontology, unlocks subclass-closure and OWL-defined-class views',
|
|
236
|
+
)
|
|
237
|
+
gv_parser.add_argument(
|
|
238
|
+
'--output', '-o', default='views.sql',
|
|
239
|
+
help='Path to write the generated SQL script (default: ./views.sql)',
|
|
240
|
+
)
|
|
241
|
+
gv_parser.add_argument(
|
|
242
|
+
'--dialect', choices=['databricks_sql', 'spark', 'postgresql', 'sqlserver'],
|
|
243
|
+
default='databricks_sql',
|
|
244
|
+
help='SQL dialect for identifier quoting (default: databricks_sql)',
|
|
245
|
+
)
|
|
246
|
+
gv_parser.add_argument(
|
|
247
|
+
'--target-catalog', default=None,
|
|
248
|
+
help='Catalog prefix for view names (required for Unity Catalog; omit for Postgres)',
|
|
249
|
+
)
|
|
250
|
+
gv_parser.add_argument(
|
|
251
|
+
'--target-schema', default='prograph_views',
|
|
252
|
+
help='Schema/namespace for generated views (default: prograph_views)',
|
|
253
|
+
)
|
|
254
|
+
gv_parser.add_argument(
|
|
255
|
+
'--include-label', action='append', default=None,
|
|
256
|
+
help='Only generate views for these labels (may be repeated)',
|
|
257
|
+
)
|
|
258
|
+
gv_parser.add_argument(
|
|
259
|
+
'--exclude-label', action='append', default=None,
|
|
260
|
+
help='Skip these labels (may be repeated)',
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
# --- emit-genie-space ---
|
|
264
|
+
egs_parser = subparsers.add_parser(
|
|
265
|
+
'emit-genie-space',
|
|
266
|
+
help='Emit a complete Genie Space YAML config for import into Databricks',
|
|
267
|
+
)
|
|
268
|
+
egs_parser.add_argument(
|
|
269
|
+
'--config', '-c', required=True,
|
|
270
|
+
help='Path to the ProGraph YAML schema config',
|
|
271
|
+
)
|
|
272
|
+
egs_parser.add_argument(
|
|
273
|
+
'--ontology', default=None,
|
|
274
|
+
help='Optional path to OWL/TTL ontology, unlocks subclass-closure + OWL-defined views',
|
|
275
|
+
)
|
|
276
|
+
egs_parser.add_argument(
|
|
277
|
+
'--output', '-o', default='genie-space.yaml',
|
|
278
|
+
help='Path to write the Genie Space YAML (default: ./genie-space.yaml)',
|
|
279
|
+
)
|
|
280
|
+
egs_parser.add_argument(
|
|
281
|
+
'--name', default=None,
|
|
282
|
+
help='Display name for the Genie Space (default: derived from schema name)',
|
|
283
|
+
)
|
|
284
|
+
egs_parser.add_argument(
|
|
285
|
+
'--description', default='',
|
|
286
|
+
help='Free-form Space description',
|
|
287
|
+
)
|
|
288
|
+
egs_parser.add_argument(
|
|
289
|
+
'--dialect', choices=['databricks_sql', 'spark', 'postgresql', 'sqlserver'],
|
|
290
|
+
default='databricks_sql',
|
|
291
|
+
help='SQL dialect for view bodies (default: databricks_sql)',
|
|
292
|
+
)
|
|
293
|
+
egs_parser.add_argument(
|
|
294
|
+
'--target-catalog', default=None,
|
|
295
|
+
help='Catalog prefix for view names (required for Unity Catalog)',
|
|
296
|
+
)
|
|
297
|
+
egs_parser.add_argument(
|
|
298
|
+
'--target-schema', default='prograph_views',
|
|
299
|
+
help='Schema/namespace for generated views (default: prograph_views)',
|
|
300
|
+
)
|
|
301
|
+
egs_parser.add_argument(
|
|
302
|
+
'--include-label', action='append', default=None,
|
|
303
|
+
help='Only include these labels (may be repeated)',
|
|
304
|
+
)
|
|
305
|
+
egs_parser.add_argument(
|
|
306
|
+
'--exclude-label', action='append', default=None,
|
|
307
|
+
help='Skip these labels (may be repeated)',
|
|
308
|
+
)
|
|
309
|
+
egs_parser.add_argument(
|
|
310
|
+
'--summary-instructions', action='store_true', default=False,
|
|
311
|
+
help='Emit just the overview section in instructions (token-friendly)',
|
|
312
|
+
)
|
|
313
|
+
egs_parser.add_argument(
|
|
314
|
+
'--max-sample-queries', type=int, default=25,
|
|
315
|
+
help='Cap on the number of NL→SQL sample queries (default: 25)',
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
# --- upload-genie-space ---
|
|
319
|
+
ugs_parser = subparsers.add_parser(
|
|
320
|
+
'upload-genie-space',
|
|
321
|
+
help='Create or update a Genie Space on Databricks via the Genie API',
|
|
322
|
+
)
|
|
323
|
+
ugs_parser.add_argument(
|
|
324
|
+
'--config', '-c', required=True,
|
|
325
|
+
help='Path to the ProGraph YAML schema config',
|
|
326
|
+
)
|
|
327
|
+
ugs_parser.add_argument(
|
|
328
|
+
'--warehouse-id', required=True,
|
|
329
|
+
help='Databricks SQL Warehouse ID Genie should query (e.g. 06bb2559f22f28f4)',
|
|
330
|
+
)
|
|
331
|
+
ugs_parser.add_argument(
|
|
332
|
+
'--ontology', default=None,
|
|
333
|
+
help='Optional path to OWL/TTL ontology, unlocks OWL-defined and closure views',
|
|
334
|
+
)
|
|
335
|
+
ugs_parser.add_argument(
|
|
336
|
+
'--name', default=None,
|
|
337
|
+
help='Display name for the Space (default: derived from schema name)',
|
|
338
|
+
)
|
|
339
|
+
ugs_parser.add_argument(
|
|
340
|
+
'--description', default='',
|
|
341
|
+
help='Free-form Space description',
|
|
342
|
+
)
|
|
343
|
+
ugs_parser.add_argument(
|
|
344
|
+
'--dialect', choices=['databricks_sql', 'spark', 'postgresql', 'sqlserver'],
|
|
345
|
+
default='databricks_sql',
|
|
346
|
+
help='SQL dialect for view bodies (default: databricks_sql)',
|
|
347
|
+
)
|
|
348
|
+
ugs_parser.add_argument(
|
|
349
|
+
'--target-catalog', default=None,
|
|
350
|
+
help='Catalog prefix for view names (required for Unity Catalog)',
|
|
351
|
+
)
|
|
352
|
+
ugs_parser.add_argument(
|
|
353
|
+
'--target-schema', default='prograph_views',
|
|
354
|
+
help='Schema/namespace for generated views (default: prograph_views)',
|
|
355
|
+
)
|
|
356
|
+
ugs_parser.add_argument(
|
|
357
|
+
'--include-label', action='append', default=None,
|
|
358
|
+
help='Only include these labels (may be repeated)',
|
|
359
|
+
)
|
|
360
|
+
ugs_parser.add_argument(
|
|
361
|
+
'--exclude-label', action='append', default=None,
|
|
362
|
+
help='Skip these labels (may be repeated)',
|
|
363
|
+
)
|
|
364
|
+
ugs_parser.add_argument(
|
|
365
|
+
'--summary-instructions', action='store_true', default=False,
|
|
366
|
+
help='Emit just the overview section in instructions (token-friendly)',
|
|
367
|
+
)
|
|
368
|
+
ugs_parser.add_argument(
|
|
369
|
+
'--max-sample-queries', type=int, default=25,
|
|
370
|
+
help='Cap on the number of NL→SQL sample queries (default: 25)',
|
|
371
|
+
)
|
|
372
|
+
ugs_parser.add_argument(
|
|
373
|
+
'--host', default=None,
|
|
374
|
+
help='Databricks workspace host (default: $DATABRICKS_HOST)',
|
|
375
|
+
)
|
|
376
|
+
ugs_parser.add_argument(
|
|
377
|
+
'--token', default=None,
|
|
378
|
+
help='Databricks PAT (default: $DATABRICKS_TOKEN)',
|
|
379
|
+
)
|
|
380
|
+
ugs_parser.add_argument(
|
|
381
|
+
'--parent-path', default=None,
|
|
382
|
+
help='Workspace path under which to create the Space (default: Databricks chooses)',
|
|
383
|
+
)
|
|
384
|
+
ugs_parser.add_argument(
|
|
385
|
+
'--update-if-exists', action='store_true', default=False,
|
|
386
|
+
help='Update an existing Space in place instead of creating a new one '
|
|
387
|
+
'(requires --existing-space-id)',
|
|
388
|
+
)
|
|
389
|
+
ugs_parser.add_argument(
|
|
390
|
+
'--existing-space-id', default=None,
|
|
391
|
+
help='Space ID to update (only meaningful with --update-if-exists)',
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
# --- serve (the SPARQL endpoint) ---
|
|
395
|
+
from prograph.endpoint.cli import add_parser as add_serve_parser
|
|
396
|
+
add_serve_parser(subparsers)
|
|
397
|
+
|
|
398
|
+
args = parser.parse_args(argv)
|
|
399
|
+
|
|
400
|
+
if args.command is None:
|
|
401
|
+
parser.print_help()
|
|
402
|
+
sys.exit(1)
|
|
403
|
+
|
|
404
|
+
if args.command == 'load-ttl':
|
|
405
|
+
_cmd_load_ttl(args)
|
|
406
|
+
elif args.command == 'infer-schema':
|
|
407
|
+
_cmd_infer_schema(args)
|
|
408
|
+
elif args.command == 'generate-views':
|
|
409
|
+
_cmd_generate_views(args)
|
|
410
|
+
elif args.command == 'emit-genie-space':
|
|
411
|
+
_cmd_emit_genie_space(args)
|
|
412
|
+
elif args.command == 'upload-genie-space':
|
|
413
|
+
_cmd_upload_genie_space(args)
|
|
414
|
+
elif args.command == 'serve':
|
|
415
|
+
from prograph.endpoint.cli import run as serve
|
|
416
|
+
serve(args)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
if __name__ == '__main__':
|
|
420
|
+
main()
|