prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/__init__.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""ProGraph - Multi-language graph query engine for Python."""
|
|
2
|
+
|
|
3
|
+
from prograph._version import __version__
|
|
4
|
+
from prograph.engine import ProGraph, CypherEngine
|
|
5
|
+
from prograph.hints import QueryHints, parse_hints
|
|
6
|
+
from prograph.plan.ir.semantics import Semantics
|
|
7
|
+
from prograph import logging as logging # noqa: Structured logging module
|
|
8
|
+
from prograph import config # YAML configuration module
|
|
9
|
+
from prograph.execution import QueryContext, QueryCancelledException
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"ProGraph",
|
|
13
|
+
"CypherEngine",
|
|
14
|
+
"__version__",
|
|
15
|
+
"logging",
|
|
16
|
+
"config",
|
|
17
|
+
"QueryHints",
|
|
18
|
+
"Semantics",
|
|
19
|
+
"parse_hints",
|
|
20
|
+
"QueryContext",
|
|
21
|
+
"QueryCancelledException",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# Lazy import for NLP module to avoid requiring openai dependency
|
|
26
|
+
def __getattr__(name: str):
|
|
27
|
+
if name == "nlp":
|
|
28
|
+
from prograph import nlp
|
|
29
|
+
return nlp
|
|
30
|
+
elif name == "NaturalLanguageInterface":
|
|
31
|
+
from prograph.nlp import NaturalLanguageInterface
|
|
32
|
+
return NaturalLanguageInterface
|
|
33
|
+
elif name == "r2rml":
|
|
34
|
+
from prograph import r2rml
|
|
35
|
+
return r2rml
|
|
36
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
prograph/__main__.py
ADDED
prograph/_version.py
ADDED
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
"""Version information for ProGraph."""
|
|
2
|
+
|
|
3
|
+
__version__ = "1.3.3"
|
|
4
|
+
|
|
5
|
+
# Version history:
|
|
6
|
+
# 1.3.3 - (2026-11-22) Published on PyPI: pip install "prograph[duckdb]"; requires Python 3.11 or later;
|
|
7
|
+
# the demo apps stay in the repository, outside the installed package
|
|
8
|
+
# 1.3.2 - (2026-11-21) An empty Databricks answer over SEA keeps its columns instead of failing
|
|
9
|
+
# 1.3.1 - (2026-11-20) Fixes a SPARQL UNION that lost the rows of a branch no constant narrows
|
|
10
|
+
# 1.3.0 - (2026-11-19) Cypher applies more of the ontology and computes decimals exactly; ask
|
|
11
|
+
# writes Cypher; SPARQL matches literals by value; requires Python 3.10 or later
|
|
12
|
+
# 1.2.0 - (2026-11-16) Cypher is supported on every engine over a YAML mapping, with the class and
|
|
13
|
+
# property hierarchies of an ontology and writes into the owned store
|
|
14
|
+
# 1.1.1 - (2026-11-16) Fixes to the shipped examples: the turbofan SHACL demo's decimals; the Foundry
|
|
15
|
+
# example's equipment query and stable/1.1 install
|
|
16
|
+
# 1.1.0 - (2026-11-16) OWL 2 QL (by rewriting) and OWL 2 RL (materialized) entailment,
|
|
17
|
+
# inconsistency detection, stored facts read only where provably current,
|
|
18
|
+
# owl:imports from local files only
|
|
19
|
+
# 1.0.4 - (2026-11-13) Security: string values with backslashes or ${ escaped on PostgreSQL and Spark
|
|
20
|
+
# 1.0.3 - (2026-11-13) Fixes: RDF term columns are structs on PostgreSQL and Lakebase
|
|
21
|
+
# 1.0.2 - (2026-11-12) Property paths on Spark and Databricks walked by levels first;
|
|
22
|
+
# IRI templates over string columns; fewer catalog reads on a SQL warehouse
|
|
23
|
+
# 1.0.1 - (2026-11-12) Fixes: relationship ends over a label of two node mappings;
|
|
24
|
+
# link classes in the schema description
|
|
25
|
+
# 1.0.0 - (2026-11-12) One relational plan
|
|
26
|
+
# for SPARQL, built on source maps, from tag pre-1.0.0
|
|
27
|
+
# 0.6.0 - The hardening phase's work since 0.5.2 (docs/changelog.md, 0.6.0)
|
|
28
|
+
# - Results: the database's own Arrow, types set in the generated SQL
|
|
29
|
+
# - SPARQL to the W3C specifications first; SPARQL 1.1 Protocol
|
|
30
|
+
# endpoint (prograph serve) with MCP for SPARQL at /mcp
|
|
31
|
+
# - Accepted, not built: one core over source maps; OWL 2 QL by
|
|
32
|
+
# T-mapping under the entailment regime
|
|
33
|
+
#
|
|
34
|
+
# 0.5.2 - Managed-edge performance investigation + source-table-direct baseline
|
|
35
|
+
# - 201/201 LDBC battery on Databricks SF10 (source-table direct)
|
|
36
|
+
# - PK/FK RELY constraints on managed-edge tables (optimizer hints)
|
|
37
|
+
# - Liquid Clustering: CLUSTER BY (rel_type, src_id) on edges,
|
|
38
|
+
# CLUSTER BY (prograph_id, source_table) on node_id_map
|
|
39
|
+
# - ANALYZE TABLE COMPUTE STATISTICS + DELTA STATISTICS after init
|
|
40
|
+
# - SHACL bridge Oracle fix: lowercase column names + dialect-aware boolean
|
|
41
|
+
# - W3C SQL tests: 25/25 across all 5 SQL backends (OWL + SHACL)
|
|
42
|
+
# - Source-table-direct battery runner (no managed edges overhead)
|
|
43
|
+
# - Performance finding: managed-edge views add 2 extra JOINs per
|
|
44
|
+
# rel access; FK elimination blocked by CAST opacity through views;
|
|
45
|
+
# source tables are strictly better for read-only analytics
|
|
46
|
+
#
|
|
47
|
+
# 0.5.1 - Genie packaging + turbofan column-name fix
|
|
48
|
+
# - pyproject.toml: new [genie] extras (databricks-sdk>=0.40,
|
|
49
|
+
# databricks-sql-connector, rdflib, pandas, pyyaml) so
|
|
50
|
+
# `pip install prograph[genie]` works as advertised.
|
|
51
|
+
# - turbofan_schema.py + graph.yaml + _turbofan_seed.py DDL:
|
|
52
|
+
# health_state → has_health_state (matches the canonical
|
|
53
|
+
# :hasHealthState TTL predicate after camel_to_snake).
|
|
54
|
+
# - genie_qa_notebook.py: catalog widget label clarifies that
|
|
55
|
+
# the value is a single-part name.
|
|
56
|
+
# - Misc Genie cleanup: deploy_space(apply_views=True),
|
|
57
|
+
# SHACL violation views, prograph.mcp Databricks App recipe,
|
|
58
|
+
# cross-model NLI battery dashboard, Path B integration-guide
|
|
59
|
+
# rewrite around the supported MCP shape.
|
|
60
|
+
#
|
|
61
|
+
# 0.5.0 - SQL-first backends + managed edge storage
|
|
62
|
+
# Phase 4: 9 backends (memory, pandas, spark, databricks_sql, duckdb,
|
|
63
|
+
# mysql, postgresql, sqlserver, oracle) with 95-test conformance battery
|
|
64
|
+
# - Dialect abstraction: DialectConfig (data) + Dialect (behavior) with
|
|
65
|
+
# per-backend subclasses (OracleDialect, DuckDBDialect, etc.)
|
|
66
|
+
# - OracleSerializer, DuckDB/MySQL/SQL Server dialect DDL overrides
|
|
67
|
+
# - AdjacencyStore Protocol: 7 implementations (Delta, DuckDB, Postgres,
|
|
68
|
+
# MySQL, SQL Server, Oracle, ManagedEdges)
|
|
69
|
+
# - OWL 2 RL+ SQLReasoner: all 12 rules with SQL forms, verified on DuckDB
|
|
70
|
+
# - Reasoner ABC: DataFrameReasoner + SQLReasoner (hard cutover)
|
|
71
|
+
# - SHACL bridge: cursor dispatch for SQL-only backends
|
|
72
|
+
# - execution_mode='sql' now strict (raises, no silent fallback)
|
|
73
|
+
# - SET/DELETE mutations via SQL DML on all SQL backends
|
|
74
|
+
# - count(node_var) SQL pushdown fix (hub node elimination)
|
|
75
|
+
# - SPARQL BIND fix (missing MATCH for DATATYPE property bindings)
|
|
76
|
+
# - Managed edge storage: ProGraph-owned edges table + node_id_map
|
|
77
|
+
# with integer surrogate keys, zero-build adjacency, Databricks
|
|
78
|
+
# scale-tested (800K nodes, 800K edges)
|
|
79
|
+
# - filter_predicate on RelationshipMapping (shared-table support)
|
|
80
|
+
# - Docker conformance infrastructure (Postgres, SQL Server, Oracle, MySQL)
|
|
81
|
+
# - LDBC SNB managed edges setup (notebook + CLI)
|
|
82
|
+
# - param_style on DialectConfig (%s, ?, :1)
|
|
83
|
+
# 0.4.21 - CLI commands, turbofan demo, exception handling cleanup
|
|
84
|
+
# - CLI commands: load-ttl (TTL → CSV/Parquet DataFrames),
|
|
85
|
+
# infer-schema (OWL ontology → YAML config + SQL DDL)
|
|
86
|
+
# - Turbofan engine predictive maintenance demo schema
|
|
87
|
+
# (10 node tables, 38 labels via filter_predicate, 17 relationship tables)
|
|
88
|
+
# - Replace ~55 silent except Exception handlers across 19 files with
|
|
89
|
+
# specific exception types and structured debug logging
|
|
90
|
+
# 0.4.20 - filter_predicate on NodeMapping + CYCLE detection for recursive CTEs
|
|
91
|
+
# - filter_predicate: Dict[str, Any] on NodeMapping for row-level
|
|
92
|
+
# discriminator filters (e.g., filter_predicate={'type': 'Bearing'})
|
|
93
|
+
# - Maps multiple graph labels to the same base table
|
|
94
|
+
# - Injected on reads via _apply_filter_predicate() at 5 sites:
|
|
95
|
+
# _bind_node, _bind_unlabeled_node, _bind_relationship end node,
|
|
96
|
+
# VL handler start node, VL handler end nodes
|
|
97
|
+
# - Naturally skipped on writes (DeltaMutationBuilder bypasses tree builder)
|
|
98
|
+
# - YAML config support: filter_predicate key on node mappings
|
|
99
|
+
# - CYCLE clause for PostgreSQL recursive CTEs (PostgreSQL 14+):
|
|
100
|
+
# - Appends CYCLE "current_id" SET "is_cycle" USING "_path" to
|
|
101
|
+
# recursive CTEs, preventing infinite loops on cyclic graphs
|
|
102
|
+
# - Body filter adds NOT is_cycle alongside depth guard
|
|
103
|
+
# - cycle_columns field on CTEDef, preserved through optimizer passes
|
|
104
|
+
# (ProjectionPruning and RedundantSortElimination)
|
|
105
|
+
# - supports_cycle_detection on DialectConfig (True for PostgreSQL)
|
|
106
|
+
# - 26 new tests across 4 test files
|
|
107
|
+
# 0.4.19 - Multi-dialect SQL generation + PostgreSQL and SQL Server backends
|
|
108
|
+
# - Dialect abstraction: DialectConfig captures ~15 SQL engine differences
|
|
109
|
+
# - PostgreSQL backend:
|
|
110
|
+
# - Double-quote identifier quoting
|
|
111
|
+
# - LEFT JOIN ... WHERE IS NULL (anti-join), WHERE EXISTS (semi-join)
|
|
112
|
+
# - CROSS JOIN LATERAL unnest() (lateral explode)
|
|
113
|
+
# - array_agg, date subtraction, EXTRACT-based MONTHS_BETWEEN
|
|
114
|
+
# - SQLAlchemy + psycopg2 connection support
|
|
115
|
+
# - SQL Server backend:
|
|
116
|
+
# - Bracket identifier quoting ([name])
|
|
117
|
+
# - LIMIT → TOP (no ORDER BY) or OFFSET FETCH (with ORDER BY)
|
|
118
|
+
# - LEFT JOIN ... WHERE IS NULL (anti-join), WHERE EXISTS (semi-join)
|
|
119
|
+
# - CROSS APPLY OPENJSON (lateral explode)
|
|
120
|
+
# - STRING_AGG, DATEDIFF(DAY/MONTH, start, end), CAST(GETDATE() AS DATE)
|
|
121
|
+
# - Type mappings: NVARCHAR(MAX), FLOAT, BIT
|
|
122
|
+
# - SQLAlchemy + pyodbc connection support
|
|
123
|
+
# - to_sql(dialect='postgresql'|'sqlserver'|'mssql') for SQL generation
|
|
124
|
+
# - backend='postgresql'|'sqlserver' for end-to-end execution
|
|
125
|
+
# - _assemble_sql() hook for dialect-specific SQL assembly
|
|
126
|
+
# - SQLGraph base class for SQL-database-backed graphs
|
|
127
|
+
# - 67 new tests (31 SQL Server + 36 PostgreSQL serializer)
|
|
128
|
+
# 0.4.18 - Warehouse polling + timeout for long-running queries
|
|
129
|
+
# - Queries exceeding the 50s API wait_timeout are now polled to
|
|
130
|
+
# completion instead of silently returning empty results
|
|
131
|
+
# - New warehouse_timeout parameter on ProGraph, SparkExecutor, SparkGraph
|
|
132
|
+
# cancels the warehouse statement and raises TimeoutError when exceeded
|
|
133
|
+
# - TimeoutError is a hard error (no Spark fallback); warehouse is the
|
|
134
|
+
# faster path, so falling back to Spark would be slower
|
|
135
|
+
# - Poll loop checks QueryContext for cancellation between iterations
|
|
136
|
+
# - Failed/cancelled warehouse statements raise RuntimeError with details
|
|
137
|
+
# - Structured logging: warehouse_poll_start, warehouse_poll_complete events
|
|
138
|
+
# 0.4.17 - Live schema update for ProGraph engine
|
|
139
|
+
# - update_schema() mutates schema in place without recreating the engine
|
|
140
|
+
# - Supports add_nodes, remove_node_labels, replace_nodes,
|
|
141
|
+
# add_relationships, remove_rel_types, add_dataframes, add_table_stats
|
|
142
|
+
# - Selective cache invalidation: only rebuilds VL indexes when
|
|
143
|
+
# relationship mappings change
|
|
144
|
+
# - node cache, cached tables, adjacency builder caches
|
|
145
|
+
# selectively cleared based on SchemaChangeSet
|
|
146
|
+
# - GraphSchema mutation methods: remove_node_mapping(),
|
|
147
|
+
# remove_relationship_mapping(), replace_node_mapping()
|
|
148
|
+
# - Convenience methods: add_mapping(), remove_mapping()
|
|
149
|
+
# - SchemaChangeSet dataclass tracks changes and drives invalidation
|
|
150
|
+
# - Dependent relationship check prevents removing nodes with active rels
|
|
151
|
+
# - Memory, Pandas, and Spark backends all supported
|
|
152
|
+
# 0.4.16 - Query battery 183/183 passing, 0 timeouts (SF10, 157s total)
|
|
153
|
+
# - Selective anti-join deferral past WITH when CTE has folded LIMIT
|
|
154
|
+
# - Anti-joins only defer when ORDER+LIMIT folding has bounded the CTE output
|
|
155
|
+
# - Fixes C161, C163, C164, C170 timeouts (previously 50s+ each)
|
|
156
|
+
# - Fix unlabeled-node UNION branch joins preventing SemiJoin conversion
|
|
157
|
+
# - Set rel_type on branch joins so JoinElimination preserves INNER JOIN
|
|
158
|
+
# - Fixes C149 UNRESOLVED_COLUMN error (FK columns inaccessible after SEMI JOIN)
|
|
159
|
+
# - LabelCheckExpr translation in ExprTranslator (content:Post → __label__ = 'Post')
|
|
160
|
+
# - Remove blanket anti-join deferral pass (replaced by selective deferral)
|
|
161
|
+
# - Fix anti-join ON condition to include end-node FK when both variables bound
|
|
162
|
+
# - Fix SemiJoin elimination for joins with alias mismatch
|
|
163
|
+
# - Fix circular re-inference in transitive predicate pushdown
|
|
164
|
+
# - Transitive predicate inference through equi-joins
|
|
165
|
+
# - SemiJoin IR node and join elimination for unused endpoints
|
|
166
|
+
# - Native LEFT ANTI JOIN in tree serializer
|
|
167
|
+
# - Consolidate SQL generation routing, delete ~3600 lines
|
|
168
|
+
# - Multi-match fusion updates passthrough WITH projections for merged variables
|
|
169
|
+
# 0.4.15 - Tree-based SQL pipeline as default backend
|
|
170
|
+
# - New sql_backend parameter on ProGraph: 'tree' (default), 'interpreter', 'auto'
|
|
171
|
+
# - Tree pipeline (TreeBuilder → OptimizationPipeline → SparkSQLSerializer) is now
|
|
172
|
+
# the primary SQL generation path, replacing SparkSQLInterpreter
|
|
173
|
+
# - SPARQL parity for tree pipeline:
|
|
174
|
+
# - Unlabeled node binding via SetOperation(UNION_ALL) across all node tables
|
|
175
|
+
# - Multi-label relationship handling (5 methods for start/end label combos)
|
|
176
|
+
# - BIND support (return_all=True registers aliases without CTE)
|
|
177
|
+
# - SPARQL function mappings: REGEX→RLIKE, STRLEN→LENGTH, UCASE→UPPER,
|
|
178
|
+
# LCASE→LOWER, STRSTARTS/STRENDS→LIKE
|
|
179
|
+
# - COUNT(unlabeled_var) → COUNT(__id__)
|
|
180
|
+
# - SetOperation rendering in SQL serializer (FROM clause + JOIN target)
|
|
181
|
+
# - RLIKE operator support in expression serializer
|
|
182
|
+
# - PROGRAPH_TREE_SERIALIZER env var still works as override
|
|
183
|
+
# 0.4.14 - SPARQL SQL pushdown fixes for old interpreter
|
|
184
|
+
# - Unlabeled node UNION ALL, multi-label relationships, BIND passthrough
|
|
185
|
+
# - SPARQL function mappings in interpreter path
|
|
186
|
+
# 0.4.13 - Disable SQLGlot optimization (eliminate_ctes breaks CTE column references)
|
|
187
|
+
# - Fix NAMED_STRUCT alias shadowing in passthrough WITH (C64)
|
|
188
|
+
# - Fix hub node persistence across CTE boundaries (C145)
|
|
189
|
+
# - Fix COUNT of post-WITH node variables (C197)
|
|
190
|
+
# - Fix UNWIND struct rejoin field extraction (C199)
|
|
191
|
+
# 0.4.12 - Predicate pushdown through WITH optimizer pass
|
|
192
|
+
# - Filters on preserved GROUP BY key variables move before WITH
|
|
193
|
+
# - Reduces rows entering GROUP BY (core bottleneck for C161, retail queries)
|
|
194
|
+
# - Safety: only moves when all filter vars are preserved node vars
|
|
195
|
+
# - Mixed filters (preserved vars + aliases) stay in place
|
|
196
|
+
# - Runs as Pass 5 after anti-join deferral
|
|
197
|
+
# 0.4.11 - Composite key support for multi-column node identifiers
|
|
198
|
+
# - NodeMapping accepts id_column as Union[str, List[str]]
|
|
199
|
+
# - Added id_columns property (always returns List[str])
|
|
200
|
+
# - Added is_composite_key property
|
|
201
|
+
# - RelationshipMapping: start_id_column/end_id_column accept lists
|
|
202
|
+
# - Added start_id_columns, end_id_columns, has_composite_start_key, has_composite_end_key
|
|
203
|
+
# - SparkGraph/PandasGraph: _get_row_id returns tuple for composite keys
|
|
204
|
+
# - DeltaMutationBuilder: _build_id_where handles composite keys
|
|
205
|
+
# - YAML config: id_column accepts list of column names
|
|
206
|
+
# - Use case: TPC-H lineitem table (l_orderkey + l_linenumber)
|
|
207
|
+
# 0.4.10 - Table qualifications map for explicit table name mappings
|
|
208
|
+
# - GraphSchema accepts table_qualifications dict parameter
|
|
209
|
+
# - ProGraph accepts table_qualifications parameter (sets on schema)
|
|
210
|
+
# - qualified_source_table checks map first, then defaults
|
|
211
|
+
# - Example: {"accounts": "bg.demo.my_accounts"}
|
|
212
|
+
# 0.4.9 - Unity Catalog table name qualification
|
|
213
|
+
# - Added TableMapping base class with qualified_source_table property
|
|
214
|
+
# - NodeMapping and RelationshipMapping inherit from TableMapping
|
|
215
|
+
# - GraphSchema accepts default_catalog and default_schema parameters
|
|
216
|
+
# - ProGraph can set default_catalog/default_schema (sets them on schema)
|
|
217
|
+
# - SQL generation uses qualified table names automatically
|
|
218
|
+
# - Improved error message when neither dataframes nor spark provided
|
|
219
|
+
# 0.4.8 - Fix _load_dataframes_from_schema iteration
|
|
220
|
+
# - Fixed bug where schema.node_mappings/relationship_mappings were
|
|
221
|
+
# incorrectly accessed with .values() (they are lists, not dicts)
|
|
222
|
+
# 0.4.7 - Optional spark parameter for simpler API
|
|
223
|
+
# - Added spark= parameter to ProGraph.__init__()
|
|
224
|
+
# - Auto-loads DataFrames from Unity Catalog when spark + schema provided
|
|
225
|
+
# - Simplifies Databricks Apps integration (no manual dataframes dict needed)
|
|
226
|
+
# 0.4.6 - Gremlin valueMap TinkerPop compliance + MATCH node returns + Query hints
|
|
227
|
+
# - valueMap() now returns values wrapped in lists for TinkerPop compliance
|
|
228
|
+
# - e.g., {name: ['Alice']} instead of {name: 'Alice'}
|
|
229
|
+
# - Added wrap_in_list flag to Projection class
|
|
230
|
+
# - Applied in both Python executor and SQL pushdown paths
|
|
231
|
+
# - Unlabeled MATCH (m) support:
|
|
232
|
+
# - SQL path: UNION ALL across all node tables (columns aligned with NULL)
|
|
233
|
+
# - DataFrame path: Each node has only its own properties (no NULL padding)
|
|
234
|
+
# - Fixed MATCH (p:Person) RETURN p to return full node struct, not just ID
|
|
235
|
+
# - Added to_dict() to Node, PandasNode, SparkNode for consistent output
|
|
236
|
+
# - Node dicts include __id__, __label__, and all mapped properties
|
|
237
|
+
# - Query hints for per-query execution control:
|
|
238
|
+
# - /*+ execution_mode=dataframe */ MATCH (m) RETURN m
|
|
239
|
+
# - Multiline hints supported with key=value and function(args) syntax
|
|
240
|
+
# - Hints: execution_mode, index_layer, vl_mode, broadcast(tables), timeout
|
|
241
|
+
# - execute() and explain_physical() accept execution_mode parameter override
|
|
242
|
+
# 0.4.5 - ProAct UI workflow_id tracking and entity timeline enrichment
|
|
243
|
+
# - Added workflow_id field to Workflow and WorkflowInstance classes
|
|
244
|
+
# - Workflow instances now track definition ID to survive name changes
|
|
245
|
+
# - Entity timeline enriched with SCORED, WORKFLOW, TASK events
|
|
246
|
+
# - Event type normalization (TASK_CREATED → TASK, etc.)
|
|
247
|
+
# - Fixed workflow count queries to use correct status values
|
|
248
|
+
# - Fixed Lakebase sync JSON handling for JSONB columns
|
|
249
|
+
# - Real Databricks data integration with ProGraph inference
|
|
250
|
+
# - ProAct branded logo and graph API route
|
|
251
|
+
# - Demo mode with action execution and missing templates
|
|
252
|
+
# - Modal save, row templates, and demo data persistence fixes
|
|
253
|
+
# 0.4.4 - ProAct UI and Dynamic Layer demo
|
|
254
|
+
# - FastAPI web UI with Tailwind CSS + DaisyUI + HTMX
|
|
255
|
+
# - Dashboard, Investigation, Tasks, Rules, Triggers, Workflows pages
|
|
256
|
+
# - Cytoscape.js graph visualization for entity networks
|
|
257
|
+
# - Two-tier data access: Lakebase (~300ms) + ProGraph (~500-700ms)
|
|
258
|
+
# - ProGraph warm-up on startup to avoid cold-start latency
|
|
259
|
+
# - Dynamic Layer notebook: mutations with mutable=True schemas
|
|
260
|
+
# - Action types: HoldPayment, ReleasePayment, BlockPayment, FlagAccount, FlagEntity
|
|
261
|
+
# - Closed-loop demo: action → graph change → re-inference → score changes
|
|
262
|
+
# - Delta time travel for full audit trail
|
|
263
|
+
# 0.4.3 - ProAct inference adapters
|
|
264
|
+
# - SqlRulesAdapter: SQL-based inference with automatic SQL Warehouse routing
|
|
265
|
+
# - EnsembleAdapter: Combine multiple adapters with weighted_average, max, noisy_or, voting
|
|
266
|
+
# - SparqlRulesAdapter: SPARQL-based inference rules
|
|
267
|
+
# - SQL Warehouse integration provides 3-5x speedup on complex queries
|
|
268
|
+
# - Standardized documentation naming to lowercase-kebab-case
|
|
269
|
+
# 0.4.2 - ProAct action layer enhancements
|
|
270
|
+
# - WorkflowEngine now extracts Spark from ProGraph for Delta persistence
|
|
271
|
+
# - Fix CANNOT_DETERMINE_TYPE errors with explicit schemas for workflow/task persistence
|
|
272
|
+
# - Added comprehensive ProAct documentation in docs/proact/
|
|
273
|
+
# - WorkflowEngine demonstration added to Spark notebook
|
|
274
|
+
# 0.4.1 - Fix inline property filters on relationship end nodes in Spark
|
|
275
|
+
# - Filters like (m:Movie {title: 'X'}) now work correctly in both SQL and DataFrame modes
|
|
276
|
+
# - Added right_properties parameter to _join_relationship_df
|
|
277
|
+
# - SQL path now applies end_node.properties to WHERE clause
|
|
278
|
+
# 0.4.0 - Mutation support milestone release
|
|
279
|
+
# SPARQL 1.1 Update:
|
|
280
|
+
# - INSERT DATA: create nodes with properties from RDF triples
|
|
281
|
+
# - DELETE DATA: delete nodes by explicit URI
|
|
282
|
+
# - DELETE/INSERT WHERE: pattern-based property updates (MODIFY)
|
|
283
|
+
# - INSERT WHERE: create relationships from matched patterns
|
|
284
|
+
# - DELETE WHERE: delete matched patterns
|
|
285
|
+
# - CLEAR GRAPH <uri>: delete all nodes of a label
|
|
286
|
+
# - DROP GRAPH <uri>: same as CLEAR GRAPH
|
|
287
|
+
# Cypher Mutations:
|
|
288
|
+
# - CREATE nodes and relationships
|
|
289
|
+
# - SET properties on nodes and relationships
|
|
290
|
+
# - DELETE and DETACH DELETE
|
|
291
|
+
# - MERGE with ON CREATE/ON MATCH
|
|
292
|
+
# - REMOVE properties
|
|
293
|
+
# Backend Support:
|
|
294
|
+
# - Memory backend: full mutation support
|
|
295
|
+
# - Pandas backend: mutable=True mappings enable writes
|
|
296
|
+
# - Spark backend: Delta Lake tables with mutable=True mappings
|
|
297
|
+
# Infrastructure:
|
|
298
|
+
# - MutableDataFrameMixin for Pandas/Spark mutation operations
|
|
299
|
+
# - PandasNode/PandasRelationship classes extending base types
|
|
300
|
+
# - ID generation strategies (uuid, sequence)
|
|
301
|
+
# 0.3.103 - 100% SQL pushdown coverage for query battery (100/100 queries)
|
|
302
|
+
# - IS NULL / IS NOT NULL filter support in pushdown builder and SQL generation
|
|
303
|
+
# - Passthrough WITH detection: simple variable refs merged across MATCH clauses
|
|
304
|
+
# - WITH alias references in function projections (size(friendNames) -> SIZE(COLLECT_LIST(...)))
|
|
305
|
+
# - Function projections passed through execution path to join_and_aggregate_sql
|
|
306
|
+
# - Fixes C58 (COLLECT + size), C64 (passthrough WITH), C79 (IS NOT NULL)
|
|
307
|
+
# 0.3.102 - SPARQL PredicateMapping for RDF/LPG bridge
|
|
308
|
+
# - PredicateKind.DATATYPE maps RDF predicates to node properties
|
|
309
|
+
# - PredicateKind.OBJECT maps RDF predicates to relationships
|
|
310
|
+
# - schema.auto_generate_predicate_mappings() from existing node/relationship mappings
|
|
311
|
+
# - Variable predicate expansion: ?s ?p ?o expands to UNION over all predicates
|
|
312
|
+
# - BIND passthrough fix: return_all=True preserves existing bindings
|
|
313
|
+
# - SPARQL comments: # to end of line now supported
|
|
314
|
+
# - W3C BIND tests: bind01, bind02 passing
|
|
315
|
+
# 0.3.101 - SPARQL ontology expansion (class hierarchy via VALUES, property paths for equivalents)
|
|
316
|
+
# - New to_sql() method for query transpilation (Cypher to SQL without execution)
|
|
317
|
+
# - SQL generation for NODE_SCAN and SINGLE_HOP pushdown types
|
|
318
|
+
# - SQLGenerationError exception for unsupported patterns
|
|
319
|
+
# 0.3.100 - ORDER BY now supports function projection aliases (e.g., upper(p.name) AS x ORDER BY x)
|
|
320
|
+
# - _build_order_spec now tracks function projection aliases alongside aggregation aliases
|
|
321
|
+
# - Enables pushdown for queries with computed expressions in RETURN referenced by ORDER BY
|
|
322
|
+
# 0.3.99 - Fix ORDER BY with computed expression aliases (e.g., conversion_rate)
|
|
323
|
+
# - _apply_order_by now handles VariableRef expressions (RETURN aliases)
|
|
324
|
+
# - Fix Spark aggregation order for unpushed predicates (NOT pattern predicates)
|
|
325
|
+
# - When unpushed predicates exist, Spark returns raw rows for Python filtering
|
|
326
|
+
# - Aggregation applied in Python AFTER predicate filtering (correct order)
|
|
327
|
+
# - Collaborative filtering queries now work correctly on Spark backend
|
|
328
|
+
# 0.3.98 - NOT pattern predicate filtering in pushdown execution
|
|
329
|
+
# - Support for NOT (a)-[:REL]->(b) patterns in WHERE clauses
|
|
330
|
+
# - Enables collaborative filtering queries that exclude existing relationships
|
|
331
|
+
# - Pattern predicates evaluated in Python post-filtering after pushdown
|
|
332
|
+
# - Automatic projection augmentation for pattern predicate variables
|
|
333
|
+
# - Works with both Pandas and Spark backends
|
|
334
|
+
# 0.3.97 - Gremlin null/NaN/Infinity semantics (TinkerPop two-valued logic)
|
|
335
|
+
# - TinkerPop semantics: null == null → True (vs Cypher's unknown)
|
|
336
|
+
# - TinkerPop semantics: NaN == NaN → False (IEEE 754)
|
|
337
|
+
# - TinkerPop semantics: null <= null → True, null < null → False
|
|
338
|
+
# - Added NaN, Infinity, -Infinity parsing in Gremlin
|
|
339
|
+
# - ExecutionPlan tracks source_language for semantic differences
|
|
340
|
+
# - inject() tracking for proper empty result handling
|
|
341
|
+
# - Gremlin TinkerPop pass rate: 677/1989 (34.0%)
|
|
342
|
+
# 0.3.96 - Multi-hop SQL aggregation with proper column selection (44x faster)
|
|
343
|
+
# - Fixed SELECT * bug in aggregation queries that caused massive data shuffling
|
|
344
|
+
# - Complex patterns with GROUP BY now select only needed columns
|
|
345
|
+
# - Shared variables handled with WHERE clause instead of duplicate JOIN
|
|
346
|
+
# - Query Battery (50 queries): 100% passing on SF100 (270M nodes, 1.2B rels)
|
|
347
|
+
# - C30 (3-relationship + aggregation): 249s → 5.6s with SQL Warehouse
|
|
348
|
+
# - SQL Warehouse benchmark: 2.78x faster overall on SF100
|
|
349
|
+
# - Average query time with warehouse: 3.1s (was 9.8s serverless-only)
|
|
350
|
+
# - Test environment: Databricks Serverless + SQL Warehouse (Photon)
|
|
351
|
+
# 0.3.95 - OPTIONAL MATCH pushdown with LEFT JOIN support
|
|
352
|
+
# - Queries with OPTIONAL MATCH now use LEFT JOIN instead of Python fallback
|
|
353
|
+
# - Query battery now achieves 100% pushdown coverage (50/50 queries)
|
|
354
|
+
# - Fixed bug: _extract_operations was returning last MATCH instead of first
|
|
355
|
+
# - Added --cluster argument to explain_battery.py for dedicated cluster execution
|
|
356
|
+
# 0.3.94 - Python explode optimization for adjacency VL (additional 20% faster)
|
|
357
|
+
# - Collect raw adjacency rows and explode in Python instead of Spark
|
|
358
|
+
# - Avoids Spark explode() operator overhead
|
|
359
|
+
# - SF10 KNOWS*1..2: 1.4s → 1.3s
|
|
360
|
+
# - SF100 KNOWS*1..3: 3.5s → 2.8s (21% faster)
|
|
361
|
+
# - Adjacency now 2.6x faster than distributed (was 2.4x)
|
|
362
|
+
# 0.3.93 - Adjacency Index VL optimization (5x faster)
|
|
363
|
+
# - Removed per-hop count() actions that added 2-3s overhead
|
|
364
|
+
# - Python fast path: skip Spark distinct(), collect results directly
|
|
365
|
+
# - Set-based visited filtering (O(1) vs list comprehension)
|
|
366
|
+
# - SF10 KNOWS*1..2: 7.5s → 1.4s (5.4x faster)
|
|
367
|
+
# - SF100 KNOWS*1..3: 5.1s → 3.5s (1.5x faster)
|
|
368
|
+
# - Adjacency now 2.4x faster than distributed (was 0.8x slower)
|
|
369
|
+
# - Three-tier routing verified: Index Layer + Adjacency + Distributed
|
|
370
|
+
# - Test cluster: 8x i3.2xlarge workers, i3.8xlarge driver, DBR 16.4
|
|
371
|
+
# 0.3.92 - Pattern comprehension and full existential subquery support
|
|
372
|
+
# - Pattern comprehension: [(n)-[r]->(b) | expr] syntax
|
|
373
|
+
# - Iterates over pattern matches and evaluates map expression
|
|
374
|
+
# - Full existential subquery: EXISTS { MATCH ... WHERE ... RETURN ... }
|
|
375
|
+
# - Nested EXISTS with proper context variable passing
|
prograph/cli/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""ProGraph CLI, data-prep commands for TTL loading and schema inference."""
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Shared IRI utility functions for the ProGraph CLI.
|
|
2
|
+
|
|
3
|
+
Extracted from sparql_plan_builder.py for reuse across CLI commands.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from prograph.schema.naming import column_from_local_name
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
from rdflib import RDF, RDFS, OWL
|
|
12
|
+
from rdflib.term import Node
|
|
13
|
+
RDFLIB_AVAILABLE = True
|
|
14
|
+
except ImportError:
|
|
15
|
+
RDFLIB_AVAILABLE = False
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# OWL/RDFS predicates that define schema (TBox), not instance data (ABox)
|
|
19
|
+
_ONTOLOGY_PREDICATES = {
|
|
20
|
+
'http://www.w3.org/2000/01/rdf-schema#subClassOf',
|
|
21
|
+
'http://www.w3.org/2000/01/rdf-schema#subPropertyOf',
|
|
22
|
+
'http://www.w3.org/2000/01/rdf-schema#domain',
|
|
23
|
+
'http://www.w3.org/2000/01/rdf-schema#range',
|
|
24
|
+
'http://www.w3.org/2002/07/owl#equivalentClass',
|
|
25
|
+
'http://www.w3.org/2002/07/owl#inverseOf',
|
|
26
|
+
'http://www.w3.org/2002/07/owl#propertyChainAxiom',
|
|
27
|
+
'http://www.w3.org/2002/07/owl#onProperty',
|
|
28
|
+
'http://www.w3.org/2002/07/owl#someValuesFrom',
|
|
29
|
+
'http://www.w3.org/2002/07/owl#allValuesFrom',
|
|
30
|
+
'http://www.w3.org/2002/07/owl#minCardinality',
|
|
31
|
+
'http://www.w3.org/2002/07/owl#maxCardinality',
|
|
32
|
+
'http://www.w3.org/2002/07/owl#cardinality',
|
|
33
|
+
'http://www.w3.org/2002/07/owl#onDatatype',
|
|
34
|
+
'http://www.w3.org/2002/07/owl#withRestrictions',
|
|
35
|
+
'http://www.w3.org/2002/07/owl#intersectionOf',
|
|
36
|
+
'http://www.w3.org/2002/07/owl#unionOf',
|
|
37
|
+
'http://www.w3.org/2002/07/owl#versionInfo',
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
# OWL/RDFS types that indicate schema definitions
|
|
41
|
+
_ONTOLOGY_TYPES = {
|
|
42
|
+
'http://www.w3.org/2002/07/owl#Class',
|
|
43
|
+
'http://www.w3.org/2002/07/owl#ObjectProperty',
|
|
44
|
+
'http://www.w3.org/2002/07/owl#DatatypeProperty',
|
|
45
|
+
'http://www.w3.org/2002/07/owl#FunctionalProperty',
|
|
46
|
+
'http://www.w3.org/2002/07/owl#TransitiveProperty',
|
|
47
|
+
'http://www.w3.org/2002/07/owl#SymmetricProperty',
|
|
48
|
+
'http://www.w3.org/2002/07/owl#Ontology',
|
|
49
|
+
'http://www.w3.org/2002/07/owl#Restriction',
|
|
50
|
+
'http://www.w3.org/2000/01/rdf-schema#Datatype',
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def extract_local_name(iri: str, uppercase: bool = False) -> str:
|
|
55
|
+
"""Extract the local name from an IRI.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
iri: The IRI to extract from (e.g., 'http://example.org/Person')
|
|
59
|
+
uppercase: If True, uppercase the result (for relationship types)
|
|
60
|
+
|
|
61
|
+
Examples:
|
|
62
|
+
'http://example.org/Person' -> 'Person'
|
|
63
|
+
'http://example.org/ontology#knows' -> 'knows' (or 'KNOWS')
|
|
64
|
+
"""
|
|
65
|
+
s = str(iri)
|
|
66
|
+
# Remove angle brackets if present
|
|
67
|
+
if s.startswith('<') and s.endswith('>'):
|
|
68
|
+
s = s[1:-1]
|
|
69
|
+
|
|
70
|
+
# Find the last # or / to get local name
|
|
71
|
+
local = s
|
|
72
|
+
for sep in ('#', '/'):
|
|
73
|
+
if sep in s:
|
|
74
|
+
local = s.rsplit(sep, 1)[-1]
|
|
75
|
+
break
|
|
76
|
+
|
|
77
|
+
return local.upper() if uppercase else local
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def is_ontology_triple(subj: Any, pred: Any, obj: Any) -> bool:
|
|
81
|
+
"""Check if a triple is an OWL/RDFS schema definition (not instance data).
|
|
82
|
+
|
|
83
|
+
Filters out triples like:
|
|
84
|
+
- :Person a owl:Class
|
|
85
|
+
- :knows rdfs:domain :Person
|
|
86
|
+
- owl:equivalentClass / owl:intersectionOf / etc.
|
|
87
|
+
"""
|
|
88
|
+
pred_str = str(pred)
|
|
89
|
+
obj_str = str(obj)
|
|
90
|
+
|
|
91
|
+
# Check if predicate is a schema-defining predicate
|
|
92
|
+
if pred_str in _ONTOLOGY_PREDICATES:
|
|
93
|
+
return True
|
|
94
|
+
|
|
95
|
+
# Check if this is an rdf:type with an ontology class as the object
|
|
96
|
+
if pred_str == str(RDF.type) if RDFLIB_AVAILABLE else pred_str == 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type':
|
|
97
|
+
if obj_str in _ONTOLOGY_TYPES:
|
|
98
|
+
return True
|
|
99
|
+
|
|
100
|
+
# Check if predicate is owl:* or from OWL restriction patterns
|
|
101
|
+
if pred_str.startswith('http://www.w3.org/2002/07/owl#'):
|
|
102
|
+
return True
|
|
103
|
+
|
|
104
|
+
# rdf:first / rdf:rest (list patterns used in OWL)
|
|
105
|
+
if pred_str in (
|
|
106
|
+
'http://www.w3.org/1999/02/22-rdf-syntax-ns#first',
|
|
107
|
+
'http://www.w3.org/1999/02/22-rdf-syntax-ns#rest',
|
|
108
|
+
):
|
|
109
|
+
return True
|
|
110
|
+
|
|
111
|
+
return False
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def iri_to_rel_table_name(iri: str) -> str:
|
|
117
|
+
"""Convert an object property IRI to a relationship table name.
|
|
118
|
+
|
|
119
|
+
Examples:
|
|
120
|
+
'http://example.org/isPartOf' -> 'is_part_of'
|
|
121
|
+
'http://example.org/monitors' -> 'monitors'
|
|
122
|
+
"""
|
|
123
|
+
local = extract_local_name(iri)
|
|
124
|
+
return column_from_local_name(local)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""``prograph emit-genie-space``, emit a complete Genie Space YAML config.
|
|
2
|
+
|
|
3
|
+
Bundles the W1+W2 view generation, W4's instruction + sample-query
|
|
4
|
+
generation, and optional MCP server registration into one
|
|
5
|
+
:class:`GenieSpace` and writes it to YAML.
|
|
6
|
+
|
|
7
|
+
The output is importable into Databricks Genie either by manual paste
|
|
8
|
+
into the Genie Space UI or by a future programmatic uploader (W6+).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any, Optional
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def emit_genie_space_command(
|
|
19
|
+
config_path: str,
|
|
20
|
+
ontology_path: Optional[str] = None,
|
|
21
|
+
output_path: str = "genie-space.yaml",
|
|
22
|
+
space_name: Optional[str] = None,
|
|
23
|
+
space_description: str = "",
|
|
24
|
+
target_catalog: Optional[str] = None,
|
|
25
|
+
target_schema: str = "prograph_views",
|
|
26
|
+
dialect_name: str = "databricks_sql",
|
|
27
|
+
include_labels: Optional[list[str]] = None,
|
|
28
|
+
exclude_labels: Optional[list[str]] = None,
|
|
29
|
+
summary_instructions: bool = False,
|
|
30
|
+
max_sample_queries: int = 25,
|
|
31
|
+
) -> dict[str, Any]:
|
|
32
|
+
"""Execute the ``emit-genie-space`` command.
|
|
33
|
+
|
|
34
|
+
Decoupled from argparse so it's testable directly.
|
|
35
|
+
"""
|
|
36
|
+
from prograph.config import load_config, validate_config
|
|
37
|
+
from prograph.genie import emit_genie_space, write_space_yaml
|
|
38
|
+
from prograph.plan.sql import dialect as dialect_module
|
|
39
|
+
|
|
40
|
+
config = load_config(Path(config_path))
|
|
41
|
+
warnings = validate_config(config)
|
|
42
|
+
if warnings:
|
|
43
|
+
for w in warnings:
|
|
44
|
+
print(f"warning: {w}", file=sys.stderr)
|
|
45
|
+
|
|
46
|
+
schema = config.to_schema(by_table=True)
|
|
47
|
+
ontology = _load_ontology(ontology_path)
|
|
48
|
+
dialect = _resolve_dialect(dialect_name, dialect_module)
|
|
49
|
+
|
|
50
|
+
space = emit_genie_space(
|
|
51
|
+
schema=schema,
|
|
52
|
+
ontology=ontology,
|
|
53
|
+
name=space_name,
|
|
54
|
+
description=space_description,
|
|
55
|
+
target_catalog=target_catalog,
|
|
56
|
+
target_schema=target_schema,
|
|
57
|
+
dialect=dialect,
|
|
58
|
+
include_labels=include_labels,
|
|
59
|
+
exclude_labels=exclude_labels,
|
|
60
|
+
summary_instructions=summary_instructions,
|
|
61
|
+
max_sample_queries=max_sample_queries,
|
|
62
|
+
source_config=str(Path(config_path).resolve()),
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
output = write_space_yaml(space, output_path)
|
|
66
|
+
|
|
67
|
+
return {
|
|
68
|
+
"output": str(output.resolve()),
|
|
69
|
+
"space_name": space.name,
|
|
70
|
+
"table_count": len(space.tables),
|
|
71
|
+
"view_count": len(space.views),
|
|
72
|
+
"sample_query_count": len(space.sample_queries),
|
|
73
|
+
"instruction_chars": len(space.instructions.rendered),
|
|
74
|
+
"target_namespace": (
|
|
75
|
+
f"{target_catalog}.{target_schema}"
|
|
76
|
+
if target_catalog else target_schema
|
|
77
|
+
),
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# ─── Shared helpers (parallel to generate_views.py) ─────────────────
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _load_ontology(ontology_path: Optional[str]):
|
|
85
|
+
if ontology_path is None:
|
|
86
|
+
return None
|
|
87
|
+
from prograph.ontology import OntologyLayer
|
|
88
|
+
return OntologyLayer.from_owl(ontology_path)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _resolve_dialect(name: str, dialect_module: Any):
|
|
92
|
+
mapping = {
|
|
93
|
+
"databricks_sql": dialect_module.SPARK_DIALECT,
|
|
94
|
+
"spark": dialect_module.SPARK_DIALECT,
|
|
95
|
+
"postgresql": dialect_module.POSTGRESQL_DIALECT,
|
|
96
|
+
}
|
|
97
|
+
sqlserver = getattr(dialect_module, "SQLSERVER_DIALECT", None)
|
|
98
|
+
if sqlserver is not None:
|
|
99
|
+
mapping["sqlserver"] = sqlserver
|
|
100
|
+
if name not in mapping:
|
|
101
|
+
raise ValueError(
|
|
102
|
+
f"Unknown dialect {name!r}. Known: {', '.join(sorted(mapping))}"
|
|
103
|
+
)
|
|
104
|
+
return mapping[name]
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
__all__ = ["emit_genie_space_command"]
|