prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,848 @@
|
|
|
1
|
+
"""YAML Configuration for ProGraph.
|
|
2
|
+
|
|
3
|
+
Provides declarative configuration of graph schemas and data sources using YAML,
|
|
4
|
+
inspired by Databricks Metric Views and Asset Bundles (DABs) patterns.
|
|
5
|
+
|
|
6
|
+
Example YAML:
|
|
7
|
+
|
|
8
|
+
version: "1.0"
|
|
9
|
+
|
|
10
|
+
graph:
|
|
11
|
+
name: social_network
|
|
12
|
+
comment: "LDBC SNB social network schema"
|
|
13
|
+
# decimal_text: database # a DECIMAL column's text as the database writes it (Ontop's); default canonical
|
|
14
|
+
|
|
15
|
+
sources:
|
|
16
|
+
person:
|
|
17
|
+
table: catalog.schema.person
|
|
18
|
+
knows:
|
|
19
|
+
file: ./data/person_knows_person.csv
|
|
20
|
+
options:
|
|
21
|
+
delimiter: "|"
|
|
22
|
+
header: true
|
|
23
|
+
|
|
24
|
+
nodes:
|
|
25
|
+
- label: Person
|
|
26
|
+
source: person
|
|
27
|
+
id_column: id
|
|
28
|
+
properties:
|
|
29
|
+
firstName: firstName
|
|
30
|
+
lastName: lastName
|
|
31
|
+
|
|
32
|
+
relationships:
|
|
33
|
+
- type: KNOWS
|
|
34
|
+
source: knows
|
|
35
|
+
start_node:
|
|
36
|
+
label: Person
|
|
37
|
+
column: Person.id
|
|
38
|
+
end_node:
|
|
39
|
+
label: Person
|
|
40
|
+
column: Person.id.1
|
|
41
|
+
|
|
42
|
+
Usage:
|
|
43
|
+
|
|
44
|
+
from prograph import ProGraph
|
|
45
|
+
from prograph.config import load_config
|
|
46
|
+
|
|
47
|
+
config = load_config('graph_schema.yaml')
|
|
48
|
+
engine = ProGraph.from_config(config, spark=spark)
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
import os
|
|
52
|
+
import re
|
|
53
|
+
from dataclasses import dataclass, field
|
|
54
|
+
from pathlib import Path
|
|
55
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
56
|
+
|
|
57
|
+
import yaml
|
|
58
|
+
|
|
59
|
+
from prograph.schema.mapping import policy_name
|
|
60
|
+
from prograph.plan.ir.keys import key_trust_name
|
|
61
|
+
from prograph.schema import GraphSchema, NodeMapping, RelationshipMapping
|
|
62
|
+
from prograph.rules import home
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# =============================================================================
|
|
66
|
+
# Configuration Data Classes
|
|
67
|
+
# =============================================================================
|
|
68
|
+
|
|
69
|
+
@dataclass
|
|
70
|
+
class SourceConfig:
|
|
71
|
+
"""Configuration for a data source.
|
|
72
|
+
|
|
73
|
+
Attributes:
|
|
74
|
+
name: Source identifier (key in sources mapping)
|
|
75
|
+
table: Unity Catalog table reference (catalog.schema.table)
|
|
76
|
+
file: File path (CSV, Parquet, JSON)
|
|
77
|
+
query: SQL query to define the source
|
|
78
|
+
options: Reader options (delimiter, header, inferSchema, etc.)
|
|
79
|
+
comment: Description of the source
|
|
80
|
+
"""
|
|
81
|
+
name: str
|
|
82
|
+
table: Optional[str] = None
|
|
83
|
+
file: Optional[str] = None
|
|
84
|
+
query: Optional[str] = None
|
|
85
|
+
options: Dict[str, Any] = field(default_factory=dict)
|
|
86
|
+
comment: Optional[str] = None
|
|
87
|
+
|
|
88
|
+
def __post_init__(self):
|
|
89
|
+
sources_defined = sum(1 for s in [self.table, self.file, self.query] if s)
|
|
90
|
+
if sources_defined == 0:
|
|
91
|
+
raise ValueError(f"Source '{self.name}' must define one of: table, file, or query")
|
|
92
|
+
if sources_defined > 1:
|
|
93
|
+
raise ValueError(f"Source '{self.name}' must define only one of: table, file, or query")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass
|
|
97
|
+
class NodeConfig:
|
|
98
|
+
"""Configuration for a node mapping.
|
|
99
|
+
|
|
100
|
+
Attributes:
|
|
101
|
+
label: Node label (e.g., 'Person', 'Post')
|
|
102
|
+
source: Reference to a source name
|
|
103
|
+
id_column: Column(s) containing node IDs. Can be a single column name
|
|
104
|
+
or a list of column names for composite keys.
|
|
105
|
+
properties: Mapping of property names to column names
|
|
106
|
+
comment: Description of the node type
|
|
107
|
+
mutable: If True, ``engine.set_properties`` may write this label's
|
|
108
|
+
rows (prograph.writes); nothing else writes a mapped table
|
|
109
|
+
graph: Optional named graph IRI for this node mapping
|
|
110
|
+
filter_predicate: Optional dict of column=value discriminator filters
|
|
111
|
+
iri_template: R2RML-style template deciding this node's IRI, e.g.
|
|
112
|
+
'http://example.org/person/{person_id}'. Without it the IRI is
|
|
113
|
+
synthesized from the table and id.
|
|
114
|
+
class_iri: The class this node's rows are typed with, how a mapping
|
|
115
|
+
binds to an externally-authored ontology instead of an invented
|
|
116
|
+
class. Defaults to '{vocab}{Label}'.
|
|
117
|
+
rdf_class: False for a label that only groups rows: no rdf:type
|
|
118
|
+
triple for it. Defaults to True.
|
|
119
|
+
on_duplicate: What a Cypher statement does when two rows hold one
|
|
120
|
+
key value: 'refuse' (the engine's default) or 'trust'
|
|
121
|
+
(prograph.node_keys). None leaves it to the engine.
|
|
122
|
+
keys: 'trusted' (the user asserts the key unique, so a plan may
|
|
123
|
+
rely on it) or 'check' (verified per data version; not built
|
|
124
|
+
yet). None relies on it only where the database enforces it
|
|
125
|
+
(one-plan.md 2.4).
|
|
126
|
+
"""
|
|
127
|
+
label: Optional[str]
|
|
128
|
+
source: str
|
|
129
|
+
id_column: Union[str, List[str]]
|
|
130
|
+
properties: Dict[str, str] = field(default_factory=dict)
|
|
131
|
+
comment: Optional[str] = None
|
|
132
|
+
mutable: bool = False
|
|
133
|
+
graph: Optional[str] = None
|
|
134
|
+
filter_predicate: Optional[Dict[str, Any]] = None
|
|
135
|
+
iri_template: Optional[str] = None
|
|
136
|
+
class_iri: Optional[str] = None
|
|
137
|
+
rdf_class: bool = True
|
|
138
|
+
on_duplicate: Optional[str] = None
|
|
139
|
+
keys: Optional[str] = None
|
|
140
|
+
identity: Optional[str] = None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@dataclass
|
|
144
|
+
class RelationshipEndpoint:
|
|
145
|
+
"""Configuration for a relationship endpoint (start or end node).
|
|
146
|
+
|
|
147
|
+
Attributes:
|
|
148
|
+
label: The node label for this endpoint; None for an end given by
|
|
149
|
+
an identity.
|
|
150
|
+
column: Column(s) for the foreign key reference. Can be a single column
|
|
151
|
+
name or a list of column names for composite keys.
|
|
152
|
+
identity: An end given by an identity instead of a label: the node of
|
|
153
|
+
its key in any node mapping that declares `identity: <name>`.
|
|
154
|
+
"""
|
|
155
|
+
label: Optional[str]
|
|
156
|
+
column: Union[str, List[str]]
|
|
157
|
+
identity: Optional[str] = None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@dataclass
|
|
161
|
+
class RelationshipConfig:
|
|
162
|
+
"""Configuration for a relationship mapping.
|
|
163
|
+
|
|
164
|
+
Attributes:
|
|
165
|
+
type: Relationship type (e.g., 'KNOWS', 'LIKES')
|
|
166
|
+
source: Reference to a source name
|
|
167
|
+
start_node: Start node configuration (label and column)
|
|
168
|
+
end_node: End node configuration (label and column)
|
|
169
|
+
properties: Mapping of property names to column names
|
|
170
|
+
comment: Description of the relationship type
|
|
171
|
+
mutable: If True, OWL RL may write inferred edges into this table; nothing else does
|
|
172
|
+
graph: Optional named graph IRI for this relationship mapping
|
|
173
|
+
filter_predicate: Optional dict of column=value discriminator filters,
|
|
174
|
+
for several relationship types sharing one table
|
|
175
|
+
reify: ``true`` is refused (``NotBuilt``): reified edge statements are not built on the
|
|
176
|
+
SPARQL graph; they return as RDF 1.2 reifiers. Unset or ``false`` load.
|
|
177
|
+
relation_class_iri: The class a reified statement node is typed with.
|
|
178
|
+
Defaults to '{vocab}{PascalCaseRelType}'.
|
|
179
|
+
referential_integrity: Every key in the table has a node row, so SQL
|
|
180
|
+
may drop node joins that only connect relationships.
|
|
181
|
+
keys: 'trusted' (no two rows hold one start and end, asserted) or
|
|
182
|
+
'check' (not built yet); None relies only on what the database
|
|
183
|
+
enforces.
|
|
184
|
+
id_column: The column (or columns) that names one relationship, its
|
|
185
|
+
id; unset, a relationship is its row (Cypher tells two rows of one
|
|
186
|
+
start and end apart by their number among them).
|
|
187
|
+
"""
|
|
188
|
+
type: str
|
|
189
|
+
source: str
|
|
190
|
+
start_node: RelationshipEndpoint
|
|
191
|
+
end_node: RelationshipEndpoint
|
|
192
|
+
properties: Dict[str, str] = field(default_factory=dict)
|
|
193
|
+
comment: Optional[str] = None
|
|
194
|
+
mutable: bool = False
|
|
195
|
+
graph: Optional[str] = None
|
|
196
|
+
filter_predicate: Optional[Dict[str, Any]] = None
|
|
197
|
+
reify: Optional[bool] = None
|
|
198
|
+
relation_class_iri: Optional[str] = None
|
|
199
|
+
referential_integrity: bool = False
|
|
200
|
+
keys: Optional[str] = None
|
|
201
|
+
id_column: Optional[Union[str, List[str]]] = None
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
@dataclass
|
|
205
|
+
class RDFConfig:
|
|
206
|
+
"""RDF/R2RML configuration for SPARQL and ontology integration.
|
|
207
|
+
|
|
208
|
+
Attributes:
|
|
209
|
+
base_iri: Base IRI for node IRIs, set as the schema's ``base_iri``:
|
|
210
|
+
a node mapping with no iri_template is
|
|
211
|
+
``{base_iri}{source_table}/{id}``
|
|
212
|
+
vocab_iri: Vocabulary IRI: a node mapping with no class_iri gets
|
|
213
|
+
``{vocab_iri}{Label}``, and auto_generate mints its predicates
|
|
214
|
+
in this namespace (the default vocabulary,
|
|
215
|
+
``prograph.schema.iri.DEFAULT_VOCAB_IRI``, when unset)
|
|
216
|
+
graph_iri: Optional default named graph IRI
|
|
217
|
+
r2rml: Path to an R2RML Turtle file to load and compile
|
|
218
|
+
auto_generate: If True, auto-generate predicate mappings from schema
|
|
219
|
+
prefixes: Additional namespace prefix -> IRI mappings
|
|
220
|
+
"""
|
|
221
|
+
base_iri: Optional[str] = None
|
|
222
|
+
vocab_iri: Optional[str] = None
|
|
223
|
+
graph_iri: Optional[str] = None
|
|
224
|
+
r2rml: Optional[str] = None
|
|
225
|
+
auto_generate: bool = False
|
|
226
|
+
prefixes: Dict[str, str] = field(default_factory=dict)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@dataclass
|
|
230
|
+
class GraphConfig:
|
|
231
|
+
"""Complete graph configuration.
|
|
232
|
+
|
|
233
|
+
Attributes:
|
|
234
|
+
version: Configuration version (default: "1.0")
|
|
235
|
+
name: Graph name
|
|
236
|
+
comment: Description of the graph
|
|
237
|
+
sources: Data source configurations
|
|
238
|
+
nodes: Node mapping configurations
|
|
239
|
+
relationships: Relationship mapping configurations
|
|
240
|
+
variables: Variable substitutions
|
|
241
|
+
rdf: Optional RDF/R2RML configuration
|
|
242
|
+
decimal_text: ``graph: decimal_text:``, how a DECIMAL column's value
|
|
243
|
+
is written in a term (``GraphSchema``'s ``decimal_text``):
|
|
244
|
+
``canonical`` (the default) or ``database`` (Ontop's text)
|
|
245
|
+
"""
|
|
246
|
+
version: str = "1.0"
|
|
247
|
+
name: str = "prograph"
|
|
248
|
+
comment: Optional[str] = None
|
|
249
|
+
sources: Dict[str, SourceConfig] = field(default_factory=dict)
|
|
250
|
+
nodes: List[NodeConfig] = field(default_factory=list)
|
|
251
|
+
relationships: List[RelationshipConfig] = field(default_factory=list)
|
|
252
|
+
variables: Dict[str, Any] = field(default_factory=dict)
|
|
253
|
+
rdf: Optional[RDFConfig] = None
|
|
254
|
+
# The owned store's labels (`store: labels: <Label>: {key: [properties]}`): the key Cypher's MERGE creates
|
|
255
|
+
# a node of that label by, its values giving the node's IRI.
|
|
256
|
+
store_keys: Dict[str, Tuple[str, ...]] = field(default_factory=dict)
|
|
257
|
+
decimal_text: Optional[str] = None
|
|
258
|
+
# `procedures:` (Cypher): declared procedures, `cypher_procedures.Procedure`s.
|
|
259
|
+
procedures: List[Any] = field(default_factory=list)
|
|
260
|
+
|
|
261
|
+
@home("config-source-tables", "A mapping's table from a config file",
|
|
262
|
+
terms=("source_table", "by_table", "to_schema", "YAML config", "Genie"))
|
|
263
|
+
def to_schema(self, by_table: bool = False) -> GraphSchema:
|
|
264
|
+
"""Convert configuration to a GraphSchema.
|
|
265
|
+
|
|
266
|
+
Args:
|
|
267
|
+
by_table: What a mapping's ``source_table`` is. By default it is
|
|
268
|
+
the source's name (its key under ``sources:``), which is how
|
|
269
|
+
``load_dataframes`` keys the frames it loads; the source's
|
|
270
|
+
declared table is kept on the schema (``declared_tables``), and
|
|
271
|
+
an engine reads it there unless it is given a frame of that
|
|
272
|
+
name (``engine.apply_declared_tables``). With
|
|
273
|
+
``by_table=True`` it is the table the source declares
|
|
274
|
+
(``sources.<name>.table``, such as
|
|
275
|
+
``catalog.schema.table``), for a schema whose SQL runs in a
|
|
276
|
+
database that knows the table by that name: a SQL
|
|
277
|
+
warehouse, generated views. A source with no ``table:`` (a
|
|
278
|
+
file) keeps its name either way.
|
|
279
|
+
|
|
280
|
+
Returns:
|
|
281
|
+
GraphSchema with node and relationship mappings
|
|
282
|
+
"""
|
|
283
|
+
schema = GraphSchema(name=self.name, decimal_text=self.decimal_text)
|
|
284
|
+
|
|
285
|
+
def table_of(source_name: str) -> str:
|
|
286
|
+
declared = self.sources[source_name].table if by_table else None
|
|
287
|
+
return declared or source_name
|
|
288
|
+
|
|
289
|
+
def named(mapping, source_name: str):
|
|
290
|
+
# A synthesized IRI carries the source's name with or without by_table, so one source mints
|
|
291
|
+
# one IRI (`iri.iri_segment`); by_table only chooses the table read.
|
|
292
|
+
if table_of(source_name) != source_name:
|
|
293
|
+
mapping.iri_segment = source_name
|
|
294
|
+
return mapping
|
|
295
|
+
|
|
296
|
+
# Each source's declared table, where the mappings name the source: an engine that reads
|
|
297
|
+
# tables (a SQL connection, a Spark session) reads it there, one given a frame of that name
|
|
298
|
+
# reads the frame (`ProGraph`, `apply_declared_tables`).
|
|
299
|
+
if not by_table:
|
|
300
|
+
schema.declared_tables = {name: src.table for name, src in self.sources.items()
|
|
301
|
+
if getattr(src, "table", None) and src.table != name}
|
|
302
|
+
# The sources that are files (no table): a SQL backend refuses them by name when the engine starts.
|
|
303
|
+
# The owned store's MERGE keys, one record every writer reads; a mapped label has its mapping's key.
|
|
304
|
+
schema.file_sources = {name for name, src in self.sources.items()
|
|
305
|
+
if getattr(src, "file", None) and not getattr(src, "table", None)}
|
|
306
|
+
|
|
307
|
+
# Add node mappings
|
|
308
|
+
for node in self.nodes:
|
|
309
|
+
source = self.sources.get(node.source)
|
|
310
|
+
if not source:
|
|
311
|
+
raise ValueError(f"Node '{node.label}' references unknown source '{node.source}'")
|
|
312
|
+
|
|
313
|
+
schema.add_node_mapping(named(NodeMapping(
|
|
314
|
+
source_table=table_of(node.source),
|
|
315
|
+
label=node.label,
|
|
316
|
+
id_column=node.id_column,
|
|
317
|
+
property_columns=node.properties,
|
|
318
|
+
mutable=node.mutable,
|
|
319
|
+
graph_iri=node.graph,
|
|
320
|
+
filter_predicate=node.filter_predicate,
|
|
321
|
+
iri_template=node.iri_template,
|
|
322
|
+
class_iri=node.class_iri,
|
|
323
|
+
rdf_class=node.rdf_class,
|
|
324
|
+
on_duplicate=node.on_duplicate,
|
|
325
|
+
keys=node.keys,
|
|
326
|
+
identity=node.identity,
|
|
327
|
+
), node.source))
|
|
328
|
+
|
|
329
|
+
# Add relationship mappings
|
|
330
|
+
for rel in self.relationships:
|
|
331
|
+
source = self.sources.get(rel.source)
|
|
332
|
+
if not source:
|
|
333
|
+
raise ValueError(f"Relationship '{rel.type}' references unknown source '{rel.source}'")
|
|
334
|
+
|
|
335
|
+
schema.add_relationship_mapping(named(RelationshipMapping(
|
|
336
|
+
source_table=table_of(rel.source),
|
|
337
|
+
rel_type=rel.type,
|
|
338
|
+
start_node_label=rel.start_node.label,
|
|
339
|
+
start_node_identity=rel.start_node.identity,
|
|
340
|
+
end_node_identity=rel.end_node.identity,
|
|
341
|
+
start_id_column=rel.start_node.column,
|
|
342
|
+
end_node_label=rel.end_node.label,
|
|
343
|
+
end_id_column=rel.end_node.column,
|
|
344
|
+
property_columns=rel.properties,
|
|
345
|
+
mutable=rel.mutable,
|
|
346
|
+
graph_iri=rel.graph,
|
|
347
|
+
filter_predicate=rel.filter_predicate,
|
|
348
|
+
reify=rel.reify,
|
|
349
|
+
relation_class_iri=rel.relation_class_iri,
|
|
350
|
+
referential_integrity=rel.referential_integrity,
|
|
351
|
+
keys=rel.keys,
|
|
352
|
+
id_column=rel.id_column,
|
|
353
|
+
), rel.source))
|
|
354
|
+
|
|
355
|
+
# RDF/R2RML integration
|
|
356
|
+
if self.rdf:
|
|
357
|
+
# Load R2RML mapping and merge into schema
|
|
358
|
+
if self.rdf.r2rml:
|
|
359
|
+
# The document's triples maps join the schema's source maps, read by SPARQL with
|
|
360
|
+
# no property-graph labels.
|
|
361
|
+
from prograph.r2rml.parser import R2RMLParser
|
|
362
|
+
from prograph.r2rml.schema import include
|
|
363
|
+
include(schema, R2RMLParser().parse_file(self.rdf.r2rml))
|
|
364
|
+
|
|
365
|
+
# Namespaces. The base IRI of instance IRIs has one home, the
|
|
366
|
+
# schema; a class IRI fills what a mapping does not declare.
|
|
367
|
+
if self.rdf.base_iri:
|
|
368
|
+
schema.base_iri = self.rdf.base_iri
|
|
369
|
+
for nm in schema.node_mappings:
|
|
370
|
+
if self.rdf.vocab_iri and not nm.class_iri and nm.label is not None:
|
|
371
|
+
nm.class_iri = f"{self.rdf.vocab_iri}{nm.label}"
|
|
372
|
+
|
|
373
|
+
# Auto-generate predicate mappings
|
|
374
|
+
if self.rdf.auto_generate:
|
|
375
|
+
if self.rdf.vocab_iri:
|
|
376
|
+
schema.auto_generate_predicate_mappings(
|
|
377
|
+
namespace=self.rdf.vocab_iri)
|
|
378
|
+
else:
|
|
379
|
+
schema.auto_generate_predicate_mappings()
|
|
380
|
+
|
|
381
|
+
# Register prefixes
|
|
382
|
+
for prefix, ns in self.rdf.prefixes.items():
|
|
383
|
+
schema.add_prefix(prefix, ns)
|
|
384
|
+
|
|
385
|
+
# The owned store's MERGE keys, one record every writer reads (`GraphSchema.set_store_keys`).
|
|
386
|
+
schema.set_store_keys(dict(self.store_keys))
|
|
387
|
+
return schema
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
# =============================================================================
|
|
391
|
+
# Variable Substitution
|
|
392
|
+
# =============================================================================
|
|
393
|
+
|
|
394
|
+
def substitute_variables(value: Any, variables: Dict[str, Any]) -> Any:
|
|
395
|
+
"""Substitute ${var.name} patterns in strings.
|
|
396
|
+
|
|
397
|
+
Follows Databricks Asset Bundles variable substitution pattern.
|
|
398
|
+
|
|
399
|
+
Args:
|
|
400
|
+
value: Value to process (string, dict, or list)
|
|
401
|
+
variables: Variable definitions
|
|
402
|
+
|
|
403
|
+
Returns:
|
|
404
|
+
Value with variables substituted
|
|
405
|
+
"""
|
|
406
|
+
if isinstance(value, str):
|
|
407
|
+
# Pattern: ${var.variable_name}
|
|
408
|
+
pattern = r'\$\{var\.(\w+)\}'
|
|
409
|
+
|
|
410
|
+
def replace(match):
|
|
411
|
+
var_name = match.group(1)
|
|
412
|
+
if var_name not in variables:
|
|
413
|
+
raise ValueError(f"Undefined variable: {var_name}")
|
|
414
|
+
var_value = variables[var_name]
|
|
415
|
+
# Handle variable objects with 'default' key
|
|
416
|
+
if isinstance(var_value, dict) and 'default' in var_value:
|
|
417
|
+
return str(var_value['default'])
|
|
418
|
+
return str(var_value)
|
|
419
|
+
|
|
420
|
+
return re.sub(pattern, replace, value)
|
|
421
|
+
|
|
422
|
+
elif isinstance(value, dict):
|
|
423
|
+
return {k: substitute_variables(v, variables) for k, v in value.items()}
|
|
424
|
+
|
|
425
|
+
elif isinstance(value, list):
|
|
426
|
+
return [substitute_variables(item, variables) for item in value]
|
|
427
|
+
|
|
428
|
+
return value
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
# =============================================================================
|
|
432
|
+
# YAML Parsing
|
|
433
|
+
# =============================================================================
|
|
434
|
+
|
|
435
|
+
def _parse_source(name: str, data: Dict[str, Any]) -> SourceConfig:
|
|
436
|
+
"""Parse a source configuration from YAML data."""
|
|
437
|
+
return SourceConfig(
|
|
438
|
+
name=name,
|
|
439
|
+
table=data.get('table'),
|
|
440
|
+
file=data.get('file'),
|
|
441
|
+
query=data.get('query'),
|
|
442
|
+
options=data.get('options', {}),
|
|
443
|
+
comment=data.get('comment')
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
# Keys that once made a mapping writable through a query. ProGraph is
|
|
448
|
+
# read-only over the tables it maps but for `set_properties` on a
|
|
449
|
+
# node mapping marked `mutable`, and OWL RL into a node or relationship
|
|
450
|
+
# mapping marked `mutable` (decided 2026-11-04; prograph/read_only.py):
|
|
451
|
+
# `mutable` is read, and a configuration that sets `id_generator` is told so
|
|
452
|
+
# rather than having it ignored.
|
|
453
|
+
_NODE_WRITE_KEYS = ("id_generator",)
|
|
454
|
+
_RELATIONSHIP_WRITE_KEYS = ("id_generator",)
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _refuse_write_keys(data: Dict[str, Any], what: str, keys: Tuple[str, ...]) -> None:
|
|
458
|
+
found = [k for k in keys if k in data]
|
|
459
|
+
if found:
|
|
460
|
+
raise ValueError(
|
|
461
|
+
f"{what}: {', '.join(found)} {'is' if len(found) == 1 else 'are'} no longer a "
|
|
462
|
+
f"mapping setting; ProGraph is read-only over the tables it maps, "
|
|
463
|
+
f"and a mapping's mutable opens only set_properties and OWL RL's inferred rows. "
|
|
464
|
+
f"Remove {'it' if len(found) == 1 else 'them'}.")
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _flag(data: Dict[str, Any], key: str, what: str) -> bool:
|
|
468
|
+
"""A yes/no setting: true or false, absent being false. A string such
|
|
469
|
+
as "false" is refused, not read as true."""
|
|
470
|
+
value = data.get(key, False)
|
|
471
|
+
if not isinstance(value, bool):
|
|
472
|
+
raise ValueError(f"{what}: {key} is true or false, got {value!r}")
|
|
473
|
+
return value
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _parse_node(data: Dict[str, Any]) -> NodeConfig:
|
|
477
|
+
"""Parse a node configuration from YAML data."""
|
|
478
|
+
_refuse_write_keys(data, f"node {data.get('label')!r}", _NODE_WRITE_KEYS)
|
|
479
|
+
return NodeConfig(
|
|
480
|
+
label=data.get('label'),
|
|
481
|
+
source=data['source'],
|
|
482
|
+
id_column=data['id_column'],
|
|
483
|
+
properties=data.get('properties', {}),
|
|
484
|
+
comment=data.get('comment'),
|
|
485
|
+
mutable=_flag(data, 'mutable', f"node {data.get('label')!r}"),
|
|
486
|
+
graph=data.get('graph'),
|
|
487
|
+
filter_predicate=data.get('filter_predicate'),
|
|
488
|
+
iri_template=data.get('iri_template'),
|
|
489
|
+
class_iri=data.get('class_iri'),
|
|
490
|
+
rdf_class=data.get('rdf_class', True),
|
|
491
|
+
on_duplicate=policy_name(data.get('on_duplicate'), f"node {data.get('label')!r}: on_duplicate"),
|
|
492
|
+
keys=key_trust_name(data.get('keys'), f"node {data.get('label')!r}: keys"),
|
|
493
|
+
identity=data.get('identity'),
|
|
494
|
+
)
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _parse_relationship(data: Dict[str, Any]) -> RelationshipConfig:
|
|
498
|
+
"""Parse a relationship configuration from YAML data."""
|
|
499
|
+
_refuse_write_keys(data, f"relationship {data.get('type')!r}", _RELATIONSHIP_WRITE_KEYS)
|
|
500
|
+
start_node = data['start_node']
|
|
501
|
+
end_node = data['end_node']
|
|
502
|
+
|
|
503
|
+
return RelationshipConfig(
|
|
504
|
+
type=data['type'],
|
|
505
|
+
source=data['source'],
|
|
506
|
+
start_node=RelationshipEndpoint(
|
|
507
|
+
label=start_node.get('label'),
|
|
508
|
+
column=start_node['column'],
|
|
509
|
+
identity=start_node.get('identity'),
|
|
510
|
+
),
|
|
511
|
+
end_node=RelationshipEndpoint(
|
|
512
|
+
label=end_node.get('label'),
|
|
513
|
+
column=end_node['column'],
|
|
514
|
+
identity=end_node.get('identity'),
|
|
515
|
+
),
|
|
516
|
+
properties=data.get('properties', {}),
|
|
517
|
+
comment=data.get('comment'),
|
|
518
|
+
mutable=_flag(data, 'mutable', f"relationship {data.get('type')!r}"),
|
|
519
|
+
graph=data.get('graph'),
|
|
520
|
+
filter_predicate=data.get('filter_predicate'),
|
|
521
|
+
reify=data.get('reify'),
|
|
522
|
+
relation_class_iri=data.get('relation_class_iri'),
|
|
523
|
+
referential_integrity=bool(data.get('referential_integrity', False)),
|
|
524
|
+
keys=key_trust_name(data.get('keys'), f"relationship {data.get('type')!r}: keys"),
|
|
525
|
+
id_column=data.get('id_column'),
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def _parse_config(data: Dict[str, Any],
|
|
530
|
+
overrides: Optional[Dict[str, Any]] = None) -> GraphConfig:
|
|
531
|
+
"""Parse complete configuration from YAML data."""
|
|
532
|
+
# Extract variables first for substitution; a caller's values win.
|
|
533
|
+
variables = {**(data.get('variables') or {}), **(overrides or {})}
|
|
534
|
+
|
|
535
|
+
# Apply variable substitution to the entire config
|
|
536
|
+
data = substitute_variables(data, variables)
|
|
537
|
+
|
|
538
|
+
# Parse graph metadata
|
|
539
|
+
graph_data = data.get('graph', {})
|
|
540
|
+
version = data.get('version', '1.0')
|
|
541
|
+
name = graph_data.get('name', 'prograph')
|
|
542
|
+
comment = graph_data.get('comment')
|
|
543
|
+
|
|
544
|
+
# Parse sources
|
|
545
|
+
sources = {}
|
|
546
|
+
for source_name, source_data in data.get('sources', {}).items():
|
|
547
|
+
sources[source_name] = _parse_source(source_name, source_data)
|
|
548
|
+
|
|
549
|
+
# Parse nodes
|
|
550
|
+
nodes = []
|
|
551
|
+
for node_data in data.get('nodes', []):
|
|
552
|
+
nodes.append(_parse_node(node_data))
|
|
553
|
+
|
|
554
|
+
# Parse relationships
|
|
555
|
+
relationships = []
|
|
556
|
+
for rel_data in data.get('relationships', []):
|
|
557
|
+
relationships.append(_parse_relationship(rel_data))
|
|
558
|
+
|
|
559
|
+
store_keys = _parse_store(data.get('store'))
|
|
560
|
+
|
|
561
|
+
# Parse RDF config
|
|
562
|
+
rdf_config = None
|
|
563
|
+
rdf_data = data.get('rdf')
|
|
564
|
+
if rdf_data:
|
|
565
|
+
rdf_config = RDFConfig(
|
|
566
|
+
base_iri=rdf_data.get('base_iri'),
|
|
567
|
+
vocab_iri=rdf_data.get('vocab_iri'),
|
|
568
|
+
graph_iri=rdf_data.get('graph_iri'),
|
|
569
|
+
r2rml=rdf_data.get('r2rml'),
|
|
570
|
+
auto_generate=rdf_data.get('auto_generate', False),
|
|
571
|
+
prefixes=rdf_data.get('prefixes', {}),
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
return GraphConfig(
|
|
575
|
+
version=version,
|
|
576
|
+
name=name,
|
|
577
|
+
comment=comment,
|
|
578
|
+
sources=sources,
|
|
579
|
+
nodes=nodes,
|
|
580
|
+
relationships=relationships,
|
|
581
|
+
variables=variables,
|
|
582
|
+
rdf=rdf_config,
|
|
583
|
+
store_keys=store_keys,
|
|
584
|
+
decimal_text=graph_data.get('decimal_text'),
|
|
585
|
+
procedures=_parse_procedures(data.get('procedures')),
|
|
586
|
+
)
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def _parse_procedures(data: Any) -> List[Any]:
|
|
590
|
+
"""`procedures:` a list of {name, inputs: [{name, type}], outputs: [{name, type}], and rows: [[...]] or
|
|
591
|
+
source: <table or view> (with an optional columns: {name: column})}, each declared through
|
|
592
|
+
`cypher_procedures.declare`, so a YAML section and `engine.declare_procedure` check alike."""
|
|
593
|
+
from prograph.plan.ir.cypher_procedures import declare
|
|
594
|
+
if not data:
|
|
595
|
+
return []
|
|
596
|
+
if not isinstance(data, list):
|
|
597
|
+
raise ValueError("procedures: is a list of procedures")
|
|
598
|
+
found = []
|
|
599
|
+
for spec in data:
|
|
600
|
+
if not isinstance(spec, dict) or not isinstance(spec.get('name'), str):
|
|
601
|
+
raise ValueError("procedures: each procedure has a name")
|
|
602
|
+
name = spec['name']
|
|
603
|
+
extra = set(spec) - {'name', 'inputs', 'outputs', 'rows', 'source', 'columns'}
|
|
604
|
+
if extra:
|
|
605
|
+
raise ValueError(f"procedure {name}: unknown key {', '.join(sorted(map(str, extra)))}")
|
|
606
|
+
|
|
607
|
+
def pairs(key):
|
|
608
|
+
out = []
|
|
609
|
+
for item in spec.get(key) or []:
|
|
610
|
+
if not isinstance(item, dict) or set(item) != {'name', 'type'} \
|
|
611
|
+
or not isinstance(item['name'], str) or not isinstance(item['type'], str):
|
|
612
|
+
raise ValueError(f"procedure {name}: {key} are {{name: ..., type: ...}} entries")
|
|
613
|
+
out.append((item['name'], item['type']))
|
|
614
|
+
return out
|
|
615
|
+
rows = spec.get('rows')
|
|
616
|
+
if rows is not None and not (isinstance(rows, list) and all(isinstance(r, list) for r in rows)):
|
|
617
|
+
raise ValueError(f"procedure {name}: rows is a list of rows, each a list")
|
|
618
|
+
columns = spec.get('columns')
|
|
619
|
+
if columns is not None and not isinstance(columns, dict):
|
|
620
|
+
raise ValueError(f"procedure {name}: columns maps an input or output name to a column")
|
|
621
|
+
found.append(declare(name, pairs('inputs'), pairs('outputs'), rows, spec.get('source'), columns))
|
|
622
|
+
return found
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def _parse_store(data: Any) -> Dict[str, Tuple[str, ...]]:
|
|
626
|
+
"""`store: labels: <Label>: {key: [properties]}`: each label's MERGE key (a list of property names)."""
|
|
627
|
+
if not data:
|
|
628
|
+
return {}
|
|
629
|
+
labels = data.get('labels') if isinstance(data, dict) else None
|
|
630
|
+
if not isinstance(labels, dict) or set(data) - {'labels'}:
|
|
631
|
+
raise ValueError("store: holds labels: <Label>: {key: [properties]}")
|
|
632
|
+
keys = {}
|
|
633
|
+
for label, spec in labels.items():
|
|
634
|
+
key = spec.get('key') if isinstance(spec, dict) else None
|
|
635
|
+
if isinstance(key, str):
|
|
636
|
+
key = [key]
|
|
637
|
+
if not key or not all(isinstance(k, str) and k for k in key) or set(spec) - {'key'}:
|
|
638
|
+
raise ValueError(f"store: labels: {label}: needs key: [properties], the properties MERGE creates "
|
|
639
|
+
f"a {label} by")
|
|
640
|
+
keys[str(label)] = tuple(key)
|
|
641
|
+
return keys
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def load_config(path: Union[str, Path],
|
|
645
|
+
variables: Optional[Dict[str, Any]] = None) -> GraphConfig:
|
|
646
|
+
"""Load graph configuration from a YAML file.
|
|
647
|
+
|
|
648
|
+
Args:
|
|
649
|
+
path: Path to YAML configuration file
|
|
650
|
+
variables: Values for the file's ``${var.name}`` placeholders, over
|
|
651
|
+
the file's own ``variables``. A mapping says which tables become
|
|
652
|
+
which labels; whoever loads it says where those tables are (a
|
|
653
|
+
catalog and schema), so one file serves several places.
|
|
654
|
+
|
|
655
|
+
Returns:
|
|
656
|
+
Parsed GraphConfig
|
|
657
|
+
|
|
658
|
+
Raises:
|
|
659
|
+
FileNotFoundError: If file doesn't exist
|
|
660
|
+
ValueError: If configuration is invalid
|
|
661
|
+
"""
|
|
662
|
+
path = Path(path)
|
|
663
|
+
if not path.exists():
|
|
664
|
+
raise FileNotFoundError(f"Configuration file not found: {path}")
|
|
665
|
+
|
|
666
|
+
with open(path, 'r') as f:
|
|
667
|
+
data = yaml.safe_load(f)
|
|
668
|
+
|
|
669
|
+
return _parse_config(data, variables)
|
|
670
|
+
|
|
671
|
+
|
|
672
|
+
def load_config_from_string(yaml_string: str) -> GraphConfig:
|
|
673
|
+
"""Load graph configuration from a YAML string.
|
|
674
|
+
|
|
675
|
+
Args:
|
|
676
|
+
yaml_string: YAML configuration as string
|
|
677
|
+
|
|
678
|
+
Returns:
|
|
679
|
+
Parsed GraphConfig
|
|
680
|
+
"""
|
|
681
|
+
data = yaml.safe_load(yaml_string)
|
|
682
|
+
return _parse_config(data)
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
# =============================================================================
|
|
686
|
+
# Validation
|
|
687
|
+
# =============================================================================
|
|
688
|
+
|
|
689
|
+
def validate_config(config: GraphConfig) -> List[str]:
|
|
690
|
+
"""Validate a graph configuration.
|
|
691
|
+
|
|
692
|
+
Args:
|
|
693
|
+
config: Configuration to validate
|
|
694
|
+
|
|
695
|
+
Returns:
|
|
696
|
+
List of validation errors (empty if valid)
|
|
697
|
+
"""
|
|
698
|
+
errors = []
|
|
699
|
+
|
|
700
|
+
# Check that all node sources exist
|
|
701
|
+
for node in config.nodes:
|
|
702
|
+
if node.source not in config.sources:
|
|
703
|
+
errors.append(f"Node '{node.label}' references unknown source '{node.source}'")
|
|
704
|
+
|
|
705
|
+
# Check that all relationship sources exist
|
|
706
|
+
for rel in config.relationships:
|
|
707
|
+
if rel.source not in config.sources:
|
|
708
|
+
errors.append(f"Relationship '{rel.type}' references unknown source '{rel.source}'")
|
|
709
|
+
|
|
710
|
+
# Check that relationship endpoints reference defined node labels
|
|
711
|
+
defined_labels = {node.label for node in config.nodes}
|
|
712
|
+
for rel in config.relationships:
|
|
713
|
+
if rel.start_node.identity is None and rel.start_node.label is not None \
|
|
714
|
+
and rel.start_node.label not in defined_labels:
|
|
715
|
+
errors.append(
|
|
716
|
+
f"Relationship '{rel.type}' start_node references "
|
|
717
|
+
f"undefined label '{rel.start_node.label}'"
|
|
718
|
+
)
|
|
719
|
+
if rel.end_node.identity is None and rel.end_node.label is not None \
|
|
720
|
+
and rel.end_node.label not in defined_labels:
|
|
721
|
+
errors.append(
|
|
722
|
+
f"Relationship '{rel.type}' end_node references "
|
|
723
|
+
f"undefined label '{rel.end_node.label}'"
|
|
724
|
+
)
|
|
725
|
+
|
|
726
|
+
# An end gives a label or an identity, one of them; an identity some node declares (as to_schema refuses)
|
|
727
|
+
declared = {node.identity for node in config.nodes if getattr(node, "identity", None)}
|
|
728
|
+
for rel in config.relationships:
|
|
729
|
+
for side, end in (("start_node", rel.start_node), ("end_node", rel.end_node)):
|
|
730
|
+
if end.label is not None and end.identity is not None:
|
|
731
|
+
errors.append(f"Relationship '{rel.type}' {side} gives both a label and an identity: give one")
|
|
732
|
+
elif end.label is None and end.identity is None:
|
|
733
|
+
errors.append(f"Relationship '{rel.type}' {side} gives no label: give a label, or an identity for "
|
|
734
|
+
f"any node of one identity")
|
|
735
|
+
elif end.identity in ("global", "distinct"):
|
|
736
|
+
errors.append(f"Relationship '{rel.type}' {side} gives the identity '{end.identity}', which names "
|
|
737
|
+
f"no one set of nodes")
|
|
738
|
+
elif end.identity is not None and end.identity not in declared:
|
|
739
|
+
errors.append(f"Relationship '{rel.type}' {side} gives the identity '{end.identity}', which no "
|
|
740
|
+
f"node declares")
|
|
741
|
+
|
|
742
|
+
# Check for duplicate node labels
|
|
743
|
+
labels = [node.label for node in config.nodes if node.label is not None]
|
|
744
|
+
seen = set()
|
|
745
|
+
for label in labels:
|
|
746
|
+
if label in seen:
|
|
747
|
+
errors.append(f"Duplicate node label: '{label}'")
|
|
748
|
+
seen.add(label)
|
|
749
|
+
|
|
750
|
+
# Check for duplicate relationship types (warning, not error)
|
|
751
|
+
# Multiple relationships of same type are allowed (e.g., KNOWS from different sources)
|
|
752
|
+
|
|
753
|
+
return errors
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
# =============================================================================
|
|
757
|
+
# DataFrame Loading
|
|
758
|
+
# =============================================================================
|
|
759
|
+
|
|
760
|
+
def load_dataframes(
|
|
761
|
+
config: GraphConfig,
|
|
762
|
+
spark: Optional[Any] = None,
|
|
763
|
+
base_path: Optional[Union[str, Path]] = None
|
|
764
|
+
) -> Dict[str, Any]:
|
|
765
|
+
"""Load DataFrames from configured sources.
|
|
766
|
+
|
|
767
|
+
Args:
|
|
768
|
+
config: Graph configuration
|
|
769
|
+
spark: SparkSession (optional, uses pandas if not provided)
|
|
770
|
+
base_path: Base path for relative file paths
|
|
771
|
+
|
|
772
|
+
Returns:
|
|
773
|
+
Dict mapping source names to DataFrames
|
|
774
|
+
"""
|
|
775
|
+
dataframes = {}
|
|
776
|
+
base_path = Path(base_path) if base_path else Path('.')
|
|
777
|
+
|
|
778
|
+
for name, source in config.sources.items():
|
|
779
|
+
if source.table:
|
|
780
|
+
# Load from Unity Catalog table
|
|
781
|
+
if spark is None:
|
|
782
|
+
raise ValueError(f"SparkSession required to load table '{source.table}'")
|
|
783
|
+
dataframes[name] = spark.table(source.table)
|
|
784
|
+
|
|
785
|
+
elif source.file:
|
|
786
|
+
# Load from file
|
|
787
|
+
file_path = Path(source.file)
|
|
788
|
+
if not file_path.is_absolute():
|
|
789
|
+
file_path = base_path / file_path
|
|
790
|
+
|
|
791
|
+
if spark is not None:
|
|
792
|
+
# Use Spark
|
|
793
|
+
options = source.options.copy()
|
|
794
|
+
file_str = str(file_path)
|
|
795
|
+
|
|
796
|
+
if file_str.endswith('.csv'):
|
|
797
|
+
df = spark.read.csv(
|
|
798
|
+
file_str,
|
|
799
|
+
header=options.pop('header', True),
|
|
800
|
+
inferSchema=options.pop('inferSchema', True),
|
|
801
|
+
sep=options.pop('delimiter', ','),
|
|
802
|
+
**options
|
|
803
|
+
)
|
|
804
|
+
elif file_str.endswith('.parquet'):
|
|
805
|
+
df = spark.read.parquet(file_str, **options)
|
|
806
|
+
elif file_str.endswith('.json'):
|
|
807
|
+
df = spark.read.json(file_str, **options)
|
|
808
|
+
else:
|
|
809
|
+
raise ValueError(f"Unsupported file format: {file_path.suffix}")
|
|
810
|
+
|
|
811
|
+
dataframes[name] = df
|
|
812
|
+
else:
|
|
813
|
+
# Use pandas
|
|
814
|
+
import pandas as pd
|
|
815
|
+
options = source.options.copy()
|
|
816
|
+
file_str = str(file_path)
|
|
817
|
+
|
|
818
|
+
if file_str.endswith('.csv'):
|
|
819
|
+
# The same options as the Spark reader: header and
|
|
820
|
+
# inferSchema are booleans there, which pandas spells
|
|
821
|
+
# header=0/None and dtype=str.
|
|
822
|
+
header = options.pop('header', True)
|
|
823
|
+
if isinstance(header, bool):
|
|
824
|
+
header = 0 if header else None
|
|
825
|
+
if options.pop('inferSchema', True) is False:
|
|
826
|
+
options.setdefault('dtype', str)
|
|
827
|
+
df = pd.read_csv(
|
|
828
|
+
file_str,
|
|
829
|
+
header=header,
|
|
830
|
+
delimiter=options.pop('delimiter', ','),
|
|
831
|
+
**options
|
|
832
|
+
)
|
|
833
|
+
elif file_str.endswith('.parquet'):
|
|
834
|
+
df = pd.read_parquet(file_str, **options)
|
|
835
|
+
elif file_str.endswith('.json'):
|
|
836
|
+
df = pd.read_json(file_str, **options)
|
|
837
|
+
else:
|
|
838
|
+
raise ValueError(f"Unsupported file format: {file_path.suffix}")
|
|
839
|
+
|
|
840
|
+
dataframes[name] = df
|
|
841
|
+
|
|
842
|
+
elif source.query:
|
|
843
|
+
# Load from SQL query
|
|
844
|
+
if spark is None:
|
|
845
|
+
raise ValueError(f"SparkSession required to execute query for source '{name}'")
|
|
846
|
+
dataframes[name] = spark.sql(source.query)
|
|
847
|
+
|
|
848
|
+
return dataframes
|