prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,848 @@
1
+ """YAML Configuration for ProGraph.
2
+
3
+ Provides declarative configuration of graph schemas and data sources using YAML,
4
+ inspired by Databricks Metric Views and Asset Bundles (DABs) patterns.
5
+
6
+ Example YAML:
7
+
8
+ version: "1.0"
9
+
10
+ graph:
11
+ name: social_network
12
+ comment: "LDBC SNB social network schema"
13
+ # decimal_text: database # a DECIMAL column's text as the database writes it (Ontop's); default canonical
14
+
15
+ sources:
16
+ person:
17
+ table: catalog.schema.person
18
+ knows:
19
+ file: ./data/person_knows_person.csv
20
+ options:
21
+ delimiter: "|"
22
+ header: true
23
+
24
+ nodes:
25
+ - label: Person
26
+ source: person
27
+ id_column: id
28
+ properties:
29
+ firstName: firstName
30
+ lastName: lastName
31
+
32
+ relationships:
33
+ - type: KNOWS
34
+ source: knows
35
+ start_node:
36
+ label: Person
37
+ column: Person.id
38
+ end_node:
39
+ label: Person
40
+ column: Person.id.1
41
+
42
+ Usage:
43
+
44
+ from prograph import ProGraph
45
+ from prograph.config import load_config
46
+
47
+ config = load_config('graph_schema.yaml')
48
+ engine = ProGraph.from_config(config, spark=spark)
49
+ """
50
+
51
+ import os
52
+ import re
53
+ from dataclasses import dataclass, field
54
+ from pathlib import Path
55
+ from typing import Any, Dict, List, Optional, Tuple, Union
56
+
57
+ import yaml
58
+
59
+ from prograph.schema.mapping import policy_name
60
+ from prograph.plan.ir.keys import key_trust_name
61
+ from prograph.schema import GraphSchema, NodeMapping, RelationshipMapping
62
+ from prograph.rules import home
63
+
64
+
65
+ # =============================================================================
66
+ # Configuration Data Classes
67
+ # =============================================================================
68
+
69
+ @dataclass
70
+ class SourceConfig:
71
+ """Configuration for a data source.
72
+
73
+ Attributes:
74
+ name: Source identifier (key in sources mapping)
75
+ table: Unity Catalog table reference (catalog.schema.table)
76
+ file: File path (CSV, Parquet, JSON)
77
+ query: SQL query to define the source
78
+ options: Reader options (delimiter, header, inferSchema, etc.)
79
+ comment: Description of the source
80
+ """
81
+ name: str
82
+ table: Optional[str] = None
83
+ file: Optional[str] = None
84
+ query: Optional[str] = None
85
+ options: Dict[str, Any] = field(default_factory=dict)
86
+ comment: Optional[str] = None
87
+
88
+ def __post_init__(self):
89
+ sources_defined = sum(1 for s in [self.table, self.file, self.query] if s)
90
+ if sources_defined == 0:
91
+ raise ValueError(f"Source '{self.name}' must define one of: table, file, or query")
92
+ if sources_defined > 1:
93
+ raise ValueError(f"Source '{self.name}' must define only one of: table, file, or query")
94
+
95
+
96
+ @dataclass
97
+ class NodeConfig:
98
+ """Configuration for a node mapping.
99
+
100
+ Attributes:
101
+ label: Node label (e.g., 'Person', 'Post')
102
+ source: Reference to a source name
103
+ id_column: Column(s) containing node IDs. Can be a single column name
104
+ or a list of column names for composite keys.
105
+ properties: Mapping of property names to column names
106
+ comment: Description of the node type
107
+ mutable: If True, ``engine.set_properties`` may write this label's
108
+ rows (prograph.writes); nothing else writes a mapped table
109
+ graph: Optional named graph IRI for this node mapping
110
+ filter_predicate: Optional dict of column=value discriminator filters
111
+ iri_template: R2RML-style template deciding this node's IRI, e.g.
112
+ 'http://example.org/person/{person_id}'. Without it the IRI is
113
+ synthesized from the table and id.
114
+ class_iri: The class this node's rows are typed with, how a mapping
115
+ binds to an externally-authored ontology instead of an invented
116
+ class. Defaults to '{vocab}{Label}'.
117
+ rdf_class: False for a label that only groups rows: no rdf:type
118
+ triple for it. Defaults to True.
119
+ on_duplicate: What a Cypher statement does when two rows hold one
120
+ key value: 'refuse' (the engine's default) or 'trust'
121
+ (prograph.node_keys). None leaves it to the engine.
122
+ keys: 'trusted' (the user asserts the key unique, so a plan may
123
+ rely on it) or 'check' (verified per data version; not built
124
+ yet). None relies on it only where the database enforces it
125
+ (one-plan.md 2.4).
126
+ """
127
+ label: Optional[str]
128
+ source: str
129
+ id_column: Union[str, List[str]]
130
+ properties: Dict[str, str] = field(default_factory=dict)
131
+ comment: Optional[str] = None
132
+ mutable: bool = False
133
+ graph: Optional[str] = None
134
+ filter_predicate: Optional[Dict[str, Any]] = None
135
+ iri_template: Optional[str] = None
136
+ class_iri: Optional[str] = None
137
+ rdf_class: bool = True
138
+ on_duplicate: Optional[str] = None
139
+ keys: Optional[str] = None
140
+ identity: Optional[str] = None
141
+
142
+
143
+ @dataclass
144
+ class RelationshipEndpoint:
145
+ """Configuration for a relationship endpoint (start or end node).
146
+
147
+ Attributes:
148
+ label: The node label for this endpoint; None for an end given by
149
+ an identity.
150
+ column: Column(s) for the foreign key reference. Can be a single column
151
+ name or a list of column names for composite keys.
152
+ identity: An end given by an identity instead of a label: the node of
153
+ its key in any node mapping that declares `identity: <name>`.
154
+ """
155
+ label: Optional[str]
156
+ column: Union[str, List[str]]
157
+ identity: Optional[str] = None
158
+
159
+
160
+ @dataclass
161
+ class RelationshipConfig:
162
+ """Configuration for a relationship mapping.
163
+
164
+ Attributes:
165
+ type: Relationship type (e.g., 'KNOWS', 'LIKES')
166
+ source: Reference to a source name
167
+ start_node: Start node configuration (label and column)
168
+ end_node: End node configuration (label and column)
169
+ properties: Mapping of property names to column names
170
+ comment: Description of the relationship type
171
+ mutable: If True, OWL RL may write inferred edges into this table; nothing else does
172
+ graph: Optional named graph IRI for this relationship mapping
173
+ filter_predicate: Optional dict of column=value discriminator filters,
174
+ for several relationship types sharing one table
175
+ reify: ``true`` is refused (``NotBuilt``): reified edge statements are not built on the
176
+ SPARQL graph; they return as RDF 1.2 reifiers. Unset or ``false`` load.
177
+ relation_class_iri: The class a reified statement node is typed with.
178
+ Defaults to '{vocab}{PascalCaseRelType}'.
179
+ referential_integrity: Every key in the table has a node row, so SQL
180
+ may drop node joins that only connect relationships.
181
+ keys: 'trusted' (no two rows hold one start and end, asserted) or
182
+ 'check' (not built yet); None relies only on what the database
183
+ enforces.
184
+ id_column: The column (or columns) that names one relationship, its
185
+ id; unset, a relationship is its row (Cypher tells two rows of one
186
+ start and end apart by their number among them).
187
+ """
188
+ type: str
189
+ source: str
190
+ start_node: RelationshipEndpoint
191
+ end_node: RelationshipEndpoint
192
+ properties: Dict[str, str] = field(default_factory=dict)
193
+ comment: Optional[str] = None
194
+ mutable: bool = False
195
+ graph: Optional[str] = None
196
+ filter_predicate: Optional[Dict[str, Any]] = None
197
+ reify: Optional[bool] = None
198
+ relation_class_iri: Optional[str] = None
199
+ referential_integrity: bool = False
200
+ keys: Optional[str] = None
201
+ id_column: Optional[Union[str, List[str]]] = None
202
+
203
+
204
+ @dataclass
205
+ class RDFConfig:
206
+ """RDF/R2RML configuration for SPARQL and ontology integration.
207
+
208
+ Attributes:
209
+ base_iri: Base IRI for node IRIs, set as the schema's ``base_iri``:
210
+ a node mapping with no iri_template is
211
+ ``{base_iri}{source_table}/{id}``
212
+ vocab_iri: Vocabulary IRI: a node mapping with no class_iri gets
213
+ ``{vocab_iri}{Label}``, and auto_generate mints its predicates
214
+ in this namespace (the default vocabulary,
215
+ ``prograph.schema.iri.DEFAULT_VOCAB_IRI``, when unset)
216
+ graph_iri: Optional default named graph IRI
217
+ r2rml: Path to an R2RML Turtle file to load and compile
218
+ auto_generate: If True, auto-generate predicate mappings from schema
219
+ prefixes: Additional namespace prefix -> IRI mappings
220
+ """
221
+ base_iri: Optional[str] = None
222
+ vocab_iri: Optional[str] = None
223
+ graph_iri: Optional[str] = None
224
+ r2rml: Optional[str] = None
225
+ auto_generate: bool = False
226
+ prefixes: Dict[str, str] = field(default_factory=dict)
227
+
228
+
229
+ @dataclass
230
+ class GraphConfig:
231
+ """Complete graph configuration.
232
+
233
+ Attributes:
234
+ version: Configuration version (default: "1.0")
235
+ name: Graph name
236
+ comment: Description of the graph
237
+ sources: Data source configurations
238
+ nodes: Node mapping configurations
239
+ relationships: Relationship mapping configurations
240
+ variables: Variable substitutions
241
+ rdf: Optional RDF/R2RML configuration
242
+ decimal_text: ``graph: decimal_text:``, how a DECIMAL column's value
243
+ is written in a term (``GraphSchema``'s ``decimal_text``):
244
+ ``canonical`` (the default) or ``database`` (Ontop's text)
245
+ """
246
+ version: str = "1.0"
247
+ name: str = "prograph"
248
+ comment: Optional[str] = None
249
+ sources: Dict[str, SourceConfig] = field(default_factory=dict)
250
+ nodes: List[NodeConfig] = field(default_factory=list)
251
+ relationships: List[RelationshipConfig] = field(default_factory=list)
252
+ variables: Dict[str, Any] = field(default_factory=dict)
253
+ rdf: Optional[RDFConfig] = None
254
+ # The owned store's labels (`store: labels: <Label>: {key: [properties]}`): the key Cypher's MERGE creates
255
+ # a node of that label by, its values giving the node's IRI.
256
+ store_keys: Dict[str, Tuple[str, ...]] = field(default_factory=dict)
257
+ decimal_text: Optional[str] = None
258
+ # `procedures:` (Cypher): declared procedures, `cypher_procedures.Procedure`s.
259
+ procedures: List[Any] = field(default_factory=list)
260
+
261
+ @home("config-source-tables", "A mapping's table from a config file",
262
+ terms=("source_table", "by_table", "to_schema", "YAML config", "Genie"))
263
+ def to_schema(self, by_table: bool = False) -> GraphSchema:
264
+ """Convert configuration to a GraphSchema.
265
+
266
+ Args:
267
+ by_table: What a mapping's ``source_table`` is. By default it is
268
+ the source's name (its key under ``sources:``), which is how
269
+ ``load_dataframes`` keys the frames it loads; the source's
270
+ declared table is kept on the schema (``declared_tables``), and
271
+ an engine reads it there unless it is given a frame of that
272
+ name (``engine.apply_declared_tables``). With
273
+ ``by_table=True`` it is the table the source declares
274
+ (``sources.<name>.table``, such as
275
+ ``catalog.schema.table``), for a schema whose SQL runs in a
276
+ database that knows the table by that name: a SQL
277
+ warehouse, generated views. A source with no ``table:`` (a
278
+ file) keeps its name either way.
279
+
280
+ Returns:
281
+ GraphSchema with node and relationship mappings
282
+ """
283
+ schema = GraphSchema(name=self.name, decimal_text=self.decimal_text)
284
+
285
+ def table_of(source_name: str) -> str:
286
+ declared = self.sources[source_name].table if by_table else None
287
+ return declared or source_name
288
+
289
+ def named(mapping, source_name: str):
290
+ # A synthesized IRI carries the source's name with or without by_table, so one source mints
291
+ # one IRI (`iri.iri_segment`); by_table only chooses the table read.
292
+ if table_of(source_name) != source_name:
293
+ mapping.iri_segment = source_name
294
+ return mapping
295
+
296
+ # Each source's declared table, where the mappings name the source: an engine that reads
297
+ # tables (a SQL connection, a Spark session) reads it there, one given a frame of that name
298
+ # reads the frame (`ProGraph`, `apply_declared_tables`).
299
+ if not by_table:
300
+ schema.declared_tables = {name: src.table for name, src in self.sources.items()
301
+ if getattr(src, "table", None) and src.table != name}
302
+ # The sources that are files (no table): a SQL backend refuses them by name when the engine starts.
303
+ # The owned store's MERGE keys, one record every writer reads; a mapped label has its mapping's key.
304
+ schema.file_sources = {name for name, src in self.sources.items()
305
+ if getattr(src, "file", None) and not getattr(src, "table", None)}
306
+
307
+ # Add node mappings
308
+ for node in self.nodes:
309
+ source = self.sources.get(node.source)
310
+ if not source:
311
+ raise ValueError(f"Node '{node.label}' references unknown source '{node.source}'")
312
+
313
+ schema.add_node_mapping(named(NodeMapping(
314
+ source_table=table_of(node.source),
315
+ label=node.label,
316
+ id_column=node.id_column,
317
+ property_columns=node.properties,
318
+ mutable=node.mutable,
319
+ graph_iri=node.graph,
320
+ filter_predicate=node.filter_predicate,
321
+ iri_template=node.iri_template,
322
+ class_iri=node.class_iri,
323
+ rdf_class=node.rdf_class,
324
+ on_duplicate=node.on_duplicate,
325
+ keys=node.keys,
326
+ identity=node.identity,
327
+ ), node.source))
328
+
329
+ # Add relationship mappings
330
+ for rel in self.relationships:
331
+ source = self.sources.get(rel.source)
332
+ if not source:
333
+ raise ValueError(f"Relationship '{rel.type}' references unknown source '{rel.source}'")
334
+
335
+ schema.add_relationship_mapping(named(RelationshipMapping(
336
+ source_table=table_of(rel.source),
337
+ rel_type=rel.type,
338
+ start_node_label=rel.start_node.label,
339
+ start_node_identity=rel.start_node.identity,
340
+ end_node_identity=rel.end_node.identity,
341
+ start_id_column=rel.start_node.column,
342
+ end_node_label=rel.end_node.label,
343
+ end_id_column=rel.end_node.column,
344
+ property_columns=rel.properties,
345
+ mutable=rel.mutable,
346
+ graph_iri=rel.graph,
347
+ filter_predicate=rel.filter_predicate,
348
+ reify=rel.reify,
349
+ relation_class_iri=rel.relation_class_iri,
350
+ referential_integrity=rel.referential_integrity,
351
+ keys=rel.keys,
352
+ id_column=rel.id_column,
353
+ ), rel.source))
354
+
355
+ # RDF/R2RML integration
356
+ if self.rdf:
357
+ # Load R2RML mapping and merge into schema
358
+ if self.rdf.r2rml:
359
+ # The document's triples maps join the schema's source maps, read by SPARQL with
360
+ # no property-graph labels.
361
+ from prograph.r2rml.parser import R2RMLParser
362
+ from prograph.r2rml.schema import include
363
+ include(schema, R2RMLParser().parse_file(self.rdf.r2rml))
364
+
365
+ # Namespaces. The base IRI of instance IRIs has one home, the
366
+ # schema; a class IRI fills what a mapping does not declare.
367
+ if self.rdf.base_iri:
368
+ schema.base_iri = self.rdf.base_iri
369
+ for nm in schema.node_mappings:
370
+ if self.rdf.vocab_iri and not nm.class_iri and nm.label is not None:
371
+ nm.class_iri = f"{self.rdf.vocab_iri}{nm.label}"
372
+
373
+ # Auto-generate predicate mappings
374
+ if self.rdf.auto_generate:
375
+ if self.rdf.vocab_iri:
376
+ schema.auto_generate_predicate_mappings(
377
+ namespace=self.rdf.vocab_iri)
378
+ else:
379
+ schema.auto_generate_predicate_mappings()
380
+
381
+ # Register prefixes
382
+ for prefix, ns in self.rdf.prefixes.items():
383
+ schema.add_prefix(prefix, ns)
384
+
385
+ # The owned store's MERGE keys, one record every writer reads (`GraphSchema.set_store_keys`).
386
+ schema.set_store_keys(dict(self.store_keys))
387
+ return schema
388
+
389
+
390
+ # =============================================================================
391
+ # Variable Substitution
392
+ # =============================================================================
393
+
394
+ def substitute_variables(value: Any, variables: Dict[str, Any]) -> Any:
395
+ """Substitute ${var.name} patterns in strings.
396
+
397
+ Follows Databricks Asset Bundles variable substitution pattern.
398
+
399
+ Args:
400
+ value: Value to process (string, dict, or list)
401
+ variables: Variable definitions
402
+
403
+ Returns:
404
+ Value with variables substituted
405
+ """
406
+ if isinstance(value, str):
407
+ # Pattern: ${var.variable_name}
408
+ pattern = r'\$\{var\.(\w+)\}'
409
+
410
+ def replace(match):
411
+ var_name = match.group(1)
412
+ if var_name not in variables:
413
+ raise ValueError(f"Undefined variable: {var_name}")
414
+ var_value = variables[var_name]
415
+ # Handle variable objects with 'default' key
416
+ if isinstance(var_value, dict) and 'default' in var_value:
417
+ return str(var_value['default'])
418
+ return str(var_value)
419
+
420
+ return re.sub(pattern, replace, value)
421
+
422
+ elif isinstance(value, dict):
423
+ return {k: substitute_variables(v, variables) for k, v in value.items()}
424
+
425
+ elif isinstance(value, list):
426
+ return [substitute_variables(item, variables) for item in value]
427
+
428
+ return value
429
+
430
+
431
+ # =============================================================================
432
+ # YAML Parsing
433
+ # =============================================================================
434
+
435
+ def _parse_source(name: str, data: Dict[str, Any]) -> SourceConfig:
436
+ """Parse a source configuration from YAML data."""
437
+ return SourceConfig(
438
+ name=name,
439
+ table=data.get('table'),
440
+ file=data.get('file'),
441
+ query=data.get('query'),
442
+ options=data.get('options', {}),
443
+ comment=data.get('comment')
444
+ )
445
+
446
+
447
+ # Keys that once made a mapping writable through a query. ProGraph is
448
+ # read-only over the tables it maps but for `set_properties` on a
449
+ # node mapping marked `mutable`, and OWL RL into a node or relationship
450
+ # mapping marked `mutable` (decided 2026-11-04; prograph/read_only.py):
451
+ # `mutable` is read, and a configuration that sets `id_generator` is told so
452
+ # rather than having it ignored.
453
+ _NODE_WRITE_KEYS = ("id_generator",)
454
+ _RELATIONSHIP_WRITE_KEYS = ("id_generator",)
455
+
456
+
457
+ def _refuse_write_keys(data: Dict[str, Any], what: str, keys: Tuple[str, ...]) -> None:
458
+ found = [k for k in keys if k in data]
459
+ if found:
460
+ raise ValueError(
461
+ f"{what}: {', '.join(found)} {'is' if len(found) == 1 else 'are'} no longer a "
462
+ f"mapping setting; ProGraph is read-only over the tables it maps, "
463
+ f"and a mapping's mutable opens only set_properties and OWL RL's inferred rows. "
464
+ f"Remove {'it' if len(found) == 1 else 'them'}.")
465
+
466
+
467
+ def _flag(data: Dict[str, Any], key: str, what: str) -> bool:
468
+ """A yes/no setting: true or false, absent being false. A string such
469
+ as "false" is refused, not read as true."""
470
+ value = data.get(key, False)
471
+ if not isinstance(value, bool):
472
+ raise ValueError(f"{what}: {key} is true or false, got {value!r}")
473
+ return value
474
+
475
+
476
+ def _parse_node(data: Dict[str, Any]) -> NodeConfig:
477
+ """Parse a node configuration from YAML data."""
478
+ _refuse_write_keys(data, f"node {data.get('label')!r}", _NODE_WRITE_KEYS)
479
+ return NodeConfig(
480
+ label=data.get('label'),
481
+ source=data['source'],
482
+ id_column=data['id_column'],
483
+ properties=data.get('properties', {}),
484
+ comment=data.get('comment'),
485
+ mutable=_flag(data, 'mutable', f"node {data.get('label')!r}"),
486
+ graph=data.get('graph'),
487
+ filter_predicate=data.get('filter_predicate'),
488
+ iri_template=data.get('iri_template'),
489
+ class_iri=data.get('class_iri'),
490
+ rdf_class=data.get('rdf_class', True),
491
+ on_duplicate=policy_name(data.get('on_duplicate'), f"node {data.get('label')!r}: on_duplicate"),
492
+ keys=key_trust_name(data.get('keys'), f"node {data.get('label')!r}: keys"),
493
+ identity=data.get('identity'),
494
+ )
495
+
496
+
497
+ def _parse_relationship(data: Dict[str, Any]) -> RelationshipConfig:
498
+ """Parse a relationship configuration from YAML data."""
499
+ _refuse_write_keys(data, f"relationship {data.get('type')!r}", _RELATIONSHIP_WRITE_KEYS)
500
+ start_node = data['start_node']
501
+ end_node = data['end_node']
502
+
503
+ return RelationshipConfig(
504
+ type=data['type'],
505
+ source=data['source'],
506
+ start_node=RelationshipEndpoint(
507
+ label=start_node.get('label'),
508
+ column=start_node['column'],
509
+ identity=start_node.get('identity'),
510
+ ),
511
+ end_node=RelationshipEndpoint(
512
+ label=end_node.get('label'),
513
+ column=end_node['column'],
514
+ identity=end_node.get('identity'),
515
+ ),
516
+ properties=data.get('properties', {}),
517
+ comment=data.get('comment'),
518
+ mutable=_flag(data, 'mutable', f"relationship {data.get('type')!r}"),
519
+ graph=data.get('graph'),
520
+ filter_predicate=data.get('filter_predicate'),
521
+ reify=data.get('reify'),
522
+ relation_class_iri=data.get('relation_class_iri'),
523
+ referential_integrity=bool(data.get('referential_integrity', False)),
524
+ keys=key_trust_name(data.get('keys'), f"relationship {data.get('type')!r}: keys"),
525
+ id_column=data.get('id_column'),
526
+ )
527
+
528
+
529
+ def _parse_config(data: Dict[str, Any],
530
+ overrides: Optional[Dict[str, Any]] = None) -> GraphConfig:
531
+ """Parse complete configuration from YAML data."""
532
+ # Extract variables first for substitution; a caller's values win.
533
+ variables = {**(data.get('variables') or {}), **(overrides or {})}
534
+
535
+ # Apply variable substitution to the entire config
536
+ data = substitute_variables(data, variables)
537
+
538
+ # Parse graph metadata
539
+ graph_data = data.get('graph', {})
540
+ version = data.get('version', '1.0')
541
+ name = graph_data.get('name', 'prograph')
542
+ comment = graph_data.get('comment')
543
+
544
+ # Parse sources
545
+ sources = {}
546
+ for source_name, source_data in data.get('sources', {}).items():
547
+ sources[source_name] = _parse_source(source_name, source_data)
548
+
549
+ # Parse nodes
550
+ nodes = []
551
+ for node_data in data.get('nodes', []):
552
+ nodes.append(_parse_node(node_data))
553
+
554
+ # Parse relationships
555
+ relationships = []
556
+ for rel_data in data.get('relationships', []):
557
+ relationships.append(_parse_relationship(rel_data))
558
+
559
+ store_keys = _parse_store(data.get('store'))
560
+
561
+ # Parse RDF config
562
+ rdf_config = None
563
+ rdf_data = data.get('rdf')
564
+ if rdf_data:
565
+ rdf_config = RDFConfig(
566
+ base_iri=rdf_data.get('base_iri'),
567
+ vocab_iri=rdf_data.get('vocab_iri'),
568
+ graph_iri=rdf_data.get('graph_iri'),
569
+ r2rml=rdf_data.get('r2rml'),
570
+ auto_generate=rdf_data.get('auto_generate', False),
571
+ prefixes=rdf_data.get('prefixes', {}),
572
+ )
573
+
574
+ return GraphConfig(
575
+ version=version,
576
+ name=name,
577
+ comment=comment,
578
+ sources=sources,
579
+ nodes=nodes,
580
+ relationships=relationships,
581
+ variables=variables,
582
+ rdf=rdf_config,
583
+ store_keys=store_keys,
584
+ decimal_text=graph_data.get('decimal_text'),
585
+ procedures=_parse_procedures(data.get('procedures')),
586
+ )
587
+
588
+
589
+ def _parse_procedures(data: Any) -> List[Any]:
590
+ """`procedures:` a list of {name, inputs: [{name, type}], outputs: [{name, type}], and rows: [[...]] or
591
+ source: <table or view> (with an optional columns: {name: column})}, each declared through
592
+ `cypher_procedures.declare`, so a YAML section and `engine.declare_procedure` check alike."""
593
+ from prograph.plan.ir.cypher_procedures import declare
594
+ if not data:
595
+ return []
596
+ if not isinstance(data, list):
597
+ raise ValueError("procedures: is a list of procedures")
598
+ found = []
599
+ for spec in data:
600
+ if not isinstance(spec, dict) or not isinstance(spec.get('name'), str):
601
+ raise ValueError("procedures: each procedure has a name")
602
+ name = spec['name']
603
+ extra = set(spec) - {'name', 'inputs', 'outputs', 'rows', 'source', 'columns'}
604
+ if extra:
605
+ raise ValueError(f"procedure {name}: unknown key {', '.join(sorted(map(str, extra)))}")
606
+
607
+ def pairs(key):
608
+ out = []
609
+ for item in spec.get(key) or []:
610
+ if not isinstance(item, dict) or set(item) != {'name', 'type'} \
611
+ or not isinstance(item['name'], str) or not isinstance(item['type'], str):
612
+ raise ValueError(f"procedure {name}: {key} are {{name: ..., type: ...}} entries")
613
+ out.append((item['name'], item['type']))
614
+ return out
615
+ rows = spec.get('rows')
616
+ if rows is not None and not (isinstance(rows, list) and all(isinstance(r, list) for r in rows)):
617
+ raise ValueError(f"procedure {name}: rows is a list of rows, each a list")
618
+ columns = spec.get('columns')
619
+ if columns is not None and not isinstance(columns, dict):
620
+ raise ValueError(f"procedure {name}: columns maps an input or output name to a column")
621
+ found.append(declare(name, pairs('inputs'), pairs('outputs'), rows, spec.get('source'), columns))
622
+ return found
623
+
624
+
625
+ def _parse_store(data: Any) -> Dict[str, Tuple[str, ...]]:
626
+ """`store: labels: <Label>: {key: [properties]}`: each label's MERGE key (a list of property names)."""
627
+ if not data:
628
+ return {}
629
+ labels = data.get('labels') if isinstance(data, dict) else None
630
+ if not isinstance(labels, dict) or set(data) - {'labels'}:
631
+ raise ValueError("store: holds labels: <Label>: {key: [properties]}")
632
+ keys = {}
633
+ for label, spec in labels.items():
634
+ key = spec.get('key') if isinstance(spec, dict) else None
635
+ if isinstance(key, str):
636
+ key = [key]
637
+ if not key or not all(isinstance(k, str) and k for k in key) or set(spec) - {'key'}:
638
+ raise ValueError(f"store: labels: {label}: needs key: [properties], the properties MERGE creates "
639
+ f"a {label} by")
640
+ keys[str(label)] = tuple(key)
641
+ return keys
642
+
643
+
644
+ def load_config(path: Union[str, Path],
645
+ variables: Optional[Dict[str, Any]] = None) -> GraphConfig:
646
+ """Load graph configuration from a YAML file.
647
+
648
+ Args:
649
+ path: Path to YAML configuration file
650
+ variables: Values for the file's ``${var.name}`` placeholders, over
651
+ the file's own ``variables``. A mapping says which tables become
652
+ which labels; whoever loads it says where those tables are (a
653
+ catalog and schema), so one file serves several places.
654
+
655
+ Returns:
656
+ Parsed GraphConfig
657
+
658
+ Raises:
659
+ FileNotFoundError: If file doesn't exist
660
+ ValueError: If configuration is invalid
661
+ """
662
+ path = Path(path)
663
+ if not path.exists():
664
+ raise FileNotFoundError(f"Configuration file not found: {path}")
665
+
666
+ with open(path, 'r') as f:
667
+ data = yaml.safe_load(f)
668
+
669
+ return _parse_config(data, variables)
670
+
671
+
672
+ def load_config_from_string(yaml_string: str) -> GraphConfig:
673
+ """Load graph configuration from a YAML string.
674
+
675
+ Args:
676
+ yaml_string: YAML configuration as string
677
+
678
+ Returns:
679
+ Parsed GraphConfig
680
+ """
681
+ data = yaml.safe_load(yaml_string)
682
+ return _parse_config(data)
683
+
684
+
685
+ # =============================================================================
686
+ # Validation
687
+ # =============================================================================
688
+
689
+ def validate_config(config: GraphConfig) -> List[str]:
690
+ """Validate a graph configuration.
691
+
692
+ Args:
693
+ config: Configuration to validate
694
+
695
+ Returns:
696
+ List of validation errors (empty if valid)
697
+ """
698
+ errors = []
699
+
700
+ # Check that all node sources exist
701
+ for node in config.nodes:
702
+ if node.source not in config.sources:
703
+ errors.append(f"Node '{node.label}' references unknown source '{node.source}'")
704
+
705
+ # Check that all relationship sources exist
706
+ for rel in config.relationships:
707
+ if rel.source not in config.sources:
708
+ errors.append(f"Relationship '{rel.type}' references unknown source '{rel.source}'")
709
+
710
+ # Check that relationship endpoints reference defined node labels
711
+ defined_labels = {node.label for node in config.nodes}
712
+ for rel in config.relationships:
713
+ if rel.start_node.identity is None and rel.start_node.label is not None \
714
+ and rel.start_node.label not in defined_labels:
715
+ errors.append(
716
+ f"Relationship '{rel.type}' start_node references "
717
+ f"undefined label '{rel.start_node.label}'"
718
+ )
719
+ if rel.end_node.identity is None and rel.end_node.label is not None \
720
+ and rel.end_node.label not in defined_labels:
721
+ errors.append(
722
+ f"Relationship '{rel.type}' end_node references "
723
+ f"undefined label '{rel.end_node.label}'"
724
+ )
725
+
726
+ # An end gives a label or an identity, one of them; an identity some node declares (as to_schema refuses)
727
+ declared = {node.identity for node in config.nodes if getattr(node, "identity", None)}
728
+ for rel in config.relationships:
729
+ for side, end in (("start_node", rel.start_node), ("end_node", rel.end_node)):
730
+ if end.label is not None and end.identity is not None:
731
+ errors.append(f"Relationship '{rel.type}' {side} gives both a label and an identity: give one")
732
+ elif end.label is None and end.identity is None:
733
+ errors.append(f"Relationship '{rel.type}' {side} gives no label: give a label, or an identity for "
734
+ f"any node of one identity")
735
+ elif end.identity in ("global", "distinct"):
736
+ errors.append(f"Relationship '{rel.type}' {side} gives the identity '{end.identity}', which names "
737
+ f"no one set of nodes")
738
+ elif end.identity is not None and end.identity not in declared:
739
+ errors.append(f"Relationship '{rel.type}' {side} gives the identity '{end.identity}', which no "
740
+ f"node declares")
741
+
742
+ # Check for duplicate node labels
743
+ labels = [node.label for node in config.nodes if node.label is not None]
744
+ seen = set()
745
+ for label in labels:
746
+ if label in seen:
747
+ errors.append(f"Duplicate node label: '{label}'")
748
+ seen.add(label)
749
+
750
+ # Check for duplicate relationship types (warning, not error)
751
+ # Multiple relationships of same type are allowed (e.g., KNOWS from different sources)
752
+
753
+ return errors
754
+
755
+
756
+ # =============================================================================
757
+ # DataFrame Loading
758
+ # =============================================================================
759
+
760
+ def load_dataframes(
761
+ config: GraphConfig,
762
+ spark: Optional[Any] = None,
763
+ base_path: Optional[Union[str, Path]] = None
764
+ ) -> Dict[str, Any]:
765
+ """Load DataFrames from configured sources.
766
+
767
+ Args:
768
+ config: Graph configuration
769
+ spark: SparkSession (optional, uses pandas if not provided)
770
+ base_path: Base path for relative file paths
771
+
772
+ Returns:
773
+ Dict mapping source names to DataFrames
774
+ """
775
+ dataframes = {}
776
+ base_path = Path(base_path) if base_path else Path('.')
777
+
778
+ for name, source in config.sources.items():
779
+ if source.table:
780
+ # Load from Unity Catalog table
781
+ if spark is None:
782
+ raise ValueError(f"SparkSession required to load table '{source.table}'")
783
+ dataframes[name] = spark.table(source.table)
784
+
785
+ elif source.file:
786
+ # Load from file
787
+ file_path = Path(source.file)
788
+ if not file_path.is_absolute():
789
+ file_path = base_path / file_path
790
+
791
+ if spark is not None:
792
+ # Use Spark
793
+ options = source.options.copy()
794
+ file_str = str(file_path)
795
+
796
+ if file_str.endswith('.csv'):
797
+ df = spark.read.csv(
798
+ file_str,
799
+ header=options.pop('header', True),
800
+ inferSchema=options.pop('inferSchema', True),
801
+ sep=options.pop('delimiter', ','),
802
+ **options
803
+ )
804
+ elif file_str.endswith('.parquet'):
805
+ df = spark.read.parquet(file_str, **options)
806
+ elif file_str.endswith('.json'):
807
+ df = spark.read.json(file_str, **options)
808
+ else:
809
+ raise ValueError(f"Unsupported file format: {file_path.suffix}")
810
+
811
+ dataframes[name] = df
812
+ else:
813
+ # Use pandas
814
+ import pandas as pd
815
+ options = source.options.copy()
816
+ file_str = str(file_path)
817
+
818
+ if file_str.endswith('.csv'):
819
+ # The same options as the Spark reader: header and
820
+ # inferSchema are booleans there, which pandas spells
821
+ # header=0/None and dtype=str.
822
+ header = options.pop('header', True)
823
+ if isinstance(header, bool):
824
+ header = 0 if header else None
825
+ if options.pop('inferSchema', True) is False:
826
+ options.setdefault('dtype', str)
827
+ df = pd.read_csv(
828
+ file_str,
829
+ header=header,
830
+ delimiter=options.pop('delimiter', ','),
831
+ **options
832
+ )
833
+ elif file_str.endswith('.parquet'):
834
+ df = pd.read_parquet(file_str, **options)
835
+ elif file_str.endswith('.json'):
836
+ df = pd.read_json(file_str, **options)
837
+ else:
838
+ raise ValueError(f"Unsupported file format: {file_path.suffix}")
839
+
840
+ dataframes[name] = df
841
+
842
+ elif source.query:
843
+ # Load from SQL query
844
+ if spark is None:
845
+ raise ValueError(f"SparkSession required to execute query for source '{name}'")
846
+ dataframes[name] = spark.sql(source.query)
847
+
848
+ return dataframes