prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/provenance.py ADDED
@@ -0,0 +1,222 @@
1
+ """Where a mapped graph's triples come from, as PROV-O (https://www.w3.org/TR/prov-o/).
2
+
3
+ Mapping-level provenance: read from the source maps alone, with no data. One `prov:Entity` per source
4
+ (a table or a query, with its columns as `prov:hadMember`), per source map and per pattern (a map's
5
+ subject term, one predicate and its object term); each pattern is `prov:wasDerivedFrom` its source and the
6
+ columns its terms read. The dataset (the IRI `prograph.describe` gives it) is `prov:wasDerivedFrom` every
7
+ source and `prov:wasGeneratedBy` a `prov:Activity` that `prov:used` the mapping, a `prov:Plan` whose
8
+ `dcterms:hasVersion` is a content hash of the compiled maps (`mapping_hash`): it changes when what the
9
+ mapping means changes, and not otherwise (not by reordering a node's predicate-object maps or properties).
10
+ PROV-O's canonical shape for a plan is a qualified `prov:Association` with `prov:hadPlan` on the activity's
11
+ agent; no agent is modelled here, so the activity says `prov:used` the plan.
12
+
13
+ A pattern is derived from the columns its terms read, a per-row language tag's column (`TermMap.language_column`,
14
+ which `TermMap.columns` leaves out: it builds the term's text) and the columns of the source's row filter,
15
+ which the source map entity states as `urn:prograph:prov:filter`.
16
+
17
+ A pattern carries `urn:prograph:prov:predicate` (and `urn:prograph:prov:object` for a constant object), so "where does ?p come from"
18
+ is a SPARQL query over this graph.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import dataclasses
24
+ import hashlib
25
+ from typing import Any, Dict, Iterable, List, Optional, Tuple
26
+ from urllib.parse import quote
27
+
28
+ from prograph.rules import module_home
29
+ from prograph.schema.source_map import IRI, LITERAL
30
+
31
+ module_home("mapping-provenance", "A mapped graph's sources, source maps and triple patterns as PROV-O",
32
+ terms=("provenance", "PROV-O", "prov:wasDerivedFrom", "where does a predicate come from",
33
+ "mapping hash", "mapping version", "engine.provenance"))
34
+
35
+ PROV = "http://www.w3.org/ns/prov#"
36
+ RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
37
+ DEFAULT_BASE = "urn:prograph:prov:"
38
+ # A pattern's predicate and constant object. rdf:predicate and rdf:object have the domain rdf:Statement, which a
39
+ # pattern is not (it stands for every triple of a kind), so ProGraph's own properties say it.
40
+ PATTERN_PREDICATE = "urn:prograph:prov:predicate"
41
+ PATTERN_OBJECT = "urn:prograph:prov:object"
42
+ MAP_FILTER = "urn:prograph:prov:filter"
43
+
44
+
45
+ def _read(tm: Any) -> List[str]:
46
+ """The columns a term map reads: its text's and its per-row language tag's."""
47
+ if tm is None:
48
+ return []
49
+ return [*tm.columns(), *([tm.language_column] if tm.language_column else [])]
50
+
51
+
52
+ def mapping_hash(maps: Any) -> str:
53
+ """The sha256 (hex) of a canonical form of the maps: sets in sorted order and no identity of an object,
54
+ so one mapping hashes alike in any process and at any path (an R2RML map's id is read relative to its
55
+ document), and a changed map, term or source changes it."""
56
+ ids = [m.id for m in (*maps.nodes, *maps.edges)]
57
+ # An R2RML map's id is its document's IRI and a fragment: the document's place is not the mapping's content.
58
+ documents = tuple(sorted({i[:i.rindex("#") + 1] for i in ids if "://" in i and "#" in i}, key=len, reverse=True))
59
+ return hashlib.sha256(repr(_canon(maps, documents)).encode("utf-8")).hexdigest()
60
+
61
+
62
+ # Fields whose order is not the mapping's meaning (a node's predicate-object maps, a YAML node's properties).
63
+ _UNORDERED = frozenset({"predicate_objects", "property_columns", "classes", "graph_maps", "predicate_declarations",
64
+ "property_predicates", "relationship_predicates", "representatives", "prefixes"})
65
+
66
+
67
+ def _canon(x: Any, documents: Tuple[str, ...] = ()) -> Any:
68
+ if isinstance(x, str):
69
+ for d in documents:
70
+ if d in x:
71
+ x = x.replace(d, "#")
72
+ return x
73
+ if dataclasses.is_dataclass(x) and not isinstance(x, type):
74
+ def field(f: Any) -> Any:
75
+ value = _canon(getattr(x, f.name), documents)
76
+ return (f.name, tuple(sorted(value, key=repr)) if f.name in _UNORDERED else value)
77
+ return (type(x).__name__, tuple(field(f) for f in dataclasses.fields(x) if f.compare))
78
+ if isinstance(x, (frozenset, set)):
79
+ return ("set", tuple(sorted(repr(_canon(i, documents)) for i in x)))
80
+ if isinstance(x, dict):
81
+ return ("dict", tuple(sorted((repr(_canon(k, documents)), _canon(v, documents)) for k, v in x.items())))
82
+ if isinstance(x, (list, tuple)):
83
+ return tuple(_canon(i, documents) for i in x)
84
+ return x
85
+
86
+
87
+ def provenance(maps: Any, *, dataset_iri: Optional[str] = None, base: str = DEFAULT_BASE) -> Any:
88
+ """The provenance, an rdflib Graph.
89
+
90
+ Args:
91
+ maps: The `SourceMaps` described.
92
+ dataset_iri: The generated graph's IRI (`describe_dataset`'s: `<endpoint>#dataset`); default `<base>dataset/<hash>`.
93
+ base: The prefix of every IRI this mints.
94
+ """
95
+ from rdflib import Graph, Literal, Namespace, URIRef
96
+ from rdflib.namespace import DCTERMS, RDF, RDFS, XSD
97
+ prov = Namespace(PROV)
98
+ g = Graph()
99
+ for prefix, ns in (("prov", prov), ("dcterms", DCTERMS), ("rdfs", RDFS)):
100
+ g.bind(prefix, ns)
101
+
102
+ digest = mapping_hash(maps)
103
+ short = digest[:16]
104
+ mapping = URIRef(f"{base}mapping/{short}")
105
+ activity = URIRef(f"{base}activity/{short}")
106
+ dataset = URIRef(dataset_iri) if dataset_iri else URIRef(f"{base}dataset/{short}")
107
+ for t in (prov.Entity, prov.Collection, prov.Plan):
108
+ g.add((mapping, RDF.type, t))
109
+ g.add((mapping, DCTERMS.hasVersion, Literal(digest)))
110
+ if maps.settings.name:
111
+ g.add((mapping, DCTERMS.title, Literal(maps.settings.name)))
112
+ g.add((activity, RDF.type, prov.Activity))
113
+ g.add((activity, RDFS.label, Literal("mapping evaluation")))
114
+ g.add((activity, prov.used, mapping))
115
+ g.add((dataset, RDF.type, prov.Entity))
116
+ g.add((dataset, prov.wasGeneratedBy, activity))
117
+
118
+ sources: Dict[str, Any] = {}
119
+
120
+ def source_of(src: Any) -> Tuple[Any, str]:
121
+ key = src.qualified if not src.query else f"query-{hashlib.sha256(src.query.encode()).hexdigest()[:16]}"
122
+ node = URIRef(f"{base}source/{quote(key, safe='')}")
123
+ if key not in sources:
124
+ sources[key] = node
125
+ for t in (prov.Entity, prov.Collection):
126
+ g.add((node, RDF.type, t))
127
+ g.add((node, RDFS.label, Literal(src.table if src.query else src.qualified)))
128
+ if src.query:
129
+ g.add((node, prov.value, Literal(src.query)))
130
+ g.add((activity, prov.used, node))
131
+ g.add((dataset, prov.wasDerivedFrom, node))
132
+ return node, key
133
+
134
+ def column_of(src: Any, name: str) -> Any:
135
+ node, key = source_of(src)
136
+ col = URIRef(f"{base}column/{quote(key, safe='')}/{quote(name, safe='')}")
137
+ if (node, prov.hadMember, col) not in g:
138
+ g.add((col, RDF.type, prov.Entity))
139
+ g.add((col, RDFS.label, Literal(name)))
140
+ g.add((node, prov.hadMember, col))
141
+ return col
142
+
143
+ def term_of(tm: Any) -> Optional[Any]:
144
+ if tm is None or tm.constant is None:
145
+ return None
146
+ if tm.term_type == IRI:
147
+ return URIRef(tm.constant)
148
+ if tm.term_type == LITERAL:
149
+ return Literal(tm.constant, lang=tm.language, datatype=URIRef(tm.datatype) if tm.datatype else None)
150
+ return None
151
+
152
+ def pattern(owner: Any, owner_id: str, n: int, label: str, predicate: Optional[str], obj: Any,
153
+ derived: Iterable[Any]) -> None:
154
+ pat = URIRef(f"{base}pattern/{quote(owner_id, safe='')}/{n}")
155
+ g.add((pat, RDF.type, prov.Entity))
156
+ g.add((pat, RDFS.label, Literal(label)))
157
+ g.add((owner, prov.hadMember, pat))
158
+ g.add((pat, prov.wasGeneratedBy, activity))
159
+ if predicate:
160
+ g.add((pat, URIRef(PATTERN_PREDICATE), URIRef(predicate)))
161
+ if obj is not None:
162
+ g.add((pat, URIRef(PATTERN_OBJECT), obj))
163
+ for d in derived:
164
+ g.add((pat, prov.wasDerivedFrom, d))
165
+
166
+ def derived_of(*groups: Tuple[Any, Iterable[str]]) -> List[Any]:
167
+ out: List[Any] = []
168
+ for src, names in groups:
169
+ out.append(source_of(src)[0])
170
+ out.extend(column_of(src, c) for c in dict.fromkeys([*names, *(c for c, _ in src.filter or ())]))
171
+ return out
172
+
173
+ def map_node(m: Any) -> Any:
174
+ node = URIRef(f"{base}map/{quote(m.id, safe='')}")
175
+ for t in (prov.Entity, prov.Collection):
176
+ g.add((node, RDF.type, t))
177
+ g.add((node, RDFS.label, Literal(getattr(m, "label", None) or m.id)))
178
+ g.add((node, DCTERMS.identifier, Literal(m.id)))
179
+ g.add((mapping, prov.hadMember, node))
180
+ filters = [src.filter for src in (m.source, getattr(getattr(m, "join", None), "parent", None)) if src and src.filter]
181
+ for pairs in filters:
182
+ g.add((node, URIRef(MAP_FILTER), Literal(" AND ".join(
183
+ f"{c} IS NULL" if v is None else f"{c} = {v!r}" for c, v in pairs))))
184
+ return node
185
+
186
+ def graph_columns(m: Any, extra: Iterable[Any] = ()) -> List[str]:
187
+ return [c for tm in (*m.graph_maps, *extra) for c in _read(tm)]
188
+
189
+ for m in maps.nodes:
190
+ node = map_node(m)
191
+ subject = _read(m.subject)
192
+ n = 0
193
+ for cls in m.classes:
194
+ pattern(node, m.id, n, f"{m.id}: a <{cls}>", RDF_TYPE, URIRef(cls),
195
+ derived_of((m.source, subject + graph_columns(m))))
196
+ n += 1
197
+ for po in m.predicate_objects:
198
+ cols = subject + _read(po.object) + graph_columns(po, [po.predicate_map]) + graph_columns(m)
199
+ pattern(node, m.id, n, f"{m.id}: {po.predicate or 'a predicate made from the row'}",
200
+ po.predicate, term_of(po.object), derived_of((m.source, cols)))
201
+ n += 1
202
+
203
+ for e in maps.edges:
204
+ node = map_node(e)
205
+ child = list(e.start.columns) + list(e.end.columns) + graph_columns(e, [e.predicate_map])
206
+ if e.start.term is not None:
207
+ child += _read(e.start.term)
208
+ parent: List[str] = []
209
+ if e.end.parent_columns:
210
+ parent = list(e.end.parent_columns)
211
+ elif e.end.term is not None:
212
+ child += _read(e.end.term)
213
+ groups = [(e.source, child)]
214
+ if e.join is not None:
215
+ child += [c for c, _ in e.join.conditions]
216
+ parent += [p for _, p in e.join.conditions]
217
+ if e.end.term is not None and e.end.parent_columns:
218
+ parent += _read(e.end.term)
219
+ groups.append((e.join.parent, parent))
220
+ pattern(node, e.id, 0, f"{e.id}: {e.predicate or 'a predicate made from the row'}", e.predicate, None,
221
+ derived_of(*groups))
222
+ return g
@@ -0,0 +1,48 @@
1
+ """R2RML integration for ProGraph.
2
+
3
+ Parse W3C R2RML mapping documents and compile them bidirectionally
4
+ to/from ProGraph's GraphSchema.
5
+
6
+ Reads mappings with rdflib, a dependency of prograph.
7
+ """
8
+
9
+ try:
10
+ import rdflib # noqa: F401
11
+ RDFLIB_AVAILABLE = True
12
+ except ImportError:
13
+ RDFLIB_AVAILABLE = False
14
+
15
+ from prograph.r2rml.model import (
16
+ R2RMLMapping,
17
+ TriplesMap,
18
+ SubjectMap,
19
+ ObjectMap,
20
+ PredicateObjectMap,
21
+ PredicateMap,
22
+ GraphMap,
23
+ LogicalTable,
24
+ JoinCondition,
25
+ )
26
+
27
+ # The parser requires rdflib at runtime, but we still export it
28
+ # so that import errors surface at instantiation, not at module import.
29
+ from prograph.r2rml.parser import R2RMLParser, R2RMLParseError
30
+ from prograph.r2rml.schema import PropertyGraphViewError, R2RMLSchema, load
31
+
32
+ __all__ = [
33
+ "R2RMLParser",
34
+ "R2RMLParseError",
35
+ "R2RMLSchema",
36
+ "PropertyGraphViewError",
37
+ "load",
38
+ "R2RMLMapping",
39
+ "TriplesMap",
40
+ "SubjectMap",
41
+ "ObjectMap",
42
+ "PredicateObjectMap",
43
+ "PredicateMap",
44
+ "GraphMap",
45
+ "LogicalTable",
46
+ "JoinCondition",
47
+ "RDFLIB_AVAILABLE",
48
+ ]
@@ -0,0 +1,259 @@
1
+ """A database's tables as the Direct Mapping reads them (https://www.w3.org/TR/rdb-direct-mapping/ 3):
2
+ each table's columns in order, its declared PRIMARY KEY in key order, and its FOREIGN KEYs with the
3
+ referenced table and columns, paired by position. Read from the catalog of DuckDB, PostgreSQL and
4
+ Databricks (Unity Catalog); another database is refused by name, and a Spark session or a frame, which
5
+ declares no constraint, has every table keyless. Only base tables are read (the Recommendation maps
6
+ tables): a view, a materialized view or a streaming table is left out; map one with R2RML.
7
+
8
+ The Databricks reader pairs a foreign key's child column with the parent key's column named by
9
+ `key_column_usage.position_in_unique_constraint` (the parent key being the constraint
10
+ `referential_constraints` names), so a foreign key that lists its columns in another order than the parent
11
+ key declares them pairs correctly; it has run on a warehouse (docs/conformance/r2rml.md). DuckDB's and
12
+ PostgreSQL's readers are tested by the W3C cases.
13
+
14
+ The declared keys are read, as the standard has it, whether or not the database enforces them (a
15
+ Databricks key is informational): the trust a plan may place in a key is `plan.ir.keys.trusted_keys`,
16
+ another question. The queries here are the ordered, paired form of the ones `keys._read_constraints`
17
+ asks as unordered sets; the quoting of a literal is the dialect's (`Dialect.literal`).
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ from dataclasses import dataclass
23
+ from typing import Any, Dict, List, Optional, Sequence, Tuple
24
+
25
+ DIALECTS = ("duckdb", "postgresql", "databricks_sql")
26
+
27
+
28
+ class DirectMappingError(ValueError):
29
+ """A database the Direct Mapping cannot be read from, or a mapping it cannot make, by name."""
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class ForeignKey:
34
+ """One FOREIGN KEY: the child's columns and the parent table's, paired by position."""
35
+ columns: Tuple[str, ...]
36
+ parent: str
37
+ parent_columns: Tuple[str, ...]
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class Table:
42
+ """One table: its columns in order, its primary key (() for none) and its foreign keys."""
43
+ name: str
44
+ columns: Tuple[str, ...]
45
+ primary_key: Tuple[str, ...] = ()
46
+ foreign_keys: Tuple[ForeignKey, ...] = ()
47
+
48
+
49
+ def _rows(connection: Any, dialect: str, sql: str) -> List[Tuple[Any, ...]]:
50
+ if dialect == "duckdb":
51
+ return [tuple(r) for r in connection.execute(sql).fetchall()]
52
+ cursor = connection.cursor()
53
+ try:
54
+ cursor.execute(sql)
55
+ return [tuple(r) for r in cursor.fetchall()]
56
+ finally:
57
+ cursor.close()
58
+
59
+
60
+ def read_tables(connection: Any, dialect: str, schema: Optional[str] = None,
61
+ tables: Optional[Sequence[str]] = None) -> List[Table]:
62
+ """The tables of `schema` (the connection's current one where None; `catalog.schema` on
63
+ Databricks), or just `tables`, as `Table`s in name order. A foreign key to a table not in the set
64
+ is refused by name."""
65
+ if dialect not in DIALECTS:
66
+ raise DirectMappingError(
67
+ f"the Direct Mapping reads the catalog of DuckDB, PostgreSQL and Databricks; {dialect!r} has no "
68
+ f"reader yet: write the mapping in R2RML or YAML, or pass the tables as a mapping")
69
+ from prograph.plan.sql.dialect import get_dialect_instance
70
+ lit = get_dialect_instance(dialect).literal
71
+ read = {"duckdb": _duckdb, "postgresql": _postgres, "databricks_sql": _databricks}[dialect]
72
+ found = read(connection, lit, schema)
73
+ if tables is not None:
74
+ missing = [t for t in tables if t not in found]
75
+ if missing:
76
+ raise DirectMappingError(f"no such table in {schema or 'the current schema'}: {', '.join(missing)}")
77
+ found = {t: found[t] for t in tables}
78
+ for table in found.values():
79
+ for fk in table.foreign_keys:
80
+ if fk.parent not in found:
81
+ raise DirectMappingError(
82
+ f"table {table.name}: a foreign key references {fk.parent}, which is not among the "
83
+ f"tables mapped; map it too (or leave the table out)")
84
+ return [found[name] for name in sorted(found)]
85
+
86
+
87
+ def _assemble(columns: Dict[str, List[str]], keys: Dict[str, List[str]],
88
+ fks: Dict[str, Dict[str, Tuple[str, List[str], List[str]]]]) -> Dict[str, Table]:
89
+ out = {}
90
+ for name, cols in columns.items():
91
+ foreign = tuple(ForeignKey(tuple(c), parent, tuple(pc))
92
+ for _, (parent, c, pc) in sorted(fks.get(name, {}).items()))
93
+ out[name] = Table(name, tuple(cols), tuple(keys.get(name, ())), foreign)
94
+ return out
95
+
96
+
97
+ def _duckdb(con: Any, lit: Any, schema: Optional[str]) -> Dict[str, Table]:
98
+ where = f"schema_name = {lit(schema) if schema else 'current_schema()'} AND database_name = current_database()"
99
+ names = [r[0] for r in _rows(con, "duckdb", f"SELECT table_name FROM duckdb_tables() WHERE {where}")]
100
+ columns = {n: [] for n in names}
101
+ for t, c in _rows(con, "duckdb", f"SELECT table_name, column_name FROM duckdb_columns() WHERE {where} "
102
+ "ORDER BY table_name, column_index"):
103
+ if t in columns:
104
+ columns[t].append(c)
105
+ keys: Dict[str, List[str]] = {}
106
+ fks: Dict[str, Dict[str, Any]] = {}
107
+ for t, kind, cols, parent, pcols, idx in _rows(
108
+ con, "duckdb", "SELECT table_name, constraint_type, constraint_column_names, referenced_table, "
109
+ f"referenced_column_names, constraint_index FROM duckdb_constraints() WHERE {where} "
110
+ "AND constraint_type IN ('PRIMARY KEY', 'FOREIGN KEY') ORDER BY constraint_index"):
111
+ if kind == "PRIMARY KEY":
112
+ keys[t] = list(cols)
113
+ else:
114
+ fks.setdefault(t, {})[f"{idx:08d}"] = (parent, list(cols), list(pcols))
115
+ return _assemble(columns, keys, fks)
116
+
117
+
118
+ def _postgres(con: Any, lit: Any, schema: Optional[str]) -> Dict[str, Table]:
119
+ sch = lit(schema) if schema else "current_schema()"
120
+ sql = ("SELECT table_name FROM information_schema.tables WHERE "
121
+ f"table_schema = {sch} AND table_type = 'BASE TABLE'")
122
+ names = [r[0] for r in _rows(con, "postgresql", sql)]
123
+ columns = {n: [] for n in names}
124
+ for t, c in _rows(con, "postgresql", "SELECT table_name, column_name FROM information_schema.columns WHERE "
125
+ f"table_schema = {sch} ORDER BY table_name, ordinal_position"):
126
+ if t in columns:
127
+ columns[t].append(c)
128
+ keys: Dict[str, List[str]] = {}
129
+ fks: Dict[str, Dict[str, Any]] = {}
130
+ sql = ("SELECT t.relname, c.conname, c.contype::text, a.attname, rt.relname, rn.nspname, ra.attname, n.nspname "
131
+ "FROM pg_constraint c JOIN pg_class t ON t.oid = c.conrelid JOIN pg_namespace n ON n.oid = t.relnamespace "
132
+ "CROSS JOIN LATERAL unnest(c.conkey, COALESCE(c.confkey, c.conkey)) WITH ORDINALITY AS k(att, ratt, ord) "
133
+ "JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.att "
134
+ "LEFT JOIN pg_class rt ON rt.oid = c.confrelid LEFT JOIN pg_namespace rn ON rn.oid = rt.relnamespace "
135
+ "LEFT JOIN pg_attribute ra ON ra.attrelid = c.confrelid AND ra.attnum = k.ratt "
136
+ f"WHERE c.contype IN ('p', 'f') AND n.nspname = {sch} ORDER BY t.relname, c.conname, k.ord")
137
+ for t, name, kind, col, parent, pschema, pcol, tschema in _rows(con, "postgresql", sql):
138
+ if kind == "p":
139
+ keys.setdefault(t, []).append(col)
140
+ else:
141
+ if pschema != tschema:
142
+ raise DirectMappingError(f"table {t}: foreign key {name} references {pschema}.{parent}, "
143
+ f"another schema than the one mapped")
144
+ entry = fks.setdefault(t, {}).setdefault(name, (parent, [], []))
145
+ entry[1].append(col)
146
+ entry[2].append(pcol)
147
+ return _assemble(columns, keys, fks)
148
+
149
+
150
+ def _databricks(con: Any, lit: Any, schema: Optional[str]) -> Dict[str, Table]:
151
+ catalog, _, name = (schema or "").rpartition(".")
152
+ if schema is None:
153
+ (catalog, name), = _rows(con, "databricks_sql", "SELECT current_catalog(), current_schema()")
154
+ cat = catalog or _rows(con, "databricks_sql", "SELECT current_catalog()")[0][0]
155
+ from prograph.plan.sql.dialect import get_dialect_instance
156
+ info = f"{get_dialect_instance('databricks_sql').quote(cat)}.information_schema"
157
+ where = f"table_schema = {lit(name)}"
158
+ sql = (f"SELECT table_name FROM {info}.tables WHERE {where} AND "
159
+ "table_type NOT IN ('VIEW', 'MATERIALIZED_VIEW', 'STREAMING_TABLE')")
160
+ names = [r[0] for r in _rows(con, "databricks_sql", sql)]
161
+ columns = {n: [] for n in names}
162
+ for t, c in _rows(con, "databricks_sql", f"SELECT table_name, column_name FROM {info}.columns WHERE {where} "
163
+ "ORDER BY table_name, ordinal_position"):
164
+ if t in columns:
165
+ columns[t].append(c)
166
+ keys: Dict[str, List[str]] = {}
167
+ fks: Dict[str, Dict[str, Any]] = {}
168
+ kinds = {r[0]: (r[1], r[2]) for r in _rows(
169
+ con, "databricks_sql", "SELECT constraint_name, constraint_type, table_name FROM "
170
+ f"{info}.table_constraints WHERE {where}")}
171
+ usage = _rows(con, "databricks_sql",
172
+ "SELECT constraint_name, table_name, column_name, ordinal_position, position_in_unique_constraint "
173
+ f"FROM {info}.key_column_usage WHERE {where} ORDER BY constraint_name, ordinal_position")
174
+ parent_of = {r[0]: (r[1], r[2]) for r in _rows(
175
+ con, "databricks_sql", "SELECT constraint_name, unique_constraint_schema, unique_constraint_name FROM "
176
+ f"{info}.referential_constraints WHERE constraint_schema = {lit(name)}")}
177
+ for t, key in _databricks_keys(kinds, usage, parent_of, name):
178
+ if key[0] == "PRIMARY KEY":
179
+ keys[t] = key[1]
180
+ else:
181
+ fks.setdefault(t, {})[key[1]] = key[2]
182
+ return _assemble(columns, keys, fks)
183
+
184
+
185
+ def _databricks_keys(kinds: Dict[str, Tuple[str, str]], usage: Sequence[Tuple[Any, ...]],
186
+ parent_of: Dict[str, Tuple[str, str]], schema: str):
187
+ """The keys of one Databricks schema from its information_schema rows, as (table, ('PRIMARY KEY', columns))
188
+ and (table, ('FOREIGN KEY', constraint name, (parent table, child columns, parent columns))).
189
+
190
+ `kinds`: constraint name -> (type, table); `usage`: key_column_usage rows (constraint name, table, column,
191
+ ordinal_position, position_in_unique_constraint); `parent_of`: a foreign key's name -> (schema, name) of
192
+ the key it references. A foreign key's child column i is paired with the column of the parent key at
193
+ `position_in_unique_constraint`, not with the parent's i-th column: the two orders differ when the
194
+ foreign key lists its columns in another order than the parent key declares them."""
195
+ used: Dict[str, List[Tuple[int, str, str, Optional[int]]]] = {}
196
+ for cname, t, col, ordinal, parent_pos in usage:
197
+ used.setdefault(cname, []).append((int(ordinal), t, col, None if parent_pos is None else int(parent_pos)))
198
+ for cname, (kind, t) in kinds.items():
199
+ mine = sorted(used.get(cname, []))
200
+ if kind == "PRIMARY KEY":
201
+ yield t, ("PRIMARY KEY", [c for _, _, c, _ in mine])
202
+ elif kind == "FOREIGN KEY":
203
+ pschema, pname = parent_of.get(cname, (None, ""))
204
+ target = sorted(used.get(pname, []))
205
+ if not target:
206
+ raise DirectMappingError(f"table {t}: foreign key {cname} references no key the catalog shows")
207
+ if pschema is not None and pschema != schema:
208
+ raise DirectMappingError(f"table {t}: foreign key {cname} references a key in schema {pschema}, "
209
+ f"another schema than the one mapped")
210
+ by_pos = {ordinal: col for ordinal, _, col, _ in target}
211
+ parents = []
212
+ for ordinal, _, col, parent_pos in mine:
213
+ pos = parent_pos if parent_pos is not None else ordinal
214
+ if pos not in by_pos:
215
+ raise DirectMappingError(f"table {t}: foreign key {cname}: column {col} is paired with "
216
+ f"position {pos} of a key that has {len(target)} columns")
217
+ parents.append(by_pos[pos])
218
+ yield t, ("FOREIGN KEY", cname, (target[0][1], [c for _, _, c, _ in mine], parents))
219
+
220
+
221
+ def read_not_null(connection: Any, dialect: str, schema: Optional[str] = None) -> Dict[str, frozenset]:
222
+ """The NOT NULL columns of each base table of `schema` (the connection's current one where None;
223
+ `catalog.schema` on Databricks): table name -> its columns that hold no NULL by constraint (a PRIMARY
224
+ KEY column is one). A table with none is absent; so is a view, whose columns declare nothing. Read as
225
+ the database states it, enforced or not (a Databricks NOT NULL is enforced; its keys are not): the
226
+ shapes generated from it (`prograph.shacl.generate`) say where each fact came from."""
227
+ if dialect not in DIALECTS:
228
+ raise DirectMappingError(f"no catalog reader for {dialect!r}")
229
+ from prograph.plan.sql.dialect import get_dialect_instance
230
+ lit = get_dialect_instance(dialect).literal
231
+ if dialect == "duckdb":
232
+ sch = lit(schema) if schema else "current_schema()"
233
+ rows = _rows(connection, dialect,
234
+ "SELECT c.table_name, c.column_name FROM duckdb_columns() c JOIN duckdb_tables() t "
235
+ "ON t.database_name = c.database_name AND t.schema_name = c.schema_name "
236
+ "AND t.table_name = c.table_name "
237
+ f"WHERE c.schema_name = {sch} AND c.database_name = current_database() AND NOT c.is_nullable")
238
+ elif dialect == "postgresql":
239
+ sch = lit(schema) if schema else "current_schema()"
240
+ rows = _rows(connection, dialect,
241
+ "SELECT c.table_name, c.column_name FROM information_schema.columns c "
242
+ "JOIN information_schema.tables t ON t.table_schema = c.table_schema "
243
+ "AND t.table_name = c.table_name AND t.table_type = 'BASE TABLE' "
244
+ f"WHERE c.table_schema = {sch} AND c.is_nullable = 'NO'")
245
+ else:
246
+ catalog, _, name = (schema or "").rpartition(".")
247
+ if schema is None:
248
+ (catalog, name), = _rows(connection, dialect, "SELECT current_catalog(), current_schema()")
249
+ cat = catalog or _rows(connection, dialect, "SELECT current_catalog()")[0][0]
250
+ info = f"{get_dialect_instance(dialect).quote(cat)}.information_schema"
251
+ rows = _rows(connection, dialect,
252
+ f"SELECT c.table_name, c.column_name FROM {info}.columns c JOIN {info}.tables t "
253
+ "ON t.table_schema = c.table_schema AND t.table_name = c.table_name "
254
+ "AND t.table_type NOT IN ('VIEW', 'MATERIALIZED_VIEW', 'STREAMING_TABLE') "
255
+ f"WHERE c.table_schema = {lit(name)} AND c.is_nullable = 'NO'")
256
+ out: Dict[str, set] = {}
257
+ for table, column in rows:
258
+ out.setdefault(table, set()).add(column)
259
+ return {t: frozenset(c) for t, c in out.items()}