prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,559 @@
1
+ """One instance's whole graph: the root, the nodes that belong to it, and the facts about them, as one document
2
+ (Turtle, JSON-LD, or nested JSON).
3
+
4
+ Experimental: an extra outside ProGraph's engine API. Its arguments and output may change between releases.
5
+
6
+ Where such a document ends is not something RDF or OWL decides: OWL is open-world and has no notion of a
7
+ document's boundary. So the caller says where the payload stops, in one of two ways:
8
+
9
+ - ``follow``: one or more SPARQL 1.1 property paths (section 9) from the root. The walk applies every path to
10
+ every member, breadth first, and each value it reaches is a member, expanded in turn: its own facts and
11
+ outgoing links are in the payload. A step on a property the bound ontology declares ``owl:TransitiveProperty``
12
+ is walked to its closure (as the path ``p+``); any other step is walked once, or ``depth`` times. A member's
13
+ facts are what the engine answers: in 1.0, which applies no ``owl:TransitiveProperty``, its stated links (a
14
+ blade part of its module, not of the engine); a transitive answer is asked for by a path (``:isPartOf+``),
15
+ and OWL RL, after 1.0, entails it. No ontology is needed: without one, every step is a single step. A link to
16
+ a node outside the payload is kept as its IRI, never expanded.
17
+ - ``shapes``: a SHACL node shape (SHACL 2.1) for the root, read as a description of what one record holds. Each
18
+ ``sh:property`` with an ``sh:path`` is included; a property shape with ``sh:node`` embeds its values with that
19
+ node shape, recursively; one without keeps them as IRIs or literals. A node is expanded once, so cycles end.
20
+
21
+ Only ProGraph's public API is used: ``engine.execute(..., language="sparql")``, a CONSTRUCT for each step of the
22
+ walk and one for the members' facts. Nothing here changes the engine or adds an option to it.
23
+
24
+ from prograph.extras.instance_graph import instance_graph
25
+ text = instance_graph(engine, "http://example.org/i#thing-1", follow=["ex:hasPart", "^ex:attachedTo"],
26
+ prefixes={"ex": "http://example.org/o#"}, format="turtle")
27
+ """
28
+ from __future__ import annotations
29
+
30
+ import json
31
+ import math
32
+ import re
33
+ from dataclasses import dataclass, field
34
+ from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
35
+
36
+ import rdflib
37
+ from rdflib import BNode, Graph, Literal, URIRef
38
+ from rdflib.collection import Collection
39
+ from rdflib.namespace import OWL, RDF
40
+
41
+ SH = rdflib.Namespace("http://www.w3.org/ns/shacl#")
42
+ FORMATS = ("graph", "turtle", "json-ld", "json")
43
+
44
+
45
+ class InstanceGraphError(ValueError):
46
+ """The request cannot be answered as asked: a path or a shape construct this module does not read, an IRI
47
+ that cannot be written in SPARQL, an unknown prefix, a root with no shape, an argument out of range."""
48
+
49
+
50
+ # ------------------------------------------------------------------------------------------- IRIs in SPARQL text
51
+
52
+ # SPARQL 1.1, 19.8 [139] IRIREF: '<' ([^<>"{}|^`\]-[#x00-#x20])* '>'
53
+ _IRIREF_FORBIDDEN = re.compile(r'[<>"{}|^`\\\x00-\x20]')
54
+
55
+
56
+ def _iri(value: str) -> str:
57
+ """`value` as a SPARQL IRI reference, `<value>`: an absolute IRI with none of the characters IRIREF
58
+ excludes, refused otherwise (it is never escaped into something else)."""
59
+ text = str(value)
60
+ if not text or _IRIREF_FORBIDDEN.search(text) or ":" not in text:
61
+ raise InstanceGraphError(f"not an absolute IRI that SPARQL can write: {text!r}")
62
+ return f"<{text}>"
63
+
64
+
65
+ # ------------------------------------------------------------------------------------------------ shapes, read
66
+
67
+ @dataclass
68
+ class _Step:
69
+ predicate: URIRef
70
+ inverse: bool = False
71
+
72
+
73
+ @dataclass
74
+ class _PropertyShape:
75
+ steps: List[_Step] # one step, or a sequence path's steps
76
+ node: Optional[URIRef] = None # sh:node: embed the values with this node shape
77
+
78
+
79
+ @dataclass
80
+ class _NodeShape:
81
+ iri: URIRef
82
+ target_classes: List[URIRef] = field(default_factory=list)
83
+ properties: List[_PropertyShape] = field(default_factory=list)
84
+
85
+
86
+ # What a node shape or property shape may hold here, and what is refused by name (SHACL 2.1, 2.2, 2.3, 4).
87
+ _NODE_READ = {RDF.type, SH.targetClass, SH.property, SH.name, SH.description, SH.order, SH.group,
88
+ rdflib.RDFS.label, rdflib.RDFS.comment}
89
+ _PROPERTY_READ = {RDF.type, SH.path, SH.node, SH.name, SH.description, SH.order, SH.group, rdflib.RDFS.label,
90
+ rdflib.RDFS.comment, SH["class"], SH.datatype, SH.nodeKind, SH.minCount, SH.maxCount}
91
+ # sh:class, sh:datatype, sh:nodeKind and the counts are validation constraints: they do not change which
92
+ # values a record holds, so they are read past (a payload shape may be a validation shape's copy).
93
+
94
+
95
+ def _refuse_unknown(g: Graph, node: Any, allowed: set, where: str) -> None:
96
+ for p in sorted(set(g.predicates(node, None)), key=str):
97
+ if p not in allowed:
98
+ raise InstanceGraphError(f"{where} uses {p.n3(g.namespace_manager)}, which this module does not "
99
+ f"read; it is refused rather than ignored")
100
+
101
+
102
+ def _path(g: Graph, path: Any, where: str) -> List[_Step]:
103
+ """A property path (SHACL 2.3.1): a predicate IRI, an inverse path of one, or a sequence (an RDF list) of
104
+ those. Alternative, zero-or-more, one-or-more and zero-or-one paths are refused."""
105
+ if isinstance(path, URIRef):
106
+ return [_Step(path)]
107
+ if isinstance(path, BNode):
108
+ inverse = g.value(path, SH.inversePath)
109
+ if inverse is not None:
110
+ if not isinstance(inverse, URIRef):
111
+ raise InstanceGraphError(f"{where}: an inverse path of a path that is not one IRI (SHACL 2.3.1.3) "
112
+ f"is not read here")
113
+ return [_Step(inverse, inverse=True)]
114
+ for kind, section in ((SH.alternativePath, "2.3.1.2"), (SH.zeroOrMorePath, "2.3.1.4"),
115
+ (SH.oneOrMorePath, "2.3.1.5"), (SH.zeroOrOnePath, "2.3.1.6")):
116
+ if g.value(path, kind) is not None:
117
+ raise InstanceGraphError(f"{where}: {kind.n3(g.namespace_manager)} (SHACL {section}) is not read "
118
+ f"here; name each step, or walk a transitive property with follow=")
119
+ if g.value(path, RDF.first) is not None:
120
+ steps: List[_Step] = []
121
+ for item in Collection(g, path):
122
+ inner = _path(g, item, where)
123
+ if len(inner) != 1:
124
+ raise InstanceGraphError(f"{where}: a sequence inside a sequence path is not read here")
125
+ steps.extend(inner)
126
+ if len(steps) < 2:
127
+ raise InstanceGraphError(f"{where}: a sequence path has at least two members (SHACL 2.3.1.1)")
128
+ return steps
129
+ raise InstanceGraphError(f"{where}: sh:path {path!r} is not a path this module reads (SHACL 2.3.1)")
130
+
131
+
132
+ def _node_shape(g: Graph, shape: URIRef) -> _NodeShape:
133
+ if (shape, None, None) not in g:
134
+ raise InstanceGraphError(f"no node shape {shape} in the shapes graph")
135
+ where = f"node shape {shape.n3(g.namespace_manager)}"
136
+ _refuse_unknown(g, shape, _NODE_READ, where)
137
+ out = _NodeShape(shape, target_classes=sorted(o for o in g.objects(shape, SH.targetClass)))
138
+ for ps in sorted(g.objects(shape, SH.property), key=lambda n: (str(g.value(n, SH.order) or ""), str(n))):
139
+ pwhere = f"{where}, property shape {ps.n3(g.namespace_manager)}"
140
+ _refuse_unknown(g, ps, _PROPERTY_READ, pwhere)
141
+ if set(g.objects(ps, RDF.type)) - {SH.PropertyShape}:
142
+ raise InstanceGraphError(f"{pwhere} is typed other than sh:PropertyShape")
143
+ paths, nodes = list(g.objects(ps, SH.path)), list(g.objects(ps, SH.node))
144
+ if len(paths) != 1:
145
+ raise InstanceGraphError(f"{pwhere} has {len(paths)} sh:path values; a property shape has one "
146
+ f"(SHACL 2.3.1), and none is picked")
147
+ if len(nodes) > 1:
148
+ raise InstanceGraphError(f"{pwhere} has {len(nodes)} sh:node values; embedding with several shapes is "
149
+ f"not read here, and none is picked")
150
+ path, node = paths[0], (nodes[0] if nodes else None)
151
+ if node is not None and not isinstance(node, URIRef):
152
+ raise InstanceGraphError(f"{pwhere}: sh:node is a node shape's IRI here")
153
+ out.properties.append(_PropertyShape(_path(g, path, pwhere), node))
154
+ return out
155
+
156
+
157
+ # ------------------------------------------------------------------------------------------------- the payload
158
+
159
+ def _rows(engine: Any, query: str) -> List[Dict[str, Any]]:
160
+ return engine.execute(query, language="sparql", return_type="list")
161
+
162
+
163
+ def _types(engine: Any, node: URIRef) -> List[URIRef]:
164
+ rows = _rows(engine, f"SELECT DISTINCT ?t WHERE {{ {_iri(node)} a ?t }}")
165
+ return [URIRef(r["t"]) for r in rows if r.get("t")]
166
+
167
+
168
+ def _transitive_properties(engine: Any) -> set:
169
+ """The properties the bound ontology declares owl:TransitiveProperty; none without an ontology."""
170
+ ontology = getattr(engine, "ontology", None)
171
+ if ontology is None:
172
+ return set()
173
+ return set(ontology.rdf_graph.subjects(RDF.type, OWL.TransitiveProperty))
174
+
175
+
176
+ def _construct_of(engine: Any, query: str) -> Graph:
177
+ result = engine.execute(query, language="sparql")
178
+ if not isinstance(result, Graph): # a CONSTRUCT gives an rdflib Graph
179
+ raise InstanceGraphError(f"the engine did not return a graph for a CONSTRUCT ({type(result).__name__})")
180
+ return result
181
+
182
+
183
+ # A SPARQL 1.1 property path (section 9) in the forms read here: an IRI or a prefixed name, `a`, `^path`,
184
+ # `path/path`, `path|path` and parentheses. The repetition and negation forms are refused: whether a step repeats
185
+ # is the ontology's to say (a transitive property), or the caller's through depth.
186
+ _PATH_TOKEN = re.compile(r"\s*(<[^<>\s]*>|\^|/|\||\(|\)|\*|\+|\?|!|\{|[A-Za-z_][\w.-]*:[\w.%-]*|:[\w.%-]*|a(?=[\s/|)]|$))")
187
+ _STANDARD_PREFIXES = {"rdf": str(RDF), "rdfs": str(rdflib.RDFS), "owl": str(OWL), "xsd": str(rdflib.XSD)}
188
+
189
+
190
+ def _follow_paths(text: str, prefixes: Dict[str, str]) -> List[List[_Step]]:
191
+ """`text` as the step sequences it may take: an alternative gives one sequence per branch, an inverse
192
+ reverses its sequence and flips each step."""
193
+ tokens, at = [], 0
194
+ while at < len(text.rstrip()):
195
+ m = _PATH_TOKEN.match(text, at)
196
+ if not m:
197
+ raise InstanceGraphError(f"follow path {text!r}: cannot read it at {text[at:]!r}")
198
+ tokens.append(m.group(1))
199
+ at = m.end()
200
+ for t in tokens:
201
+ if t in ("*", "+", "?", "!", "{"):
202
+ raise InstanceGraphError(f"follow path {text!r}: {t!r} (SPARQL 1.1, 9.1) is not read here; a step on a "
203
+ f"property the ontology declares transitive is followed to its closure, "
204
+ f"and depth= repeats the others")
205
+ pos = 0
206
+
207
+ def peek():
208
+ return tokens[pos] if pos < len(tokens) else None
209
+
210
+ def take():
211
+ nonlocal pos
212
+ pos += 1
213
+ return tokens[pos - 1]
214
+
215
+ def alternative():
216
+ out = sequence()
217
+ while peek() == "|":
218
+ take()
219
+ out += sequence()
220
+ return out
221
+
222
+ def sequence():
223
+ out = element()
224
+ while peek() == "/":
225
+ take()
226
+ right = element()
227
+ out = [a + b for a in out for b in right]
228
+ return out
229
+
230
+ def element():
231
+ if peek() == "^":
232
+ take()
233
+ return [[_Step(st.predicate, not st.inverse) for st in reversed(seq)] for seq in element()]
234
+ t = take() if peek() is not None else None
235
+ if t == "(":
236
+ inner = alternative()
237
+ if peek() != ")":
238
+ raise InstanceGraphError(f"follow path {text!r}: a '(' is not closed")
239
+ take()
240
+ return inner
241
+ if t is None or t in (")", "/", "|"):
242
+ raise InstanceGraphError(f"follow path {text!r}: a step is missing")
243
+ if t == "a":
244
+ return [[_Step(RDF.type)]]
245
+ if t.startswith("<"):
246
+ return [[_Step(URIRef(t[1:-1]))]]
247
+ prefix, local = t.split(":", 1)
248
+ if prefix not in prefixes:
249
+ raise InstanceGraphError(f"follow path {text!r}: the prefix {prefix + ':'!r} is not known; give it in "
250
+ f"prefixes=")
251
+ return [[_Step(URIRef(prefixes[prefix] + local))]]
252
+
253
+ paths = alternative()
254
+ if pos != len(tokens):
255
+ raise InstanceGraphError(f"follow path {text!r}: cannot read it from {' '.join(tokens[pos:])!r}")
256
+ return paths
257
+
258
+
259
+ # The most IRIs one query's VALUES list holds: a level with more is asked in several queries, whose results are
260
+ # united. It keeps a statement within what a warehouse takes (Databricks SQL caps a statement's size); the
261
+ # answer does not depend on it.
262
+ VALUES_CHUNK = 2000
263
+
264
+
265
+ def _values_clause(nodes: Sequence[Any]) -> str:
266
+ return f"VALUES ?n {{ {' '.join(_iri(n) for n in nodes)} }}"
267
+
268
+
269
+ def _construct_over(engine: Any, nodes: Sequence[Any], template: str, pattern: str) -> Graph:
270
+ """CONSTRUCT { template } WHERE { VALUES ?n { nodes } pattern }, VALUES_CHUNK nodes per query."""
271
+ out = Graph()
272
+ for start in range(0, len(nodes), VALUES_CHUNK):
273
+ chunk = nodes[start:start + VALUES_CHUNK]
274
+ out += _construct_of(engine, f"CONSTRUCT {{ {template} }} WHERE {{ {_values_clause(chunk)} {pattern} }}")
275
+ return out
276
+
277
+
278
+ def _step_batch(engine: Any, nodes: Sequence[Any], step: _Step, out: Graph) -> Dict[Any, List[Any]]:
279
+ """The nodes one step reaches from each of `nodes`, in one query (VALUES), the step's triples added to
280
+ `out`. A literal ends the step; a blank node cannot be named in the next query, so it ends it too."""
281
+ if not nodes:
282
+ return {}
283
+ pattern = _step_pattern("?n", step, "?v")
284
+ found = _construct_over(engine, list(nodes), pattern, pattern)
285
+ reached: Dict[Any, set] = {}
286
+ asked = set(nodes)
287
+ for s, p, o in found:
288
+ out.add((s, p, o))
289
+ source, value = (o, s) if step.inverse else (s, o)
290
+ if source in asked and isinstance(value, URIRef):
291
+ reached.setdefault(source, set()).add(value)
292
+ return {k: sorted(v, key=str) for k, v in reached.items()}
293
+
294
+
295
+ def _by_follow(engine: Any, root: URIRef, follow: Sequence[str], depth: int, prefixes: Dict[str, str]):
296
+ """The walk, one level at a time. Within a level, the paths that are one step on a transitive property are
297
+ followed until they reach nothing new (their closure: the level does not grow, since the closure is the
298
+ relation); then, below `depth`, every other path is applied once to the level's members, and what it reaches
299
+ is the next level. So a node's level is the fewest applications of non-transitive paths it takes to reach,
300
+ whatever the order of the paths. Each step is one query for all the nodes it starts from.
301
+ Returns the graph (the members' facts and the paths' triples) and each member's parent in the walk: the
302
+ member it was first reached from and the step, for a one-step path; None for the root and for a member at
303
+ the end of a longer path."""
304
+ transitive = _transitive_properties(engine)
305
+ closures, others = [], []
306
+ for text in follow:
307
+ sequences = _follow_paths(text, prefixes)
308
+ if len(sequences) == 1 and len(sequences[0]) == 1 and sequences[0][0].predicate in transitive:
309
+ closures.append(sequences[0][0])
310
+ else:
311
+ others.append(sequences)
312
+ out = Graph()
313
+ level: Dict[Any, int] = {root: 0}
314
+ parents: Dict[Any, Optional[Tuple[URIRef, URIRef, bool]]] = {root: None}
315
+
316
+ def reach(value, d, parent):
317
+ if value not in level:
318
+ level[value] = d
319
+ parents[value] = parent
320
+ return True
321
+ return False
322
+
323
+ current, d = [root], 0
324
+ while current:
325
+ members, frontier = list(current), list(current)
326
+ while frontier: # the closures, within the level
327
+ new = []
328
+ for step in closures:
329
+ for source, values in sorted(_step_batch(engine, frontier, step, out).items(), key=lambda kv: str(kv[0])):
330
+ for v in values:
331
+ if reach(v, d, (source, step.predicate, step.inverse)):
332
+ new.append(v)
333
+ members += new
334
+ frontier = new
335
+ nxt: List[Any] = []
336
+ if d < depth:
337
+ for sequences in others:
338
+ for steps in sequences:
339
+ # (origin, node) pairs along the path; a transitive step inside a longer path takes its closure
340
+ pairs = {(m, m) for m in members}
341
+ for step in steps:
342
+ ends: set = set()
343
+ todo = pairs
344
+ while todo:
345
+ nodes = sorted({n for _, n in todo}, key=str)
346
+ found = _step_batch(engine, nodes, step, out)
347
+ step_out = {(o, v) for o, n in todo for v in found.get(n, [])}
348
+ new = step_out - ends
349
+ ends |= new
350
+ todo = new if (step.predicate in transitive and len(steps) > 1) else set()
351
+ pairs = ends
352
+ for origin, v in sorted(pairs, key=lambda t: (str(t[0]), str(t[1]))):
353
+ parent = (origin, steps[0].predicate, steps[0].inverse) if len(steps) == 1 else None
354
+ if reach(v, d + 1, parent):
355
+ nxt.append(v)
356
+ current, d = sorted(set(nxt), key=str), d + 1
357
+ out += _construct_over(engine, sorted(level, key=str), "?n ?p ?o", "?n ?p ?o")
358
+ return out, parents
359
+
360
+
361
+ def _step_pattern(subject: str, step: _Step, obj: str) -> str:
362
+ return f"{obj} {_iri(step.predicate)} {subject}" if step.inverse else f"{subject} {_iri(step.predicate)} {obj}"
363
+
364
+
365
+ def _by_shape(engine: Any, root: URIRef, shapes: Graph, shape: Optional[str]):
366
+ """Walk the shape from the root, one level at a time: for each node shape and each of its property shapes,
367
+ one CONSTRUCT of the path's triples for every node of the level expanded with that shape (VALUES). A node is
368
+ expanded once, with the first shape that reaches it (by IRI: a node reached through two node shapes gets the
369
+ first one's properties). Returns the triples and, for each expanded node, its parent in the tree: the node
370
+ whose one-step property shape embedded it first (breadth first, so a parent is always nearer the root than
371
+ its child), or None (the root, or a node embedded at the end of a sequence path)."""
372
+ if shape is None:
373
+ types = set(_types(engine, root))
374
+ candidates = sorted({s for s, c in shapes.subject_objects(SH.targetClass) if c in types})
375
+ if len(candidates) != 1:
376
+ raise InstanceGraphError(f"{len(candidates)} node shapes target a class of {root} "
377
+ f"({', '.join(map(str, candidates)) or 'none'}); name one with shape=")
378
+ shape = candidates[0]
379
+ read: Dict[URIRef, _NodeShape] = {}
380
+
381
+ def node_shape(iri: URIRef) -> _NodeShape:
382
+ if iri not in read:
383
+ read[iri] = _node_shape(shapes, iri)
384
+ return read[iri]
385
+
386
+ out = Graph()
387
+ parents: Dict[URIRef, Optional[Tuple[URIRef, URIRef, bool]]] = {}
388
+ level: List[Tuple[URIRef, URIRef, Optional[Tuple[URIRef, URIRef, bool]]]] = [(root, URIRef(shape), None)]
389
+ while level:
390
+ by_shape: Dict[URIRef, List[URIRef]] = {}
391
+ for node, ns_iri, parent in level:
392
+ if node not in parents:
393
+ parents[node] = parent
394
+ by_shape.setdefault(ns_iri, []).append(node)
395
+ next_level = []
396
+ expanded = sorted({n for nodes in by_shape.values() for n in nodes}, key=str)
397
+ if expanded:
398
+ out += _construct_over(engine, expanded, "?n a ?t", "?n a ?t")
399
+ for ns_iri, nodes in sorted(by_shape.items(), key=lambda kv: str(kv[0])):
400
+ nodes = sorted(nodes, key=str)
401
+ for ps in node_shape(ns_iri).properties:
402
+ names = [f"?v{i}" for i in range(len(ps.steps))]
403
+ chain, subject = [], "?n"
404
+ for step, name in zip(ps.steps, names):
405
+ chain.append(_step_pattern(subject, step, name))
406
+ subject = name
407
+ pattern = " . ".join(chain)
408
+ found = _construct_over(engine, nodes, pattern, pattern)
409
+ out += found
410
+ if ps.node is None:
411
+ continue
412
+ via = len(ps.steps) == 1
413
+ for node in nodes: # the path's end values from this node, walked over the triples returned
414
+ ends = {node}
415
+ for step in ps.steps:
416
+ ends = {s if step.inverse else o
417
+ for e in ends
418
+ for s, _, o in (found.triples((None, step.predicate, e)) if step.inverse
419
+ else found.triples((e, step.predicate, None)))}
420
+ for v in sorted((e for e in ends if isinstance(e, URIRef)), key=str):
421
+ next_level.append((v, ps.node, (node, ps.steps[0].predicate, ps.steps[0].inverse) if via else None))
422
+ level = next_level
423
+ return out, parents
424
+
425
+
426
+ # ------------------------------------------------------------------------------------------------ nested JSON
427
+
428
+ _XSD = rdflib.namespace.XSD
429
+
430
+
431
+ def _node_id(term: Any) -> str:
432
+ return f"_:{term}" if isinstance(term, BNode) else str(term)
433
+
434
+
435
+ def _json_literal(o: Literal) -> Any:
436
+ """A literal as JSON-LD reads it back as the same term (JSON-LD 1.1, 8.6 and the RDF conversion): a native
437
+ JSON value only for a plain string, an xsd:boolean, an xsd:integer, and an xsd:double with a fractional
438
+ part (a whole number in JSON reads as an xsd:integer); anything else as a value object with its datatype or
439
+ its language tag."""
440
+ if o.language:
441
+ return {"@value": str(o), "@language": o.language}
442
+ dt = o.datatype
443
+ if dt is None or dt == _XSD.string:
444
+ return str(o)
445
+ v = o.toPython()
446
+ if dt == _XSD.boolean and isinstance(v, bool):
447
+ return v
448
+ if dt == _XSD.integer and isinstance(v, int) and not isinstance(v, bool):
449
+ return v
450
+ if dt == _XSD.double and isinstance(v, float) and math.isfinite(v) and v % 1 != 0 and abs(v) < 1e21:
451
+ return v
452
+ return {"@value": str(o), "@type": str(dt)}
453
+
454
+
455
+ def _json_tree(graph: Graph, root: URIRef, parent_of: Dict[URIRef, Tuple[URIRef, URIRef, bool]]) -> Dict[str, Any]:
456
+ """The graph as one object tree rooted at `root` (JSON-LD 1.1): every object has @id, and @type where it has
457
+ types; a child sits under its parent, through the property (or, for a child that points at its parent,
458
+ under the parent's "@reverse", 4.8); every other node is {"@id": ...}. Nothing is dropped: a triple whose
459
+ subject is not in the tree goes under its object's "@reverse" when the object is, else in "@included"
460
+ (4.7), one object per subject. Read back, the tree is the graph."""
461
+ children: Dict[URIRef, List[Tuple[URIRef, URIRef, bool]]] = {}
462
+ for child, (parent, predicate, inverse) in parent_of.items():
463
+ children.setdefault(parent, []).append((predicate, child, inverse))
464
+ objects: Dict[Any, Dict[str, Any]] = {}
465
+ covered = set()
466
+
467
+ def value(term: Any) -> Any:
468
+ return _json_literal(term) if isinstance(term, Literal) else {"@id": _node_id(term)}
469
+
470
+ def write(node: Any, triples_of=None) -> Dict[str, Any]:
471
+ obj: Dict[str, Any] = {"@id": _node_id(node)}
472
+ objects.setdefault(node, obj)
473
+ mine = sorted(graph.triples((node, None, None)) if triples_of is None else triples_of, key=lambda t: (str(t[1]), str(t[2])))
474
+ types = [o for _, p, o in mine if p == RDF.type]
475
+ if types:
476
+ obj["@type"] = [_node_id(t) for t in types]
477
+ forward = {(p, c) for p, c, inverse in children.get(node, []) if not inverse}
478
+ for s, p, o in mine:
479
+ covered.add((s, p, o))
480
+ if p == RDF.type:
481
+ continue
482
+ obj.setdefault(str(p), []).append(write(o) if (p, o) in forward and o not in objects else value(o))
483
+ for p, c, inverse in sorted(children.get(node, []), key=lambda x: (str(x[0]), str(x[1]))):
484
+ if inverse and c not in objects:
485
+ obj.setdefault("@reverse", {}).setdefault(str(p), []).append(write(c))
486
+ return obj
487
+
488
+ tree = write(root)
489
+ rest: Dict[Any, List[Tuple[Any, Any, Any]]] = {}
490
+ for t in sorted((t for t in graph if t not in covered), key=lambda t: tuple(map(str, t))):
491
+ rest.setdefault(t[0], []).append(t)
492
+ included: Dict[Any, List[Tuple[Any, Any, Any]]] = {}
493
+ for s, triples in rest.items():
494
+ (_, p, o), = triples if len(triples) == 1 else [(None, None, None)]
495
+ if len(triples) == 1 and o in objects and p != RDF.type:
496
+ # a node outside the tree whose one triple links into it: under that node's @reverse
497
+ objects[o].setdefault("@reverse", {}).setdefault(str(p), []).append({"@id": _node_id(s)})
498
+ else:
499
+ included[s] = triples
500
+ if included:
501
+ tree["@included"] = [write(s, triples) for s, triples in sorted(included.items(), key=lambda kv: str(kv[0]))]
502
+ return tree
503
+
504
+
505
+ # ------------------------------------------------------------------------------------------------------ entry
506
+
507
+ def instance_graph(engine: Any, root: str, *, follow: Union[None, str, Sequence[str]] = None, depth: int = 1,
508
+ prefixes: Optional[Dict[str, str]] = None, shapes: Union[None, str, Graph] = None,
509
+ shape: Optional[str] = None, format: str = "graph") -> Union[Graph, str, Dict[str, Any]]:
510
+ """The whole graph of the instance `root`.
511
+
512
+ Args:
513
+ engine: a ProGraph engine.
514
+ root: the root's IRI.
515
+ follow: a SPARQL 1.1 property path, or several: the payload is the root and every node the walk reaches
516
+ through them (the module's docstring says how).
517
+ depth: how many times the walk applies a path that is not one step on a transitive property (1 by
518
+ default: the values of the root and of what the closures reach from it). The closures are not
519
+ counted: with depth=0 the payload is the root and what its transitive steps reach, and with no
520
+ transitive step, the root alone.
521
+ prefixes: prefix names for the paths, beside rdf, rdfs, owl, xsd and the bound ontology's own.
522
+ shapes: a SHACL shapes graph (an rdflib Graph or a Turtle file's path): the payload is what the node
523
+ shape for `root` describes. Give `follow` or `shapes`, not both.
524
+ shape: the node shape's IRI; without it, the one node shape whose sh:targetClass is a class of `root`.
525
+ format: "graph" (an rdflib Graph), "turtle", "json-ld" (rdflib's serializations), or "json" (one object
526
+ tree: each member under the member the walk first reached it from; it reads as JSON-LD too).
527
+ """
528
+ if format not in FORMATS:
529
+ raise InstanceGraphError(f"format is one of {', '.join(FORMATS)}, not {format!r}")
530
+ if (follow is None) == (shapes is None):
531
+ raise InstanceGraphError("give follow (property paths to walk) or shapes (a SHACL shapes graph)")
532
+ if not isinstance(depth, int) or isinstance(depth, bool) or depth < 0:
533
+ raise InstanceGraphError(f"depth is a whole number, 0 or more, not {depth!r}")
534
+ root_iri = URIRef(root)
535
+ _iri(root)
536
+ if follow is not None:
537
+ known = dict(_STANDARD_PREFIXES)
538
+ ontology = getattr(engine, "ontology", None)
539
+ if ontology is not None:
540
+ known.update({p: str(n) for p, n in ontology.rdf_graph.namespaces()})
541
+ known.update(prefixes or {})
542
+ graph, parents = _by_follow(engine, root_iri, [follow] if isinstance(follow, str) else list(follow),
543
+ depth, known)
544
+ else:
545
+ shapes_graph = shapes if isinstance(shapes, Graph) else Graph().parse(shapes)
546
+ graph, parents = _by_shape(engine, root_iri, shapes_graph, shape)
547
+ parent_of = {n: p for n, p in parents.items() if p is not None}
548
+ ontology = getattr(engine, "ontology", None)
549
+ if ontology is not None: # the ontology's own prefixes, for Turtle a reader recognises
550
+ for prefix, namespace in ontology.rdf_graph.namespaces():
551
+ graph.bind(prefix, namespace, override=False, replace=False)
552
+ if format == "graph":
553
+ return graph
554
+ if format == "turtle":
555
+ return graph.serialize(format="turtle")
556
+ if format == "json-ld":
557
+ return graph.serialize(format="json-ld")
558
+ return _json_tree(graph, root_iri, parent_of)
559
+
@@ -0,0 +1,83 @@
1
+ """ProGraph integration with Databricks Genie.
2
+
3
+ Exposes ProGraph's semantic work (OWL-entailed classes, SHACL
4
+ violations, schema-derived subclass closures, future metric
5
+ definitions) to Genie via three layered paths:
6
+
7
+ 1. **Generated SQL views**, ``prograph_views.*`` projections Genie
8
+ consumes natively as tables.
9
+ 2. **MCP server escape hatch**, the Phase 0 MCP server with
10
+ Genie-tuned tool descriptions for graph-shaped queries that don't
11
+ fit in SQL.
12
+ 3. **`prograph emit-genie-space` CLI**, turnkey: schema in, complete
13
+ Genie Space config out.
14
+
15
+ See ``docs/guides/genie.md`` for the user-facing workflow.
16
+
17
+ Status: package scaffold. Public types are defined here; the
18
+ generators (W1+) and the CLI (W3, W5) land in subsequent commits.
19
+ """
20
+
21
+ from prograph.genie.formats import (
22
+ GenieSpace,
23
+ InstructionsDoc,
24
+ InstructionsSection,
25
+ McpServerRegistration,
26
+ SampleQuery,
27
+ ViewDefinition,
28
+ ViewSource,
29
+ )
30
+ from prograph.genie.instruction_generator import (
31
+ DEFAULT_CHAR_BUDGET,
32
+ generate_instructions,
33
+ )
34
+ from prograph.genie.sample_query_generator import (
35
+ DEFAULT_MAX_QUERIES,
36
+ generate_sample_queries,
37
+ )
38
+ from prograph.genie.databricks_uploader import (
39
+ SERIALIZED_SPACE_VERSION,
40
+ to_databricks_serialized_space,
41
+ upload_space,
42
+ )
43
+ from prograph.genie.deploy import deploy_space
44
+ from prograph.genie.space_emitter import (
45
+ emit_genie_space,
46
+ load_space_yaml,
47
+ render_space_yaml,
48
+ write_space_yaml,
49
+ )
50
+ from prograph.genie.view_generator import generate_views
51
+ from prograph.genie.shacl_views import generate_shacl_violation_views
52
+ from prograph.genie.view_writer import (
53
+ render_view_sql,
54
+ render_views_script,
55
+ write_views,
56
+ )
57
+
58
+ __all__ = [
59
+ "ViewDefinition",
60
+ "ViewSource",
61
+ "SampleQuery",
62
+ "InstructionsDoc",
63
+ "InstructionsSection",
64
+ "McpServerRegistration",
65
+ "GenieSpace",
66
+ "generate_views",
67
+ "generate_shacl_violation_views",
68
+ "render_view_sql",
69
+ "render_views_script",
70
+ "write_views",
71
+ "generate_instructions",
72
+ "generate_sample_queries",
73
+ "emit_genie_space",
74
+ "render_space_yaml",
75
+ "write_space_yaml",
76
+ "load_space_yaml",
77
+ "DEFAULT_CHAR_BUDGET",
78
+ "DEFAULT_MAX_QUERIES",
79
+ "to_databricks_serialized_space",
80
+ "upload_space",
81
+ "deploy_space",
82
+ "SERIALIZED_SPACE_VERSION",
83
+ ]