prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/read_only.py ADDED
@@ -0,0 +1,122 @@
1
+ """ProGraph is read-only over the data it maps, with two
2
+ exceptions Brice decided on 2026-11-04 (opt-in writes, ProAct's needs).
3
+
4
+ The one home of the rule: what counts as a write, which engines hold data
5
+ ProGraph owns, what may write a user's table, and the refusal's words.
6
+ Every entry point that could write a user's table asks here before
7
+ anything is read or written: `execute()` for a query's plan (Cypher's
8
+ CREATE, MERGE, SET, REMOVE, DELETE and DETACH DELETE, and the same
9
+ operations from any other language; SPARQL Update), `set_properties`, the
10
+ endpoint's `/update` and the MCP server's update form. No query form that
11
+ writes is built (`prograph.plan.ir`), so a query never writes.
12
+
13
+ What writes a user's table:
14
+
15
+ - `set_properties` (`prograph.writes`), on a node mapping marked
16
+ ``mutable`` (``mutable: true`` in YAML, ``NodeMapping(mutable=True)``),
17
+ only the rows that mapping owns and never an inferred row; a mapping not
18
+ marked mutable is refused (:func:`refuse_not_mutable`);
19
+
20
+ Nothing else does: a query's write clauses and every SPARQL Update form
21
+ are refused, a graph's and an executor's
22
+ mutators refuse, a mapping's ``mutable`` opens `set_properties` (a
23
+ node's) only.
24
+
25
+ ProGraph still writes data it owns: generated views and telemetry (principles.md, "Operational artifacts").
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ from typing import Any
31
+
32
+ from prograph.utils.exceptions import ReadOnlyGraphError
33
+ from prograph.rules import module_home
34
+
35
+ module_home("read-only-writes", "A write ProGraph refuses",
36
+ terms=("read only", "write", "update", "ReadOnlyGraphError", "set_properties", "INSERT"))
37
+
38
+ READ_ONLY = ("ProGraph is read-only over the tables it maps: {what} would write "
39
+ "to them, so it is refused and nothing was read or written. Query the data with "
40
+ "MATCH ... RETURN or SELECT; change it in the database or DataFrame that holds it.")
41
+
42
+ NOT_MUTABLE = ("set_properties writes only to a node mapping marked mutable (mutable: true in "
43
+ "the YAML, NodeMapping(mutable=True)); the mapping of {label!r} is not mutable, so the "
44
+ "write is refused and nothing was read or written. ProGraph is read-only over "
45
+ "the other tables it maps.")
46
+
47
+ ASK_WRITE = ("ask is read-only: the Cypher query writes ({clauses}), so it was refused and nothing was "
48
+ "run. Answer the question with a query that only reads: MATCH ... RETURN.")
49
+
50
+
51
+ _UPDATING = ("DETACH DELETE", "CREATE", "MERGE", "SET", "REMOVE", "DELETE")
52
+
53
+
54
+ def cypher_write_clauses(query: str) -> list:
55
+ """The updating clauses a Cypher query holds (CREATE, MERGE, SET, REMOVE, DELETE), in order, by their first
56
+ word; [] for a query that only reads. A query that does not parse holds none here: its parse error is the
57
+ planner's to raise."""
58
+ from prograph.plan.ir import cypher_front
59
+ from prograph.parser.cypher.CypherParser import CypherParser as P
60
+ try:
61
+ tree = cypher_front.parse(query)
62
+ except Exception: # noqa: BLE001 - not this rule's to report
63
+ return []
64
+ found = []
65
+ for c in cypher_front._walk(tree):
66
+ if isinstance(c, P.OC_UpdatingClauseContext):
67
+ # The clause's text with its whitespace taken out (getText keeps a space between DETACH and DELETE
68
+ # where the query has one), compared with each clause's words run together.
69
+ text = "".join(c.getText().upper().split())
70
+ word = next((w for w in _UPDATING if text.startswith(w.replace(" ", ""))), "an updating clause")
71
+ if word not in found:
72
+ found.append(word)
73
+ return found
74
+
75
+
76
+ def refuse_ask_write(query: str) -> None:
77
+ """Raise the refusal of a Cypher query that writes, for ask, which only reads (the owned store would take
78
+ the write otherwise)."""
79
+ clauses = cypher_write_clauses(query)
80
+ if clauses:
81
+ raise ReadOnlyGraphError(ASK_WRITE.format(clauses=", ".join(clauses)))
82
+
83
+
84
+ def message(what: str) -> str:
85
+ return READ_ONLY.format(what=what)
86
+
87
+
88
+ def refuse(what: str) -> None:
89
+ """Raise the refusal for a write `what` names."""
90
+ raise ReadOnlyGraphError(message(what))
91
+
92
+
93
+ def refuse_not_mutable(label: str) -> None:
94
+ """Raise the refusal of `set_properties` on a mapping not marked mutable."""
95
+ raise ReadOnlyGraphError(NOT_MUTABLE.format(label=label))
96
+
97
+
98
+ # The mutators of a graph and of an executor over a user's data: every
99
+ # graph and executor takes these, which refuse. They come
100
+ # first among a class's bases, so nothing behind them answers instead.
101
+ GRAPH_MUTATORS = ("add_node", "add_relationship", "remove_node", "remove_relationship",
102
+ "set_node_property", "set_node_properties", "add_node_label",
103
+ "remove_node_label", "set_relationship_property")
104
+ EXECUTOR_MUTATORS = ("create_node", "create_relationship", "delete_node", "delete_relationship",
105
+ "set_property", "set_properties", "remove_property", "add_label",
106
+ "remove_label")
107
+
108
+
109
+ def _refusing(name: str):
110
+ def mutator(self, *args: Any, **kwargs: Any) -> None:
111
+ refuse(f"{type(self).__name__}.{name}")
112
+ mutator.__name__ = name
113
+ mutator.__doc__ = "Refused: ProGraph is read-only over the tables it maps."
114
+ return mutator
115
+
116
+
117
+ GraphRefusesWrites = type("GraphRefusesWrites", (), {
118
+ "__doc__": "A graph over a user's tables: every mutator refuses.",
119
+ **{name: _refusing(name) for name in GRAPH_MUTATORS}})
120
+ ExecutorRefusesWrites = type("ExecutorRefusesWrites", (), {
121
+ "__doc__": "An executor over a user's tables: every mutator refuses.",
122
+ **{name: _refusing(name) for name in EXECUTOR_MUTATORS}})
prograph/results.py ADDED
@@ -0,0 +1,157 @@
1
+ """Query results as Arrow.
2
+
3
+ execute() returns a pyarrow.Table: what the warehouse sent, with no
4
+ per-row Python in between. Databricks hands its results over as Arrow
5
+ (cursor.fetchall_arrow()), DuckDB natively; a result built from rows is converted once at the end. return_type="list" is to_pylist() of the
6
+ same table, for small results and plain Python.
7
+
8
+ ProGraph does nothing to a result after the database returns it: types are
9
+ set in the generated SQL (on SQL Server a predicate is cast to BIT, which the
10
+ driver returns as a boolean). Where a database cannot express a type (MySQL,
11
+ and Oracle before 23ai, have no boolean result type, so a predicate comes back
12
+ as 0/1), the result is what it returns.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from typing import Any, Dict, Iterable, List, Optional
18
+
19
+ import pyarrow as pa
20
+ from prograph.rules import module_home
21
+
22
+ module_home("result-conversion", "Result conversion",
23
+ terms=("Arrow table", "rows to Arrow", "result conversion", "node map", "relationship map"))
24
+
25
+
26
+ # ---------------------------------------------------------------------------
27
+ # Python rows to Arrow
28
+ # ---------------------------------------------------------------------------
29
+
30
+
31
+ def rows_to_arrow(rows: List[Dict[str, Any]], columns: Optional[Iterable[str]] = None,
32
+ types: Optional[Dict[str, pa.DataType]] = None) -> pa.Table:
33
+ """Python result rows as a table. `columns` names the columns when there
34
+ are no rows (the RETURN's names), so an empty result still has them.
35
+ `types` gives a column's type where the plan sets it, as the SQL sets
36
+ it on the other path (a SPARQL term struct): a field that is
37
+ null in every row keeps its type.
38
+
39
+ Cypher lets a column, or a list, hold values of different types
40
+ (`UNWIND [1, 'a'] AS x RETURN x`, `RETURN [1, 'a']`). Such values become a
41
+ dense union, Arrow's type for a value that is one of several types, so
42
+ every result is a table; to_pylist() gives the values back as they were.
43
+ """
44
+ types = types or {}
45
+ if not rows:
46
+ names = list(columns or [])
47
+ return pa.table({name: pa.array([], type=types.get(name, pa.null())) for name in names})
48
+ names = list(rows[0].keys())
49
+ return pa.table({name: (pa.array([row.get(name) for row in rows], type=types[name])
50
+ if name in types else
51
+ _array([row.get(name) for row in rows]))
52
+ for name in names})
53
+
54
+
55
+ # The kinds a dense union keeps apart, in a fixed order so equal results
56
+ # get equal types.
57
+ _KINDS = ("bool", "int", "float", "str", "bytes", "list", "map", "other", "decimal")
58
+
59
+ # The one definition of Arrow's dense union for values of several types: its members are kinds of
60
+ # `_KINDS`, in this order, each with the type code of its place there, so a value of a kind has one code
61
+ # whichever path built the union (`_array` from Python values, `cypher_mixed.unwrap_table` from a
62
+ # column of tagged structs). A Cypher mixed value is one of the first four, always all four members, so
63
+ # every batch of an answer has the one type whatever values it holds.
64
+ MIXED_MEMBERS = _KINDS[:4]
65
+ # A property map whose values include an exact decimal (a mapped DECIMAL column beside other types) holds
66
+ # a fifth member, the decimal at the columns' common precision and scale: integers, floats and decimals
67
+ # stay apart (an exact decimal read as a float would lose its exactness).
68
+ DECIMAL_MEMBER = "decimal"
69
+
70
+
71
+ def dense_union(kinds: List[str], type_codes: Any, offsets: Any, children: List[Any]) -> pa.Array:
72
+ """A dense union of `children` (one per kind of `kinds`, in `_KINDS` order), each row's member given by
73
+ `type_codes` (the kinds' `_KINDS` indexes) and its place in that child by `offsets`."""
74
+ return pa.UnionArray.from_dense(type_codes, offsets, children, list(kinds),
75
+ [_KINDS.index(k) for k in kinds])
76
+
77
+
78
+ def _kind(value: Any) -> str:
79
+ if isinstance(value, bool):
80
+ return "bool"
81
+ if isinstance(value, int):
82
+ return "int"
83
+ if isinstance(value, float):
84
+ return "float"
85
+ if isinstance(value, str):
86
+ return "str"
87
+ if isinstance(value, (bytes, bytearray)):
88
+ return "bytes"
89
+ if isinstance(value, (list, tuple)):
90
+ return "list"
91
+ if isinstance(value, dict):
92
+ return "map"
93
+ return "other"
94
+
95
+
96
+ def _array(values: List[Any]) -> pa.Array:
97
+ """`values` as one Arrow array: its own type where Arrow infers one,
98
+ else a list or struct of arrays built the same way, else a dense union
99
+ of one child per kind of value."""
100
+ try:
101
+ return pa.array(values)
102
+ except (pa.ArrowInvalid, pa.ArrowTypeError, pa.ArrowNotImplementedError):
103
+ pass
104
+ present = [v for v in values if v is not None]
105
+ kinds = {_kind(v) for v in present}
106
+ if kinds == {"list"}:
107
+ offsets, flat = [0], []
108
+ for v in values:
109
+ if v is not None:
110
+ flat.extend(v)
111
+ offsets.append(len(flat))
112
+ mask = pa.array([v is None for v in values])
113
+ return pa.ListArray.from_arrays(pa.array(offsets, type=pa.int32()), _array(flat), mask=mask)
114
+ if kinds == {"map"}:
115
+ keys = list(dict.fromkeys(k for v in present for k in v))
116
+ children = [_array([None if v is None else v.get(k) for v in values]) for k in keys]
117
+ mask = pa.array([v is None for v in values])
118
+ return pa.StructArray.from_arrays(children, names=keys, mask=mask)
119
+ if kinds == {"other"}:
120
+ # Arrow has no type for these values and there is no kind to split
121
+ # them by: splitting again would never end.
122
+ raise TypeError(
123
+ f"a result value of type {type(present[0]).__name__} has no Arrow type")
124
+ order = [k for k in _KINDS if k in kinds] or ["other"]
125
+ groups: Dict[str, List[Any]] = {k: [] for k in order}
126
+ type_ids, offsets = [], []
127
+ for v in values:
128
+ # A null goes in the first child; the union keeps its position.
129
+ kind = order[0] if v is None else _kind(v)
130
+ type_ids.append(_KINDS.index(kind))
131
+ offsets.append(len(groups[kind]))
132
+ groups[kind].append(v)
133
+ return dense_union(order, pa.array(type_ids, type=pa.int8()), pa.array(offsets, type=pa.int32()),
134
+ [_array(groups[k]) for k in order])
135
+
136
+
137
+ def table_to_pandas(table: pa.Table):
138
+ """`table` as a pandas DataFrame. pandas has no column type for an
139
+ Arrow union, so a column holding one becomes an object column of the
140
+ same values; every other column converts as Arrow converts it."""
141
+ import pandas as pd
142
+
143
+ def has_union(t: pa.DataType) -> bool:
144
+ # A map (a node's properties) reads back as a dict, as the list answer gives it, not as pairs.
145
+ if pa.types.is_union(t) or pa.types.is_map(t):
146
+ return True
147
+ if pa.types.is_list(t) or pa.types.is_large_list(t):
148
+ return has_union(t.value_type)
149
+ return any(has_union(t.field(i).type) for i in range(t.num_fields))
150
+
151
+ unions = [f.name for f in table.schema if has_union(f.type)]
152
+ if not unions:
153
+ return table.to_pandas()
154
+ frame = table.drop_columns(unions).to_pandas()
155
+ for name in unions:
156
+ frame[name] = pd.Series(table.column(name).to_pylist(maps_as_pydicts="strict"), dtype=object)
157
+ return frame[table.column_names]
@@ -0,0 +1,168 @@
1
+ """The registry of rules and their homes (docs/architecture/rules.md).
2
+
3
+ A rule that more than one layer needs (when two cells are one RDF term, how
4
+ SQL quotes a string, a term's text) lives in one home, and every other
5
+ module calls it. Before writing code, look the rule up here:
6
+
7
+ python -m prograph.rules term identity # rules whose id, title or text match
8
+ python -m prograph.rules # every rule
9
+
10
+ A home enters the registry by a marker that holds the rule's id, its title
11
+ and the terms it goes by. A marker does nothing: it records, and a static
12
+ scan of the package reads it (nothing is imported to answer a query; later
13
+ code may read the markers too):
14
+
15
+ @home("term-identity-in-sql", "When two cells are one RDF term, in SQL",
16
+ terms=("sameTerm", "DISTINCT", "GROUP BY", "COUNT(DISTINCT)"))
17
+ def identity_key(cell): ...
18
+
19
+ module_home("xsd-datatypes", "XSD datatypes") # a whole module's rule
20
+
21
+ Each rule has one home, and one row in docs/architecture/rules.md that says
22
+ what it owns (`tests/architecture/test_rules_registry.py` holds the two to
23
+ each other). A rule found nowhere here is new: give it a home and a row.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import ast
29
+ import re
30
+ import sys
31
+ from dataclasses import dataclass
32
+ from pathlib import Path
33
+ from typing import Callable, Dict, List, Optional, Tuple, TypeVar
34
+
35
+ F = TypeVar("F", bound=Callable)
36
+
37
+ PACKAGE = Path(__file__).resolve().parents[1]
38
+ DOC = PACKAGE.parent / "docs" / "architecture" / "rules.md"
39
+ _ID = re.compile(r"[a-z][a-z0-9]*(-[a-z0-9]+)*")
40
+
41
+
42
+ def home(rule: str, title: str, *, terms: Tuple[str, ...] = ()) -> Callable[[F], F]:
43
+ """Mark the function (or class) that is `rule`'s home, with the terms
44
+ the rule goes by. A holder only: the function is returned as it is."""
45
+ return lambda fn: fn
46
+
47
+
48
+ def module_home(rule: str, title: str, *, terms: Tuple[str, ...] = ()) -> None:
49
+ """Mark the calling module as `rule`'s home, where the whole module is
50
+ the rule (a table of datatypes, a dialect's quoting). A holder only."""
51
+
52
+
53
+ @dataclass(frozen=True)
54
+ class Home:
55
+ """One registered home: the rule's id and title, the module, and the
56
+ function or class (None for a whole module)."""
57
+ rule: str
58
+ title: str
59
+ module: str
60
+ name: Optional[str]
61
+ line: int
62
+ terms: Tuple[str, ...] = ()
63
+
64
+ @property
65
+ def where(self) -> str:
66
+ return f"{self.module}::{self.name}" if self.name else self.module
67
+
68
+
69
+ def _marker(call: ast.expr, names) -> Optional[ast.Call]:
70
+ if isinstance(call, ast.Call):
71
+ f = call.func
72
+ name = f.id if isinstance(f, ast.Name) else f.attr if isinstance(f, ast.Attribute) else None
73
+ if name in names:
74
+ return call
75
+ return None
76
+
77
+
78
+ def _args(call: ast.Call):
79
+ """The marker's id, title and terms, or None where they are not
80
+ literals (an id is lower-case words joined by '-')."""
81
+ try:
82
+ args = [ast.literal_eval(a) for a in call.args[:2]]
83
+ terms = next((tuple(ast.literal_eval(k.value)) for k in call.keywords
84
+ if k.arg == "terms"), ())
85
+ except ValueError:
86
+ return None
87
+ if len(args) != 2 or not _ID.fullmatch(str(args[0])):
88
+ return None
89
+ return args + [terms]
90
+
91
+
92
+ def registry(root: Path = PACKAGE) -> List[Home]:
93
+ """Every marked home under `root`, by a static scan."""
94
+ out: List[Home] = []
95
+ for path in sorted(root.rglob("*.py")):
96
+ if "__pycache__" in path.parts or "parser" in path.parts:
97
+ continue
98
+ try:
99
+ tree = ast.parse(path.read_text())
100
+ except SyntaxError:
101
+ continue
102
+ module = path.relative_to(root.parent).as_posix()
103
+ for node in ast.walk(tree):
104
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
105
+ for d in node.decorator_list:
106
+ call = _marker(d, {"home"})
107
+ args = _args(call) if call else None
108
+ if args:
109
+ out.append(Home(args[0], args[1], module, node.name, node.lineno,
110
+ args[2]))
111
+ elif isinstance(node, ast.Expr):
112
+ call = _marker(node.value, {"module_home"})
113
+ args = _args(call) if call else None
114
+ if args:
115
+ out.append(Home(args[0], args[1], module, None, node.lineno, args[2]))
116
+ return out
117
+
118
+
119
+ def rows(doc: Path = DOC) -> Dict[str, Dict[str, str]]:
120
+ """rules.md's rows of rules with a home, by id: title, home, what it owns, its tests."""
121
+ out: Dict[str, Dict[str, str]] = {}
122
+ section = None
123
+ for line in doc.read_text().splitlines():
124
+ if line.startswith("## "):
125
+ section = line[3:].strip()
126
+ continue
127
+ if section != "Rules with a home" or not line.startswith("| `"):
128
+ continue
129
+ cells = [c.strip() for c in line.strip().strip("|").split("|")]
130
+ m = re.match(r"`([a-z0-9-]+)`", cells[0])
131
+ if m and len(cells) >= 4:
132
+ out[m.group(1)] = {"title": cells[1], "home": cells[2], "owns": cells[3],
133
+ "tests": cells[4] if len(cells) > 4 else ""}
134
+ return out
135
+
136
+
137
+ def _norm(text: str) -> str:
138
+ """Text as the lookup compares it: lower case, `_` and `-` as spaces."""
139
+ return " ".join(re.sub(r"[_\-]+", " ", text.lower()).split())
140
+
141
+
142
+ def _text(h: "Home", table: Dict[str, Dict[str, str]]) -> str:
143
+ row = table.get(h.rule, {})
144
+ return _norm(" ".join([h.rule, h.title, *h.terms, row.get("owns", ""), row.get("tests", "")]))
145
+
146
+
147
+ def _holds(text: str, word: str) -> bool:
148
+ """`word` begins a word of `text`, as written or without a plural `s` or
149
+ `es` (aliases finds alias, and alias finds aliases)."""
150
+ w = _norm(word)
151
+ forms = {w, re.sub(r"s$", "", w), re.sub(r"es$", "", w)} - {""}
152
+ return any(re.search(r"(?<![a-z0-9])" + re.escape(f), text) for f in forms if len(f) > 2 or f == w)
153
+
154
+
155
+ def find(words: List[str]) -> List[Home]:
156
+ """The homes whose rule id, title, terms or rules.md row hold every word
157
+ (case, plurals, `-` and `_` aside; a word matches from a word's start)."""
158
+ table = rows()
159
+ return [h for h in registry() if all(_holds(_text(h, table), w) for w in words)]
160
+
161
+
162
+ def closest(words: List[str], limit: int = 5) -> List[Tuple[Home, int]]:
163
+ """Where no rule holds every word: the rules holding the most of them,
164
+ with how many, for the reader to judge (a rule may go by other words)."""
165
+ table = rows()
166
+ scored = [(h, sum(_holds(_text(h, table), w) for w in words)) for h in registry()]
167
+ scored = [(h, n) for h, n in scored if n]
168
+ return sorted(scored, key=lambda x: (-x[1], x[0].rule))[:limit]
@@ -0,0 +1,32 @@
1
+ """python -m prograph.rules [words ...]: the rules whose id, title or
2
+ rules.md text hold every word, and each one's home."""
3
+
4
+ import sys
5
+ from typing import List, Optional
6
+
7
+ from prograph.rules import closest, find, rows
8
+
9
+
10
+ def main(argv: Optional[List[str]] = None) -> int:
11
+ words = list(sys.argv[1:] if argv is None else argv)
12
+ table = rows()
13
+ hits = find(words)
14
+ for h in sorted(hits, key=lambda h: h.rule):
15
+ print(f"{h.rule}: {h.title}\n home: {h.where} (line {h.line})")
16
+ if h.terms:
17
+ print(f" terms: {', '.join(h.terms)}")
18
+ owns = table.get(h.rule, {}).get("owns")
19
+ if owns:
20
+ print(f" owns: {owns[:300]}{'...' if len(owns) > 300 else ''}")
21
+ if not hits:
22
+ near = closest(words) if len(words) > 1 else []
23
+ if near:
24
+ print(f"no rule holds all {len(words)} words; the closest (read their rows before deciding):")
25
+ for h, n in near:
26
+ print(f" {h.rule} ({n} of {len(words)}): {h.title}\n home: {h.where} (line {h.line})")
27
+ print("if no rule above decides it, it is a new rule;"
28
+ " give it a home (@home) and a row in docs/architecture/rules.md")
29
+ return 0
30
+
31
+
32
+ sys.exit(main())
@@ -0,0 +1,60 @@
1
+ """Schema module for graph schema mapping.
2
+
3
+ This module provides classes for mapping external data sources
4
+ (Pandas DataFrames, Spark tables) to virtual graph structures.
5
+ """
6
+
7
+ from prograph.schema.mapping import (
8
+ TableMapping,
9
+ TableStats,
10
+ NodeMapping,
11
+ RelationshipMapping,
12
+ GraphSchema,
13
+ PredicateMapping,
14
+ PredicateKind,
15
+ )
16
+
17
+ from prograph.schema.change_set import SchemaChangeSet
18
+
19
+ from prograph.schema.context import (
20
+ PropertyDescriptor,
21
+ NodeDescriptor,
22
+ RelationshipDescriptor,
23
+ PredicateDescriptor,
24
+ SchemaDescriptor,
25
+ describe_schema,
26
+ render_markdown,
27
+ render_compact,
28
+ render_prompt,
29
+ )
30
+
31
+ from prograph.schema.examples import (
32
+ QueryExample,
33
+ generate_examples,
34
+ render_examples,
35
+ )
36
+
37
+ __all__ = [
38
+ 'TableMapping',
39
+ 'TableStats',
40
+ 'NodeMapping',
41
+ 'RelationshipMapping',
42
+ 'GraphSchema',
43
+ 'PredicateMapping',
44
+ 'PredicateKind',
45
+ 'SchemaChangeSet',
46
+ # Descriptor / renderers
47
+ 'PropertyDescriptor',
48
+ 'NodeDescriptor',
49
+ 'RelationshipDescriptor',
50
+ 'PredicateDescriptor',
51
+ 'SchemaDescriptor',
52
+ 'describe_schema',
53
+ 'render_markdown',
54
+ 'render_compact',
55
+ 'render_prompt',
56
+ # Examples
57
+ 'QueryExample',
58
+ 'generate_examples',
59
+ 'render_examples',
60
+ ]
@@ -0,0 +1,50 @@
1
+ """Schema change tracking for live schema updates.
2
+
3
+ Provides a lightweight dataclass that records what changed in a schema
4
+ update, enabling selective cache invalidation in graph backends.
5
+ """
6
+
7
+ from dataclasses import dataclass, field
8
+ from typing import Set
9
+
10
+
11
+ @dataclass
12
+ class SchemaChangeSet:
13
+ """Tracks what changed during a schema update.
14
+
15
+ Used by graph backends to selectively invalidate caches.
16
+
17
+ Example:
18
+ >>> changes = SchemaChangeSet(
19
+ ... added_node_labels={'Product'},
20
+ ... removed_rel_types={'PURCHASED'},
21
+ ... )
22
+ >>> changes.has_relationship_changes
23
+ True
24
+ """
25
+
26
+ added_node_labels: Set[str] = field(default_factory=set)
27
+ removed_node_labels: Set[str] = field(default_factory=set)
28
+ replaced_node_labels: Set[str] = field(default_factory=set)
29
+ added_rel_types: Set[str] = field(default_factory=set)
30
+ removed_rel_types: Set[str] = field(default_factory=set)
31
+ added_tables: Set[str] = field(default_factory=set)
32
+ removed_tables: Set[str] = field(default_factory=set)
33
+
34
+ @property
35
+ def has_changes(self) -> bool:
36
+ """True if any schema change was recorded."""
37
+ return bool(
38
+ self.added_node_labels
39
+ or self.removed_node_labels
40
+ or self.replaced_node_labels
41
+ or self.added_rel_types
42
+ or self.removed_rel_types
43
+ or self.added_tables
44
+ or self.removed_tables
45
+ )
46
+
47
+ @property
48
+ def has_relationship_changes(self) -> bool:
49
+ """True if relationship mappings were added or removed."""
50
+ return bool(self.added_rel_types or self.removed_rel_types)