prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/read_only.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""ProGraph is read-only over the data it maps, with two
|
|
2
|
+
exceptions Brice decided on 2026-11-04 (opt-in writes, ProAct's needs).
|
|
3
|
+
|
|
4
|
+
The one home of the rule: what counts as a write, which engines hold data
|
|
5
|
+
ProGraph owns, what may write a user's table, and the refusal's words.
|
|
6
|
+
Every entry point that could write a user's table asks here before
|
|
7
|
+
anything is read or written: `execute()` for a query's plan (Cypher's
|
|
8
|
+
CREATE, MERGE, SET, REMOVE, DELETE and DETACH DELETE, and the same
|
|
9
|
+
operations from any other language; SPARQL Update), `set_properties`, the
|
|
10
|
+
endpoint's `/update` and the MCP server's update form. No query form that
|
|
11
|
+
writes is built (`prograph.plan.ir`), so a query never writes.
|
|
12
|
+
|
|
13
|
+
What writes a user's table:
|
|
14
|
+
|
|
15
|
+
- `set_properties` (`prograph.writes`), on a node mapping marked
|
|
16
|
+
``mutable`` (``mutable: true`` in YAML, ``NodeMapping(mutable=True)``),
|
|
17
|
+
only the rows that mapping owns and never an inferred row; a mapping not
|
|
18
|
+
marked mutable is refused (:func:`refuse_not_mutable`);
|
|
19
|
+
|
|
20
|
+
Nothing else does: a query's write clauses and every SPARQL Update form
|
|
21
|
+
are refused, a graph's and an executor's
|
|
22
|
+
mutators refuse, a mapping's ``mutable`` opens `set_properties` (a
|
|
23
|
+
node's) only.
|
|
24
|
+
|
|
25
|
+
ProGraph still writes data it owns: generated views and telemetry (principles.md, "Operational artifacts").
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
from prograph.utils.exceptions import ReadOnlyGraphError
|
|
33
|
+
from prograph.rules import module_home
|
|
34
|
+
|
|
35
|
+
module_home("read-only-writes", "A write ProGraph refuses",
|
|
36
|
+
terms=("read only", "write", "update", "ReadOnlyGraphError", "set_properties", "INSERT"))
|
|
37
|
+
|
|
38
|
+
READ_ONLY = ("ProGraph is read-only over the tables it maps: {what} would write "
|
|
39
|
+
"to them, so it is refused and nothing was read or written. Query the data with "
|
|
40
|
+
"MATCH ... RETURN or SELECT; change it in the database or DataFrame that holds it.")
|
|
41
|
+
|
|
42
|
+
NOT_MUTABLE = ("set_properties writes only to a node mapping marked mutable (mutable: true in "
|
|
43
|
+
"the YAML, NodeMapping(mutable=True)); the mapping of {label!r} is not mutable, so the "
|
|
44
|
+
"write is refused and nothing was read or written. ProGraph is read-only over "
|
|
45
|
+
"the other tables it maps.")
|
|
46
|
+
|
|
47
|
+
ASK_WRITE = ("ask is read-only: the Cypher query writes ({clauses}), so it was refused and nothing was "
|
|
48
|
+
"run. Answer the question with a query that only reads: MATCH ... RETURN.")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
_UPDATING = ("DETACH DELETE", "CREATE", "MERGE", "SET", "REMOVE", "DELETE")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def cypher_write_clauses(query: str) -> list:
|
|
55
|
+
"""The updating clauses a Cypher query holds (CREATE, MERGE, SET, REMOVE, DELETE), in order, by their first
|
|
56
|
+
word; [] for a query that only reads. A query that does not parse holds none here: its parse error is the
|
|
57
|
+
planner's to raise."""
|
|
58
|
+
from prograph.plan.ir import cypher_front
|
|
59
|
+
from prograph.parser.cypher.CypherParser import CypherParser as P
|
|
60
|
+
try:
|
|
61
|
+
tree = cypher_front.parse(query)
|
|
62
|
+
except Exception: # noqa: BLE001 - not this rule's to report
|
|
63
|
+
return []
|
|
64
|
+
found = []
|
|
65
|
+
for c in cypher_front._walk(tree):
|
|
66
|
+
if isinstance(c, P.OC_UpdatingClauseContext):
|
|
67
|
+
# The clause's text with its whitespace taken out (getText keeps a space between DETACH and DELETE
|
|
68
|
+
# where the query has one), compared with each clause's words run together.
|
|
69
|
+
text = "".join(c.getText().upper().split())
|
|
70
|
+
word = next((w for w in _UPDATING if text.startswith(w.replace(" ", ""))), "an updating clause")
|
|
71
|
+
if word not in found:
|
|
72
|
+
found.append(word)
|
|
73
|
+
return found
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def refuse_ask_write(query: str) -> None:
|
|
77
|
+
"""Raise the refusal of a Cypher query that writes, for ask, which only reads (the owned store would take
|
|
78
|
+
the write otherwise)."""
|
|
79
|
+
clauses = cypher_write_clauses(query)
|
|
80
|
+
if clauses:
|
|
81
|
+
raise ReadOnlyGraphError(ASK_WRITE.format(clauses=", ".join(clauses)))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def message(what: str) -> str:
|
|
85
|
+
return READ_ONLY.format(what=what)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def refuse(what: str) -> None:
|
|
89
|
+
"""Raise the refusal for a write `what` names."""
|
|
90
|
+
raise ReadOnlyGraphError(message(what))
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def refuse_not_mutable(label: str) -> None:
|
|
94
|
+
"""Raise the refusal of `set_properties` on a mapping not marked mutable."""
|
|
95
|
+
raise ReadOnlyGraphError(NOT_MUTABLE.format(label=label))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# The mutators of a graph and of an executor over a user's data: every
|
|
99
|
+
# graph and executor takes these, which refuse. They come
|
|
100
|
+
# first among a class's bases, so nothing behind them answers instead.
|
|
101
|
+
GRAPH_MUTATORS = ("add_node", "add_relationship", "remove_node", "remove_relationship",
|
|
102
|
+
"set_node_property", "set_node_properties", "add_node_label",
|
|
103
|
+
"remove_node_label", "set_relationship_property")
|
|
104
|
+
EXECUTOR_MUTATORS = ("create_node", "create_relationship", "delete_node", "delete_relationship",
|
|
105
|
+
"set_property", "set_properties", "remove_property", "add_label",
|
|
106
|
+
"remove_label")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _refusing(name: str):
|
|
110
|
+
def mutator(self, *args: Any, **kwargs: Any) -> None:
|
|
111
|
+
refuse(f"{type(self).__name__}.{name}")
|
|
112
|
+
mutator.__name__ = name
|
|
113
|
+
mutator.__doc__ = "Refused: ProGraph is read-only over the tables it maps."
|
|
114
|
+
return mutator
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
GraphRefusesWrites = type("GraphRefusesWrites", (), {
|
|
118
|
+
"__doc__": "A graph over a user's tables: every mutator refuses.",
|
|
119
|
+
**{name: _refusing(name) for name in GRAPH_MUTATORS}})
|
|
120
|
+
ExecutorRefusesWrites = type("ExecutorRefusesWrites", (), {
|
|
121
|
+
"__doc__": "An executor over a user's tables: every mutator refuses.",
|
|
122
|
+
**{name: _refusing(name) for name in EXECUTOR_MUTATORS}})
|
prograph/results.py
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""Query results as Arrow.
|
|
2
|
+
|
|
3
|
+
execute() returns a pyarrow.Table: what the warehouse sent, with no
|
|
4
|
+
per-row Python in between. Databricks hands its results over as Arrow
|
|
5
|
+
(cursor.fetchall_arrow()), DuckDB natively; a result built from rows is converted once at the end. return_type="list" is to_pylist() of the
|
|
6
|
+
same table, for small results and plain Python.
|
|
7
|
+
|
|
8
|
+
ProGraph does nothing to a result after the database returns it: types are
|
|
9
|
+
set in the generated SQL (on SQL Server a predicate is cast to BIT, which the
|
|
10
|
+
driver returns as a boolean). Where a database cannot express a type (MySQL,
|
|
11
|
+
and Oracle before 23ai, have no boolean result type, so a predicate comes back
|
|
12
|
+
as 0/1), the result is what it returns.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from typing import Any, Dict, Iterable, List, Optional
|
|
18
|
+
|
|
19
|
+
import pyarrow as pa
|
|
20
|
+
from prograph.rules import module_home
|
|
21
|
+
|
|
22
|
+
module_home("result-conversion", "Result conversion",
|
|
23
|
+
terms=("Arrow table", "rows to Arrow", "result conversion", "node map", "relationship map"))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# ---------------------------------------------------------------------------
|
|
27
|
+
# Python rows to Arrow
|
|
28
|
+
# ---------------------------------------------------------------------------
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def rows_to_arrow(rows: List[Dict[str, Any]], columns: Optional[Iterable[str]] = None,
|
|
32
|
+
types: Optional[Dict[str, pa.DataType]] = None) -> pa.Table:
|
|
33
|
+
"""Python result rows as a table. `columns` names the columns when there
|
|
34
|
+
are no rows (the RETURN's names), so an empty result still has them.
|
|
35
|
+
`types` gives a column's type where the plan sets it, as the SQL sets
|
|
36
|
+
it on the other path (a SPARQL term struct): a field that is
|
|
37
|
+
null in every row keeps its type.
|
|
38
|
+
|
|
39
|
+
Cypher lets a column, or a list, hold values of different types
|
|
40
|
+
(`UNWIND [1, 'a'] AS x RETURN x`, `RETURN [1, 'a']`). Such values become a
|
|
41
|
+
dense union, Arrow's type for a value that is one of several types, so
|
|
42
|
+
every result is a table; to_pylist() gives the values back as they were.
|
|
43
|
+
"""
|
|
44
|
+
types = types or {}
|
|
45
|
+
if not rows:
|
|
46
|
+
names = list(columns or [])
|
|
47
|
+
return pa.table({name: pa.array([], type=types.get(name, pa.null())) for name in names})
|
|
48
|
+
names = list(rows[0].keys())
|
|
49
|
+
return pa.table({name: (pa.array([row.get(name) for row in rows], type=types[name])
|
|
50
|
+
if name in types else
|
|
51
|
+
_array([row.get(name) for row in rows]))
|
|
52
|
+
for name in names})
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# The kinds a dense union keeps apart, in a fixed order so equal results
|
|
56
|
+
# get equal types.
|
|
57
|
+
_KINDS = ("bool", "int", "float", "str", "bytes", "list", "map", "other", "decimal")
|
|
58
|
+
|
|
59
|
+
# The one definition of Arrow's dense union for values of several types: its members are kinds of
|
|
60
|
+
# `_KINDS`, in this order, each with the type code of its place there, so a value of a kind has one code
|
|
61
|
+
# whichever path built the union (`_array` from Python values, `cypher_mixed.unwrap_table` from a
|
|
62
|
+
# column of tagged structs). A Cypher mixed value is one of the first four, always all four members, so
|
|
63
|
+
# every batch of an answer has the one type whatever values it holds.
|
|
64
|
+
MIXED_MEMBERS = _KINDS[:4]
|
|
65
|
+
# A property map whose values include an exact decimal (a mapped DECIMAL column beside other types) holds
|
|
66
|
+
# a fifth member, the decimal at the columns' common precision and scale: integers, floats and decimals
|
|
67
|
+
# stay apart (an exact decimal read as a float would lose its exactness).
|
|
68
|
+
DECIMAL_MEMBER = "decimal"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def dense_union(kinds: List[str], type_codes: Any, offsets: Any, children: List[Any]) -> pa.Array:
|
|
72
|
+
"""A dense union of `children` (one per kind of `kinds`, in `_KINDS` order), each row's member given by
|
|
73
|
+
`type_codes` (the kinds' `_KINDS` indexes) and its place in that child by `offsets`."""
|
|
74
|
+
return pa.UnionArray.from_dense(type_codes, offsets, children, list(kinds),
|
|
75
|
+
[_KINDS.index(k) for k in kinds])
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _kind(value: Any) -> str:
|
|
79
|
+
if isinstance(value, bool):
|
|
80
|
+
return "bool"
|
|
81
|
+
if isinstance(value, int):
|
|
82
|
+
return "int"
|
|
83
|
+
if isinstance(value, float):
|
|
84
|
+
return "float"
|
|
85
|
+
if isinstance(value, str):
|
|
86
|
+
return "str"
|
|
87
|
+
if isinstance(value, (bytes, bytearray)):
|
|
88
|
+
return "bytes"
|
|
89
|
+
if isinstance(value, (list, tuple)):
|
|
90
|
+
return "list"
|
|
91
|
+
if isinstance(value, dict):
|
|
92
|
+
return "map"
|
|
93
|
+
return "other"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _array(values: List[Any]) -> pa.Array:
|
|
97
|
+
"""`values` as one Arrow array: its own type where Arrow infers one,
|
|
98
|
+
else a list or struct of arrays built the same way, else a dense union
|
|
99
|
+
of one child per kind of value."""
|
|
100
|
+
try:
|
|
101
|
+
return pa.array(values)
|
|
102
|
+
except (pa.ArrowInvalid, pa.ArrowTypeError, pa.ArrowNotImplementedError):
|
|
103
|
+
pass
|
|
104
|
+
present = [v for v in values if v is not None]
|
|
105
|
+
kinds = {_kind(v) for v in present}
|
|
106
|
+
if kinds == {"list"}:
|
|
107
|
+
offsets, flat = [0], []
|
|
108
|
+
for v in values:
|
|
109
|
+
if v is not None:
|
|
110
|
+
flat.extend(v)
|
|
111
|
+
offsets.append(len(flat))
|
|
112
|
+
mask = pa.array([v is None for v in values])
|
|
113
|
+
return pa.ListArray.from_arrays(pa.array(offsets, type=pa.int32()), _array(flat), mask=mask)
|
|
114
|
+
if kinds == {"map"}:
|
|
115
|
+
keys = list(dict.fromkeys(k for v in present for k in v))
|
|
116
|
+
children = [_array([None if v is None else v.get(k) for v in values]) for k in keys]
|
|
117
|
+
mask = pa.array([v is None for v in values])
|
|
118
|
+
return pa.StructArray.from_arrays(children, names=keys, mask=mask)
|
|
119
|
+
if kinds == {"other"}:
|
|
120
|
+
# Arrow has no type for these values and there is no kind to split
|
|
121
|
+
# them by: splitting again would never end.
|
|
122
|
+
raise TypeError(
|
|
123
|
+
f"a result value of type {type(present[0]).__name__} has no Arrow type")
|
|
124
|
+
order = [k for k in _KINDS if k in kinds] or ["other"]
|
|
125
|
+
groups: Dict[str, List[Any]] = {k: [] for k in order}
|
|
126
|
+
type_ids, offsets = [], []
|
|
127
|
+
for v in values:
|
|
128
|
+
# A null goes in the first child; the union keeps its position.
|
|
129
|
+
kind = order[0] if v is None else _kind(v)
|
|
130
|
+
type_ids.append(_KINDS.index(kind))
|
|
131
|
+
offsets.append(len(groups[kind]))
|
|
132
|
+
groups[kind].append(v)
|
|
133
|
+
return dense_union(order, pa.array(type_ids, type=pa.int8()), pa.array(offsets, type=pa.int32()),
|
|
134
|
+
[_array(groups[k]) for k in order])
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def table_to_pandas(table: pa.Table):
|
|
138
|
+
"""`table` as a pandas DataFrame. pandas has no column type for an
|
|
139
|
+
Arrow union, so a column holding one becomes an object column of the
|
|
140
|
+
same values; every other column converts as Arrow converts it."""
|
|
141
|
+
import pandas as pd
|
|
142
|
+
|
|
143
|
+
def has_union(t: pa.DataType) -> bool:
|
|
144
|
+
# A map (a node's properties) reads back as a dict, as the list answer gives it, not as pairs.
|
|
145
|
+
if pa.types.is_union(t) or pa.types.is_map(t):
|
|
146
|
+
return True
|
|
147
|
+
if pa.types.is_list(t) or pa.types.is_large_list(t):
|
|
148
|
+
return has_union(t.value_type)
|
|
149
|
+
return any(has_union(t.field(i).type) for i in range(t.num_fields))
|
|
150
|
+
|
|
151
|
+
unions = [f.name for f in table.schema if has_union(f.type)]
|
|
152
|
+
if not unions:
|
|
153
|
+
return table.to_pandas()
|
|
154
|
+
frame = table.drop_columns(unions).to_pandas()
|
|
155
|
+
for name in unions:
|
|
156
|
+
frame[name] = pd.Series(table.column(name).to_pylist(maps_as_pydicts="strict"), dtype=object)
|
|
157
|
+
return frame[table.column_names]
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""The registry of rules and their homes (docs/architecture/rules.md).
|
|
2
|
+
|
|
3
|
+
A rule that more than one layer needs (when two cells are one RDF term, how
|
|
4
|
+
SQL quotes a string, a term's text) lives in one home, and every other
|
|
5
|
+
module calls it. Before writing code, look the rule up here:
|
|
6
|
+
|
|
7
|
+
python -m prograph.rules term identity # rules whose id, title or text match
|
|
8
|
+
python -m prograph.rules # every rule
|
|
9
|
+
|
|
10
|
+
A home enters the registry by a marker that holds the rule's id, its title
|
|
11
|
+
and the terms it goes by. A marker does nothing: it records, and a static
|
|
12
|
+
scan of the package reads it (nothing is imported to answer a query; later
|
|
13
|
+
code may read the markers too):
|
|
14
|
+
|
|
15
|
+
@home("term-identity-in-sql", "When two cells are one RDF term, in SQL",
|
|
16
|
+
terms=("sameTerm", "DISTINCT", "GROUP BY", "COUNT(DISTINCT)"))
|
|
17
|
+
def identity_key(cell): ...
|
|
18
|
+
|
|
19
|
+
module_home("xsd-datatypes", "XSD datatypes") # a whole module's rule
|
|
20
|
+
|
|
21
|
+
Each rule has one home, and one row in docs/architecture/rules.md that says
|
|
22
|
+
what it owns (`tests/architecture/test_rules_registry.py` holds the two to
|
|
23
|
+
each other). A rule found nowhere here is new: give it a home and a row.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import ast
|
|
29
|
+
import re
|
|
30
|
+
import sys
|
|
31
|
+
from dataclasses import dataclass
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
from typing import Callable, Dict, List, Optional, Tuple, TypeVar
|
|
34
|
+
|
|
35
|
+
F = TypeVar("F", bound=Callable)
|
|
36
|
+
|
|
37
|
+
PACKAGE = Path(__file__).resolve().parents[1]
|
|
38
|
+
DOC = PACKAGE.parent / "docs" / "architecture" / "rules.md"
|
|
39
|
+
_ID = re.compile(r"[a-z][a-z0-9]*(-[a-z0-9]+)*")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def home(rule: str, title: str, *, terms: Tuple[str, ...] = ()) -> Callable[[F], F]:
|
|
43
|
+
"""Mark the function (or class) that is `rule`'s home, with the terms
|
|
44
|
+
the rule goes by. A holder only: the function is returned as it is."""
|
|
45
|
+
return lambda fn: fn
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def module_home(rule: str, title: str, *, terms: Tuple[str, ...] = ()) -> None:
|
|
49
|
+
"""Mark the calling module as `rule`'s home, where the whole module is
|
|
50
|
+
the rule (a table of datatypes, a dialect's quoting). A holder only."""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class Home:
|
|
55
|
+
"""One registered home: the rule's id and title, the module, and the
|
|
56
|
+
function or class (None for a whole module)."""
|
|
57
|
+
rule: str
|
|
58
|
+
title: str
|
|
59
|
+
module: str
|
|
60
|
+
name: Optional[str]
|
|
61
|
+
line: int
|
|
62
|
+
terms: Tuple[str, ...] = ()
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def where(self) -> str:
|
|
66
|
+
return f"{self.module}::{self.name}" if self.name else self.module
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _marker(call: ast.expr, names) -> Optional[ast.Call]:
|
|
70
|
+
if isinstance(call, ast.Call):
|
|
71
|
+
f = call.func
|
|
72
|
+
name = f.id if isinstance(f, ast.Name) else f.attr if isinstance(f, ast.Attribute) else None
|
|
73
|
+
if name in names:
|
|
74
|
+
return call
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _args(call: ast.Call):
|
|
79
|
+
"""The marker's id, title and terms, or None where they are not
|
|
80
|
+
literals (an id is lower-case words joined by '-')."""
|
|
81
|
+
try:
|
|
82
|
+
args = [ast.literal_eval(a) for a in call.args[:2]]
|
|
83
|
+
terms = next((tuple(ast.literal_eval(k.value)) for k in call.keywords
|
|
84
|
+
if k.arg == "terms"), ())
|
|
85
|
+
except ValueError:
|
|
86
|
+
return None
|
|
87
|
+
if len(args) != 2 or not _ID.fullmatch(str(args[0])):
|
|
88
|
+
return None
|
|
89
|
+
return args + [terms]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def registry(root: Path = PACKAGE) -> List[Home]:
|
|
93
|
+
"""Every marked home under `root`, by a static scan."""
|
|
94
|
+
out: List[Home] = []
|
|
95
|
+
for path in sorted(root.rglob("*.py")):
|
|
96
|
+
if "__pycache__" in path.parts or "parser" in path.parts:
|
|
97
|
+
continue
|
|
98
|
+
try:
|
|
99
|
+
tree = ast.parse(path.read_text())
|
|
100
|
+
except SyntaxError:
|
|
101
|
+
continue
|
|
102
|
+
module = path.relative_to(root.parent).as_posix()
|
|
103
|
+
for node in ast.walk(tree):
|
|
104
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
105
|
+
for d in node.decorator_list:
|
|
106
|
+
call = _marker(d, {"home"})
|
|
107
|
+
args = _args(call) if call else None
|
|
108
|
+
if args:
|
|
109
|
+
out.append(Home(args[0], args[1], module, node.name, node.lineno,
|
|
110
|
+
args[2]))
|
|
111
|
+
elif isinstance(node, ast.Expr):
|
|
112
|
+
call = _marker(node.value, {"module_home"})
|
|
113
|
+
args = _args(call) if call else None
|
|
114
|
+
if args:
|
|
115
|
+
out.append(Home(args[0], args[1], module, None, node.lineno, args[2]))
|
|
116
|
+
return out
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def rows(doc: Path = DOC) -> Dict[str, Dict[str, str]]:
|
|
120
|
+
"""rules.md's rows of rules with a home, by id: title, home, what it owns, its tests."""
|
|
121
|
+
out: Dict[str, Dict[str, str]] = {}
|
|
122
|
+
section = None
|
|
123
|
+
for line in doc.read_text().splitlines():
|
|
124
|
+
if line.startswith("## "):
|
|
125
|
+
section = line[3:].strip()
|
|
126
|
+
continue
|
|
127
|
+
if section != "Rules with a home" or not line.startswith("| `"):
|
|
128
|
+
continue
|
|
129
|
+
cells = [c.strip() for c in line.strip().strip("|").split("|")]
|
|
130
|
+
m = re.match(r"`([a-z0-9-]+)`", cells[0])
|
|
131
|
+
if m and len(cells) >= 4:
|
|
132
|
+
out[m.group(1)] = {"title": cells[1], "home": cells[2], "owns": cells[3],
|
|
133
|
+
"tests": cells[4] if len(cells) > 4 else ""}
|
|
134
|
+
return out
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _norm(text: str) -> str:
|
|
138
|
+
"""Text as the lookup compares it: lower case, `_` and `-` as spaces."""
|
|
139
|
+
return " ".join(re.sub(r"[_\-]+", " ", text.lower()).split())
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _text(h: "Home", table: Dict[str, Dict[str, str]]) -> str:
|
|
143
|
+
row = table.get(h.rule, {})
|
|
144
|
+
return _norm(" ".join([h.rule, h.title, *h.terms, row.get("owns", ""), row.get("tests", "")]))
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _holds(text: str, word: str) -> bool:
|
|
148
|
+
"""`word` begins a word of `text`, as written or without a plural `s` or
|
|
149
|
+
`es` (aliases finds alias, and alias finds aliases)."""
|
|
150
|
+
w = _norm(word)
|
|
151
|
+
forms = {w, re.sub(r"s$", "", w), re.sub(r"es$", "", w)} - {""}
|
|
152
|
+
return any(re.search(r"(?<![a-z0-9])" + re.escape(f), text) for f in forms if len(f) > 2 or f == w)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def find(words: List[str]) -> List[Home]:
|
|
156
|
+
"""The homes whose rule id, title, terms or rules.md row hold every word
|
|
157
|
+
(case, plurals, `-` and `_` aside; a word matches from a word's start)."""
|
|
158
|
+
table = rows()
|
|
159
|
+
return [h for h in registry() if all(_holds(_text(h, table), w) for w in words)]
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def closest(words: List[str], limit: int = 5) -> List[Tuple[Home, int]]:
|
|
163
|
+
"""Where no rule holds every word: the rules holding the most of them,
|
|
164
|
+
with how many, for the reader to judge (a rule may go by other words)."""
|
|
165
|
+
table = rows()
|
|
166
|
+
scored = [(h, sum(_holds(_text(h, table), w) for w in words)) for h in registry()]
|
|
167
|
+
scored = [(h, n) for h, n in scored if n]
|
|
168
|
+
return sorted(scored, key=lambda x: (-x[1], x[0].rule))[:limit]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""python -m prograph.rules [words ...]: the rules whose id, title or
|
|
2
|
+
rules.md text hold every word, and each one's home."""
|
|
3
|
+
|
|
4
|
+
import sys
|
|
5
|
+
from typing import List, Optional
|
|
6
|
+
|
|
7
|
+
from prograph.rules import closest, find, rows
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
11
|
+
words = list(sys.argv[1:] if argv is None else argv)
|
|
12
|
+
table = rows()
|
|
13
|
+
hits = find(words)
|
|
14
|
+
for h in sorted(hits, key=lambda h: h.rule):
|
|
15
|
+
print(f"{h.rule}: {h.title}\n home: {h.where} (line {h.line})")
|
|
16
|
+
if h.terms:
|
|
17
|
+
print(f" terms: {', '.join(h.terms)}")
|
|
18
|
+
owns = table.get(h.rule, {}).get("owns")
|
|
19
|
+
if owns:
|
|
20
|
+
print(f" owns: {owns[:300]}{'...' if len(owns) > 300 else ''}")
|
|
21
|
+
if not hits:
|
|
22
|
+
near = closest(words) if len(words) > 1 else []
|
|
23
|
+
if near:
|
|
24
|
+
print(f"no rule holds all {len(words)} words; the closest (read their rows before deciding):")
|
|
25
|
+
for h, n in near:
|
|
26
|
+
print(f" {h.rule} ({n} of {len(words)}): {h.title}\n home: {h.where} (line {h.line})")
|
|
27
|
+
print("if no rule above decides it, it is a new rule;"
|
|
28
|
+
" give it a home (@home) and a row in docs/architecture/rules.md")
|
|
29
|
+
return 0
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
sys.exit(main())
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Schema module for graph schema mapping.
|
|
2
|
+
|
|
3
|
+
This module provides classes for mapping external data sources
|
|
4
|
+
(Pandas DataFrames, Spark tables) to virtual graph structures.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from prograph.schema.mapping import (
|
|
8
|
+
TableMapping,
|
|
9
|
+
TableStats,
|
|
10
|
+
NodeMapping,
|
|
11
|
+
RelationshipMapping,
|
|
12
|
+
GraphSchema,
|
|
13
|
+
PredicateMapping,
|
|
14
|
+
PredicateKind,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
from prograph.schema.change_set import SchemaChangeSet
|
|
18
|
+
|
|
19
|
+
from prograph.schema.context import (
|
|
20
|
+
PropertyDescriptor,
|
|
21
|
+
NodeDescriptor,
|
|
22
|
+
RelationshipDescriptor,
|
|
23
|
+
PredicateDescriptor,
|
|
24
|
+
SchemaDescriptor,
|
|
25
|
+
describe_schema,
|
|
26
|
+
render_markdown,
|
|
27
|
+
render_compact,
|
|
28
|
+
render_prompt,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
from prograph.schema.examples import (
|
|
32
|
+
QueryExample,
|
|
33
|
+
generate_examples,
|
|
34
|
+
render_examples,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
__all__ = [
|
|
38
|
+
'TableMapping',
|
|
39
|
+
'TableStats',
|
|
40
|
+
'NodeMapping',
|
|
41
|
+
'RelationshipMapping',
|
|
42
|
+
'GraphSchema',
|
|
43
|
+
'PredicateMapping',
|
|
44
|
+
'PredicateKind',
|
|
45
|
+
'SchemaChangeSet',
|
|
46
|
+
# Descriptor / renderers
|
|
47
|
+
'PropertyDescriptor',
|
|
48
|
+
'NodeDescriptor',
|
|
49
|
+
'RelationshipDescriptor',
|
|
50
|
+
'PredicateDescriptor',
|
|
51
|
+
'SchemaDescriptor',
|
|
52
|
+
'describe_schema',
|
|
53
|
+
'render_markdown',
|
|
54
|
+
'render_compact',
|
|
55
|
+
'render_prompt',
|
|
56
|
+
# Examples
|
|
57
|
+
'QueryExample',
|
|
58
|
+
'generate_examples',
|
|
59
|
+
'render_examples',
|
|
60
|
+
]
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Schema change tracking for live schema updates.
|
|
2
|
+
|
|
3
|
+
Provides a lightweight dataclass that records what changed in a schema
|
|
4
|
+
update, enabling selective cache invalidation in graph backends.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Set
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class SchemaChangeSet:
|
|
13
|
+
"""Tracks what changed during a schema update.
|
|
14
|
+
|
|
15
|
+
Used by graph backends to selectively invalidate caches.
|
|
16
|
+
|
|
17
|
+
Example:
|
|
18
|
+
>>> changes = SchemaChangeSet(
|
|
19
|
+
... added_node_labels={'Product'},
|
|
20
|
+
... removed_rel_types={'PURCHASED'},
|
|
21
|
+
... )
|
|
22
|
+
>>> changes.has_relationship_changes
|
|
23
|
+
True
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
added_node_labels: Set[str] = field(default_factory=set)
|
|
27
|
+
removed_node_labels: Set[str] = field(default_factory=set)
|
|
28
|
+
replaced_node_labels: Set[str] = field(default_factory=set)
|
|
29
|
+
added_rel_types: Set[str] = field(default_factory=set)
|
|
30
|
+
removed_rel_types: Set[str] = field(default_factory=set)
|
|
31
|
+
added_tables: Set[str] = field(default_factory=set)
|
|
32
|
+
removed_tables: Set[str] = field(default_factory=set)
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def has_changes(self) -> bool:
|
|
36
|
+
"""True if any schema change was recorded."""
|
|
37
|
+
return bool(
|
|
38
|
+
self.added_node_labels
|
|
39
|
+
or self.removed_node_labels
|
|
40
|
+
or self.replaced_node_labels
|
|
41
|
+
or self.added_rel_types
|
|
42
|
+
or self.removed_rel_types
|
|
43
|
+
or self.added_tables
|
|
44
|
+
or self.removed_tables
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def has_relationship_changes(self) -> bool:
|
|
49
|
+
"""True if relationship mappings were added or removed."""
|
|
50
|
+
return bool(self.added_rel_types or self.removed_rel_types)
|