prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,643 @@
|
|
|
1
|
+
"""The dataset a query sees (SPARQL 1.1 Query 13): its default graph and its named graphs, decided here alone.
|
|
2
|
+
|
|
3
|
+
- **Default graph**: the mapped data per `default_graph` (a map with no graph map, or `rr:defaultGraph`, is in it;
|
|
4
|
+
under 'union' every mapped triple is), the ontology's triples (`ontology_triples`: closed under the RDFS core,
|
|
5
|
+
read-only, with what the data's vocabulary adds where it has triples), the consequences
|
|
6
|
+
the plan rewrites (the saturated maps and the typing branches, `entry.saturated`), and the owned store's stored
|
|
7
|
+
facts (OWL RL's materialized ones) where they were made under the engine's own cache key (`cache_key`).
|
|
8
|
+
- **Named graphs**: the mapped graph maps, and the owned store's writer graphs (Cypher's, ProAct's). A writer
|
|
9
|
+
graph is asserted data: each statement is in its writer's graph and in the default graph (the store's
|
|
10
|
+
generated mapping puts both graph maps on it, as an R2RML triples map may, 6.4), so a write is read back with
|
|
11
|
+
or without GRAPH, under either `default_graph`. Stored derived facts are never in a named graph.
|
|
12
|
+
- **FROM / FROM NAMED** (13.2) replace the data part (`dataset_choices`): the stored facts were derived over the
|
|
13
|
+
configured default graph, so a default graph a query builds with FROM holds none of them. owl:sameAs matches
|
|
14
|
+
through the components of the graph a pattern reads (`active_graph`): GRAPH <g>'s, the one FROM graph's, or
|
|
15
|
+
the default graph's.
|
|
16
|
+
- **One set.** Every source above gives its triples through maps, and a triple pattern is the Distinct of their
|
|
17
|
+
union (`Translator.matched`): a fact mapped, rewritten and stored at once matches once.
|
|
18
|
+
- **Stored facts are a cache** (`cache_key`): the profile, the ontology's digest (imports included: the graph
|
|
19
|
+
the engine reads), the mapping and `default_graph`, and the datatype map; not the sameas mode, an output switch
|
|
20
|
+
under which the same facts and components are stored. The store records
|
|
21
|
+
the key of its last run (`owl_rl.last_run`) in the run's own write step. The key is the engine's
|
|
22
|
+
(`engine_key`), never its schema's, which several engines may share. Whether a query reads the stored facts
|
|
23
|
+
is decided when it is planned, from one statement read then (`store_now`, never a `Store` object's cached
|
|
24
|
+
state: the store's highest seq, the record of its last run and its latest alias generation): their maps join
|
|
25
|
+
only where the store's last run was made under the engine's key (`maps_seen`), and owl:sameAs reads the latest
|
|
26
|
+
alias generation only where its marker and the record both carry that key (`identity.for_engine` takes the
|
|
27
|
+
answer, comparing nothing). Under the default freshness the same planning also proves the facts current
|
|
28
|
+
(`freshness`): the sources' versions the run recorded against theirs now, and the store's seq against the
|
|
29
|
+
run's step; a change refreshes first, a source no plan can prove refuses by name. The facts themselves are read when the query runs, so a retraction
|
|
30
|
+
is seen at once. A condition in each scan instead was repeated at every step of a path: a path over stored
|
|
31
|
+
facts on Spark took 70 s against 42 s, and the alias rows' grew Spark's plans past its driver's memory. An OWL RL run reads as premises the store's
|
|
32
|
+
asserted statements and its own derivations, never what the store held derived (`premise_statements`).
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
from dataclasses import dataclass, field
|
|
38
|
+
from typing import Any, Dict, FrozenSet, List, Optional, Sequence, Tuple
|
|
39
|
+
|
|
40
|
+
from prograph.plan.ir.terms import IRITerm
|
|
41
|
+
from prograph.rules import home, module_home
|
|
42
|
+
from prograph.schema.source_map import RR_DEFAULT_GRAPH
|
|
43
|
+
|
|
44
|
+
module_home("dataset-seen", "The dataset a query sees: default and named graphs from mapped data, ontology "
|
|
45
|
+
"triples, rewritten consequences and stored facts",
|
|
46
|
+
terms=("dataset a query sees", "union default graph", "default_graph", "stored facts",
|
|
47
|
+
"derived facts visible", "cache key", "RL graph visibility", "writer graph", "store maps",
|
|
48
|
+
"generated mapping", "ontology triples", "one set", "premise statements", "rr:defaultGraph"))
|
|
49
|
+
|
|
50
|
+
# The datatype map a run's facts were derived under: RDF term identity (D-entailment is not built).
|
|
51
|
+
DATATYPES = "term"
|
|
52
|
+
|
|
53
|
+
# The OWL profiles an ontology is reasoned under: 'ql' (the default) by rewriting alone, 'rl' with stored facts too.
|
|
54
|
+
PROFILES = ("ql", "rl")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@home("entailment-profile", "Whether an engine reasons, and under which OWL profile",
|
|
58
|
+
terms=("profile", "OWL profile", "ql", "rl", "ontology turns reasoning on", "simple matching",
|
|
59
|
+
"simple entailment", "reasoning on"))
|
|
60
|
+
def engine_profile(ontology: Any, profile: Optional[str]) -> Optional[str]:
|
|
61
|
+
"""The profile an engine reasons under: None with no ontology (a query matches simply), else `profile`,
|
|
62
|
+
'ql' when none is given. `profile` without an ontology, or outside `PROFILES`, raises ValueError."""
|
|
63
|
+
if ontology is None:
|
|
64
|
+
if profile is not None:
|
|
65
|
+
raise ValueError(f"profile={profile!r} picks the OWL profile an ontology is reasoned under: give the "
|
|
66
|
+
f"engine an ontology= too (with no ontology a query matches simply)")
|
|
67
|
+
return None
|
|
68
|
+
if profile is None:
|
|
69
|
+
return "ql"
|
|
70
|
+
if profile not in PROFILES:
|
|
71
|
+
raise ValueError(f"profile is one of {', '.join(repr(p) for p in PROFILES)}, not {profile!r}")
|
|
72
|
+
return profile
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def derived_graphs() -> Tuple[str, ...]:
|
|
76
|
+
"""The owned store's graphs that hold stored derived facts (a cache, never asserted data)."""
|
|
77
|
+
from prograph.store import RL_GRAPH
|
|
78
|
+
return (RL_GRAPH,)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def property_graph_graphs() -> Tuple[str, ...]:
|
|
82
|
+
"""The owned store's graphs the property-graph view (Cypher) reads: Cypher's own writes."""
|
|
83
|
+
from prograph.store import CYPHER_GRAPH
|
|
84
|
+
return (CYPHER_GRAPH,)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# -- the stored facts' cache key ----------------------------------------------------------------------------
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def cache_key(engine: Any, rules: Any) -> str:
|
|
91
|
+
"""The key stored facts are made and read under: the profile (`profile=`), the ontology's digest
|
|
92
|
+
(`rules.digest`, of the graph the engine reads, its imports included), the engine's own mapping (its compiled
|
|
93
|
+
source maps, the owned store's generated maps aside: those change with what a run writes), `default_graph`,
|
|
94
|
+
and the datatype map (not the sameas mode: both modes store the same facts and components). The store keeps the key of its last run (`owl_rl.last_run`); a run on
|
|
95
|
+
open is skipped where it is the engine's and its sources are unchanged (`stale`)."""
|
|
96
|
+
import hashlib
|
|
97
|
+
parts = (f"profile={getattr(engine, '_profile', None)}", f"ontology={rules.digest}",
|
|
98
|
+
f"mapping={canonical(own_maps(engine))}", f"default_graph={getattr(engine, 'default_graph', '')}",
|
|
99
|
+
f"datatypes={DATATYPES}")
|
|
100
|
+
return hashlib.sha256("\n".join(parts).encode("utf-8")).hexdigest()
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def own_maps(engine: Any) -> Any:
|
|
104
|
+
"""The engine's own mapping, compiled: its source maps without the owned store's generated ones (those
|
|
105
|
+
change with what a run writes). What the cache key covers, and whose tables a run's record versions."""
|
|
106
|
+
from prograph.r2rml.compile import compile_r2rml
|
|
107
|
+
schema = engine._schema
|
|
108
|
+
r2rml = schema.__dict__.get("r2rml_source")
|
|
109
|
+
if r2rml is not None:
|
|
110
|
+
maps = compile_r2rml(r2rml[0], r2rml[1])
|
|
111
|
+
else:
|
|
112
|
+
from prograph.schema.compile_schema import compile_schema
|
|
113
|
+
maps = compile_schema(schema)
|
|
114
|
+
included = schema.__dict__.get("included_r2rml")
|
|
115
|
+
if included:
|
|
116
|
+
maps = maps.joined(compile_r2rml(*included))
|
|
117
|
+
return maps
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def canonical(x: Any) -> str:
|
|
121
|
+
"""`x` (compiled maps: dataclasses, tuples, sets, dicts, enums and plain values) as text that does not depend
|
|
122
|
+
on the process: a set's members and a dict's keys sorted by their own canonical text (a frozenset's
|
|
123
|
+
iteration order follows PYTHONHASHSEED)."""
|
|
124
|
+
import dataclasses
|
|
125
|
+
import enum
|
|
126
|
+
if dataclasses.is_dataclass(x) and not isinstance(x, type):
|
|
127
|
+
fields = ", ".join(f"{f.name}={canonical(getattr(x, f.name))}" for f in dataclasses.fields(x))
|
|
128
|
+
return f"{type(x).__name__}({fields})"
|
|
129
|
+
if isinstance(x, (set, frozenset)):
|
|
130
|
+
return "{" + ", ".join(sorted(canonical(v) for v in x)) + "}"
|
|
131
|
+
if isinstance(x, dict):
|
|
132
|
+
return "{" + ", ".join(sorted(f"{canonical(k)}: {canonical(v)}" for k, v in x.items())) + "}"
|
|
133
|
+
if isinstance(x, (tuple, list)):
|
|
134
|
+
return "(" + ", ".join(canonical(v) for v in x) + ")"
|
|
135
|
+
if isinstance(x, enum.Enum):
|
|
136
|
+
return f"{type(x).__name__}.{x.name}"
|
|
137
|
+
return repr(x)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def engine_key(engine: Any, onto: Any = None) -> Optional[str]:
|
|
141
|
+
"""The key the engine reads stored facts under (`cache_key`), kept on the engine and made again when its
|
|
142
|
+
ontology's content changes (`entry.ontology_of`; `onto` its result where the caller holds it); None where the
|
|
143
|
+
engine stores no fact (no owned store, or a profile but 'rl'), so it sees none."""
|
|
144
|
+
if getattr(engine, "_profile", None) != "rl" or getattr(engine, "_store", None) is None:
|
|
145
|
+
return None
|
|
146
|
+
if onto is None:
|
|
147
|
+
from prograph.plan.ir.entry import ontology_of
|
|
148
|
+
onto = ontology_of(engine)
|
|
149
|
+
held = engine.__dict__.get("_stored_key")
|
|
150
|
+
if held is None or held[0] is not onto:
|
|
151
|
+
held = (onto, cache_key(engine, engine._rl_rules()))
|
|
152
|
+
engine.__dict__["_stored_key"] = held
|
|
153
|
+
return held[1]
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def stored_sources(engine: Any) -> Tuple[Any, ...]:
|
|
157
|
+
"""The tables the engine's stored facts are derived from (`freshness.sources` of `own_maps`), with their
|
|
158
|
+
declared version columns; made once per cache key."""
|
|
159
|
+
from prograph.plan.ir import freshness as F
|
|
160
|
+
held = engine.__dict__.get("_stored_key")
|
|
161
|
+
found = engine.__dict__.get("_stored_sources")
|
|
162
|
+
if found is None or found[0] is not held:
|
|
163
|
+
found = (held, F.sources(engine, own_maps(engine)))
|
|
164
|
+
engine.__dict__["_stored_sources"] = found
|
|
165
|
+
return found[1]
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _record_parts(store: Any, sources: bool) -> List[str]:
|
|
169
|
+
"""SQL parts (kind, a, note, generation) of the store's record of its last run: its key ('key'), time
|
|
170
|
+
('at'), and with `sources` the source versions it read ('sources') and the seq of its step ('run_seq')."""
|
|
171
|
+
from prograph.plan.ir.owl_rl import RUN_AT, RUN_DIGEST, RUN_SOURCES, RUN_SUBJECT
|
|
172
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
173
|
+
from prograph.store import ASSERT
|
|
174
|
+
d = get_dialect_instance(store.dialect)
|
|
175
|
+
text = d.type_name("STRING")
|
|
176
|
+
nulls = f"CAST(NULL AS {text}) AS note, CAST(NULL AS BIGINT) AS generation"
|
|
177
|
+
wanted = [("key", RUN_DIGEST), ("at", RUN_AT)] + ([("sources", RUN_SOURCES)] if sources else [])
|
|
178
|
+
parts = [f"SELECT '{kind}' AS kind, _k.o_text AS a, {nulls} FROM "
|
|
179
|
+
f"({store.effective_sql(derived_graphs(), about=(RUN_SUBJECT, p))}) AS _k" for kind, p in wanted]
|
|
180
|
+
if sources:
|
|
181
|
+
graphs = ", ".join(d.literal(g) for g in derived_graphs())
|
|
182
|
+
parts.append(f"SELECT 'run_seq' AS kind, CAST(max(_r.seq) AS {text}) AS a, {nulls} FROM {store._visible()} "
|
|
183
|
+
f"AS _r WHERE _r.graph IN ({graphs}) AND _r.subject = {d.literal(RUN_SUBJECT)} AND "
|
|
184
|
+
f"_r.predicate = {d.literal(RUN_DIGEST)} AND _r.op = '{ASSERT}'")
|
|
185
|
+
return parts
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _first(rows: List[tuple], kind: str) -> Optional[str]:
|
|
189
|
+
return next((r[1] for r in rows if r[0] == kind), None)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def stale(engine: Any) -> bool:
|
|
193
|
+
"""Whether the engine's stored facts need a run, read from the store and the sources now (when it is
|
|
194
|
+
built, `ProGraph._materialize_on_open`): `stale_sources` names something."""
|
|
195
|
+
return bool(stale_sources(engine))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def stale_sources(engine: Any) -> Tuple[str, ...]:
|
|
199
|
+
"""What the engine's stored facts are stale for, in the user's words, read from the store and the sources
|
|
200
|
+
now: the store holds no run under the engine's key, or a source (by its declared column, Delta version, or
|
|
201
|
+
a small table's checksum) or the store itself changed since that run (`freshness`). () where neither; a
|
|
202
|
+
source no token proves is not named (a query refuses it, or reads the last run by choice)."""
|
|
203
|
+
why, srcs, verdict = _compared(engine)
|
|
204
|
+
if why is not None:
|
|
205
|
+
return why
|
|
206
|
+
return _labelled(srcs, verdict.changed) if verdict is not None else ()
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def provably_current(engine: Any) -> bool:
|
|
210
|
+
"""Whether the store holds a run under the engine's key and every source is proven unchanged since it
|
|
211
|
+
(`freshness.Verdict.current`): a source no token proves counts as changed. Where True, a run would
|
|
212
|
+
write nothing new (`ProGraph.materialize_entailments` returns the last run's record instead)."""
|
|
213
|
+
why, _, verdict = _compared(engine)
|
|
214
|
+
return why is None and verdict is not None and verdict.current
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _compared(engine: Any) -> Tuple[Optional[Tuple[str, ...]], Tuple[Any, ...], Any]:
|
|
218
|
+
"""(why the store holds no run of this engine, the sources, the verdict of the sources now against the
|
|
219
|
+
run's record); (the reason, (), None) where no run is held under the key, and (None, (), None) where the
|
|
220
|
+
engine stores no fact."""
|
|
221
|
+
from prograph.plan.ir import freshness as F
|
|
222
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
223
|
+
key = engine_key(engine)
|
|
224
|
+
if key is None:
|
|
225
|
+
return (), (), None
|
|
226
|
+
store = engine._store
|
|
227
|
+
text = get_dialect_instance(store.dialect).type_name("STRING")
|
|
228
|
+
nulls = f"CAST(NULL AS {text}) AS note, CAST(NULL AS BIGINT) AS generation"
|
|
229
|
+
parts = [f"SELECT 'seq' AS kind, CAST(_s.m AS {text}) AS a, {nulls} FROM ({store.max_seq_sql()}) AS _s"]
|
|
230
|
+
rows = store.db.execute(" UNION ALL ".join(parts + _record_parts(store, True))).fetchall()
|
|
231
|
+
if _first(rows, "key") != key:
|
|
232
|
+
return ("the store's last run, made under another key",), (), None
|
|
233
|
+
srcs = stored_sources(engine)
|
|
234
|
+
now = F.read_now(engine, srcs)
|
|
235
|
+
run_seq = _first(rows, "run_seq")
|
|
236
|
+
moved = run_seq is None or int(_first(rows, "seq")) > int(run_seq)
|
|
237
|
+
return None, srcs, F.compare(F.recorded(_first(rows, "sources")), now, moved)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _labelled(srcs: Tuple[Any, ...], names: Tuple[str, ...]) -> Tuple[str, ...]:
|
|
241
|
+
by = {s.name: s.label or s.name for s in srcs}
|
|
242
|
+
return tuple(by.get(n, n) for n in names)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def recorded_key(store: Any) -> Optional[str]:
|
|
246
|
+
"""The key of the store's last OWL RL run, read from the store now (its record's statements alone)."""
|
|
247
|
+
from prograph.plan.ir.owl_rl import RUN_DIGEST, RUN_SUBJECT
|
|
248
|
+
rows = store.db.execute(f"SELECT o_text FROM ({store.effective_sql(derived_graphs(), about=(RUN_SUBJECT, RUN_DIGEST))}) "
|
|
249
|
+
f"AS _k").fetchall()
|
|
250
|
+
return rows[0][0] if rows else None
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def recorded_run(store: Any, predicates: Sequence[str]) -> Dict[str, Optional[str]]:
|
|
254
|
+
"""The store's record of its last OWL RL run, read now: each of `predicates` (`owl_rl.RUN_*`) with its text,
|
|
255
|
+
None where the record has none."""
|
|
256
|
+
from prograph.plan.ir.owl_rl import RUN_SUBJECT
|
|
257
|
+
held: Dict[str, Optional[str]] = {}
|
|
258
|
+
for p in predicates:
|
|
259
|
+
rows = store.db.execute(f"SELECT o_text FROM ({store.effective_sql(derived_graphs(), about=(RUN_SUBJECT, p))}) "
|
|
260
|
+
f"AS _k").fetchall()
|
|
261
|
+
held[p] = rows[0][0] if rows else None
|
|
262
|
+
return held
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
@dataclass(frozen=True)
|
|
266
|
+
class StoreNow:
|
|
267
|
+
"""What a planning reads of the owned store, in one statement (`store_now`): whether its last run was made
|
|
268
|
+
under the engine's key (its stored facts are the engine's), and the latest alias generation where it was
|
|
269
|
+
written under that key too (`identity.AliasState`, None for none). `ran_at` the time of that run, and `how`
|
|
270
|
+
what made its facts current for this planning (`freshness`): the sources' proofs, or 'last_run'."""
|
|
271
|
+
current: bool = False
|
|
272
|
+
alias: Any = None
|
|
273
|
+
ran_at: Optional[str] = None
|
|
274
|
+
how: Optional[str] = None
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def store_now(engine: Any, onto: Any = None) -> Optional[StoreNow]:
|
|
278
|
+
"""The owned store as one planning reads it, in one statement: its highest committed seq (the Store object
|
|
279
|
+
reads what it holds again where another writer moved it, as `Store.sync`), and for an engine that stores
|
|
280
|
+
facts the record of the last run and the latest alias generation (`identity.latest_sql`). None without a
|
|
281
|
+
store; nothing read in a `premises` block (an RL run's rounds read their own).
|
|
282
|
+
|
|
283
|
+
Under `freshness=None` (the default) the stored facts are read only where provably current
|
|
284
|
+
(`freshness`): the same statement reads each declared version column's maximum and count on DuckDB and
|
|
285
|
+
PostgreSQL, and the run's record of the source versions and of its step's seq; a Delta version is read
|
|
286
|
+
beside it. A source or the store changed since the run: the engine refreshes first (at most twice in one
|
|
287
|
+
planning, where writers keep changing them); a source only a checksum proves, or a store that cannot be
|
|
288
|
+
written, refuses by name (`StoredFactsNotCurrentError`). Under 'last_run' nothing but the key is read."""
|
|
289
|
+
from prograph.utils.exceptions import StoredFactsNotCurrentError
|
|
290
|
+
from prograph.plan.ir import freshness as F
|
|
291
|
+
for attempt in range(3): # two refreshes at most, then a last check
|
|
292
|
+
now, changed = _store_now(engine, onto)
|
|
293
|
+
if not changed:
|
|
294
|
+
if now is not None and now.current and not EXPLAINING.get() and not engine.__dict__.get("_rl_running"):
|
|
295
|
+
# The facts' verdict (`consistency`): under inconsistency='error' an inconsistent one refuses
|
|
296
|
+
# every query, as it refused the run that found it (a run's own planning is not refused).
|
|
297
|
+
from prograph.plan.ir.consistency import refuse_if_inconsistent
|
|
298
|
+
refuse_if_inconsistent(engine)
|
|
299
|
+
return now
|
|
300
|
+
if attempt == 2:
|
|
301
|
+
break
|
|
302
|
+
F.refresh(engine, changed)
|
|
303
|
+
raise StoredFactsNotCurrentError(
|
|
304
|
+
f"the sources of the stored OWL RL facts kept changing while this query was planned ({', '.join(changed)}"
|
|
305
|
+
f"): refreshed twice, still not current. Plan the query again, or read the facts of the last run with "
|
|
306
|
+
f"freshness='last_run'")
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
# Set while `explain()` plans a query: the freshness check is reported (`StoreNow.how`), never acted on (no
|
|
310
|
+
# refresh is written, no refusal raised); the plan reads the stored facts as they are.
|
|
311
|
+
import contextvars # noqa: E402
|
|
312
|
+
|
|
313
|
+
EXPLAINING: contextvars.ContextVar = contextvars.ContextVar("prograph_explaining_freshness", default=False)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _store_now(engine: Any, onto: Any) -> Tuple[Optional[StoreNow], Tuple[str, ...]]:
|
|
317
|
+
store = engine._schema.__dict__.get("owned_store") if getattr(engine, "_schema", None) is not None else None
|
|
318
|
+
if store is None:
|
|
319
|
+
return None, ()
|
|
320
|
+
if store._premises is not None:
|
|
321
|
+
return StoreNow(), ()
|
|
322
|
+
from prograph.plan.ir import freshness as F
|
|
323
|
+
from prograph.plan.ir import identity as I
|
|
324
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
325
|
+
d = get_dialect_instance(store.dialect)
|
|
326
|
+
text = d.type_name("STRING")
|
|
327
|
+
key = engine_key(engine, onto)
|
|
328
|
+
# The sources are checked under the default freshness, never inside a run (its own planning reads the
|
|
329
|
+
# store while the record is the last run's).
|
|
330
|
+
checking = key is not None and getattr(engine, "_freshness", None) is None \
|
|
331
|
+
and not engine.__dict__.get("_rl_running")
|
|
332
|
+
nulls = f"CAST(NULL AS {text}) AS note, CAST(NULL AS BIGINT) AS generation"
|
|
333
|
+
parts = [f"SELECT 'seq' AS kind, CAST(_s.m AS {text}) AS a, {nulls} FROM ({store.max_seq_sql()}) AS _s"]
|
|
334
|
+
meta = False
|
|
335
|
+
srcs: Tuple[Any, ...] = ()
|
|
336
|
+
if key is not None:
|
|
337
|
+
parts += _record_parts(store, checking)
|
|
338
|
+
if checking:
|
|
339
|
+
srcs = stored_sources(engine)
|
|
340
|
+
meta = _meta_held(store)
|
|
341
|
+
if meta:
|
|
342
|
+
parts.append(f"SELECT * FROM ({I.latest_sql(store)}) AS _g")
|
|
343
|
+
rows = store.db.execute(" UNION ALL ".join(parts)).fetchall()
|
|
344
|
+
seq = int(next(r[1] for r in rows if r[0] == "seq"))
|
|
345
|
+
store.saw(seq)
|
|
346
|
+
if key is None or _first(rows, "key") != key:
|
|
347
|
+
return StoreNow(), ()
|
|
348
|
+
how = "freshness='last_run': the sources are not checked"
|
|
349
|
+
if checking:
|
|
350
|
+
checks: Dict[str, Any] = {}
|
|
351
|
+
now = F.read_now(engine, srcs, checks)
|
|
352
|
+
run_seq = _first(rows, "run_seq")
|
|
353
|
+
verdict = F.compare(F.recorded(_first(rows, "sources")), now,
|
|
354
|
+
run_seq is None or seq > int(run_seq))
|
|
355
|
+
each = "; ".join(f"{s.label or s.name} by its {checks[s.name].how} ({checks[s.name].ms:.1f} ms)"
|
|
356
|
+
if now[s.name] is not None else f"{s.label or s.name}: {checks[s.name].how} "
|
|
357
|
+
f"({checks[s.name].ms:.1f} ms)" for s in srcs) or "none mapped"
|
|
358
|
+
unproven = tuple(s for s in srcs if s.name in verdict.unproven)
|
|
359
|
+
changed = _labelled(srcs, verdict.changed)
|
|
360
|
+
if unproven and not EXPLAINING.get():
|
|
361
|
+
F.refuse_unproven(unproven)
|
|
362
|
+
if changed and not EXPLAINING.get():
|
|
363
|
+
return None, changed
|
|
364
|
+
if unproven:
|
|
365
|
+
how = f"refused, a query would raise StoredFactsNotCurrentError: {F.unproven_reason(unproven)}. Each table: {each}"
|
|
366
|
+
elif changed:
|
|
367
|
+
how = f"would refresh first: {', '.join(changed)} changed since the run. Each table: {each}"
|
|
368
|
+
else:
|
|
369
|
+
how = f"proven current, the owned store unchanged since. Each table: {each}"
|
|
370
|
+
if meta:
|
|
371
|
+
alias = I.state_of([r for r in rows if r[0] in ("generation", "shape")])
|
|
372
|
+
else:
|
|
373
|
+
alias = I.read_state(store) if _meta_held(store) else None # a first generation another Store wrote
|
|
374
|
+
return StoreNow(True, alias if alias is not None and alias.key == key else None, _first(rows, "at"), how), ()
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _meta_held(store: Any) -> bool:
|
|
378
|
+
"""Whether the store's alias tables exist: once they do, always; else asked again only where the store moved
|
|
379
|
+
since it was last asked."""
|
|
380
|
+
held = store.__dict__.get("_alias_tables")
|
|
381
|
+
if held is True or (held is not None and held == store._seen_seq):
|
|
382
|
+
return held is True
|
|
383
|
+
from prograph.plan.ir.identity import meta_table
|
|
384
|
+
found = store.db.has_table(meta_table(store))
|
|
385
|
+
store.__dict__["_alias_tables"] = True if found else store._seen_seq
|
|
386
|
+
return found
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def premise_statements(store: Any) -> str:
|
|
390
|
+
"""SQL of the store's statements an OWL RL run reads as premises: every graph but the stored derived facts
|
|
391
|
+
(a stale fact never supports itself; the run's own derivations join them, `Store.premises`)."""
|
|
392
|
+
return store.effective_sql(but=derived_graphs())
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def store_mapping(store: Any, kinds: Sequence[tuple], derived_kinds: Sequence[tuple], source: str,
|
|
396
|
+
apart: bool, stored: bool = False) -> str:
|
|
397
|
+
"""The store's generated mapping as R2RML Turtle: one triples map per (predicate, object kind, datatype,
|
|
398
|
+
language) of `kinds` over `source`'s asserted statements, each statement in its writer's graph and in the
|
|
399
|
+
default graph; with `stored`, one per `derived_kinds` over its stored derived facts, in the default graph
|
|
400
|
+
alone (who reads them is `maps_seen`'s to say). `apart`: `source` may hold derived facts, which the asserted
|
|
401
|
+
maps leave out."""
|
|
402
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
403
|
+
literal = get_dialect_instance(store.dialect).literal
|
|
404
|
+
graphs = ", ".join(literal(g) for g in derived_graphs())
|
|
405
|
+
lines = ["@prefix rr: <http://www.w3.org/ns/r2rml#> .", ""]
|
|
406
|
+
families = [("prograph-store", [f"graph NOT IN ({graphs})"] if apart else [], kinds,
|
|
407
|
+
'[ rr:column "graph" ], [ rr:constant rr:defaultGraph ]')]
|
|
408
|
+
if stored:
|
|
409
|
+
families.append(("prograph-stored", [f"graph IN ({graphs})"], derived_kinds, '[ rr:constant rr:defaultGraph ]'))
|
|
410
|
+
for name, which, rows, graph_maps in families:
|
|
411
|
+
for i, (predicate, kind, datatype, lang) in enumerate(rows):
|
|
412
|
+
lines += _store_map(store, f"{name}-{i}", which, source, graph_maps, predicate, kind, datatype, lang)
|
|
413
|
+
return "\n".join(lines)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _store_map(store: Any, name: str, which: List[str], source: str, graph_maps: str, predicate: str, kind: str,
|
|
417
|
+
datatype: Optional[str], lang: Optional[str]) -> List[str]:
|
|
418
|
+
"""One generated triples map: the statements of one predicate and kind of object (the plan reads a constant
|
|
419
|
+
predicate; the mapping is regenerated after a write step, so a new one gets its map)."""
|
|
420
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
421
|
+
from prograph.store import IRI
|
|
422
|
+
literal = get_dialect_instance(store.dialect).literal
|
|
423
|
+
where = which + [f"predicate = {literal(predicate)}", f"o_kind = '{kind}'"]
|
|
424
|
+
if kind == IRI:
|
|
425
|
+
obj = '[ rr:column "o_text" ; rr:termType rr:IRI ]'
|
|
426
|
+
elif lang is not None:
|
|
427
|
+
where.append(f"o_lang = {literal(lang)}")
|
|
428
|
+
obj = f'[ rr:column "o_text" ; rr:language {_turtle_string(lang)} ]'
|
|
429
|
+
elif datatype is not None:
|
|
430
|
+
where.append(f"o_datatype = {literal(datatype)}")
|
|
431
|
+
obj = f'[ rr:column "o_text" ; rr:datatype <{datatype}> ]'
|
|
432
|
+
else:
|
|
433
|
+
where.append("o_datatype IS NULL AND o_lang IS NULL")
|
|
434
|
+
obj = '[ rr:column "o_text" ]'
|
|
435
|
+
query = f"SELECT graph, subject, o_text FROM ({source}) AS _e WHERE {' AND '.join(where)}"
|
|
436
|
+
return [f"<#{name}> rr:logicalTable [ rr:sqlQuery {_turtle_string(query)} ] ;",
|
|
437
|
+
' rr:subjectMap [ rr:column "subject" ; rr:termType rr:IRI ;',
|
|
438
|
+
f' rr:graphMap {graph_maps} ] ;',
|
|
439
|
+
f' rr:predicateObjectMap [ rr:predicate <{predicate}> ; rr:objectMap {obj} ] .',
|
|
440
|
+
""]
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _turtle_string(text: str) -> str:
|
|
444
|
+
"""A Turtle string literal (long form: an SQL query holds quotes and newlines)."""
|
|
445
|
+
return '"""' + text.replace("\\", "\\\\").replace('"""', '\\"\\"\\"') + '"""'
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
# -- the maps a query reads -----------------------------------------------------------------------------------
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
@dataclass(frozen=True)
|
|
452
|
+
class OntologyTriples:
|
|
453
|
+
"""The ontology's triples in the default graph (`ontology_triples`): `stated`, by predicate, the (subject,
|
|
454
|
+
object) pairs that hold whatever the data holds; `witnessed`, by predicate, the pairs that hold where a
|
|
455
|
+
witness has a triple: a class the maps give (("class", C): `C rdf:type rdfs:Class` and what follows) or a
|
|
456
|
+
predicate they give (("property", p): `p rdf:type rdf:Property` and what follows); `data_class`, by
|
|
457
|
+
predicate, the pairs a class the data gives from a row adds (the object of an rdf:type triple a map takes
|
|
458
|
+
from a column or template: an `rdfs:Class` by rdfs3 over rdf:type's range, and what follows), with
|
|
459
|
+
`DATA_CLASS` standing for that class: a term no stated triple types `rdfs:Class` adds them all, whatever
|
|
460
|
+
else the ontology says of it, as its other triples would make it a class already."""
|
|
461
|
+
stated: Dict[str, Tuple[Tuple[Any, Any], ...]]
|
|
462
|
+
witnessed: Dict[str, Dict[Tuple[str, str], Tuple[Tuple[Any, Any], ...]]]
|
|
463
|
+
data_class: Dict[str, Tuple[Tuple[Any, Any], ...]]
|
|
464
|
+
# Per predicate, the stated pairs only Direct semantics gives a typed variable (`typed-variables`).
|
|
465
|
+
direct: Dict[str, FrozenSet[Tuple[Any, Any]]] = field(default_factory=dict)
|
|
466
|
+
|
|
467
|
+
@property
|
|
468
|
+
def predicates(self) -> List[str]:
|
|
469
|
+
return sorted(set(self.stated) | set(self.witnessed) | set(self.data_class))
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
# Stands for a class the data gives from a row in `OntologyTriples.data_class`.
|
|
473
|
+
DATA_CLASS = IRITerm("urn:prograph:data-class")
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def ontology_triples(maps: Any, onto: Any) -> Optional[OntologyTriples]:
|
|
477
|
+
"""The ontology's own triples as the default graph holds them, read-only (`rdfs.close`: closed under the
|
|
478
|
+
RDFS core, and under profile="rl" OWL 2 RL's schema rules), and the triples the data's vocabulary
|
|
479
|
+
adds to them (`rdfs.extend`): a class or predicate the maps give is an `rdfs:Class` or an `rdf:Property`
|
|
480
|
+
where it has a triple, so those are kept by their witness. Kept on the maps for that ontology. None
|
|
481
|
+
without an ontology. Matched by the front end as constant rows beside the maps (`Translator.ontology_rows`):
|
|
482
|
+
in the default graph only, also under FROM (the ontology applies to the default graph a query builds), never
|
|
483
|
+
in a named graph."""
|
|
484
|
+
if onto is None or getattr(onto, "closure", None) is None:
|
|
485
|
+
return None
|
|
486
|
+
held = maps.__dict__.get("_ontology_triples")
|
|
487
|
+
if held is not None and held[0] is onto:
|
|
488
|
+
return held[1]
|
|
489
|
+
from prograph.plan.ir import rdfs
|
|
490
|
+
closure = onto.closure
|
|
491
|
+
stated: Dict[str, List[Tuple[Any, Any]]] = {}
|
|
492
|
+
direct: Dict[str, set] = {}
|
|
493
|
+
only = getattr(closure, "direct_only", frozenset())
|
|
494
|
+
plain = getattr(closure, "plain", None) or closure # what a witness adds, over the RDFS core alone
|
|
495
|
+
for s, p, o in sorted(closure.all()):
|
|
496
|
+
stated.setdefault(p[1], []).append((_plan_term(s), _plan_term(o)))
|
|
497
|
+
if (s, p, o) in only:
|
|
498
|
+
direct.setdefault(p[1], set()).add((_plan_term(s), _plan_term(o)))
|
|
499
|
+
witnesses: List[Tuple[Tuple[str, str], Any]] = []
|
|
500
|
+
classes = sorted({c for m in maps.nodes for c in m.classes})
|
|
501
|
+
witnesses += [(("class", c), (rdfs.iri(c), rdfs.iri(rdfs.TYPE), rdfs.iri(rdfs.CLASS))) for c in classes]
|
|
502
|
+
witnesses += [(("property", p), (rdfs.iri(p), rdfs.iri(rdfs.TYPE), rdfs.iri(rdfs.PROPERTY)))
|
|
503
|
+
for p in sorted(maps.by_predicate)]
|
|
504
|
+
witnessed: Dict[str, Dict[Tuple[str, str], List[Tuple[Any, Any]]]] = {}
|
|
505
|
+
for witness, seed in witnesses:
|
|
506
|
+
for s, p, o in sorted(rdfs.extend(plain, [seed])):
|
|
507
|
+
witnessed.setdefault(p[1], {}).setdefault(witness, []).append((_plan_term(s), _plan_term(o)))
|
|
508
|
+
# A class the data gives from a row: rdfs:Resource is `typed_resource`'s (rdfs4b over those triples).
|
|
509
|
+
data_class: Dict[str, List[Tuple[Any, Any]]] = {}
|
|
510
|
+
seed = (rdfs.iri(DATA_CLASS.text), rdfs.iri(rdfs.TYPE), rdfs.iri(rdfs.CLASS))
|
|
511
|
+
for s, p, o in sorted(rdfs.extend(plain, [seed])):
|
|
512
|
+
if seed[0] in (s, o) and (p, o) != (rdfs.iri(rdfs.TYPE), rdfs.iri(rdfs.RESOURCE)):
|
|
513
|
+
data_class.setdefault(p[1], []).append((_plan_term(s), _plan_term(o)))
|
|
514
|
+
out = OntologyTriples({p: tuple(v) for p, v in stated.items()},
|
|
515
|
+
{p: {w: tuple(v) for w, v in ws.items()} for p, ws in witnessed.items()},
|
|
516
|
+
{p: tuple(v) for p, v in data_class.items()},
|
|
517
|
+
{p: frozenset(v) for p, v in direct.items()})
|
|
518
|
+
maps.__dict__["_ontology_triples"] = (onto, out)
|
|
519
|
+
return out
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def _plan_term(t: tuple) -> Any:
|
|
523
|
+
"""`rdfs`'s term as the plan's."""
|
|
524
|
+
from prograph.plan.ir.terms import BlankTerm, LiteralTerm
|
|
525
|
+
if t[0] == "i":
|
|
526
|
+
return IRITerm(t[1])
|
|
527
|
+
if t[0] == "b":
|
|
528
|
+
return BlankTerm(t[1])
|
|
529
|
+
return LiteralTerm(t[1], t[2], t[3])
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def maps_seen(engine: Any, onto: Any, now: Optional[StoreNow] = None) -> Any:
|
|
533
|
+
"""The maps a query of `engine` reads, `onto` its ontology (`entry.ontology_of`, or None): the mapped data and
|
|
534
|
+
the store's asserted statements (`entry.source_maps`), the store's stored facts where its last run, read when
|
|
535
|
+
the query is planned, was made under the engine's key (`Store.derived_maps`, `store_now`), saturated by the
|
|
536
|
+
ontology (T-mappings: the rewritten consequences). The ontology's own triples join them at matching
|
|
537
|
+
(`ontology_triples`). `now`: the store as this planning read it (`store_now`), read here where not given."""
|
|
538
|
+
from prograph.plan.ir.entry import saturated, source_maps
|
|
539
|
+
schema = engine._schema
|
|
540
|
+
store = schema.__dict__.get("owned_store")
|
|
541
|
+
# The store read once per planning (`store_now`, never a `Store` object's cached state): first, so the maps
|
|
542
|
+
# below are of what it holds now; the stored facts where its last run was made under the engine's key.
|
|
543
|
+
if now is None and store is not None:
|
|
544
|
+
now = store_now(engine, onto)
|
|
545
|
+
maps = source_maps(schema)
|
|
546
|
+
extra = store.derived_maps() if now is not None and now.current and store.profile().derived else None
|
|
547
|
+
if extra is not None:
|
|
548
|
+
held = engine.__dict__.get("_seen_maps")
|
|
549
|
+
if held is None or held[0] is not maps or held[1] is not extra:
|
|
550
|
+
held = (maps, extra, _with_stored(schema, maps, extra))
|
|
551
|
+
engine.__dict__["_seen_maps"] = held
|
|
552
|
+
maps = held[2]
|
|
553
|
+
if onto is not None:
|
|
554
|
+
maps = saturated(maps, onto)
|
|
555
|
+
return maps
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
def _with_stored(schema: Any, maps: Any, extra: Any) -> Any:
|
|
559
|
+
"""`maps` and the stored facts' maps `extra`, one set: their query sources typed as the store's are, and
|
|
560
|
+
marked the store's own (the property-graph view reads the store through its generic maps)."""
|
|
561
|
+
from prograph.schema import types as T
|
|
562
|
+
from prograph.schema.source_map import with_decimal_text
|
|
563
|
+
out = maps.joined(with_decimal_text(extra, getattr(schema, "decimal_text", None)))
|
|
564
|
+
for source in extra.tables():
|
|
565
|
+
schema.column_types.setdefault(source.table, {c: T.of_sql_type("VARCHAR") for c in ("graph", "subject", "o_text")})
|
|
566
|
+
held = maps.__dict__.get("owned_store")
|
|
567
|
+
if held is not None:
|
|
568
|
+
out.__dict__["owned_store"] = (held[0], held[1], held[2] | frozenset(m.id for m in extra.nodes + extra.edges))
|
|
569
|
+
out.__dict__["stored_facts"] = extra # the stored derived facts' maps (Cypher's view, `lpg_view.with_derived`)
|
|
570
|
+
return out
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
# -- the graphs a triple pattern reads --------------------------------------------------------------------------
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def graph_choices(graphs: List[Any], graph: Any, dataset: Optional[Tuple[frozenset, frozenset]],
|
|
577
|
+
union_default: bool) -> List[Optional[Any]]:
|
|
578
|
+
"""The graphs a triple of the graph maps `graphs` (`source_map.graphs_of`) is read in, for the graph the
|
|
579
|
+
patterns read (`graph`: None the default graph, else GRAPH's IRI or variable; R2RML 11.1, SPARQL 13): None
|
|
580
|
+
for the default graph, else each named graph map that may give it. A triple with no graph map, or the
|
|
581
|
+
constant rr:defaultGraph, is in the default graph; with default_graph='union' (`union_default`) every
|
|
582
|
+
triple is read there too (Jena's unionDefaultGraph). Under a dataset clause, `dataset_choices`."""
|
|
583
|
+
from prograph.plan.ir.sparql_front import _may_give
|
|
584
|
+
if dataset is not None:
|
|
585
|
+
return dataset_choices(graphs, graph, dataset)
|
|
586
|
+
if graph is None:
|
|
587
|
+
if not graphs or any(g.constant == RR_DEFAULT_GRAPH for g in graphs):
|
|
588
|
+
return [None]
|
|
589
|
+
if union_default:
|
|
590
|
+
# Every triple once (the pattern's Distinct), but only a triple some graph map gives: a row with
|
|
591
|
+
# no graph value is in no graph (R2RML 11.1).
|
|
592
|
+
return [g if g.constant is None else None for g in graphs]
|
|
593
|
+
return []
|
|
594
|
+
named = [g for g in graphs if g.constant != RR_DEFAULT_GRAPH]
|
|
595
|
+
return [g for g in named if _may_give(g, graph)]
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
@home("query-dataset", "A query's dataset: FROM graphs merged as the default graph, FROM NAMED graphs for GRAPH",
|
|
599
|
+
terms=("FROM", "FROM NAMED", "dataset clause", "default-graph-uri", "named-graph-uri", "dataset",
|
|
600
|
+
"default graph", "named graph", "graph map", "merge", "blank node scope"))
|
|
601
|
+
def dataset_choices(graphs: List[Any], graph: Any, dataset: Tuple[frozenset, frozenset]) -> List[Optional[Any]]:
|
|
602
|
+
"""`graph_choices` under a dataset clause (13.2): outside GRAPH the graph maps whose graph is one of the FROM
|
|
603
|
+
graphs (a constant one kept or dropped here; a template or column one tested per row,
|
|
604
|
+
`Translator.graph_term`), the triples with no graph map in none; inside GRAPH the maps whose graph is one of
|
|
605
|
+
the FROM NAMED graphs, and for GRAPH <g> only where <g> is one."""
|
|
606
|
+
from prograph.plan.ir.sparql_front import _may_give
|
|
607
|
+
default, named = dataset
|
|
608
|
+
listed = default if graph is None else named
|
|
609
|
+
if isinstance(graph, IRITerm) and graph.text not in named:
|
|
610
|
+
return []
|
|
611
|
+
out = []
|
|
612
|
+
for g in graphs:
|
|
613
|
+
if g.constant == RR_DEFAULT_GRAPH:
|
|
614
|
+
continue
|
|
615
|
+
if g.constant is not None:
|
|
616
|
+
if g.constant in listed and (graph is None or _may_give(g, graph)):
|
|
617
|
+
out.append(g)
|
|
618
|
+
elif any(_may_give(g, IRITerm(i)) for i in listed) and (graph is None or _may_give(g, graph)):
|
|
619
|
+
out.append(g)
|
|
620
|
+
return out
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
@home("sameas-active-graph", "Whose owl:sameAs components a pattern outside GRAPH ?g matches through",
|
|
624
|
+
terms=("FROM", "default-graph-uri", "dataset", "active graph", "components of a graph"))
|
|
625
|
+
def active_graph(t: Any) -> str:
|
|
626
|
+
"""The graph whose owl:sameAs components a pattern of the translation `t` matches through (the alias table's
|
|
627
|
+
graph column), outside GRAPH ?g: GRAPH <g>'s; under a dataset clause (FROM, or the protocol's
|
|
628
|
+
default-graph-uri) the default graph is the merge of the FROM graphs, so the one FROM graph's (none for no
|
|
629
|
+
FROM graph; several are refused by name, as their merge's components are not kept); else the default
|
|
630
|
+
graph's."""
|
|
631
|
+
from prograph.plan.ir.identity import DEFAULT_GRAPH, NO_GRAPH
|
|
632
|
+
if isinstance(t.graph, IRITerm):
|
|
633
|
+
return t.graph.text
|
|
634
|
+
if t.graph is None and t.dataset is not None:
|
|
635
|
+
listed = sorted(t.dataset[0])
|
|
636
|
+
if not listed:
|
|
637
|
+
return NO_GRAPH
|
|
638
|
+
if len(listed) > 1:
|
|
639
|
+
from prograph.plan.ir.facts import NotBuilt
|
|
640
|
+
raise NotBuilt("owl:sameAs over a default graph merged from several FROM graphs (each graph's "
|
|
641
|
+
"components are kept, not their merge's): name one FROM graph, or use GRAPH")
|
|
642
|
+
return listed[0]
|
|
643
|
+
return DEFAULT_GRAPH
|