prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1119 @@
|
|
|
1
|
+
"""OWL 2 RL materialized into the owned store's RL graph (one-plan 2.7 and 2.10; docs/reference/owl-rl.md).
|
|
2
|
+
|
|
3
|
+
`materialize(engine)` computes the closure of the ontology's RL rules (`owl_rl_rules`) over the graph SPARQL
|
|
4
|
+
answers, then writes the difference from the RL graph (`store.RL_GRAPH`) as one write step.
|
|
5
|
+
|
|
6
|
+
- **Premises through the plan.** Each round, every rule's pattern is planned over the saturated maps, the
|
|
7
|
+
store's statements of every other writer, and the facts derived so far in this run (`Store.premises`:
|
|
8
|
+
the store's generated maps read that relation for the round), then lowered, so a rule reads exactly the
|
|
9
|
+
default graph a query reads, on every engine, R2RML and YAML mappings alike. Never a table by name. The RL
|
|
10
|
+
graph as stored is not a premise: a stale fact never supports itself.
|
|
11
|
+
- **Naive fixpoint.** Every rule runs each round; a round keeps the triples the default graph does not hold
|
|
12
|
+
yet (a `NOT EXISTS`, saturation's virtual facts included), staged in the engine (`StoreConnection.stage`).
|
|
13
|
+
It stops when a round derives nothing; `max_rounds` rounds without that raise
|
|
14
|
+
`MaterializationBudgetExceeded`, nothing written. A transitive property is the plan's closure `p+`
|
|
15
|
+
(`Closure`, on Spark and Databricks by the level ladder), re-run each round.
|
|
16
|
+
- **One set-level diff.** The closure is compared with the RL graph as a set of triples: asserts for triples
|
|
17
|
+
newly derived, retracts for triples no longer derived; a triple derived twice, or held by the source too,
|
|
18
|
+
is kept while one derivation stands (a source triple is never stored). Each statement's writer is
|
|
19
|
+
`rl:<rule id>`. The run's ontology digest and time are written in the same step, as the store's own
|
|
20
|
+
statements (`last_run`). On Delta a step that lost its race recomputes from the start.
|
|
21
|
+
- **Consistency** (`consistency`): the rules that conclude false derive nothing; they run once, after the
|
|
22
|
+
fixpoint, over the closure the run staged (`_Run.consistency`), and their verdict is written in the run's step
|
|
23
|
+
(`RUN_CONSISTENCY`), so an engine over current facts reads it and checks nothing again.
|
|
24
|
+
- **owl:sameAs** (`identity`): each round first closes the links (the graph's own sameAs triples, an
|
|
25
|
+
ontology's, and those prp-fp, prp-ifp, prp-key, cls-maxc2 and cls-maxqc3/4 derived so far) into
|
|
26
|
+
components, then leaves unmerged each component that is inconsistent (owl:differentFrom or
|
|
27
|
+
owl:AllDifferent between its members; a functional property's two values that are not one value) or past
|
|
28
|
+
the cap, then runs every rule with its patterns matched through the components, each solution under its
|
|
29
|
+
representative: a derived fact is stored once, on the representative. A component found inconsistent
|
|
30
|
+
after rules have read it runs the fixpoint again from its first round. The components are written as a
|
|
31
|
+
new generation of the alias table with the run's step.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import datetime as _dt
|
|
37
|
+
import uuid
|
|
38
|
+
from dataclasses import dataclass, field, replace
|
|
39
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
40
|
+
|
|
41
|
+
from prograph.rules import module_home
|
|
42
|
+
from prograph.schema import xsd
|
|
43
|
+
|
|
44
|
+
module_home("owl-rl-materialization", "OWL 2 RL materialized into the owned store: fixpoint, diff, one step",
|
|
45
|
+
terms=("materialize_entailments", "profile='rl'", "materialization", "fixpoint", "naive fixpoint",
|
|
46
|
+
"RL graph", "derived fact", "max_rounds", "MaterializationBudgetExceeded", "set-level diff",
|
|
47
|
+
"retract derived", "ontology digest", "last run", "consistency report", "rl:"))
|
|
48
|
+
|
|
49
|
+
RUN_SUBJECT = "urn:prograph:store:rl"
|
|
50
|
+
RUN_DIGEST = "urn:prograph:store:rl-ontology"
|
|
51
|
+
RUN_AT = "urn:prograph:store:rl-at"
|
|
52
|
+
# The versions of the sources the run read (`freshness.read_now`), as JSON: what a planning compares.
|
|
53
|
+
RUN_SOURCES = "urn:prograph:store:rl-sources"
|
|
54
|
+
# The rules that concluded false in the run (`consistency.as_record`), as JSON: the stored verdict.
|
|
55
|
+
RUN_CONSISTENCY = "urn:prograph:store:rl-consistency"
|
|
56
|
+
# The rounds the run took, as text: a call reusing the run checks its `max_rounds` against it.
|
|
57
|
+
RUN_ROUNDS = "urn:prograph:store:rl-rounds"
|
|
58
|
+
MAX_ROUNDS = 64
|
|
59
|
+
# The deepest transitive walk a round's closure takes after the engine's own ladder (the path_depth hint's
|
|
60
|
+
# ceiling); a longer walk is a typed error naming the depth.
|
|
61
|
+
DEEPEST = 1000
|
|
62
|
+
SAMPLE = 3
|
|
63
|
+
STATEMENT = ("subject", "predicate", "o_kind", "o_text", "o_datatype", "o_lang")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
from prograph.plan.ir.consistency import Inconsistency # noqa: E402 - the report's record (`consistency`)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True)
|
|
70
|
+
class Unmerged:
|
|
71
|
+
"""A component of owl:sameAs left unmerged: `reason` 'inconsistent' (the rule that found it: eq-diff1
|
|
72
|
+
for owl:differentFrom, eq-diff2/3 for owl:AllDifferent, prp-fp or cls-maxc2 for two values; `axiom` its axiom)
|
|
73
|
+
or 'cap' (more members than the cap), its graph ('' the default graph), its representative, size and a sample
|
|
74
|
+
of members."""
|
|
75
|
+
reason: str
|
|
76
|
+
rule: str
|
|
77
|
+
graph: str
|
|
78
|
+
representative: str
|
|
79
|
+
size: int
|
|
80
|
+
sample: Tuple[str, ...]
|
|
81
|
+
axiom: str = ""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass
|
|
85
|
+
class MaterializationReport:
|
|
86
|
+
"""What `materialize_entailments()` did: the statements asserted and retracted per rule (`rl:<rule>`
|
|
87
|
+
writers), the rounds run, the consistency rules that matched (`inconsistencies`), the axioms skipped
|
|
88
|
+
(sameAs-deriving and outside OWL 2 RL), the run's cache key (`dataset.cache_key`)
|
|
89
|
+
and the step's seq (None where nothing
|
|
90
|
+
changed but the run's record).
|
|
91
|
+
|
|
92
|
+
A call that found the store's last run current for the engine's key and tables returns that run's record
|
|
93
|
+
instead of computing again: ``seq`` None, ``rounds`` 0, nothing asserted or retracted, and ``digest``,
|
|
94
|
+
``ran_at``, ``sources``, ``inconsistencies``, ``not_run``, ``components``, ``aliases``, ``largest`` and
|
|
95
|
+
``unmerged`` those the
|
|
96
|
+
last run stored. The call's ``max_rounds`` is checked against the rounds the last run recorded: a smaller
|
|
97
|
+
budget computes the closure and raises ``MaterializationBudgetExceeded``."""
|
|
98
|
+
asserted: Dict[str, int] = field(default_factory=dict)
|
|
99
|
+
retracted: Dict[str, int] = field(default_factory=dict)
|
|
100
|
+
rounds: int = 0
|
|
101
|
+
inconsistencies: List[Inconsistency] = field(default_factory=list)
|
|
102
|
+
skipped: List[Tuple[str, str, str]] = field(default_factory=list)
|
|
103
|
+
digest: str = ""
|
|
104
|
+
seq: Optional[int] = None
|
|
105
|
+
ran_at: str = ""
|
|
106
|
+
# Each source table's versions as the run read it (`freshness.read_run`): a declared column's maximum and
|
|
107
|
+
# count or a Delta version, and a checksum of the whole table.
|
|
108
|
+
sources: Dict[str, List[str]] = field(default_factory=dict)
|
|
109
|
+
# owl:sameAs: the components merged (of two IRIs or more), their aliases (IRIs other than the
|
|
110
|
+
# representative), the largest component's size, and the components left unmerged.
|
|
111
|
+
components: int = 0
|
|
112
|
+
aliases: int = 0
|
|
113
|
+
largest: int = 0
|
|
114
|
+
unmerged: List[Unmerged] = field(default_factory=list)
|
|
115
|
+
# The consistency checks that could not run (rule, axiom, why: a data error their query met).
|
|
116
|
+
not_run: List[Tuple[str, str, str]] = field(default_factory=list)
|
|
117
|
+
|
|
118
|
+
@property
|
|
119
|
+
def consistent(self) -> bool:
|
|
120
|
+
"""No consistency rule matched and no owl:sameAs component was left unmerged as inconsistent."""
|
|
121
|
+
return not self.inconsistencies and not any(u.reason == "inconsistent" for u in self.unmerged)
|
|
122
|
+
|
|
123
|
+
def summary(self) -> str:
|
|
124
|
+
a, r = sum(self.asserted.values()), sum(self.retracted.values())
|
|
125
|
+
same = f"; {self.components} sameAs components ({self.aliases} aliases)" if self.components else ""
|
|
126
|
+
apart = f"; {len(self.unmerged)} left unmerged" if self.unmerged else ""
|
|
127
|
+
return (f"{a} asserted, {r} retracted in {self.rounds} rounds{same}{apart}; "
|
|
128
|
+
f"{'consistent' if self.consistent else 'inconsistent: ' + ', '.join(dict.fromkeys([i.rule for i in self.inconsistencies] + [u.rule for u in self.unmerged if u.reason == 'inconsistent']))}")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def last_run(store: Any) -> Optional[Tuple[str, str]]:
|
|
132
|
+
"""The (ontology digest, time) the store's RL graph was last materialized for; None where never."""
|
|
133
|
+
from prograph.store import RL_GRAPH
|
|
134
|
+
rows = store.db.execute(
|
|
135
|
+
f"SELECT predicate, o_text FROM ({store.effective_sql([RL_GRAPH])}) AS _e WHERE subject = ?",
|
|
136
|
+
[RUN_SUBJECT]).fetchall()
|
|
137
|
+
held = dict(rows)
|
|
138
|
+
if RUN_DIGEST not in held:
|
|
139
|
+
return None
|
|
140
|
+
return held[RUN_DIGEST], held.get(RUN_AT, "")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _unmerged_order(u: Unmerged) -> Tuple[Any, ...]:
|
|
144
|
+
return (u.graph, u.representative, u.rule, u.reason)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def reused_report(engine: Any, rules: Any, held: Any, max_rounds: int) -> Optional[MaterializationReport]:
|
|
148
|
+
"""The report of a call that reused the store's last run (`ProGraph.materialize_entailments`): that run's
|
|
149
|
+
key, time, source versions and consistency verdict `held` (`consistency.recorded`); no round was run. None
|
|
150
|
+
where the run recorded no rounds, or more than `max_rounds` (the call computes, and the budget is checked)."""
|
|
151
|
+
from prograph.plan.ir.dataset import recorded_run
|
|
152
|
+
rounds = recorded_run(engine._store, (RUN_ROUNDS,))[RUN_ROUNDS]
|
|
153
|
+
if rounds is None or int(rounds) > max_rounds:
|
|
154
|
+
return None
|
|
155
|
+
from prograph.plan.ir import freshness
|
|
156
|
+
from prograph.plan.ir.dataset import _first, _record_parts, cache_key
|
|
157
|
+
store = engine._store
|
|
158
|
+
rows = store.db.execute(" UNION ALL ".join(_record_parts(store, True))).fetchall()
|
|
159
|
+
report = MaterializationReport(skipped=list(rules.skipped), digest=cache_key(engine, rules),
|
|
160
|
+
ran_at=_first(rows, "at") or "", sources=freshness.recorded(_first(rows, "sources")),
|
|
161
|
+
inconsistencies=list(held.inconsistencies), not_run=list(held.not_run))
|
|
162
|
+
# owl:sameAs: the latest generation's marker and unmerged components, where it was written under this key.
|
|
163
|
+
import json
|
|
164
|
+
from prograph.plan.ir import identity as I
|
|
165
|
+
meta = I.meta_table(store)
|
|
166
|
+
if store.db.has_table(meta):
|
|
167
|
+
marker = store.db.execute(
|
|
168
|
+
f"SELECT a, note, generation FROM {meta} WHERE kind = 'generation' AND generation = "
|
|
169
|
+
f"(SELECT max(generation) FROM {meta} WHERE kind = 'generation')").fetchall()
|
|
170
|
+
if marker and marker[0][0] == report.digest:
|
|
171
|
+
note = json.loads(marker[0][1] or "{}")
|
|
172
|
+
report.components, report.aliases, report.largest = (int(note.get(k, 0)) for k in
|
|
173
|
+
("components", "aliases", "largest"))
|
|
174
|
+
for graph, rep, rule, text in store.db.execute(
|
|
175
|
+
f"SELECT graph, a, b, note FROM {meta} WHERE kind = 'unmerged' AND generation = {int(marker[0][2])}"
|
|
176
|
+
).fetchall():
|
|
177
|
+
found = json.loads(text or "{}")
|
|
178
|
+
report.unmerged.append(Unmerged(found.get("reason", ""), rule or "", graph or "", rep,
|
|
179
|
+
int(found.get("size", 0)), tuple(found.get("sample", ())),
|
|
180
|
+
found.get("axiom", "")))
|
|
181
|
+
report.unmerged.sort(key=_unmerged_order)
|
|
182
|
+
return report
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def materialize(engine: Any, rules: Any, max_rounds: int = MAX_ROUNDS) -> MaterializationReport:
|
|
186
|
+
"""Materialize `rules` (`owl_rl_rules.RuleSet`) into the engine's owned store (see the module)."""
|
|
187
|
+
from prograph.store import StepLostRace
|
|
188
|
+
store = engine._store
|
|
189
|
+
if max_rounds < 1:
|
|
190
|
+
raise ValueError(f"max_rounds is at least 1, not {max_rounds}")
|
|
191
|
+
graph = getattr(getattr(engine, "executor", None), "graph", None)
|
|
192
|
+
if getattr(graph, "_shared", False):
|
|
193
|
+
graph._prepare_local_sql() # the frames' views on the store's connection are this engine's (not another's)
|
|
194
|
+
running = engine.__dict__.get("_rl_running")
|
|
195
|
+
engine.__dict__["_rl_running"] = True # the run's own planning reads the store without a freshness check
|
|
196
|
+
try:
|
|
197
|
+
for _ in range(getattr(store, "REDO_LIMIT", 20)):
|
|
198
|
+
store.sync()
|
|
199
|
+
run = _Run(engine, store, rules, max_rounds)
|
|
200
|
+
try:
|
|
201
|
+
return run.go()
|
|
202
|
+
except StepLostRace:
|
|
203
|
+
continue # another writer's step landed: the closure is computed again from the start
|
|
204
|
+
finally:
|
|
205
|
+
run.cleanup()
|
|
206
|
+
finally:
|
|
207
|
+
engine.__dict__["_rl_running"] = running
|
|
208
|
+
from prograph.utils.exceptions import StoreWriteConflictError
|
|
209
|
+
raise StoreWriteConflictError("write contention: OWL RL materialization lost its race to other writers "
|
|
210
|
+
f"{store.REDO_LIMIT} times; nothing was written. Call it again")
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
class _Run:
|
|
214
|
+
def __init__(self, engine: Any, store: Any, rules: Any, max_rounds: int) -> None:
|
|
215
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
216
|
+
self.engine, self.store, self.rules, self.max_rounds = engine, store, rules, max_rounds
|
|
217
|
+
self.db = store.db
|
|
218
|
+
self.dialect = store.dialect
|
|
219
|
+
self.d = get_dialect_instance(self.dialect)
|
|
220
|
+
self.key = uuid.uuid4().hex[:16]
|
|
221
|
+
self.stages: List[Tuple[str, str]] = [] # (read, cleanup) per round with rows
|
|
222
|
+
self.cleanups: List[str] = []
|
|
223
|
+
self.kinds: set = set()
|
|
224
|
+
self.base: Optional[str] = None
|
|
225
|
+
self.types_before = set(engine._schema.column_types) if hasattr(engine._schema, "column_types") else set()
|
|
226
|
+
# owl:sameAs: the graph's own links (staged once), the links derived in this pass, the IRIs left
|
|
227
|
+
# unmerged, the components of the round (`identity.components`) and the identity rules read under.
|
|
228
|
+
self.stated: Optional[str] = None
|
|
229
|
+
self.link_stages: List[str] = []
|
|
230
|
+
self.poison_stages: List[str] = []
|
|
231
|
+
self.unmerged_reps: set = set()
|
|
232
|
+
self.comps: Any = None
|
|
233
|
+
self.identity: Any = None
|
|
234
|
+
self.disagree: Optional[str] = None # the staged (class, predicate) pairs of `stage_disagreeing`
|
|
235
|
+
self.component_index = 0
|
|
236
|
+
|
|
237
|
+
# -- SQL helpers -----------------------------------------------------------------------------------------
|
|
238
|
+
def lit(self, v: Optional[str]) -> str:
|
|
239
|
+
return f"CAST(NULL AS {self.d.type_name('STRING')})" if v is None else self.d.literal(v)
|
|
240
|
+
|
|
241
|
+
def text(self, sql: str) -> str:
|
|
242
|
+
return f"CAST({sql} AS {self.d.type_name('STRING')})"
|
|
243
|
+
|
|
244
|
+
def derived(self) -> str:
|
|
245
|
+
"""The rows derived so far in this run, as statements of the RL graph."""
|
|
246
|
+
from prograph.store import RL_GRAPH
|
|
247
|
+
cols = ", ".join(STATEMENT)
|
|
248
|
+
parts = [f"SELECT {self.lit(RL_GRAPH)} AS graph, {cols} FROM {read} AS _d" for read, _ in self.stages]
|
|
249
|
+
return " UNION ALL ".join(parts)
|
|
250
|
+
|
|
251
|
+
def premise_source(self) -> str:
|
|
252
|
+
"""The store's statements of every other graph (staged once per run, `stage_base`) and this run's
|
|
253
|
+
derivations: what the generated maps read in a round."""
|
|
254
|
+
cols = "graph, " + ", ".join(STATEMENT)
|
|
255
|
+
base = f"SELECT {cols} FROM {self.base} AS _o"
|
|
256
|
+
return base + (" UNION ALL " + self.derived() if self.stages else "")
|
|
257
|
+
|
|
258
|
+
def stage(self, index: int, name: str, sql: str) -> Tuple[str, str]:
|
|
259
|
+
"""Stage `sql`'s rows (`StoreConnection.stage`); (how a statement reads them, how they are removed)."""
|
|
260
|
+
read, write, drop = self.db.stage(self.store.table, f"rl-{self.key}", index, name, sql)
|
|
261
|
+
self.db.execute(write)
|
|
262
|
+
if drop not in self.cleanups:
|
|
263
|
+
self.cleanups.append(drop)
|
|
264
|
+
return read, drop
|
|
265
|
+
|
|
266
|
+
def stage_base(self) -> None:
|
|
267
|
+
"""The store's effective statements outside the RL graph, staged once: each round's maps read this
|
|
268
|
+
table, never the effective view's text again in every scan (Spark's analyzer runs out of memory on a
|
|
269
|
+
closure's levels each holding it)."""
|
|
270
|
+
from prograph.plan.ir.dataset import premise_statements
|
|
271
|
+
cols = "graph, " + ", ".join(STATEMENT)
|
|
272
|
+
self.base, _ = self.stage(0, f"_pg_rl_{self.key}_base",
|
|
273
|
+
f"SELECT {cols} FROM ({premise_statements(self.store)}) AS _o")
|
|
274
|
+
|
|
275
|
+
def stage_disagreeing(self) -> None:
|
|
276
|
+
"""What Cypher reads of this run's owl:sameAs components (`cypher_front`, one node per component), staged for
|
|
277
|
+
the generation's meta rows (`alias_statements`): each class a member of a merged component has ('merged',
|
|
278
|
+
so a Cypher pattern of another class needs no alias join), and each (class, Cypher property key) whose
|
|
279
|
+
values differ across a component's members ('disagree': the first query naming it warns). Values are
|
|
280
|
+
compared as values, type-tolerantly: `1` and `1.0`, or "10504" and 10504, are one value; a key the view
|
|
281
|
+
gives by several predicates is one key. One small query per key: a variable predicate over every map is
|
|
282
|
+
past the engines' statement limits."""
|
|
283
|
+
from prograph.plan.ir.facts import NotBuilt
|
|
284
|
+
from prograph.plan.ir.identity import SAME_AS
|
|
285
|
+
from prograph.schema.lpg_view import view_of
|
|
286
|
+
if self.comps is None or not self.comps.aliases:
|
|
287
|
+
return
|
|
288
|
+
every = {"http://www.w3.org/2000/01/rdf-schema#Resource", "http://www.w3.org/2002/07/owl#Thing"}
|
|
289
|
+
|
|
290
|
+
def plan(maps: Any, onto: Any) -> List[Tuple[str, str]]:
|
|
291
|
+
q = self.d.quote
|
|
292
|
+
out = []
|
|
293
|
+
# Under the run's identity owl:sameAs binds representatives, (rep, rep): a merged entity's; an IRI
|
|
294
|
+
# with no alias has no such triple.
|
|
295
|
+
merged = f"SELECT DISTINCT (STR(?c) AS ?k) WHERE {{ ?a <{SAME_AS}> ?b . ?a a ?c FILTER(isIRI(?c)) }}"
|
|
296
|
+
sql, _ = self.lowered(merged, maps, onto, self.identity)
|
|
297
|
+
out.append(("merged", f"SELECT {self.text('_r.' + q('k'))} AS c, {self.lit(None)} AS p FROM ({sql}) AS _r"))
|
|
298
|
+
by_key: Dict[str, set] = {}
|
|
299
|
+
view = view_of(maps)
|
|
300
|
+
for props in view.properties.values():
|
|
301
|
+
for key, pos in props.items():
|
|
302
|
+
by_key.setdefault(key, set()).update(po.predicate for po in pos if po.predicate is not None)
|
|
303
|
+
for key, preds in sorted(by_key.items()):
|
|
304
|
+
listed = " ".join(f"<{p}>" for p in sorted(preds))
|
|
305
|
+
query = (f"SELECT DISTINCT (STR(?c) AS ?k) WHERE {{ ?a <{SAME_AS}> ?b . VALUES ?p {{ {listed} }} "
|
|
306
|
+
f"VALUES ?r {{ {listed} }} ?a ?p ?v . ?a ?r ?w . ?a a ?c FILTER(isLiteral(?v) && isLiteral(?w) "
|
|
307
|
+
f"&& STR(?v) != STR(?w) && !COALESCE(?v = ?w, false) && isIRI(?c)) }}")
|
|
308
|
+
try:
|
|
309
|
+
sql, _ = self.lowered(query, maps, onto, self.identity)
|
|
310
|
+
except NotBuilt:
|
|
311
|
+
continue
|
|
312
|
+
out.append(("disagree", f"SELECT {self.text('_r.' + q('k'))} AS c, {self.lit(key)} AS p "
|
|
313
|
+
f"FROM ({sql}) AS _r"))
|
|
314
|
+
return out
|
|
315
|
+
rows = sorted({(kind, r[0], r[1]) for kind, sql in self.with_maps(plan)
|
|
316
|
+
for r in self.db.execute(sql).fetchall() if r[0] is not None and r[0] not in every})
|
|
317
|
+
if rows:
|
|
318
|
+
sql = " UNION ALL ".join(f"SELECT {self.lit(k)} AS kind, {self.lit(c)} AS c, {self.lit(p)} AS p"
|
|
319
|
+
for k, c, p in rows)
|
|
320
|
+
self.disagree, _ = self.stage(999_999_998, f"_pg_rl_{self.key}_disagree", sql)
|
|
321
|
+
|
|
322
|
+
# -- planning one rule --------------------------------------------------------------------------------
|
|
323
|
+
def rule_query(self, r: Any) -> str:
|
|
324
|
+
"""The rule's SPARQL: its derived statements' subject (and object) as text, `STR` of an IRI (a variable
|
|
325
|
+
the plan carries as a whole term, a property of IRI and literal objects, reads as its text too); only
|
|
326
|
+
IRI subjects, and an IRI object where the conclusion needs one (a literal is no premise there)."""
|
|
327
|
+
s = r.subject
|
|
328
|
+
if r.cls is not None:
|
|
329
|
+
return (f"SELECT DISTINCT (STR({s}) AS ?s) WHERE {{ {r.where} FILTER(isIRI({s})) "
|
|
330
|
+
f"FILTER NOT EXISTS {{ {s} a <{r.cls}> }} }}")
|
|
331
|
+
o = r.object
|
|
332
|
+
if r.value is not None: # a constant object (cls-hv1)
|
|
333
|
+
return (f"SELECT DISTINCT (STR({s}) AS ?s) WHERE {{ {r.where} FILTER(isIRI({s})) "
|
|
334
|
+
f"FILTER NOT EXISTS {{ {s} <{r.predicate}> {o} }} }}")
|
|
335
|
+
if r.literal:
|
|
336
|
+
return (f"SELECT DISTINCT (STR({s}) AS ?s) (STR({o}) AS ?t) (STR(DATATYPE({o})) AS ?d) (LANG({o}) AS ?l) "
|
|
337
|
+
f"WHERE {{ {r.where} FILTER(isIRI({s}) && !isBlank({o})) "
|
|
338
|
+
f"FILTER NOT EXISTS {{ {s} <{r.predicate}> {o} }} }}")
|
|
339
|
+
return (f"SELECT DISTINCT (STR({s}) AS ?s) (STR({o}) AS ?t) WHERE {{ {r.where} FILTER(isIRI({s}) && isIRI({o})) "
|
|
340
|
+
f"FILTER NOT EXISTS {{ {s} <{r.predicate}> {o} }} }}")
|
|
341
|
+
|
|
342
|
+
def lowered(self, query: str, maps: Any, onto: Any, identity: Any = None) -> str:
|
|
343
|
+
from prograph.plan.ir import lower, sparql_front
|
|
344
|
+
from prograph.plan.ir.entry import _held
|
|
345
|
+
from prograph.plan.ir.keys import trusted_keys
|
|
346
|
+
from prograph.plan.ir.rewrite import rewrite
|
|
347
|
+
graph = getattr(getattr(self.engine, "executor", None), "graph", None)
|
|
348
|
+
held = _held(self.engine._schema, self.dialect, True)
|
|
349
|
+
t = sparql_front.translate(query, maps, held, lambda m: trusted_keys(graph, m, maps),
|
|
350
|
+
union_default=getattr(self.engine, "default_graph", "union") == "union",
|
|
351
|
+
ontology=onto, identity=identity)
|
|
352
|
+
plan, _ = rewrite(t.plan, self.dialect)
|
|
353
|
+
from prograph.plan.ir.slot import Bound
|
|
354
|
+
# A column never bound (a DATATYPE of IRIs only) is written as a term's null: read as a null text.
|
|
355
|
+
never = frozenset(c for c in plan.columns if c in plan.slots and plan.slots[c].present is Bound.NEVER)
|
|
356
|
+
return lower.to_sql(plan, self.dialect, base_iri=maps.settings.base_iri), never
|
|
357
|
+
|
|
358
|
+
def wrapped(self, r: Any, lowered: Tuple[str, frozenset]) -> str:
|
|
359
|
+
from prograph.store import IRI, LITERAL
|
|
360
|
+
sql, never = lowered
|
|
361
|
+
q = self.d.quote
|
|
362
|
+
|
|
363
|
+
def col(name: str) -> str:
|
|
364
|
+
return self.lit(None) if name in never else f"_r.{q(name)}"
|
|
365
|
+
s = col("s")
|
|
366
|
+
if r.cls is not None:
|
|
367
|
+
obj = (self.lit(IRI), self.lit(r.cls), self.lit(None), self.lit(None))
|
|
368
|
+
elif r.value is not None:
|
|
369
|
+
from rdflib import Literal
|
|
370
|
+
v = r.value
|
|
371
|
+
if isinstance(v, Literal):
|
|
372
|
+
lang = getattr(v, "language", None)
|
|
373
|
+
dt = None if lang else str(v.datatype or xsd.STRING)
|
|
374
|
+
obj = (self.lit(LITERAL), self.lit(str(v)), self.lit(dt), self.lit(lang))
|
|
375
|
+
else:
|
|
376
|
+
obj = (self.lit(IRI), self.lit(str(v)), self.lit(None), self.lit(None))
|
|
377
|
+
elif r.literal:
|
|
378
|
+
t, dt, lang = col("t"), col("d"), col("l")
|
|
379
|
+
obj = (f"CASE WHEN {dt} IS NULL THEN {self.lit(IRI)} ELSE {self.lit(LITERAL)} END", self.text(t),
|
|
380
|
+
f"CASE WHEN {dt} = {self.lit(xsd.RDF_LANGSTRING)} "
|
|
381
|
+
f"THEN {self.lit(None)} ELSE {self.text(dt)} END",
|
|
382
|
+
f"CASE WHEN {lang} = '' THEN {self.lit(None)} ELSE {self.text(lang)} END")
|
|
383
|
+
else:
|
|
384
|
+
obj = (self.lit(IRI), self.text(col("t")), self.lit(None), self.lit(None))
|
|
385
|
+
return (f"SELECT {self.text(s)} AS subject, {self.lit(r.predicate)} AS predicate, {obj[0]} AS o_kind, "
|
|
386
|
+
f"{obj[1]} AS o_text, {obj[2]} AS o_datatype, {obj[3]} AS o_lang, {self.lit(r.rule)} AS rule "
|
|
387
|
+
f"FROM ({sql}) AS _r")
|
|
388
|
+
|
|
389
|
+
def planned(self, which: List[int], depth: int) -> Dict[int, str]:
|
|
390
|
+
"""The SQL of the rules `which` (indexes into `all_rules`: the rules, then the link rules), each its
|
|
391
|
+
derived statements (a link rule's, its links), planned over this round's premises with closures
|
|
392
|
+
walked to `depth`, matched through this round's components (`identity`)."""
|
|
393
|
+
rules = self.all_rules
|
|
394
|
+
|
|
395
|
+
def sql(j: int, maps: Any, onto: Any) -> str:
|
|
396
|
+
r = rules[j]
|
|
397
|
+
if j < len(self.rules.rules):
|
|
398
|
+
return self.wrapped(r, self.lowered(self.rule_query(r), maps, onto, self.identity))
|
|
399
|
+
return self.link_wrapped(r, self.lowered(self.link_query(r), maps, onto, self.identity))
|
|
400
|
+
return self.with_maps(lambda maps, onto: {j: sql(j, maps, onto) for j in which}, depth)
|
|
401
|
+
|
|
402
|
+
@property
|
|
403
|
+
def all_rules(self) -> List[Any]:
|
|
404
|
+
return list(self.rules.rules) + list(self.rules.links)
|
|
405
|
+
|
|
406
|
+
def with_maps(self, plan: Any, depth: Optional[int] = None) -> Any:
|
|
407
|
+
"""`plan(maps, onto)` over this round's premises (the saturated maps reading the store's other graphs
|
|
408
|
+
and this run's derivations), closures walked to `depth`."""
|
|
409
|
+
from prograph.plan.ir import lower
|
|
410
|
+
from prograph.plan.ir.dataset import maps_seen
|
|
411
|
+
from prograph.plan.ir.entry import ontology_of
|
|
412
|
+
onto = ontology_of(self.engine)
|
|
413
|
+
store = self.store
|
|
414
|
+
token = lower.PATH_DEPTH.set(depth or lower.DEFAULT_PATH_DEPTH)
|
|
415
|
+
try:
|
|
416
|
+
with store.premises(self.premise_source(), store._profile.kinds, self.kinds):
|
|
417
|
+
return plan(maps_seen(self.engine, onto), onto)
|
|
418
|
+
finally:
|
|
419
|
+
lower.PATH_DEPTH.reset(token)
|
|
420
|
+
|
|
421
|
+
# -- owl:sameAs (`identity`) -----------------------------------------------------------------------------
|
|
422
|
+
def link_query(self, r: Any) -> str:
|
|
423
|
+
"""A link rule's SPARQL: its two IRIs' texts, two IRIs that are not one representative."""
|
|
424
|
+
s, o = r.subject, r.object
|
|
425
|
+
return (f"SELECT DISTINCT (STR({s}) AS ?s) (STR({o}) AS ?t) WHERE {{ {r.where} "
|
|
426
|
+
f"FILTER(isIRI({s}) && isIRI({o}) && !sameTerm({s}, {o})) }}")
|
|
427
|
+
|
|
428
|
+
def link_wrapped(self, r: Any, lowered: Tuple[str, frozenset]) -> str:
|
|
429
|
+
"""A link rule's rows as links (g, a, b, source), the default graph's, none with an IRI left
|
|
430
|
+
unmerged."""
|
|
431
|
+
sql, _ = lowered
|
|
432
|
+
q = self.d.quote
|
|
433
|
+
rows = (f"SELECT {self.lit('')} AS g, {self.text('_r.' + q('s'))} AS a, {self.text('_r.' + q('t'))} AS b, "
|
|
434
|
+
f"{self.lit(f'rl:{r.rule} {r.axiom}')} AS source FROM ({sql}) AS _r")
|
|
435
|
+
if self.comps is not None and self.comps.aliases:
|
|
436
|
+
# Only a link between two components is new (a pattern's ends are representatives already, a
|
|
437
|
+
# VALUES's constants not).
|
|
438
|
+
alias = f"SELECT iri, rep FROM {self.comps.rows} AS _a WHERE g = {self.lit('')}"
|
|
439
|
+
rows = (f"SELECT _k.g, _k.a, _k.b, _k.source FROM ({rows}) AS _k LEFT JOIN ({alias}) AS _ra ON "
|
|
440
|
+
f"_ra.iri = _k.a LEFT JOIN ({alias}) AS _rb ON _rb.iri = _k.b "
|
|
441
|
+
f"WHERE coalesce(_ra.rep, _k.a) <> coalesce(_rb.rep, _k.b)")
|
|
442
|
+
return self.unpoisoned(rows)
|
|
443
|
+
|
|
444
|
+
def unpoisoned(self, links: str) -> str:
|
|
445
|
+
"""The links `links` with neither end an IRI left unmerged."""
|
|
446
|
+
poisoned = self.poisoned()
|
|
447
|
+
if poisoned is None:
|
|
448
|
+
return links
|
|
449
|
+
return (f"SELECT _l.g, _l.a, _l.b, _l.source FROM ({links}) AS _l LEFT JOIN ({poisoned}) AS _pa ON _pa.g = _l.g "
|
|
450
|
+
f"AND _pa.iri = _l.a LEFT JOIN ({poisoned}) AS _pb ON _pb.g = _l.g AND _pb.iri = _l.b "
|
|
451
|
+
f"WHERE _pa.iri IS NULL AND _pb.iri IS NULL")
|
|
452
|
+
|
|
453
|
+
def poisoned(self) -> Optional[str]:
|
|
454
|
+
"""SQL of the IRIs (g, iri) left unmerged so far, None where none is."""
|
|
455
|
+
return " UNION ALL ".join(f"SELECT g, iri FROM {p} AS _p" for p in self.poison_stages) or None
|
|
456
|
+
|
|
457
|
+
def links(self) -> Optional[str]:
|
|
458
|
+
"""SQL of every link (g, a, b, source): the graph's own and those derived in this pass; None where
|
|
459
|
+
there is none."""
|
|
460
|
+
parts = ([f"SELECT g, a, b, source FROM {self.stated} AS _s"] if self.stated else []) + \
|
|
461
|
+
[f"SELECT g, a, b, source FROM {x} AS _d" for x in self.link_stages]
|
|
462
|
+
return " UNION ALL ".join(parts) or None
|
|
463
|
+
|
|
464
|
+
def stage_component(self, name: str, sql: str) -> Tuple[str, str]:
|
|
465
|
+
"""Stage one statement of the components' computation (`identity.components`)."""
|
|
466
|
+
self.component_index += 1
|
|
467
|
+
return self.stage(1_000_000_000 + self.component_index, f"_pg_rl_{self.key}_c{self.component_index}", sql)
|
|
468
|
+
|
|
469
|
+
def stage_stated(self) -> None:
|
|
470
|
+
"""The graph's own sameAs links between two IRIs (g '' the default graph, else each named graph's),
|
|
471
|
+
read as the rules' premises are, owl:sameAs as stated; staged once, where some map or the store
|
|
472
|
+
gives owl:sameAs."""
|
|
473
|
+
from prograph.plan.ir.identity import SAME_AS
|
|
474
|
+
|
|
475
|
+
def plan(maps: Any, onto: Any) -> Optional[str]:
|
|
476
|
+
if SAME_AS not in maps.by_predicate:
|
|
477
|
+
return None
|
|
478
|
+
test = "FILTER(isIRI(?a) && isIRI(?b) && !sameTerm(?a, ?b))"
|
|
479
|
+
default, _ = self.lowered(f"SELECT DISTINCT (STR(?a) AS ?x) (STR(?b) AS ?y) WHERE {{ ?a <{SAME_AS}> ?b {test} }}",
|
|
480
|
+
maps, onto)
|
|
481
|
+
named, _ = self.lowered(f"SELECT DISTINCT (STR(?g) AS ?h) (STR(?a) AS ?x) (STR(?b) AS ?y) WHERE "
|
|
482
|
+
f"{{ GRAPH ?g {{ ?a <{SAME_AS}> ?b {test} }} }}", maps, onto)
|
|
483
|
+
q = self.d.quote
|
|
484
|
+
x, y, h = (self.text(f"_r.{q(n)}") for n in ("x", "y", "h"))
|
|
485
|
+
stated = self.lit("stated")
|
|
486
|
+
return (f"SELECT {self.lit('')} AS g, {x} AS a, {y} AS b, {stated} AS source FROM ({default}) AS _r "
|
|
487
|
+
f"UNION ALL SELECT {h} AS g, {x} AS a, {y} AS b, {stated} AS source FROM ({named}) AS _r")
|
|
488
|
+
sql = self.with_maps(plan)
|
|
489
|
+
if sql is not None:
|
|
490
|
+
self.stated, _ = self.stage(999_999_999, f"_pg_rl_{self.key}_stated", sql)
|
|
491
|
+
|
|
492
|
+
def close_links(self, report: MaterializationReport, first: bool) -> bool:
|
|
493
|
+
"""This round's components (`identity.components`), each inconsistent one or one past the cap left
|
|
494
|
+
unmerged (named in `report.unmerged`), and the identity the round's rules read under. True where a
|
|
495
|
+
component rules already read is left unmerged now: the pass runs again from its first round."""
|
|
496
|
+
from prograph.plan.ir import identity as I
|
|
497
|
+
links = self.links()
|
|
498
|
+
if links is None:
|
|
499
|
+
self.comps, self.identity = None, None
|
|
500
|
+
return False
|
|
501
|
+
self.comps = I.components(self, links, self.poisoned())
|
|
502
|
+
self.poison_capped(report)
|
|
503
|
+
bad = self.inconsistent(report)
|
|
504
|
+
if bad:
|
|
505
|
+
self.comps = I.components(self, links, self.poisoned())
|
|
506
|
+
if not first:
|
|
507
|
+
return True
|
|
508
|
+
self.identity = I.Identity(f"SELECT g, iri, rep FROM {self.comps.rows} AS _a", None, self.comps.largest,
|
|
509
|
+
"representative", run=lambda sql: self.db.execute(sql).fetchall(),
|
|
510
|
+
literal=self.d.literal, tag=f"{self.key}{self.component_index}") \
|
|
511
|
+
if self.comps.aliases else None
|
|
512
|
+
return False
|
|
513
|
+
|
|
514
|
+
def value_key(self, text: str, datatype: str, lang: str) -> str:
|
|
515
|
+
"""SQL of a literal's value as text, one text per value (a clash's test; `owl_rl_rules.same_value` is the
|
|
516
|
+
rule): an xsd:decimal or any integer type by its number, xsd:float and xsd:double each by its own number
|
|
517
|
+
(with its sign: +0.0 and -0.0 are two), an xsd:boolean by its canonical form ("1" is "true"), an
|
|
518
|
+
xsd:dateTime with a timezone by its instant (`Z` is `+00:00`) and one without by its local time
|
|
519
|
+
(`value_group` keeps the two apart: XML Schema 1.1 §3.3.7 leaves them incomparable), any other by its
|
|
520
|
+
datatype, language and lexical form."""
|
|
521
|
+
decimal = ", ".join(self.lit(xsd.XSD + t) for t in (
|
|
522
|
+
"decimal", "integer", "long", "int", "short", "byte", "nonNegativeInteger", "positiveInteger",
|
|
523
|
+
"negativeInteger", "nonPositiveInteger", "unsignedLong", "unsignedInt", "unsignedShort", "unsignedByte"))
|
|
524
|
+
cast = "CAST" if self.dialect == "postgresql" else "TRY_CAST"
|
|
525
|
+
double = "DOUBLE PRECISION" if self.dialect == "postgresql" else "DOUBLE"
|
|
526
|
+
if self.dialect == "postgresql":
|
|
527
|
+
instant = f"EXTRACT(EPOCH FROM CAST({text} AS TIMESTAMPTZ))"
|
|
528
|
+
elif self.dialect == "duckdb":
|
|
529
|
+
instant = f"epoch_us(TRY_CAST({text} AS TIMESTAMPTZ))"
|
|
530
|
+
else: # Spark SQL, Databricks SQL: a TIMESTAMP read with its offset is an instant
|
|
531
|
+
instant = f"unix_micros(TRY_CAST({text} AS TIMESTAMP))"
|
|
532
|
+
local = f"{cast}({text} AS TIMESTAMP)"
|
|
533
|
+
boolean = self.lit(xsd.BOOLEAN)
|
|
534
|
+
return (f"CASE WHEN {datatype} IN ({decimal}) THEN {self.lit('n ')} || "
|
|
535
|
+
f"{self.text(f'{cast}({text} AS DECIMAL(38, 10))')} "
|
|
536
|
+
f"WHEN {datatype} IN ({self.lit(xsd.XSD + 'float')}, {self.lit(xsd.XSD + 'double')}) THEN "
|
|
537
|
+
f"{datatype} || {self.text(f'{cast}({text} AS {double})')} "
|
|
538
|
+
f"WHEN {datatype} = {boolean} THEN CASE WHEN {text} IN ('true', '1') THEN 'b true' "
|
|
539
|
+
f"WHEN {text} IN ('false', '0') THEN 'b false' ELSE 'b ' || {text} END "
|
|
540
|
+
f"WHEN {self.value_group(text, datatype)} = 'z' THEN 'z ' || {self.text(instant)} "
|
|
541
|
+
f"WHEN {self.value_group(text, datatype)} = 'u' THEN 'u ' || {self.text(local)} "
|
|
542
|
+
f"ELSE coalesce({datatype}, '') || ' ' || coalesce({lang}, '') || ' ' || {text} END")
|
|
543
|
+
|
|
544
|
+
def value_group(self, text: str, datatype: str) -> str:
|
|
545
|
+
"""SQL: 'z' for an xsd:dateTime with a timezone, 'u' for one without, '' for any other literal."""
|
|
546
|
+
stamps = f"{self.lit(xsd.XSD + 'dateTime')}, {self.lit(xsd.XSD + 'dateTimeStamp')}"
|
|
547
|
+
return (f"CASE WHEN {datatype} IN ({stamps}) THEN CASE WHEN {text} LIKE '%Z' OR {text} LIKE '%+__:__' OR "
|
|
548
|
+
f"{text} LIKE '%-__:__' THEN 'z' ELSE 'u' END ELSE '' END")
|
|
549
|
+
|
|
550
|
+
@staticmethod
|
|
551
|
+
def two_values(key: str = "k", group: str = "g") -> str:
|
|
552
|
+
"""A HAVING of two values that are not one value: two keys among the values comparable with each other
|
|
553
|
+
(a dateTime with a timezone and one without are not)."""
|
|
554
|
+
return (f"count(DISTINCT CASE WHEN {group} <> 'u' THEN {key} END) > 1 OR "
|
|
555
|
+
f"count(DISTINCT CASE WHEN {group} <> 'z' THEN {key} END) > 1")
|
|
556
|
+
|
|
557
|
+
def poison_capped(self, report: MaterializationReport) -> None:
|
|
558
|
+
for g, rep, size in self.comps.capped:
|
|
559
|
+
if (g, rep) in self.unmerged_reps:
|
|
560
|
+
continue
|
|
561
|
+
self.unmerged_reps.add((g, rep))
|
|
562
|
+
sample = tuple(r[0] for r in self.db.execute(
|
|
563
|
+
f"SELECT x FROM {self.comps.labels} AS _l WHERE g = {self.lit(g)} AND l = {self.lit(rep)} "
|
|
564
|
+
f"ORDER BY x LIMIT {SAMPLE}").fetchall())
|
|
565
|
+
report.unmerged.append(Unmerged("cap", "", g, rep, size, sample))
|
|
566
|
+
read, _ = self.stage_component("capped", f"SELECT g, x AS iri FROM {self.comps.labels} AS _l "
|
|
567
|
+
f"WHERE g = {self.lit(g)} AND l = {self.lit(rep)}")
|
|
568
|
+
self.poison_stages.append(read)
|
|
569
|
+
|
|
570
|
+
def inconsistent(self, report: MaterializationReport) -> bool:
|
|
571
|
+
"""Leave unmerged each component of the default graph that owl:differentFrom or owl:AllDifferent
|
|
572
|
+
(eq-diff2, eq-diff3) or two values of a functional property (prp-fp, cls-maxc2: two values that are
|
|
573
|
+
not one value, dt-diff's reading) make inconsistent; a lone IRI's clash is an inconsistency of the
|
|
574
|
+
report. True where one was left unmerged now."""
|
|
575
|
+
from prograph.plan.ir import identity as I
|
|
576
|
+
if not self.comps.aliases:
|
|
577
|
+
return False
|
|
578
|
+
ident = I.Identity(f"SELECT g, iri, rep FROM {self.comps.rows} AS _a", None, self.comps.largest,
|
|
579
|
+
"representative", run=lambda sql: self.db.execute(sql).fetchall(),
|
|
580
|
+
literal=self.d.literal, tag=f"{self.key}{self.component_index}x")
|
|
581
|
+
found: Dict[str, Tuple[str, str]] = {} # rep -> (rule, axiom)
|
|
582
|
+
different = f"<{I.DIFFERENT_FROM}>"
|
|
583
|
+
checks = [(c.rule, c.axiom, True,
|
|
584
|
+
f"SELECT DISTINCT (STR(?x) AS ?r) (STR(?v) AS ?t) (STR(DATATYPE(?v)) AS ?d) (LANG(?v) AS ?l) "
|
|
585
|
+
f"WHERE {{ {c.where} FILTER(isIRI(?x) && isLiteral(?v)) }}") for c in self.rules.clashes]
|
|
586
|
+
|
|
587
|
+
def plan(maps: Any, onto: Any) -> List[Tuple[str, str, bool, str]]:
|
|
588
|
+
out = []
|
|
589
|
+
wanted = checks + ([("eq-diff1", different, False,
|
|
590
|
+
f"SELECT DISTINCT (STR(?a) AS ?r) WHERE {{ ?a <{I.DIFFERENT_FROM}> ?b "
|
|
591
|
+
f"FILTER(isIRI(?a) && sameTerm(?a, ?b)) }}")]
|
|
592
|
+
if I.DIFFERENT_FROM in maps.by_predicate else [])
|
|
593
|
+
for rule, axiom, values, query in wanted:
|
|
594
|
+
out.append((rule, axiom, values, self.lowered(query, maps, onto, ident)))
|
|
595
|
+
return out
|
|
596
|
+
q = self.d.quote
|
|
597
|
+
for rule, axiom, values, (sql, never) in self.with_maps(plan):
|
|
598
|
+
if values:
|
|
599
|
+
# Two values of one subject that are not one value (`value_key`).
|
|
600
|
+
def col(n: str) -> str:
|
|
601
|
+
return self.lit(None) if n in never else self.text(f"_r.{q(n)}")
|
|
602
|
+
key = self.value_key(col("t"), col("d"), col("l"))
|
|
603
|
+
group = self.value_group(col("t"), col("d"))
|
|
604
|
+
sql = (f"SELECT r FROM (SELECT {col('r')} AS r, {key} AS k, {group} AS g FROM ({sql}) AS _r) AS _k "
|
|
605
|
+
f"GROUP BY r HAVING {self.two_values()}")
|
|
606
|
+
else:
|
|
607
|
+
sql = f"SELECT {self.text('_r.' + q('r'))} FROM ({sql}) AS _r"
|
|
608
|
+
for (rep,) in self.db.execute(sql).fetchall():
|
|
609
|
+
found.setdefault(rep, (rule, axiom))
|
|
610
|
+
if self.rules.distinct:
|
|
611
|
+
listed = {x for a, b, _, _ in self.rules.distinct for x in (a, b)}
|
|
612
|
+
reps = dict(self.db.execute(
|
|
613
|
+
f"SELECT iri, rep FROM {self.comps.rows} AS _a WHERE g = {self.lit('')} AND iri IN "
|
|
614
|
+
f"({', '.join(self.lit(x) for x in sorted(listed))})").fetchall())
|
|
615
|
+
for a, b, axiom, rule in self.rules.distinct:
|
|
616
|
+
ra, rb = reps.get(a, a), reps.get(b, b)
|
|
617
|
+
if a != b and ra == rb:
|
|
618
|
+
found.setdefault(ra, (rule, axiom))
|
|
619
|
+
fresh = False
|
|
620
|
+
for rep, (rule, axiom) in sorted(found.items()):
|
|
621
|
+
if ("", rep) in self.unmerged_reps:
|
|
622
|
+
continue
|
|
623
|
+
members = [r[0] for r in self.db.execute(
|
|
624
|
+
f"SELECT iri FROM {self.comps.rows} AS _a WHERE g = {self.lit('')} AND rep = {self.lit(rep)} "
|
|
625
|
+
f"ORDER BY iri").fetchall()]
|
|
626
|
+
if not members: # a lone IRI: no merge to undo, an inconsistency the run's check names
|
|
627
|
+
continue
|
|
628
|
+
self.unmerged_reps.add(("", rep))
|
|
629
|
+
report.unmerged.append(Unmerged("inconsistent", rule, "", rep, len(members) + 1,
|
|
630
|
+
tuple(sorted([rep] + members)[:SAMPLE]), axiom))
|
|
631
|
+
read, _ = self.stage_component("unmerged", f"SELECT g, iri FROM {self.comps.rows} AS _a WHERE g = "
|
|
632
|
+
f"{self.lit('')} AND rep = {self.lit(rep)} UNION ALL SELECT "
|
|
633
|
+
f"{self.lit('')} AS g, {self.lit(rep)} AS iri")
|
|
634
|
+
self.poison_stages.append(read)
|
|
635
|
+
fresh = True
|
|
636
|
+
return fresh
|
|
637
|
+
|
|
638
|
+
# -- the fixpoint ----------------------------------------------------------------------------------------
|
|
639
|
+
def depths(self) -> List[int]:
|
|
640
|
+
from prograph.plan.ir import lower
|
|
641
|
+
ladder = list(lower.closure_ladder(self.dialect))
|
|
642
|
+
return ladder + [lower.DEFAULT_PATH_DEPTH, DEEPEST] if ladder else [lower.DEFAULT_PATH_DEPTH, DEEPEST]
|
|
643
|
+
|
|
644
|
+
# A round's stages: rule j's at round * RULE_STRIDE + j, the round's merged rows at round * RULE_STRIDE.
|
|
645
|
+
RULE_STRIDE = 100000
|
|
646
|
+
|
|
647
|
+
def one_round(self, n: int) -> Tuple[int, int]:
|
|
648
|
+
"""Run round `n`: each rule one statement (a transitive property's closure walked by the engine's
|
|
649
|
+
ladder: a deeper rung only where a walk is longer), then the round's triples merged into one stage
|
|
650
|
+
(a triple two rules derived once, by the first rule's id), and its links into another; how many
|
|
651
|
+
of each."""
|
|
652
|
+
from prograph.plan.ir import lower
|
|
653
|
+
from prograph.execution import checkpoint
|
|
654
|
+
pending = list(range(len(self.all_rules)))
|
|
655
|
+
reads: Dict[int, Tuple[str, str]] = {}
|
|
656
|
+
batched: Tuple[List[str], List[str]] = ([], []) # (facts, links) read of each batch (Delta)
|
|
657
|
+
depths = self.depths()
|
|
658
|
+
for i, depth in enumerate(depths):
|
|
659
|
+
checkpoint()
|
|
660
|
+
sqls = self.planned(pending, depth)
|
|
661
|
+
again = []
|
|
662
|
+
single = pending
|
|
663
|
+
if self.db.optimistic:
|
|
664
|
+
single = self.stage_batches(n, i, pending, sqls, batched)
|
|
665
|
+
for j in single:
|
|
666
|
+
try:
|
|
667
|
+
reads[j] = self.stage(n * self.RULE_STRIDE + j + 1, f"_pg_rl_{self.key}_{n}_{j}", sqls[j])
|
|
668
|
+
except Exception as exc: # noqa: BLE001 - a deeper rung answers a depth guard
|
|
669
|
+
if not lower.depth_guard(str(exc)):
|
|
670
|
+
raise
|
|
671
|
+
if i == len(depths) - 1:
|
|
672
|
+
from prograph.utils.exceptions import SQLRuntimeError
|
|
673
|
+
raise SQLRuntimeError(
|
|
674
|
+
f"OWL RL materialization: a transitive property's path is longer than {depth} steps, "
|
|
675
|
+
f"which a round walks at most", reason="path_too_deep", sql=sqls[j]) from exc
|
|
676
|
+
again.append(j)
|
|
677
|
+
pending = again
|
|
678
|
+
if not pending:
|
|
679
|
+
break
|
|
680
|
+
cols = ", ".join(STATEMENT)
|
|
681
|
+
facts = {j: x for j, x in reads.items() if j < len(self.rules.rules)}
|
|
682
|
+
linked = {j: x for j, x in reads.items() if j >= len(self.rules.rules)}
|
|
683
|
+
count = 0
|
|
684
|
+
if facts or batched[0]:
|
|
685
|
+
union = " UNION ALL ".join(
|
|
686
|
+
[f"SELECT {cols}, rule FROM {read} AS _r{j}" for j, (read, _) in sorted(facts.items())] +
|
|
687
|
+
[f"SELECT {cols}, rule FROM {read} AS _b{k}" for k, read in enumerate(batched[0])])
|
|
688
|
+
read, drop = self.stage(n * self.RULE_STRIDE, f"_pg_rl_{self.key}_{n}",
|
|
689
|
+
f"SELECT {cols}, min(rule) AS rule FROM ({union}) AS _u GROUP BY {cols}")
|
|
690
|
+
count = self.db.execute(f"SELECT count(*) FROM {read} AS _c").fetchone()[0]
|
|
691
|
+
if count:
|
|
692
|
+
self.stages.append((read, drop))
|
|
693
|
+
for row in self.db.execute(f"SELECT DISTINCT predicate, o_kind, o_datatype, o_lang FROM {read} AS _k"
|
|
694
|
+
).fetchall():
|
|
695
|
+
self.kinds.add(tuple(row))
|
|
696
|
+
links = 0
|
|
697
|
+
if linked or batched[1]:
|
|
698
|
+
union = " UNION ALL ".join(
|
|
699
|
+
[f"SELECT g, a, b, source FROM {read} AS _r{j}" for j, (read, _) in sorted(linked.items())] +
|
|
700
|
+
[f"SELECT g, a, b, source FROM {read} AS _b{k}" for k, read in enumerate(batched[1])])
|
|
701
|
+
read, _ = self.stage(n * self.RULE_STRIDE + self.RULE_STRIDE // 2, f"_pg_rl_{self.key}_{n}_links",
|
|
702
|
+
f"SELECT g, a, b, min(source) AS source FROM ({union}) AS _u GROUP BY g, a, b")
|
|
703
|
+
links = self.db.execute(f"SELECT count(*) FROM {read} AS _c").fetchone()[0]
|
|
704
|
+
if links:
|
|
705
|
+
self.link_stages.append(read)
|
|
706
|
+
if not self.db.optimistic: # the rules' own tables (on Delta every stage of the run goes at the end)
|
|
707
|
+
for _, rule_drop in reads.values():
|
|
708
|
+
self.db.execute(rule_drop)
|
|
709
|
+
self.cleanups.remove(rule_drop)
|
|
710
|
+
return count, links
|
|
711
|
+
|
|
712
|
+
# Delta: a rule's rows are one write each unless batched: at most this many rules, and this many characters
|
|
713
|
+
# of SQL (each rule inlines its premises, which a warehouse's statement limit and Spark's analyzer feel), go
|
|
714
|
+
# in one write.
|
|
715
|
+
BATCH = 40
|
|
716
|
+
BATCH_SQL_CHARS = 1_000_000
|
|
717
|
+
|
|
718
|
+
def stage_batches(self, n: int, rung: int, pending: List[int], sqls: Dict[int, str],
|
|
719
|
+
batched: Tuple[List[str], List[str]]) -> List[int]:
|
|
720
|
+
"""Delta commits every statement alone, so a round's rules are staged together: the facts rules in
|
|
721
|
+
batches of at most `BATCH` rules and `BATCH_SQL_CHARS` characters (one write each, the rule's id in
|
|
722
|
+
each row), the link rules likewise. A batch a depth guard refuses is halved until the rule is found;
|
|
723
|
+
a batch any other error refuses is left to the rules' own writes (whose error, if they fail too,
|
|
724
|
+
stands). The rules so left are returned."""
|
|
725
|
+
from prograph.plan.ir import lower
|
|
726
|
+
left: List[int] = []
|
|
727
|
+
nfacts = len(self.rules.rules)
|
|
728
|
+
cols = ", ".join(STATEMENT)
|
|
729
|
+
for links, shape in ((False, f"{cols}, rule"), (True, "g, a, b, source")):
|
|
730
|
+
ids = [j for j in pending if (j >= nfacts) == links]
|
|
731
|
+
chunks: List[List[int]] = []
|
|
732
|
+
size = 0
|
|
733
|
+
for j in ids:
|
|
734
|
+
if not chunks or len(chunks[-1]) >= self.BATCH or size + len(sqls[j]) > self.BATCH_SQL_CHARS:
|
|
735
|
+
chunks.append([])
|
|
736
|
+
size = 0
|
|
737
|
+
chunks[-1].append(j)
|
|
738
|
+
size += len(sqls[j])
|
|
739
|
+
base = n * self.RULE_STRIDE + self.RULE_STRIDE // 4 + (rung * 2 + links) * 1000
|
|
740
|
+
counter = [0]
|
|
741
|
+
|
|
742
|
+
def stage_chunk(chunk: List[int]) -> None:
|
|
743
|
+
if len(chunk) == 1:
|
|
744
|
+
left.extend(chunk)
|
|
745
|
+
return
|
|
746
|
+
union = " UNION ALL ".join(f"SELECT {shape} FROM ({sqls[j]}) AS _p{j}" for j in chunk)
|
|
747
|
+
counter[0] += 1
|
|
748
|
+
try:
|
|
749
|
+
read, _ = self.stage(base + counter[0], f"_pg_rl_{self.key}_{n}_b{rung}{int(links)}_{counter[0]}",
|
|
750
|
+
union)
|
|
751
|
+
except Exception as exc: # noqa: BLE001 - the rules' own writes answer it, or raise as before
|
|
752
|
+
if lower.depth_guard(str(exc)):
|
|
753
|
+
half = len(chunk) // 2
|
|
754
|
+
stage_chunk(chunk[:half])
|
|
755
|
+
stage_chunk(chunk[half:])
|
|
756
|
+
else:
|
|
757
|
+
left.extend(chunk)
|
|
758
|
+
return
|
|
759
|
+
batched[links].append(read)
|
|
760
|
+
for chunk in chunks:
|
|
761
|
+
stage_chunk(chunk)
|
|
762
|
+
return left
|
|
763
|
+
|
|
764
|
+
def go(self) -> MaterializationReport:
|
|
765
|
+
from prograph.plan.ir import freshness
|
|
766
|
+
from prograph.plan.ir.dataset import cache_key, stored_sources
|
|
767
|
+
report = MaterializationReport(skipped=list(self.rules.skipped), digest=cache_key(self.engine, self.rules))
|
|
768
|
+
# The sources' versions, read before the rules read them: a write during the run is seen by the next check.
|
|
769
|
+
report.sources = freshness.read_run(self.engine, stored_sources(self.engine))
|
|
770
|
+
ensure = getattr(getattr(getattr(self.engine, "executor", None), "graph", None), "_ensure_temp_views", None)
|
|
771
|
+
if callable(ensure):
|
|
772
|
+
ensure() # Spark: the engine's frames are views before its first query too (a run on open)
|
|
773
|
+
if self.rules.rules or self.rules.links or self.maybe_stated():
|
|
774
|
+
self.stage_base()
|
|
775
|
+
self.stage_stated()
|
|
776
|
+
n = 0
|
|
777
|
+
while True: # a pass; another from its first round where a component it read is left unmerged
|
|
778
|
+
again, first = False, True
|
|
779
|
+
while True:
|
|
780
|
+
n += 1
|
|
781
|
+
if n > self.max_rounds:
|
|
782
|
+
from prograph.utils.exceptions import MaterializationBudgetExceeded
|
|
783
|
+
raise MaterializationBudgetExceeded(self.max_rounds)
|
|
784
|
+
report.rounds = n
|
|
785
|
+
if self.close_links(report, first):
|
|
786
|
+
again = True
|
|
787
|
+
break
|
|
788
|
+
first = False
|
|
789
|
+
if self.one_round(n) == (0, 0):
|
|
790
|
+
break
|
|
791
|
+
if not again:
|
|
792
|
+
break
|
|
793
|
+
self.stages, self.link_stages = [], []
|
|
794
|
+
report.inconsistencies = self.consistency(report)
|
|
795
|
+
self.forget_premise_types()
|
|
796
|
+
report.ran_at = _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
|
|
797
|
+
if self.comps is not None:
|
|
798
|
+
report.components, report.aliases, report.largest = self.comps.count, self.comps.aliases, \
|
|
799
|
+
self.comps.largest
|
|
800
|
+
report.unmerged.sort(key=_unmerged_order) # the order a reused report reads them in
|
|
801
|
+
before = self.store.alias
|
|
802
|
+
seq = self.write(report)
|
|
803
|
+
report.seq = seq
|
|
804
|
+
if seq is not None:
|
|
805
|
+
self.counts(report, seq)
|
|
806
|
+
self.warn_size(report, before)
|
|
807
|
+
return report
|
|
808
|
+
|
|
809
|
+
def maybe_stated(self) -> bool:
|
|
810
|
+
"""Whether the graph may hold an owl:sameAs triple (some map, or the store, gives the predicate), or a
|
|
811
|
+
generation of the alias table is to be replaced."""
|
|
812
|
+
from prograph.plan.ir.dataset import maps_seen
|
|
813
|
+
from prograph.plan.ir.identity import SAME_AS, meta_table
|
|
814
|
+
if SAME_AS in maps_seen(self.engine, None).by_predicate:
|
|
815
|
+
return True
|
|
816
|
+
return self.store.db.has_table(meta_table(self.store))
|
|
817
|
+
|
|
818
|
+
def warn_size(self, report: MaterializationReport, before: Any) -> None:
|
|
819
|
+
"""`SameAsMergeWarning` where the largest component has `WARN_MEMBERS` members or more, or grew more
|
|
820
|
+
than tenfold since the last generation."""
|
|
821
|
+
import warnings
|
|
822
|
+
from prograph.plan.ir.identity import CAP, GROWTH, WARN_MEMBERS, SameAsMergeWarning
|
|
823
|
+
if not report.largest:
|
|
824
|
+
return
|
|
825
|
+
grew = before is not None and before.largest and report.largest > GROWTH * before.largest
|
|
826
|
+
if report.largest >= WARN_MEMBERS or grew:
|
|
827
|
+
why = (f"grew from {before.largest} to {report.largest} members" if grew else
|
|
828
|
+
f"has {report.largest} members")
|
|
829
|
+
warnings.warn(f"the largest owl:sameAs component {why}: a functional or inverse-functional property "
|
|
830
|
+
f"(or a crosswalk) may be merging entities that are not one. engine.explain_identity(a, b) "
|
|
831
|
+
f"shows why two IRIs merged; a component past {CAP} members is left unmerged",
|
|
832
|
+
SameAsMergeWarning, stacklevel=4)
|
|
833
|
+
|
|
834
|
+
# -- the alias generation (`identity`) ------------------------------------------------------------------
|
|
835
|
+
def publish(self) -> bool:
|
|
836
|
+
"""Whether this run writes a generation of the alias table: it has components or unmerged ones, or
|
|
837
|
+
one is written already (replaced, maybe by none)."""
|
|
838
|
+
from prograph.plan.ir.identity import meta_table
|
|
839
|
+
return bool(self.comps is not None and (self.comps.aliases or self.comps.capped)) or \
|
|
840
|
+
bool(self.unmerged_reps) or self.store.db.has_table(meta_table(self.store))
|
|
841
|
+
|
|
842
|
+
def alias_statements(self, report: MaterializationReport, generation: str) -> List[str]:
|
|
843
|
+
"""The statements that write this run's generation `generation` (SQL of its number): the alias rows
|
|
844
|
+
(where there are some: never an empty INSERT), then one INSERT of the meta rows (the marker, the IRI
|
|
845
|
+
shapes, the derived links, the components left unmerged)."""
|
|
846
|
+
import json
|
|
847
|
+
from prograph.plan.ir import identity as I
|
|
848
|
+
null = self.lit(None)
|
|
849
|
+
out = []
|
|
850
|
+
aliases = self.comps.aliases if self.comps is not None else 0
|
|
851
|
+
shapes: List[str] = []
|
|
852
|
+
if aliases:
|
|
853
|
+
out.append(f"INSERT INTO {I.alias_table(self.store)} (generation, graph, iri, rep) "
|
|
854
|
+
f"SELECT {generation}, g, iri, rep FROM {self.comps.rows} AS _a")
|
|
855
|
+
ns = lambda e: I.namespace_sql(self.dialect, e) # noqa: E731
|
|
856
|
+
shapes = [r[0] for r in self.db.execute(
|
|
857
|
+
f"SELECT DISTINCT n FROM (SELECT {ns('iri')} AS n FROM {self.comps.rows} AS _a UNION "
|
|
858
|
+
f"SELECT {ns('rep')} AS n FROM {self.comps.rows} AS _b) AS _n LIMIT {I.SHAPES_MAX + 1}").fetchall()]
|
|
859
|
+
every = len(shapes) > I.SHAPES_MAX
|
|
860
|
+
note = json.dumps({"aliases": aliases, "largest": report.largest, "components": report.components,
|
|
861
|
+
"all_shapes": every}, sort_keys=True)
|
|
862
|
+
# The marker's `a`: the run's cache key, so a reader takes the generation only under that key.
|
|
863
|
+
rows = [f"SELECT {generation} AS generation, 'generation' AS kind, {null} AS graph, {self.lit(report.digest)} AS a, {null} AS b, "
|
|
864
|
+
f"{self.lit(note)} AS note"]
|
|
865
|
+
rows += [f"SELECT {generation}, 'shape', {null}, {self.lit(x)}, {null}, {null}" for x in ([] if every else shapes)]
|
|
866
|
+
rows += [f"SELECT {generation}, 'link', g, a, b, source FROM {x} AS _l" for x in self.link_stages]
|
|
867
|
+
if self.disagree is not None: # Cypher's list-valued properties (`stage_disagreeing`)
|
|
868
|
+
rows.append(f"SELECT {generation}, kind, {null}, c, p, {null} FROM {self.disagree} AS _d")
|
|
869
|
+
rows += [f"SELECT {generation}, 'unmerged', {self.lit(u.graph)}, {self.lit(u.representative)}, "
|
|
870
|
+
f"{self.lit(u.rule)}, {self.lit(json.dumps({'reason': u.reason, 'size': u.size, 'sample': list(u.sample), 'axiom': u.axiom}))}"
|
|
871
|
+
for u in report.unmerged]
|
|
872
|
+
out.append(f"INSERT INTO {I.meta_table(self.store)} (generation, kind, graph, a, b, note) "
|
|
873
|
+
+ " UNION ALL ".join(rows))
|
|
874
|
+
return out
|
|
875
|
+
|
|
876
|
+
def canonical(self, rows: str, graph: bool, rule: bool) -> str:
|
|
877
|
+
"""The derived statements `rows` with their subject and IRI object under their representatives (the
|
|
878
|
+
final components'), once each: a derived fact is stored once, on the representative."""
|
|
879
|
+
if self.comps is None or not self.comps.aliases:
|
|
880
|
+
return rows
|
|
881
|
+
cols = ", ".join(STATEMENT)
|
|
882
|
+
alias = f"SELECT iri, rep FROM {self.comps.rows} AS _a WHERE g = {self.lit('')}"
|
|
883
|
+
picked = (("n.graph, " if graph else "") +
|
|
884
|
+
"coalesce(_cs.rep, n.subject) AS subject, n.predicate, n.o_kind, "
|
|
885
|
+
"CASE WHEN n.o_kind = 'iri' THEN coalesce(_co.rep, n.o_text) ELSE n.o_text END AS o_text, "
|
|
886
|
+
"n.o_datatype, n.o_lang" + (", n.rule" if rule else ""))
|
|
887
|
+
canon = (f"SELECT {picked} FROM ({rows}) AS n LEFT JOIN ({alias}) AS _cs ON _cs.iri = n.subject "
|
|
888
|
+
f"LEFT JOIN ({alias}) AS _co ON n.o_kind = 'iri' AND _co.iri = n.o_text")
|
|
889
|
+
keys = ("graph, " if graph else "") + cols
|
|
890
|
+
return (f"SELECT {keys}, min(rule) AS rule FROM ({canon}) AS _c GROUP BY {keys}" if rule
|
|
891
|
+
else f"SELECT DISTINCT {keys} FROM ({canon}) AS _c")
|
|
892
|
+
|
|
893
|
+
# -- the step --------------------------------------------------------------------------------------------
|
|
894
|
+
def write(self, report: MaterializationReport) -> Optional[int]:
|
|
895
|
+
from prograph.plan.ir import freshness
|
|
896
|
+
from prograph.plan.ir.consistency import as_record
|
|
897
|
+
from prograph.store import ASSERT, INTERNAL, LITERAL, RETRACT, RL_GRAPH
|
|
898
|
+
store = self.store
|
|
899
|
+
cols = ", ".join(STATEMENT)
|
|
900
|
+
null = self.lit(None)
|
|
901
|
+
closure = self.canonical(self.derived(), True, False) if self.stages else \
|
|
902
|
+
(f"SELECT {self.lit(RL_GRAPH)} AS graph, " + ", ".join(f"{null} AS {c}" for c in STATEMENT) + " WHERE 1 = 0")
|
|
903
|
+
rule_of = self.canonical(" UNION ALL ".join(f"SELECT {cols}, rule FROM {read} AS _d" for read, _ in self.stages),
|
|
904
|
+
False, True) if self.stages else \
|
|
905
|
+
(f"SELECT {', '.join(f'{null} AS {c}' for c in STATEMENT)}, {null} AS rule WHERE 1 = 0")
|
|
906
|
+
held = (f"SELECT {cols} FROM ({store.effective_sql([RL_GRAPH])}) AS _h "
|
|
907
|
+
f"WHERE NOT {self.db.starts_with}(predicate, {self.lit(INTERNAL)})")
|
|
908
|
+
|
|
909
|
+
def same(a: str, b: str) -> str:
|
|
910
|
+
return " AND ".join(f"coalesce({a}.{c}, '') = coalesce({b}.{c}, '')" for c in STATEMENT)
|
|
911
|
+
asserts = (f"SELECT {self.lit(RL_GRAPH)} AS graph, {', '.join('n.' + c for c in STATEMENT)}, "
|
|
912
|
+
f"{null} AS edge_id, '{ASSERT}' AS op, {self.text(self.lit('rl:') + ' || n.rule')} AS _writer, "
|
|
913
|
+
f"1 AS _o FROM ({rule_of}) AS n LEFT JOIN ({held}) AS h ON {same('n', 'h')} "
|
|
914
|
+
f"WHERE h.subject IS NULL")
|
|
915
|
+
# A retraction's writer: the rule of the statement's last assert.
|
|
916
|
+
last = (f"SELECT {cols}, writer FROM (SELECT {cols}, writer, ROW_NUMBER() OVER (PARTITION BY {cols} "
|
|
917
|
+
f"ORDER BY seq DESC, pos DESC) AS _rn FROM {store._visible()} AS _v WHERE graph = "
|
|
918
|
+
f"{self.lit(RL_GRAPH)} AND op = '{ASSERT}') AS _w WHERE _rn = 1")
|
|
919
|
+
retracts = (f"SELECT {self.lit(RL_GRAPH)} AS graph, {', '.join('h.' + c for c in STATEMENT)}, "
|
|
920
|
+
f"{null} AS edge_id, '{RETRACT}' AS op, coalesce(w.writer, 'rl') AS _writer, 0 AS _o "
|
|
921
|
+
f"FROM ({held}) AS h LEFT JOIN ({closure}) AS n ON {same('h', 'n')} "
|
|
922
|
+
f"LEFT JOIN ({last}) AS w ON {same('h', 'w')} WHERE n.subject IS NULL")
|
|
923
|
+
# The run's record: the old one retracted, the new one asserted.
|
|
924
|
+
old = (f"SELECT graph, {cols}, edge_id, '{RETRACT}' AS op, 'rl' AS _writer, 0 AS _o FROM "
|
|
925
|
+
f"({store.effective_sql([RL_GRAPH])}) AS _m WHERE subject = {self.lit(RUN_SUBJECT)}")
|
|
926
|
+
new = " UNION ALL ".join(
|
|
927
|
+
f"SELECT {self.lit(RL_GRAPH)} AS graph, {self.lit(RUN_SUBJECT)} AS subject, {self.lit(p)} AS predicate, "
|
|
928
|
+
f"{self.lit(LITERAL)} AS o_kind, {self.lit(v)} AS o_text, {self.lit(dt)} AS o_datatype, "
|
|
929
|
+
f"{null} AS o_lang, {null} AS edge_id, '{ASSERT}' AS op, 'rl' AS _writer, 2 AS _o"
|
|
930
|
+
for p, v, dt in ((RUN_DIGEST, report.digest, xsd.STRING), (RUN_AT, report.ran_at, xsd.DATETIME),
|
|
931
|
+
(RUN_SOURCES, freshness.as_record(report.sources), xsd.STRING),
|
|
932
|
+
(RUN_CONSISTENCY, as_record(report.inconsistencies, report.not_run), xsd.STRING),
|
|
933
|
+
(RUN_ROUNDS, str(report.rounds), xsd.STRING)))
|
|
934
|
+
# One INSERT of the whole diff: on DuckDB 1.5.6 an INSERT that read the table (the retractions, here
|
|
935
|
+
# none) followed by a large one in the same transaction left the table's later reads without either
|
|
936
|
+
# (LDBC SF1, 2.9 million asserts); one statement does not.
|
|
937
|
+
diff = " UNION ALL ".join(f"SELECT * FROM ({q}) AS _d{i}" for i, q in enumerate((retracts, asserts, old, new)))
|
|
938
|
+
# The alias table's new generation (owl:sameAs): on DuckDB and PostgreSQL in the step's own transaction,
|
|
939
|
+
# numbered by the step's seq; on Delta after the step's marker, under its seq, its marker row last.
|
|
940
|
+
publish = self.publish()
|
|
941
|
+
if publish:
|
|
942
|
+
self.stage_disagreeing()
|
|
943
|
+
from prograph.plan.ir import identity as I
|
|
944
|
+
prior = None
|
|
945
|
+
if publish:
|
|
946
|
+
I.ensure_tables(store)
|
|
947
|
+
prior = self.db.execute(f"SELECT max(generation) FROM {I.meta_table(store)} "
|
|
948
|
+
f"WHERE kind = 'generation'").fetchone()[0]
|
|
949
|
+
runs = [("run", x) for x in self.alias_statements(report, f"(SELECT coalesce(max(seq), 0) + 1 FROM {store.table})")] \
|
|
950
|
+
if publish and not self.db.optimistic else []
|
|
951
|
+
seq, _ = store.write_select(runs + [diff], "rl", f"rl-{uuid.uuid4().hex}", writers=True, replan=True)
|
|
952
|
+
if publish and seq is not None:
|
|
953
|
+
if self.db.optimistic:
|
|
954
|
+
for x in self.alias_statements(report, str(int(seq))):
|
|
955
|
+
self.db.execute(x)
|
|
956
|
+
if prior is not None: # the generations before the last one: no plan reads them
|
|
957
|
+
for table in (I.alias_table(store), I.meta_table(store)):
|
|
958
|
+
try:
|
|
959
|
+
self.db.execute(f"DELETE FROM {table} WHERE generation < {int(prior)}")
|
|
960
|
+
except Exception: # noqa: BLE001 - removed by the next run
|
|
961
|
+
pass
|
|
962
|
+
store._refresh()
|
|
963
|
+
return seq
|
|
964
|
+
|
|
965
|
+
def counts(self, report: MaterializationReport, seq: int) -> None:
|
|
966
|
+
from prograph.store import ASSERT, INTERNAL, RL_GRAPH
|
|
967
|
+
rows = self.db.execute(
|
|
968
|
+
f"SELECT writer, op, count(*) FROM {self.store._visible()} AS _v WHERE seq = {int(seq)} AND graph = "
|
|
969
|
+
f"{self.lit(RL_GRAPH)} AND NOT {self.db.starts_with}(predicate, {self.lit(INTERNAL)}) "
|
|
970
|
+
f"GROUP BY writer, op").fetchall()
|
|
971
|
+
for writer, op, n in rows:
|
|
972
|
+
rule = writer[3:] if writer.startswith("rl:") else writer
|
|
973
|
+
(report.asserted if op == ASSERT else report.retracted)[rule] = int(n)
|
|
974
|
+
|
|
975
|
+
# -- consistency (`consistency`) ------------------------------------------------------------------------
|
|
976
|
+
def consistency(self, report: MaterializationReport) -> List[Inconsistency]:
|
|
977
|
+
"""OWL 2 RL's rules that conclude false (Profiles §4.3, tables 4 to 9), once, over the closure this run
|
|
978
|
+
computed (the premises and its staged derivations, matched through the final components): each check's
|
|
979
|
+
solutions (cax-dw, prp-irp, ... dt-not-type), a stated `x owl:differentFrom x` (eq-diff1), the ontology's
|
|
980
|
+
owl:differentFrom or owl:AllDifferent naming one IRI twice, a subject's two values of a property it has at
|
|
981
|
+
most one of that are not one value (prp-fp, cls-maxc2, cls-maxqc3/4: `value_key`), and the owl:sameAs
|
|
982
|
+
components the run left unmerged as inconsistent. Each matched rule with a count and `SAMPLE` facts."""
|
|
983
|
+
from prograph.plan.ir import identity as I
|
|
984
|
+
from prograph.plan.ir import ql
|
|
985
|
+
from prograph.plan.ir.consistency import variables
|
|
986
|
+
from prograph.plan.ir.entry import _held
|
|
987
|
+
from prograph.plan.ir.lower import DATA_ERROR_MARK
|
|
988
|
+
from prograph.plan.ir.owl_rl_rules import Check
|
|
989
|
+
if self.base is None:
|
|
990
|
+
self.stage_base()
|
|
991
|
+
found: List[Inconsistency] = []
|
|
992
|
+
held = _held(self.engine._schema, self.dialect, True)
|
|
993
|
+
hints: Dict[int, str] = {}
|
|
994
|
+
loop = Check("eq-diff1", f"?x <{I.DIFFERENT_FROM}> ?x .", "an individual owl:differentFrom itself")
|
|
995
|
+
|
|
996
|
+
def projection(names: List[str]) -> str:
|
|
997
|
+
if not names:
|
|
998
|
+
return "(1 AS ?pgcone)"
|
|
999
|
+
return " ".join(f"(STR(?{n}) AS ?pgc{i}t) (STR(DATATYPE(?{n})) AS ?pgc{i}d) (LANG(?{n}) AS ?pgc{i}l) "
|
|
1000
|
+
f"(IF(isBlank(?{n}), \"b\", IF(isIRI(?{n}), \"i\", \"l\")) AS ?pgc{i}k)"
|
|
1001
|
+
for i, n in enumerate(names))
|
|
1002
|
+
|
|
1003
|
+
def plan(maps: Any, onto: Any) -> List[Tuple[str, Any, Any, List[str]]]:
|
|
1004
|
+
out = []
|
|
1005
|
+
checks = [c for c in self.rules.checks if c.range is None or c.range[2] or
|
|
1006
|
+
not ql.range_holds(maps, held, c.range[0], c.range[1])]
|
|
1007
|
+
checks += [loop] if I.DIFFERENT_FROM in maps.by_predicate else []
|
|
1008
|
+
for c in checks:
|
|
1009
|
+
if c.range is not None:
|
|
1010
|
+
hints[id(c)] = ql.range_hint(maps, held, c.range[0], c.range[1])
|
|
1011
|
+
names = variables(c.facts or c.where)
|
|
1012
|
+
query = f"SELECT DISTINCT {projection(names)} WHERE {{ {c.where} }}"
|
|
1013
|
+
out.append(("rows", c, self.planned_check(query, maps, onto), names))
|
|
1014
|
+
for c in self.rules.clashes:
|
|
1015
|
+
query = (f"SELECT DISTINCT (STR(?x) AS ?r) (STR(?v) AS ?t) (STR(DATATYPE(?v)) AS ?d) (LANG(?v) AS ?l) "
|
|
1016
|
+
f"WHERE {{ {c.where} FILTER(isIRI(?x) && isLiteral(?v)) }}")
|
|
1017
|
+
out.append(("values", c, self.planned_check(query, maps, onto), []))
|
|
1018
|
+
return out
|
|
1019
|
+
for how, c, lowered, names in self.with_maps(plan):
|
|
1020
|
+
try:
|
|
1021
|
+
got = self.rows_check(c, lowered, names) if how == "rows" else self.values_check(c, lowered)
|
|
1022
|
+
except Exception as exc: # noqa: BLE001 - a data error the check met (R2RML 11): not run, named
|
|
1023
|
+
text = str(exc)
|
|
1024
|
+
if DATA_ERROR_MARK not in text:
|
|
1025
|
+
raise
|
|
1026
|
+
report.not_run.append((c.rule, c.axiom, text[text.index(DATA_ERROR_MARK):].splitlines()[0]))
|
|
1027
|
+
continue
|
|
1028
|
+
if got is not None:
|
|
1029
|
+
found.append(replace(got, hint=hints.get(id(c), "")) if hints.get(id(c)) else got)
|
|
1030
|
+
for a, b, axiom, rule in self.rules.distinct:
|
|
1031
|
+
if a == b:
|
|
1032
|
+
found.append(Inconsistency(rule, axiom, ({"x": a},), 1, (f"<{a}> owl:differentFrom <{a}>",)))
|
|
1033
|
+
apart: Dict[Tuple[str, str], List[Unmerged]] = {}
|
|
1034
|
+
for u in report.unmerged:
|
|
1035
|
+
if u.reason == "inconsistent":
|
|
1036
|
+
apart.setdefault((u.rule, u.axiom), []).append(u)
|
|
1037
|
+
for (rule, axiom), us in apart.items():
|
|
1038
|
+
facts = tuple(" owl:sameAs ".join(f"<{m}>" for m in u.sample) + f" (one owl:sameAs component of {u.size} "
|
|
1039
|
+
f"IRIs, left unmerged)" for u in us[:SAMPLE])
|
|
1040
|
+
found.append(Inconsistency(rule, axiom, tuple({"x": u.representative} for u in us[:SAMPLE]), len(us),
|
|
1041
|
+
facts))
|
|
1042
|
+
return found
|
|
1043
|
+
|
|
1044
|
+
def planned_check(self, query: str, maps: Any, onto: Any) -> Tuple[str, frozenset]:
|
|
1045
|
+
"""A check's SQL over this run's premises, matched through the run's final components."""
|
|
1046
|
+
return self.lowered(query, maps, onto, self.identity)
|
|
1047
|
+
|
|
1048
|
+
def rows_check(self, c: Any, lowered: Tuple[str, frozenset], names: List[str]) -> Optional[Inconsistency]:
|
|
1049
|
+
"""A check's solutions: the first `SAMPLE` (as facts: `c.facts`, or its pattern, filled) and how many."""
|
|
1050
|
+
from prograph.plan.ir.consistency import fill, term
|
|
1051
|
+
sql, never = lowered
|
|
1052
|
+
q = self.d.quote
|
|
1053
|
+
cols = [f"pgc{i}{part}" for i in range(len(names)) for part in "tdlk"]
|
|
1054
|
+
|
|
1055
|
+
def col(n: str) -> str:
|
|
1056
|
+
return f"{self.lit(None)} AS {q(n)}" if n in never else f"{self.text('_r.' + q(n))} AS {q(n)}"
|
|
1057
|
+
picked = ", ".join(col(n) for n in cols) or "1 AS one"
|
|
1058
|
+
order = ", ".join(q(f"pgc{i}t") for i in range(len(names))) or "1"
|
|
1059
|
+
rows = self.db.execute(f"SELECT {picked} FROM ({sql}) AS _r ORDER BY {order} LIMIT {SAMPLE}").fetchall()
|
|
1060
|
+
if not rows:
|
|
1061
|
+
return None
|
|
1062
|
+
count = len(rows) if len(rows) < SAMPLE else \
|
|
1063
|
+
int(self.db.execute(f"SELECT count(*) FROM ({sql}) AS _r").fetchone()[0])
|
|
1064
|
+
sample, facts = [], []
|
|
1065
|
+
for r in rows:
|
|
1066
|
+
got = dict(zip(cols, r))
|
|
1067
|
+
terms = {n: term(got[f"pgc{i}t"], got[f"pgc{i}k"] or "l", got[f"pgc{i}d"], got[f"pgc{i}l"] or None)
|
|
1068
|
+
for i, n in enumerate(names)}
|
|
1069
|
+
sample.append({n: got[f"pgc{i}t"] for i, n in enumerate(names) if n in ("x", "y", "v", "p")})
|
|
1070
|
+
facts.append(fill(c.facts or c.where, terms))
|
|
1071
|
+
return Inconsistency(c.rule, c.axiom, tuple(sample), count, tuple(facts))
|
|
1072
|
+
|
|
1073
|
+
def values_check(self, c: Any, lowered: Tuple[str, frozenset]) -> Optional[Inconsistency]:
|
|
1074
|
+
"""A clash: the subjects with two values of the property that are not one value (`value_key`), the first
|
|
1075
|
+
`SAMPLE` with their values, and how many."""
|
|
1076
|
+
from prograph.plan.ir.consistency import fill, term
|
|
1077
|
+
sql, never = lowered
|
|
1078
|
+
q = self.d.quote
|
|
1079
|
+
|
|
1080
|
+
def col(n: str) -> str:
|
|
1081
|
+
return self.lit(None) if n in never else self.text(f"_r.{q(n)}")
|
|
1082
|
+
keyed = (f"SELECT {col('r')} AS r, {col('t')} AS t, {col('d')} AS d, {col('l')} AS l, "
|
|
1083
|
+
f"{self.value_key(col('t'), col('d'), col('l'))} AS k, {self.value_group(col('t'), col('d'))} AS g "
|
|
1084
|
+
f"FROM ({sql}) AS _r")
|
|
1085
|
+
bad = f"SELECT r FROM ({keyed}) AS _k GROUP BY r HAVING {self.two_values()}"
|
|
1086
|
+
subjects = [r[0] for r in self.db.execute(f"SELECT r FROM ({bad}) AS _b ORDER BY r LIMIT {SAMPLE}").fetchall()]
|
|
1087
|
+
if not subjects:
|
|
1088
|
+
return None
|
|
1089
|
+
count = len(subjects) if len(subjects) < SAMPLE else \
|
|
1090
|
+
int(self.db.execute(f"SELECT count(*) FROM ({bad}) AS _b").fetchone()[0])
|
|
1091
|
+
values: Dict[str, List[str]] = {}
|
|
1092
|
+
for r, t, d, l in self.db.execute(
|
|
1093
|
+
f"SELECT DISTINCT r, t, d, l FROM ({keyed}) AS _k WHERE r IN ({', '.join(self.lit(x) for x in subjects)}) "
|
|
1094
|
+
f"ORDER BY r, t").fetchall():
|
|
1095
|
+
values.setdefault(r, []).append(term(t, "l", d, l or None))
|
|
1096
|
+
facts = tuple(fill(c.facts or c.where, {"x": f"<{r}>", "v": " , ".join(values.get(r, []))}) for r in subjects)
|
|
1097
|
+
return Inconsistency(c.rule, c.axiom, tuple({"x": r} for r in subjects), count, facts)
|
|
1098
|
+
|
|
1099
|
+
def cleanup(self) -> None:
|
|
1100
|
+
for sql in self.cleanups:
|
|
1101
|
+
try:
|
|
1102
|
+
self.db.execute(sql)
|
|
1103
|
+
except Exception: # noqa: BLE001 - a temporary table dropped with its session
|
|
1104
|
+
pass
|
|
1105
|
+
self.cleanups = []
|
|
1106
|
+
self.forget_premise_types()
|
|
1107
|
+
|
|
1108
|
+
def forget_premise_types(self) -> None:
|
|
1109
|
+
"""The premise views' sources were typed for the rounds (`with_included`): forgotten after them."""
|
|
1110
|
+
types = getattr(self.engine._schema, "column_types", None)
|
|
1111
|
+
if isinstance(types, dict) and self.types_before is not None:
|
|
1112
|
+
# The store's own generated maps, as a query reads them, keep theirs: a run a query's freshness
|
|
1113
|
+
# check starts may be the first to type them, and the query's maps, cached by the store's version,
|
|
1114
|
+
# are not typed again.
|
|
1115
|
+
own = self.store.source_maps()
|
|
1116
|
+
keep = {t.table for t in own.tables()} if own is not None else set()
|
|
1117
|
+
for k in set(types) - self.types_before - keep:
|
|
1118
|
+
types.pop(k, None)
|
|
1119
|
+
self.types_before = None
|