prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,797 @@
|
|
|
1
|
+
"""owl:sameAs: one entity under several IRIs (OWL 2 RL's eq-* rules; docs/reference/owl-rl.md).
|
|
2
|
+
|
|
3
|
+
Entailment changes basic graph pattern matching only (SPARQL 1.1 Entailment Regimes 2): a triple pattern
|
|
4
|
+
matches as if eq-sym, eq-trans, eq-rep-s, eq-rep-p and eq-rep-o were materialized over the graph SPARQL
|
|
5
|
+
answers. FILTER, sameTerm, VALUES, BIND and DISTINCT and GROUP BY keys keep RDF term identity.
|
|
6
|
+
|
|
7
|
+
- **The alias table** (`{store}_alias`: generation, graph, iri, rep; one row per IRI that has another name,
|
|
8
|
+
never a reflexive one) is rebuilt whole by each OWL RL materialization (`owl_rl`), as a new generation,
|
|
9
|
+
and published with the run's write step (a marker row in `{store}_alias_meta`). A plan reads the
|
|
10
|
+
generation current when it was made; the store's version changes with it, so every plan is made again.
|
|
11
|
+
- **Components** are the connected components of the sameAs links of each graph (the default graph's, and
|
|
12
|
+
each named graph's own), by minimum-label propagation with pointer jumping in SQL (`components`), the
|
|
13
|
+
representative the least IRI by code point (the dialect's binary collation, `lower.codepoint`).
|
|
14
|
+
- **Matching** (`pattern`): each IRI position of a triple pattern that may hold an alias is compared through
|
|
15
|
+
its representative (a left join on the alias rows), the matches kept once per canonical triple, then each
|
|
16
|
+
variable joined to every member of its component (`sameas="all"`, the default) or bound to the
|
|
17
|
+
representative (`sameas="representative"`). A constant IRI is looked up at planning (one read per
|
|
18
|
+
generation and IRI) and matches every member of its component. A predicate or class IRI with aliases
|
|
19
|
+
(eq-rep-p, an rdf:type object) unions its members' maps. `?a owl:sameAs ?b` is the components' cliques
|
|
20
|
+
(no reflexive pairs) and the graph's own sameAs triples.
|
|
21
|
+
- **The skip test**: a position whose term maps cannot give an IRI of the generation's IRI shapes (the
|
|
22
|
+
namespaces its members are in) reads its IRIs as they are, with no alias join.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import hashlib
|
|
28
|
+
import json
|
|
29
|
+
from dataclasses import dataclass, field
|
|
30
|
+
from typing import Any, Callable, Dict, FrozenSet, Iterable, List, Optional, Tuple
|
|
31
|
+
|
|
32
|
+
from prograph.plan.ir.dataset import active_graph
|
|
33
|
+
from prograph.rules import home, module_home
|
|
34
|
+
|
|
35
|
+
module_home("owl-sameas", "owl:sameAs: one entity under several IRIs, at BGP matching only",
|
|
36
|
+
terms=("owl:sameAs", "sameAs", "alias", "alias table", "representative", "component", "eq-rep",
|
|
37
|
+
"eq-sym", "eq-trans", "eq-rep-p", "sameas", "all aliases", "representative mode",
|
|
38
|
+
"explain_identity", "SameAsMergeWarning", "generation", "crosswalk", "one entity"))
|
|
39
|
+
|
|
40
|
+
OWL = "http://www.w3.org/2002/07/owl#"
|
|
41
|
+
SAME_AS = OWL + "sameAs"
|
|
42
|
+
DIFFERENT_FROM = OWL + "differentFrom"
|
|
43
|
+
RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
|
|
44
|
+
DEFAULT_GRAPH = "" # the graph column of the default graph's components
|
|
45
|
+
ALIAS_MAP = "prograph-alias"
|
|
46
|
+
MODES = ("all", "representative")
|
|
47
|
+
# A component of more members than this warns (`SameAsMergeWarning`), as does one whose largest grew tenfold.
|
|
48
|
+
WARN_MEMBERS = 1000
|
|
49
|
+
GROWTH = 10
|
|
50
|
+
# A component of more members than this is left unmerged and named in the report.
|
|
51
|
+
CAP = 100_000
|
|
52
|
+
# More namespaces than this among the members: every IRI position is read through the alias rows.
|
|
53
|
+
SHAPES_MAX = 1000
|
|
54
|
+
# A pattern whose expansion may give more rows than this a triple (largest^positions) is a cost hint.
|
|
55
|
+
EXPANSION_HINT = 1000
|
|
56
|
+
# A constant with more members than this is matched by a join with the alias rows, not a VALUES list.
|
|
57
|
+
VALUES_MAX = 64
|
|
58
|
+
# The most label-propagation rounds a component computation takes before it is refused.
|
|
59
|
+
LABEL_ROUNDS = 200
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class SameAsMergeWarning(UserWarning):
|
|
63
|
+
"""A component of owl:sameAs merged many IRIs (or grew tenfold since the last run): the classic runaway
|
|
64
|
+
of a bad inverse-functional property. `engine.explain_identity(a, b)` shows why two IRIs merged."""
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class AliasState:
|
|
69
|
+
"""The current generation of a store's alias table: its number, the namespaces its members are in
|
|
70
|
+
(None: too many to list, every IRI may be an alias), its largest component, its aliases' count and the
|
|
71
|
+
cache key of the run that wrote it (`dataset.cache_key`; None for a generation written before it was kept)."""
|
|
72
|
+
generation: int
|
|
73
|
+
shapes: Optional[Tuple[str, ...]]
|
|
74
|
+
largest: int
|
|
75
|
+
aliases: int
|
|
76
|
+
key: Optional[str] = None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def alias_table(store: Any) -> str:
|
|
80
|
+
return f"{store.table}_alias"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def meta_table(store: Any) -> str:
|
|
84
|
+
return f"{store.table}_alias_meta"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def ensure_tables(store: Any) -> None:
|
|
88
|
+
"""Make the alias table and its meta table where they are missing. On DuckDB and PostgreSQL the alias
|
|
89
|
+
rows of a generation are UNIQUE by (generation, graph, iri): one representative per IRI."""
|
|
90
|
+
c, x = store.db, store.db.text
|
|
91
|
+
unique = "" if c.optimistic else ", UNIQUE (generation, graph, iri)"
|
|
92
|
+
c.create(alias_table(store), f"CREATE TABLE IF NOT EXISTS {alias_table(store)} (generation BIGINT NOT NULL, "
|
|
93
|
+
f"graph {x} NOT NULL, iri {x} NOT NULL, rep {x} NOT NULL{unique})")
|
|
94
|
+
c.create(meta_table(store), f"CREATE TABLE IF NOT EXISTS {meta_table(store)} (generation BIGINT NOT NULL, "
|
|
95
|
+
f"kind {x} NOT NULL, graph {x}, a {x}, b {x}, note {x})")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def read_state(store: Any) -> Optional[AliasState]:
|
|
99
|
+
"""The store's current alias generation (the latest marker), None where there is none or it holds no
|
|
100
|
+
alias."""
|
|
101
|
+
meta = meta_table(store)
|
|
102
|
+
try:
|
|
103
|
+
if not store.db.has_table(meta):
|
|
104
|
+
return None
|
|
105
|
+
rows = store.db.execute(latest_sql(store)).fetchall()
|
|
106
|
+
except Exception: # noqa: BLE001 - a store opened read-only before the table existed: no alias
|
|
107
|
+
return None
|
|
108
|
+
return state_of(rows)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def latest_sql(store: Any) -> str:
|
|
112
|
+
"""SQL of the latest generation's marker and IRI shapes: (kind, a, note, generation)."""
|
|
113
|
+
meta = meta_table(store)
|
|
114
|
+
return (f"SELECT kind, a, note, generation FROM {meta} WHERE generation = (SELECT max(generation) FROM {meta} "
|
|
115
|
+
f"WHERE kind = 'generation') AND kind IN ('generation', 'shape')")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def state_of(rows: Any) -> Optional[AliasState]:
|
|
119
|
+
"""The alias state of the latest generation's rows (`latest_sql`), None where it holds no alias."""
|
|
120
|
+
marker = [r for r in rows if r[0] == "generation"]
|
|
121
|
+
if not marker:
|
|
122
|
+
return None
|
|
123
|
+
note = json.loads(marker[0][2] or "{}")
|
|
124
|
+
if not note.get("aliases"):
|
|
125
|
+
return None
|
|
126
|
+
shapes = None if note.get("all_shapes") else tuple(sorted(r[1] for r in rows if r[0] == "shape"))
|
|
127
|
+
return AliasState(int(marker[0][3]), shapes, int(note.get("largest", 0)), int(note["aliases"]), marker[0][1])
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# -- the identity a plan reads ---------------------------------------------------------------------------
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@dataclass
|
|
134
|
+
class Identity:
|
|
135
|
+
"""What a plan's BGP matching reads of owl:sameAs: `rows`, SQL of the alias rows (`g`, `iri`, `rep`);
|
|
136
|
+
the IRI shapes (`shapes`, None: any IRI may be an alias); the largest component; the answer mode; and
|
|
137
|
+
`run`, which reads a statement's rows (the constants' members are looked up at planning, one statement a query)."""
|
|
138
|
+
rows: str
|
|
139
|
+
shapes: Optional[Tuple[str, ...]]
|
|
140
|
+
largest: int
|
|
141
|
+
mode: str
|
|
142
|
+
run: Callable[[str], List[tuple]]
|
|
143
|
+
literal: Callable[[str], str]
|
|
144
|
+
tag: str = ""
|
|
145
|
+
_by_iri: Dict[str, Dict[str, FrozenSet[str]]] = field(default_factory=dict)
|
|
146
|
+
# The (class, predicate) pairs whose literal values differ across a component (`owl_rl.stage_disagreeing`):
|
|
147
|
+
# Cypher reads such a property as a list (`cypher_front`).
|
|
148
|
+
disagree: FrozenSet[Tuple[str, str]] = frozenset() # (class, Cypher property key)
|
|
149
|
+
# The classes a member of a merged component has: a Cypher pattern of no such class needs no alias join.
|
|
150
|
+
merged: FrozenSet[str] = frozenset()
|
|
151
|
+
said: set = field(default_factory=set, compare=False) # Cypher's (label, key) warnings given (`warn_disagreeing`)
|
|
152
|
+
|
|
153
|
+
def __post_init__(self) -> None:
|
|
154
|
+
if self.mode not in MODES:
|
|
155
|
+
raise ValueError(f"sameas is one of {', '.join(MODES)}, not {self.mode!r}")
|
|
156
|
+
self.tag = self.tag or hashlib.sha1(self.rows.encode("utf-8")).hexdigest()[:12]
|
|
157
|
+
|
|
158
|
+
# -- the skip test ----------------------------------------------------------------------------------
|
|
159
|
+
def may_alias(self, iri: str) -> bool:
|
|
160
|
+
"""Whether a constant IRI may be an alias (it is in a namespace of the members)."""
|
|
161
|
+
return self.shapes is None or any(iri.startswith(p) for p in self.shapes)
|
|
162
|
+
|
|
163
|
+
@home("sameas-skip-test", "Where a term position needs no alias join: its maps cannot give a member's IRI",
|
|
164
|
+
terms=("skip test", "no alias join", "IRI shapes", "may meet an alias"))
|
|
165
|
+
def meets(self, slot: Any) -> bool:
|
|
166
|
+
"""Whether a column of slot `slot` may hold an IRI that has an alias: an IRI of a term map whose
|
|
167
|
+
fixed prefix and a member's namespace can be one the start of the other (`TermMap.prefix`), a
|
|
168
|
+
relative R2RML IRI, or a column of unknown provenance."""
|
|
169
|
+
from prograph.schema.source_map import IRI
|
|
170
|
+
if slot is None:
|
|
171
|
+
return True
|
|
172
|
+
if IRI not in slot.kinds:
|
|
173
|
+
return False
|
|
174
|
+
if slot.provenance is None or self.shapes is None:
|
|
175
|
+
return True
|
|
176
|
+
for tm in slot.provenance:
|
|
177
|
+
if tm.term_type != IRI:
|
|
178
|
+
continue
|
|
179
|
+
if tm.relative_iri():
|
|
180
|
+
return True
|
|
181
|
+
prefix = tm.prefix()
|
|
182
|
+
if tm.constant is not None:
|
|
183
|
+
if self.may_alias(tm.constant):
|
|
184
|
+
return True
|
|
185
|
+
elif any(prefix.startswith(p) or p.startswith(prefix) for p in self.shapes):
|
|
186
|
+
return True
|
|
187
|
+
return False
|
|
188
|
+
|
|
189
|
+
# -- planning-time lookups ----------------------------------------------------------------------------
|
|
190
|
+
@home("sameas-planning-lookups", "A constant IRI's component read at planning: one batched statement per query",
|
|
191
|
+
terms=("planning-time lookup", "constant lookup", "prefetch", "round trip", "component of a constant"))
|
|
192
|
+
def prefetch(self, iris: Iterable[str]) -> None:
|
|
193
|
+
"""Read the components of every IRI of `iris` (in every graph) not read yet that could be an alias, in one
|
|
194
|
+
statement per 400 IRIs (none where none could). The planner calls it once with the query's constants
|
|
195
|
+
(`sparql_front.translate`), where a lookup per constant sent one each."""
|
|
196
|
+
fresh = [i for i in dict.fromkeys(iris) if i not in self._by_iri]
|
|
197
|
+
for i in fresh:
|
|
198
|
+
if not self.may_alias(i):
|
|
199
|
+
self._by_iri[i] = {} # no statement: its namespace holds no member
|
|
200
|
+
wanted = [i for i in fresh if i not in self._by_iri]
|
|
201
|
+
for start in range(0, len(wanted), 400):
|
|
202
|
+
chunk = wanted[start:start + 400]
|
|
203
|
+
listed = ", ".join(self.literal(i) for i in chunk)
|
|
204
|
+
sql = (f"WITH _r AS (SELECT g, iri, rep FROM ({self.rows}) AS _a), "
|
|
205
|
+
f"_asked AS (SELECT g, iri, rep FROM _r WHERE iri IN ({listed}) "
|
|
206
|
+
f"UNION SELECT g, rep AS iri, rep FROM _r WHERE rep IN ({listed})) "
|
|
207
|
+
f"SELECT _asked.g AS graph, _asked.iri AS asked, _r.iri AS member, _asked.rep AS rep FROM _asked "
|
|
208
|
+
f"JOIN _r ON _r.rep = _asked.rep AND _r.g = _asked.g")
|
|
209
|
+
found: Dict[str, Dict[str, set]] = {i: {} for i in chunk}
|
|
210
|
+
for graph, asked, member, rep in self.run(sql): # a lookup that raises caches nothing of its chunk
|
|
211
|
+
found[asked].setdefault(graph, {rep}).add(member)
|
|
212
|
+
for i in chunk:
|
|
213
|
+
self._by_iri[i] = {g: frozenset(m) for g, m in found[i].items()}
|
|
214
|
+
|
|
215
|
+
def members(self, iris: Iterable[str], graph: str = DEFAULT_GRAPH) -> Dict[str, FrozenSet[str]]:
|
|
216
|
+
"""Each IRI of `iris` that has an alias in `graph`'s components: its component's members (itself
|
|
217
|
+
among them). Read once per IRI (`prefetch`), in every graph."""
|
|
218
|
+
listed = list(dict.fromkeys(iris))
|
|
219
|
+
self.prefetch(listed)
|
|
220
|
+
return {i: self._by_iri[i][graph] for i in listed if self._by_iri[i].get(graph)}
|
|
221
|
+
|
|
222
|
+
def members_by_graph(self, iri: str) -> Dict[str, FrozenSet[str]]:
|
|
223
|
+
"""`iri`'s component's members in each named graph where it has an alias (inside GRAPH of a variable)."""
|
|
224
|
+
self.prefetch([iri])
|
|
225
|
+
return {g: m for g, m in self._by_iri[iri].items() if g != DEFAULT_GRAPH}
|
|
226
|
+
|
|
227
|
+
# -- the alias rows as a plan ---------------------------------------------------------------------------
|
|
228
|
+
def scan(self, t: Any, iri: str, rep: str, role: str = "canon") -> Any:
|
|
229
|
+
"""The alias rows of the translator's active graph as a plan of IRI terms: `iri` and its
|
|
230
|
+
representative `rep`, and inside GRAPH ?g the graph under the active graph's column. `role` names
|
|
231
|
+
the scan's use in its map's id (an expansion's `expand<n>-k<largest>`, which the cost hint
|
|
232
|
+
`sameas-expansion` reads)."""
|
|
233
|
+
from prograph.plan.ir import expr as E, ops as O
|
|
234
|
+
from prograph.plan.ir.sparql_front import Var
|
|
235
|
+
from prograph.schema.source_map import iri_column, side_table_map
|
|
236
|
+
graph = t.graph
|
|
237
|
+
if isinstance(graph, Var):
|
|
238
|
+
where, by_graph = f" WHERE g <> {self.literal(DEFAULT_GRAPH)}", graph.name
|
|
239
|
+
else:
|
|
240
|
+
where, by_graph = f" WHERE g = {self.literal(active_graph(t))}", None
|
|
241
|
+
query = f"SELECT g, iri, rep FROM ({self.rows}) AS _a{where}"
|
|
242
|
+
column = {c: iri_column(c) for c in ("iri", "rep", "g")}
|
|
243
|
+
m = side_table_map(f"{ALIAS_MAP}-{self.tag}-{role}", query, "iri")
|
|
244
|
+
wanted = [("iri", iri), ("rep", rep)] + ([("g", by_graph)] if by_graph else [])
|
|
245
|
+
names = {c: t.fresh("c") for c, _ in wanted}
|
|
246
|
+
from prograph.schema import xsd
|
|
247
|
+
scan = O.MapScan(map=m, reads=tuple(O.ScanColumn(name=names[c], column=c, held=xsd.STRING, bytewise=True)
|
|
248
|
+
for c, _ in wanted),
|
|
249
|
+
required=tuple(names.values()))
|
|
250
|
+
x: O.Op = scan
|
|
251
|
+
for c, out in wanted:
|
|
252
|
+
x = O.Extend(x, out, E.MakeTerm(column[c], ((c, E.Col(names[c])),)))
|
|
253
|
+
return O.Project(x, tuple(out for _, out in wanted))
|
|
254
|
+
|
|
255
|
+
def members_scan(self, t: Any, rep: str, member: str) -> Any:
|
|
256
|
+
"""Each representative's members (itself and its aliases), `rep` and `member`, in the active graph:
|
|
257
|
+
the alias rows, and each representative once as its own member."""
|
|
258
|
+
from prograph.plan.ir import expr as E, ops as O
|
|
259
|
+
a = self.scan(t, member, rep)
|
|
260
|
+
own = O.Distinct(O.Project(a, tuple(c for c in a.columns if c != member)))
|
|
261
|
+
return O.Union((a, O.Project(O.Extend(own, member, E.Col(rep)), a.columns)))
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
# -- matching ---------------------------------------------------------------------------------------------
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
# The components of an empty default graph (a dataset of FROM NAMED graphs only): no alias row has it.
|
|
268
|
+
NO_GRAPH = "urn:prograph:store:no-graph"
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def predicate_members(t: Any, iri: str) -> FrozenSet[str]:
|
|
272
|
+
"""A predicate's or class's members under eq-rep-p (an rdf:type object under eq-rep-o): itself where it
|
|
273
|
+
has no alias. Components of the active graph's (`active_graph`); inside GRAPH of a variable see
|
|
274
|
+
`predicate_members_by_graph`."""
|
|
275
|
+
if not t.identity.may_alias(iri):
|
|
276
|
+
return frozenset({iri})
|
|
277
|
+
return t.identity.members([iri], active_graph(t)).get(iri, frozenset({iri}))
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def predicate_members_by_graph(t: Any, iri: str) -> Dict[str, Optional[FrozenSet[str]]]:
|
|
281
|
+
"""Inside GRAPH of a variable: each name `iri` answers under (itself, and the members of its component in
|
|
282
|
+
some named graph), with the graphs it answers in (None: every graph). A graph's entailment is its own
|
|
283
|
+
components' alone (Entailment Regimes 9), so a member answers in the graphs where it is `iri`'s member."""
|
|
284
|
+
out: Dict[str, Optional[FrozenSet[str]]] = {iri: None}
|
|
285
|
+
if not t.identity.may_alias(iri):
|
|
286
|
+
return out
|
|
287
|
+
wider: Dict[str, set] = {}
|
|
288
|
+
for graph, members in t.identity.members_by_graph(iri).items():
|
|
289
|
+
for m in members:
|
|
290
|
+
if m != iri:
|
|
291
|
+
wider.setdefault(m, set()).add(graph)
|
|
292
|
+
out.update({m: frozenset(g) for m, g in wider.items()})
|
|
293
|
+
return out
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
@home("sameas-bgp-matching", "A triple pattern matched through owl:sameAs: canonical match, then expansion",
|
|
297
|
+
terms=("canonical match", "expansion", "all aliases", "representative", "eq-rep-s", "eq-rep-o",
|
|
298
|
+
"eq-rep-p", "alias join", "constant IRI in a pattern"))
|
|
299
|
+
def pattern(t: Any, s: Any, p: Any, o: Any, quiet: bool) -> Any:
|
|
300
|
+
"""The solutions of the triple pattern (s, p, o), p an IRI, in the graph its sameAs links entail
|
|
301
|
+
(OWL 2 RL eq-*): see the module. Where no position may hold an alias, the pattern's plan is the one
|
|
302
|
+
without owl:sameAs."""
|
|
303
|
+
from prograph.plan.ir import ops as O
|
|
304
|
+
from prograph.plan.ir.terms import IRITerm
|
|
305
|
+
if p.text == SAME_AS:
|
|
306
|
+
return same_as(t, s, o, quiet)
|
|
307
|
+
from prograph.plan.ir.sparql_front import Var
|
|
308
|
+
in_var_graph = isinstance(t.graph, Var)
|
|
309
|
+
if in_var_graph:
|
|
310
|
+
# Each name answers in the graphs whose own components have it as the predicate's (class's) member.
|
|
311
|
+
if p.text == RDF_TYPE and isinstance(o, IRITerm):
|
|
312
|
+
names = predicate_members_by_graph(t, o.text)
|
|
313
|
+
parts = [_in_graphs(t, _one(t, s, p, IRITerm(c), quiet, object_done=True), graphs)
|
|
314
|
+
for c, graphs in sorted(names.items())]
|
|
315
|
+
else:
|
|
316
|
+
names = predicate_members_by_graph(t, p.text)
|
|
317
|
+
parts = [_in_graphs(t, _one(t, s, IRITerm(q), o, quiet), graphs) for q, graphs in sorted(names.items())]
|
|
318
|
+
elif p.text == RDF_TYPE and isinstance(o, IRITerm):
|
|
319
|
+
parts = [_one(t, s, p, IRITerm(cls), quiet, object_done=True) for cls in sorted(predicate_members(t, o.text))]
|
|
320
|
+
else:
|
|
321
|
+
parts = [_one(t, s, IRITerm(q), o, quiet) for q in sorted(predicate_members(t, p.text))]
|
|
322
|
+
kept = [x for x in parts if not (isinstance(x, O.Values) and not x.rows)]
|
|
323
|
+
if len(parts) == 1:
|
|
324
|
+
return parts[0]
|
|
325
|
+
if not kept:
|
|
326
|
+
return parts[0]
|
|
327
|
+
return O.Distinct(kept[0] if len(kept) == 1 else O.Union(tuple(_aligned(kept))))
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _in_graphs(t: Any, x: Any, graphs: Optional[FrozenSet[str]]) -> Any:
|
|
331
|
+
"""`x`'s rows whose graph (inside GRAPH of a variable) is one of `graphs`; all of them for None."""
|
|
332
|
+
from prograph.plan.ir import expr as E, ops as O
|
|
333
|
+
from prograph.plan.ir.terms import IRITerm
|
|
334
|
+
if graphs is None or (isinstance(x, O.Values) and not x.rows):
|
|
335
|
+
return x
|
|
336
|
+
name = E.Col(t.graph.name)
|
|
337
|
+
return O.Filter(x, E.Or(tuple(E.Compare("=", name, E.Const(IRITerm(g)), E.TERM) for g in sorted(graphs))))
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _aligned(parts: List[Any]) -> List[Any]:
|
|
341
|
+
from prograph.plan.ir import ops as O
|
|
342
|
+
cols = parts[0].columns
|
|
343
|
+
return [x if x.columns == cols else O.Project(x, cols) for x in parts]
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
PER_GRAPH = "per-graph" # a constant inside GRAPH ?g: matched through each graph's alias rows
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _one(t: Any, s: Any, p: Any, o: Any, quiet: bool, object_done: bool = False) -> Any:
|
|
350
|
+
"""One predicate's pattern (see `pattern`)."""
|
|
351
|
+
from prograph.plan.ir import expr as E, ops as O
|
|
352
|
+
from prograph.plan.ir.sparql_front import Var, _empty
|
|
353
|
+
from prograph.plan.ir.terms import IRITerm
|
|
354
|
+
identity = t.identity
|
|
355
|
+
in_var_graph = isinstance(t.graph, Var)
|
|
356
|
+
# A constant IRI with aliases: its component's members, read at planning (inside GRAPH ?g each
|
|
357
|
+
# graph's own, matched through the alias rows).
|
|
358
|
+
consts: Dict[str, Any] = {}
|
|
359
|
+
for pos, term in (("s", s), ("o", o)):
|
|
360
|
+
if not isinstance(term, IRITerm) or (pos == "o" and object_done):
|
|
361
|
+
continue
|
|
362
|
+
if in_var_graph:
|
|
363
|
+
if identity.may_alias(term.text):
|
|
364
|
+
consts[pos] = PER_GRAPH
|
|
365
|
+
elif identity.may_alias(term.text):
|
|
366
|
+
got = identity.members([term.text], active_graph(t)).get(term.text)
|
|
367
|
+
if got:
|
|
368
|
+
consts[pos] = got
|
|
369
|
+
if not consts:
|
|
370
|
+
plain = t.matched(s, p, o, quiet)
|
|
371
|
+
slots = plain.slots
|
|
372
|
+
if not any(isinstance(v, Var) and v.name in slots and identity.meets(slots[v.name]) for v in (s, o)):
|
|
373
|
+
return plain # no position may hold an alias: the plan without owl:sameAs
|
|
374
|
+
terms = {"s": s, "o": o}
|
|
375
|
+
base = {pos: Var(t.fresh("s" + pos)) if isinstance(term, Var) or pos in consts else term
|
|
376
|
+
for pos, term in terms.items()}
|
|
377
|
+
branches = [b for b in t.branches(base["s"], p, base["o"]) if b is not None]
|
|
378
|
+
if not branches:
|
|
379
|
+
return _empty(s, p, o)
|
|
380
|
+
graph_col = (t.graph.name,) if in_var_graph else ()
|
|
381
|
+
out_cols = tuple(dict.fromkeys(v.name for v in (s, o) if isinstance(v, Var))) + graph_col
|
|
382
|
+
same = isinstance(s, Var) and isinstance(o, Var) and s.name == o.name
|
|
383
|
+
variables = [pos for pos in ("s", "o") if isinstance(terms[pos], Var)]
|
|
384
|
+
rep_of = {pos: t.fresh("r" + pos) for pos in ("s", "o")}
|
|
385
|
+
plain_parts: List[Any] = []
|
|
386
|
+
canon_parts: List[Any] = []
|
|
387
|
+
expand: set = set() # the variables some branch reads through the alias rows
|
|
388
|
+
for b in branches:
|
|
389
|
+
slots = b.slots
|
|
390
|
+
meets = {pos: isinstance(base[pos], Var) and base[pos].name in slots and identity.meets(slots[base[pos].name])
|
|
391
|
+
for pos in ("s", "o")}
|
|
392
|
+
x, skip = b, False
|
|
393
|
+
for pos, members in consts.items():
|
|
394
|
+
column = base[pos].name
|
|
395
|
+
if not meets[pos]:
|
|
396
|
+
if members == PER_GRAPH: # no alias here: the constant itself
|
|
397
|
+
x = O.Filter(x, E.Compare("=", E.Col(column), E.Const(terms[pos]), E.TERM))
|
|
398
|
+
x = O.Project(x, tuple(c for c in x.columns if c != column))
|
|
399
|
+
else:
|
|
400
|
+
skip = True # no IRI of this branch is a member of the constant's component
|
|
401
|
+
elif members != PER_GRAPH:
|
|
402
|
+
x = _constant(t, x, column, members)
|
|
403
|
+
if skip:
|
|
404
|
+
continue
|
|
405
|
+
through = [pos for pos in variables if meets[pos]] + \
|
|
406
|
+
[pos for pos in consts if consts[pos] == PER_GRAPH and meets[pos]]
|
|
407
|
+
if not through:
|
|
408
|
+
y = x
|
|
409
|
+
for pos in variables:
|
|
410
|
+
if terms[pos].name not in y.columns:
|
|
411
|
+
y = O.Extend(y, terms[pos].name, E.Col(base[pos].name))
|
|
412
|
+
if same:
|
|
413
|
+
y = O.Filter(y, E.Compare("=", E.Col(base["s"].name), E.Col(base["o"].name), E.TERM))
|
|
414
|
+
plain_parts.append(O.Project(y, out_cols))
|
|
415
|
+
continue
|
|
416
|
+
expand.update(pos for pos in variables if meets[pos])
|
|
417
|
+
for pos in variables + [q for q in consts if consts[q] == PER_GRAPH and meets[q]]:
|
|
418
|
+
if meets[pos]:
|
|
419
|
+
x = _canonical(t, x, base[pos].name, rep_of[pos])
|
|
420
|
+
else:
|
|
421
|
+
x = O.Extend(x, rep_of[pos], E.Col(base[pos].name))
|
|
422
|
+
if pos in consts:
|
|
423
|
+
x = _same_entity(t, x, rep_of[pos], terms[pos])
|
|
424
|
+
canon_parts.append(O.Project(x, tuple(rep_of[pos] for pos in variables) + graph_col))
|
|
425
|
+
out: List[Any] = list(plain_parts)
|
|
426
|
+
if canon_parts:
|
|
427
|
+
cols = canon_parts[0].columns
|
|
428
|
+
canon: Any = O.Distinct(canon_parts[0] if len(canon_parts) == 1 else
|
|
429
|
+
O.Union(tuple(c if c.columns == cols else O.Project(c, cols) for c in canon_parts)))
|
|
430
|
+
if same:
|
|
431
|
+
canon = O.Filter(canon, E.Compare("=", E.Col(rep_of["s"]), E.Col(rep_of["o"]), E.TERM))
|
|
432
|
+
done = set()
|
|
433
|
+
width = len({terms[pos].name for pos in variables if pos in expand or (same and expand)})
|
|
434
|
+
for pos in variables:
|
|
435
|
+
if terms[pos].name not in done:
|
|
436
|
+
done.add(terms[pos].name)
|
|
437
|
+
if pos in expand or (same and expand):
|
|
438
|
+
canon = _expanded(t, canon, rep_of[pos], terms[pos].name, width)
|
|
439
|
+
else:
|
|
440
|
+
canon = O.Extend(canon, terms[pos].name, E.Col(rep_of[pos]))
|
|
441
|
+
out.append(O.Project(canon, out_cols))
|
|
442
|
+
if not out:
|
|
443
|
+
return _empty(s, p, o)
|
|
444
|
+
if not plain_parts:
|
|
445
|
+
return out[0] # distinct: one row per canonical triple, each expanded to members no other has
|
|
446
|
+
return O.Distinct(out[0] if len(out) == 1 else O.Union(tuple(out)))
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _constant(t: Any, x: Any, column: str, members: FrozenSet[str]) -> Any:
|
|
450
|
+
"""`x`'s rows whose `column` is one of a constant's members, `column` projected away."""
|
|
451
|
+
from prograph.plan.ir import expr as E, ops as O
|
|
452
|
+
from prograph.plan.ir.terms import IRITerm
|
|
453
|
+
if len(members) <= VALUES_MAX:
|
|
454
|
+
cond = E.Or(tuple(E.Compare("=", E.Col(column), E.Const(IRITerm(m)), E.TERM) for m in sorted(members)))
|
|
455
|
+
y = O.Filter(x, cond)
|
|
456
|
+
else:
|
|
457
|
+
rep = min(members, key=_codepoints)
|
|
458
|
+
r = t.fresh("cr")
|
|
459
|
+
m = t.identity.members_scan(t, r, column)
|
|
460
|
+
m = O.Project(O.Filter(m, E.Compare("=", E.Col(r), E.Const(IRITerm(rep)), E.TERM)), (column,))
|
|
461
|
+
y = O.SemiJoin(x, m)
|
|
462
|
+
return O.Project(y, tuple(c for c in y.columns if c != column))
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _codepoints(text: str) -> Tuple[int, ...]:
|
|
466
|
+
return tuple(ord(ch) for ch in text)
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _canonical(t: Any, x: Any, column: str, out: str) -> Any:
|
|
470
|
+
"""`x` with `out` the representative of `column`'s term: its alias row's, or itself where it has none
|
|
471
|
+
(a join and an anti-join with the alias rows, so `out` is bound in every row)."""
|
|
472
|
+
from prograph.plan.ir import expr as E, ops as O
|
|
473
|
+
r = t.fresh("ar")
|
|
474
|
+
a = t.identity.scan(t, column, r)
|
|
475
|
+
cols = x.columns + (out,)
|
|
476
|
+
aliased = O.Project(O.Extend(O.Join(x, a), out, E.Col(r)), cols)
|
|
477
|
+
own = O.Project(O.Extend(O.AntiJoin(x, O.Project(a, tuple(c for c in a.columns if c != r))), out, E.Col(column)),
|
|
478
|
+
cols)
|
|
479
|
+
return O.Union((aliased, own))
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _same_entity(t: Any, x: Any, rep: str, constant: Any) -> Any:
|
|
483
|
+
"""`x`'s rows whose representative `rep` is the constant's, in each row's graph (GRAPH ?g)."""
|
|
484
|
+
from prograph.plan.ir import expr as E, ops as O
|
|
485
|
+
from prograph.plan.ir.sparql_front import Var
|
|
486
|
+
c = Var(t.fresh("k"))
|
|
487
|
+
holder = O.Values((c.name,), ((constant,),))
|
|
488
|
+
canon = _canonical(t, holder if not isinstance(t.graph, Var) else
|
|
489
|
+
O.Join(holder, O.Distinct(O.Project(x, (t.graph.name,)))), c.name, t.fresh("kr"))
|
|
490
|
+
kr = canon.columns[-1]
|
|
491
|
+
joined = O.Join(x, O.Project(canon, tuple(col for col in canon.columns if col != c.name)))
|
|
492
|
+
return O.Project(O.Filter(joined, E.Compare("=", E.Col(rep), E.Col(kr), E.TERM)),
|
|
493
|
+
tuple(col for col in joined.columns if col != kr))
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def _expanded(t: Any, x: Any, rep: str, name: str, width: int = 1) -> Any:
|
|
497
|
+
"""`x` with `name` bound to each member of `rep`'s component (`sameas="all"`), or to `rep` itself
|
|
498
|
+
(`"representative"`). `width`: the pattern's positions expanded (up to largest^width rows a triple)."""
|
|
499
|
+
from prograph.plan.ir import expr as E, ops as O
|
|
500
|
+
own = O.Extend(x, name, E.Col(rep))
|
|
501
|
+
if t.identity.mode == "representative":
|
|
502
|
+
return own
|
|
503
|
+
r = t.fresh("er")
|
|
504
|
+
a = t.identity.scan(t, name, r, role=f"expand{width}-k{t.identity.largest}")
|
|
505
|
+
a = O.Project(O.Extend(a, rep, E.Col(r)), tuple(c for c in a.columns if c != r) + (rep,))
|
|
506
|
+
aliases = O.Join(x, a)
|
|
507
|
+
cols = own.columns
|
|
508
|
+
return O.Union((own, O.Project(aliases, cols)))
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def same_as(t: Any, s: Any, o: Any, quiet: bool) -> Any:
|
|
512
|
+
"""`s owl:sameAs o`: each ordered pair of two members of one component (the reflexive pairs aside:
|
|
513
|
+
eq-ref is not materialized), with the graph's own sameAs triples, once each. Under
|
|
514
|
+
`sameas="representative"` a variable binds each member's representative, as every pattern's does, so
|
|
515
|
+
an entity's pairs are its representative's one (r, r), and the stated triples are matched as any
|
|
516
|
+
pattern's."""
|
|
517
|
+
from prograph.plan.ir import expr as E, ops as O
|
|
518
|
+
from prograph.plan.ir.sparql_front import Var
|
|
519
|
+
from prograph.plan.ir.terms import IRITerm
|
|
520
|
+
rep_mode = t.identity.mode == "representative"
|
|
521
|
+
r, x, y = t.fresh("sr"), t.fresh("sx"), t.fresh("sy")
|
|
522
|
+
left = t.identity.members_scan(t, r, x)
|
|
523
|
+
right = t.identity.members_scan(t, r, y)
|
|
524
|
+
pairs: Any = O.Filter(O.Join(left, right), E.Not(E.Compare("=", E.Col(x), E.Col(y), E.TERM)))
|
|
525
|
+
graph_col = (t.graph.name,) if isinstance(t.graph, Var) else ()
|
|
526
|
+
|
|
527
|
+
def end(term: Any, member: str) -> Any:
|
|
528
|
+
return E.Col(r) if rep_mode and isinstance(term, Var) else E.Col(member)
|
|
529
|
+
terms = [(s, end(s, x)), (o, end(o, y))]
|
|
530
|
+
if graph_col: # the pairs' graph under a name of its own, bound to the active graph's column
|
|
531
|
+
g = t.fresh("sg")
|
|
532
|
+
pairs = O.Project(O.Extend(pairs, g, E.Col(graph_col[0])), (r, x, y, g))
|
|
533
|
+
terms.append((t.graph, E.Col(g)))
|
|
534
|
+
clique = t.bind(pairs, terms)
|
|
535
|
+
stated = _one(t, s, IRITerm(SAME_AS), o, quiet) if rep_mode else t.matched(s, IRITerm(SAME_AS), o, True)
|
|
536
|
+
parts = [clique] + ([stated] if not (isinstance(stated, O.Values) and not stated.rows) else [])
|
|
537
|
+
cols = parts[0].columns
|
|
538
|
+
return O.Distinct(parts[0] if len(parts) == 1 else O.Union(tuple(p if p.columns == cols else O.Project(p, cols)
|
|
539
|
+
for p in parts)))
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def for_engine(engine: Any, now: Any = None) -> Optional[Identity]:
|
|
543
|
+
"""The identity an engine's queries read, in the engine's answer mode (`sameas`): the alias generation the
|
|
544
|
+
dataset home gives for this planning (`dataset.store_now`, read from the store when the query is planned:
|
|
545
|
+
the latest generation, where it and the store's last run are of the engine's key); None where it gives none.
|
|
546
|
+
`now` that answer where the caller holds it."""
|
|
547
|
+
from prograph.plan.ir.dataset import store_now
|
|
548
|
+
now = now if now is not None else store_now(engine)
|
|
549
|
+
state = now.alias if now is not None else None
|
|
550
|
+
if state is None:
|
|
551
|
+
return None
|
|
552
|
+
store = engine._schema.__dict__.get("owned_store")
|
|
553
|
+
mode = getattr(engine, "_sameas", "all")
|
|
554
|
+
key = (id(store), state.generation, mode, state.key)
|
|
555
|
+
held = engine.__dict__.get("_identity")
|
|
556
|
+
if held is not None and held[0] == key:
|
|
557
|
+
return held[1]
|
|
558
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
559
|
+
rows = f"SELECT graph AS g, iri, rep FROM {alias_table(store)} WHERE generation = {int(state.generation)}"
|
|
560
|
+
try:
|
|
561
|
+
held = store.db.execute(f"SELECT kind, a, b FROM {meta_table(store)} WHERE generation = "
|
|
562
|
+
f"{int(state.generation)} AND kind IN ('disagree', 'merged')").fetchall()
|
|
563
|
+
except Exception: # noqa: BLE001 - a generation written before Cypher's rows were: every class may merge
|
|
564
|
+
held = None
|
|
565
|
+
disagree = frozenset((a, b) for k, a, b in held or () if k == "disagree")
|
|
566
|
+
# No merged class recorded (a generation written before they were, or records of no class): any may merge.
|
|
567
|
+
merged = frozenset(a for k, a, _ in held or () if k == "merged") or frozenset({"*"})
|
|
568
|
+
found = Identity(rows, state.shapes, state.largest, mode, run=lambda sql: store.db.execute(sql).fetchall(),
|
|
569
|
+
literal=get_dialect_instance(store.dialect).literal, tag=f"g{state.generation}",
|
|
570
|
+
disagree=disagree, merged=merged)
|
|
571
|
+
engine.__dict__["_identity"] = (key, found)
|
|
572
|
+
return found
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
# -- components (the fixpoint's, `owl_rl`) ------------------------------------------------------------------
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def namespace_sql(dialect: str, e: str) -> str:
|
|
579
|
+
"""SQL of the text `e`'s namespace: its text up to its last '/', '#' or ':' (the IRI shape the skip test
|
|
580
|
+
reads)."""
|
|
581
|
+
if dialect == "postgresql":
|
|
582
|
+
return f"coalesce(substring({e} from '^(.*[/#:])'), '')"
|
|
583
|
+
return f"regexp_extract({e}, '^(.*[/#:])', 1)"
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def least(dialect: str, a: str, b: str) -> str:
|
|
587
|
+
"""The lesser of two texts by code point (`lower.codepoint`'s collation)."""
|
|
588
|
+
return f"CASE WHEN {codepoint(dialect, a)} <= {codepoint(dialect, b)} THEN {a} ELSE {b} END"
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def codepoint(dialect: str, e: str) -> str:
|
|
592
|
+
"""`e` under the dialect's binary collation (the rule codepoint-string-order's CODEPOINT_TEXT)."""
|
|
593
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
594
|
+
template = get_dialect_instance(dialect).config.function_map.get("CODEPOINT_TEXT", "{0}")
|
|
595
|
+
return template.format(e)
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
@dataclass
|
|
599
|
+
class Components:
|
|
600
|
+
"""One computation of the components (`components`): the alias rows' read (`rows`: g, iri, rep, as
|
|
601
|
+
staged), the components left unmerged for their size (`capped`: (graph, rep, size)), the largest kept
|
|
602
|
+
component and the count of alias rows."""
|
|
603
|
+
rows: str
|
|
604
|
+
capped: List[Tuple[str, str, int]]
|
|
605
|
+
largest: int
|
|
606
|
+
aliases: int
|
|
607
|
+
count: int
|
|
608
|
+
labels: str = ""
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
@home("sameas-components", "owl:sameAs components: minimum-label propagation over the links, per graph",
|
|
612
|
+
terms=("connected components", "label propagation", "pointer jumping", "representative", "code point",
|
|
613
|
+
"hard cap", "left unmerged"))
|
|
614
|
+
def components(run: Any, links: str, poisoned: Optional[str], cap: Optional[int] = None) -> Components:
|
|
615
|
+
"""The connected components of the undirected links `links` (SQL of g, a, b: two IRIs' texts and the
|
|
616
|
+
graph they are linked in, '' the default graph), within each graph, each IRI of `poisoned` (SQL of g,
|
|
617
|
+
iri) left out: every node labelled with the least IRI it reaches by code point. Each round a node takes
|
|
618
|
+
the least label of its neighbours, then of its label's own label (pointer jumping), until no label
|
|
619
|
+
changes: a few statements per round, each staged (`run.stage`), on every engine alike. A component of
|
|
620
|
+
more than `cap` members is left unmerged (`Components.capped`). `run` is the materialization's run
|
|
621
|
+
(`owl_rl._Run`): its `stage`, `dialect` and `db`."""
|
|
622
|
+
d = run.dialect
|
|
623
|
+
cap = CAP if cap is None else cap
|
|
624
|
+
lt = lambda a, b: least(d, a, b) # noqa: E731
|
|
625
|
+
keep = ""
|
|
626
|
+
if poisoned:
|
|
627
|
+
keep = (f" LEFT JOIN ({poisoned}) AS _pa ON _pa.g = _l.g AND _pa.iri = _l.a "
|
|
628
|
+
f"LEFT JOIN ({poisoned}) AS _pb ON _pb.g = _l.g AND _pb.iri = _l.b WHERE _pa.iri IS NULL AND _pb.iri IS NULL")
|
|
629
|
+
kept = f"SELECT _l.g, _l.a, _l.b FROM ({links}) AS _l{keep}"
|
|
630
|
+
edges, _ = run.stage_component("edges", f"SELECT g, a AS x, b AS y FROM ({kept}) AS _k WHERE a <> b "
|
|
631
|
+
f"UNION SELECT g, b AS x, a AS y FROM ({kept}) AS _k WHERE a <> b")
|
|
632
|
+
first = "min(" + codepoint(d, "y") + ")" # not nested in the f-string: Python before 3.12 reads no inner quote
|
|
633
|
+
label, _ = run.stage_component("label0", f"SELECT g, x, {lt('x', first)} AS l FROM "
|
|
634
|
+
f"{edges} AS _e GROUP BY g, x")
|
|
635
|
+
for i in range(1, LABEL_ROUNDS + 1):
|
|
636
|
+
nbr = (f"SELECT _e.g, _e.x, min({codepoint(d, '_n.l')}) AS l FROM {edges} AS _e JOIN {label} AS _n "
|
|
637
|
+
f"ON _n.g = _e.g AND _n.x = _e.y GROUP BY _e.g, _e.x")
|
|
638
|
+
step, _ = run.stage_component(f"step{i}", f"SELECT _o.g, _o.x, {lt('_o.l', 'coalesce(_m.l, _o.l)')} AS l "
|
|
639
|
+
f"FROM {label} AS _o LEFT JOIN ({nbr}) AS _m ON _m.g = _o.g AND _m.x = _o.x")
|
|
640
|
+
jumped, _ = run.stage_component(f"jump{i}", f"SELECT _s.g, _s.x, {lt('_s.l', 'coalesce(_j.l, _s.l)')} AS l "
|
|
641
|
+
f"FROM {step} AS _s LEFT JOIN {step} AS _j ON _j.g = _s.g "
|
|
642
|
+
f"AND _j.x = _s.l")
|
|
643
|
+
changed = run.db.execute(f"SELECT count(*) FROM {jumped} AS _a JOIN {label} AS _b ON _a.g = _b.g "
|
|
644
|
+
f"AND _a.x = _b.x WHERE _a.l <> _b.l").fetchone()[0]
|
|
645
|
+
label = jumped
|
|
646
|
+
if not changed:
|
|
647
|
+
break
|
|
648
|
+
else:
|
|
649
|
+
from prograph.utils.exceptions import SQLRuntimeError
|
|
650
|
+
raise SQLRuntimeError(f"owl:sameAs components: labels still changed after {LABEL_ROUNDS} rounds",
|
|
651
|
+
reason="sameas_components")
|
|
652
|
+
sizes = run.db.execute(f"SELECT g, l, count(*) AS n FROM {label} AS _c GROUP BY g, l "
|
|
653
|
+
f"HAVING count(*) > {int(cap)}").fetchall()
|
|
654
|
+
capped = [(g, l, int(n)) for g, l, n in sizes]
|
|
655
|
+
where = ""
|
|
656
|
+
if capped:
|
|
657
|
+
listed = " OR ".join(f"(g = {run.d.literal(g)} AND l = {run.d.literal(l)})" for g, l, _ in capped)
|
|
658
|
+
where = f" AND NOT ({listed})"
|
|
659
|
+
rows, _ = run.stage_component("alias", f"SELECT g, x AS iri, l AS rep FROM {label} AS _c WHERE x <> l{where}")
|
|
660
|
+
stats = run.db.execute(f"SELECT count(*) AS components, coalesce(max(n), 0) AS largest FROM (SELECT g, rep, "
|
|
661
|
+
f"count(*) + 1 AS n "
|
|
662
|
+
f"FROM {rows} AS _r GROUP BY g, rep) AS _s").fetchone()
|
|
663
|
+
aliases = run.db.execute(f"SELECT count(*) FROM {rows} AS _r").fetchone()[0]
|
|
664
|
+
return Components(rows, capped, int(stats[1] or 0), int(aliases), int(stats[0] or 0), label)
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
# -- explaining ----------------------------------------------------------------------------------------------
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
@dataclass(frozen=True)
|
|
671
|
+
class IdentityExplanation:
|
|
672
|
+
"""Why two IRIs are one entity, or not. `merged`: one component. `path`: a shortest chain of links
|
|
673
|
+
from `a` to `b`, each (one IRI, the other, its source: 'stated by <map id>' for a sameAs triple of the
|
|
674
|
+
graph that one map gives, 'stated; may be given by <map ids>' where several may, 'rl:<rule> <axiom>' for one
|
|
675
|
+
an OWL RL rule derived). `representatives`: each IRI's representative
|
|
676
|
+
(itself where it has no alias). `unmerged`: where a component holding either was left unmerged, why."""
|
|
677
|
+
a: str
|
|
678
|
+
b: str
|
|
679
|
+
merged: bool
|
|
680
|
+
path: Tuple[Tuple[str, str, str], ...] = ()
|
|
681
|
+
representatives: Tuple[str, str] = ("", "")
|
|
682
|
+
sizes: Tuple[int, int] = (1, 1)
|
|
683
|
+
unmerged: Tuple[str, ...] = ()
|
|
684
|
+
|
|
685
|
+
def __str__(self) -> str:
|
|
686
|
+
if not self.merged:
|
|
687
|
+
why = f"; {'; '.join(self.unmerged)}" if self.unmerged else ""
|
|
688
|
+
return (f"<{self.a}> and <{self.b}> are not merged: <{self.a}> is in a component of {self.sizes[0]} "
|
|
689
|
+
f"(representative <{self.representatives[0]}>), <{self.b}> in one of {self.sizes[1]} "
|
|
690
|
+
f"(representative <{self.representatives[1]}>){why}")
|
|
691
|
+
steps = "\n".join(f" <{x}> owl:sameAs <{y}>: {src}" for x, y, src in self.path)
|
|
692
|
+
return f"<{self.a}> and <{self.b}> are one entity (representative <{self.representatives[0]}>):\n{steps}"
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
def explain(engine: Any, a: str, b: str) -> IdentityExplanation:
|
|
696
|
+
"""`engine.explain_identity(a, b)`: see `IdentityExplanation`. The default graph's components."""
|
|
697
|
+
from collections import deque
|
|
698
|
+
ident = for_engine(engine)
|
|
699
|
+
store = engine._schema.__dict__.get("owned_store")
|
|
700
|
+
unmerged = _unmerged_notes(store, (a, b))
|
|
701
|
+
if ident is None:
|
|
702
|
+
return IdentityExplanation(a, b, a == b, representatives=(a, b), unmerged=unmerged)
|
|
703
|
+
got = ident.members([a, b])
|
|
704
|
+
ma, mb = got.get(a, frozenset({a})), got.get(b, frozenset({b}))
|
|
705
|
+
rep_a, rep_b = min(ma, key=_codepoints), min(mb, key=_codepoints)
|
|
706
|
+
if ma != mb:
|
|
707
|
+
return IdentityExplanation(a, b, a == b, representatives=(rep_a, rep_b), sizes=(len(ma), len(mb)),
|
|
708
|
+
unmerged=unmerged)
|
|
709
|
+
edges = _component_links(engine, store, ident, ma)
|
|
710
|
+
graph: Dict[str, List[Tuple[str, str]]] = {}
|
|
711
|
+
for x, y, src in edges:
|
|
712
|
+
graph.setdefault(x, []).append((y, src))
|
|
713
|
+
graph.setdefault(y, []).append((x, src))
|
|
714
|
+
back: Dict[str, Tuple[str, str]] = {a: ("", "")}
|
|
715
|
+
todo = deque([a])
|
|
716
|
+
while todo:
|
|
717
|
+
x = todo.popleft()
|
|
718
|
+
if x == b:
|
|
719
|
+
break
|
|
720
|
+
for y, src in sorted(graph.get(x, ())):
|
|
721
|
+
if y not in back:
|
|
722
|
+
back[y] = (x, src)
|
|
723
|
+
todo.append(y)
|
|
724
|
+
path: List[Tuple[str, str, str]] = []
|
|
725
|
+
x = b
|
|
726
|
+
while x != a and x in back:
|
|
727
|
+
prev, src = back[x]
|
|
728
|
+
path.append((prev, x, src))
|
|
729
|
+
x = prev
|
|
730
|
+
return IdentityExplanation(a, b, True, tuple(reversed(path)), (rep_a, rep_b), (len(ma), len(mb)), unmerged)
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def _unmerged_notes(store: Any, iris: Tuple[str, ...]) -> Tuple[str, ...]:
|
|
734
|
+
if store is None or not store.db.has_table(meta_table(store)):
|
|
735
|
+
return ()
|
|
736
|
+
meta = meta_table(store)
|
|
737
|
+
rows = store.db.execute(
|
|
738
|
+
f"SELECT a, b, note FROM {meta} WHERE kind = 'unmerged' AND generation = (SELECT max(generation) FROM {meta} "
|
|
739
|
+
f"WHERE kind = 'generation')").fetchall()
|
|
740
|
+
out = []
|
|
741
|
+
for rep, rule, note in rows:
|
|
742
|
+
held = json.loads(note or "{}")
|
|
743
|
+
if any(i == rep or i in held.get("sample", ()) for i in iris):
|
|
744
|
+
reason = "inconsistent" + (f" ({rule})" if rule else "") if held.get("reason") == "inconsistent" else \
|
|
745
|
+
f"past the cap of {CAP} members"
|
|
746
|
+
out.append(f"a component of {held.get('size')} with representative <{rep}> was left unmerged: {reason}")
|
|
747
|
+
return tuple(out)
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
def _component_links(engine: Any, store: Any, ident: Identity, members: FrozenSet[str]) -> List[Tuple[str, str, str]]:
|
|
751
|
+
"""The links among `members`: the graph's own sameAs triples (by the maps that may give them) and those the
|
|
752
|
+
last run derived (its meta rows)."""
|
|
753
|
+
from prograph.plan.ir import lower, sparql_front
|
|
754
|
+
from prograph.plan.ir.dataset import maps_seen
|
|
755
|
+
from prograph.plan.ir.entry import _held, ontology_of
|
|
756
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
757
|
+
out: List[Tuple[str, str, str]] = []
|
|
758
|
+
onto = ontology_of(engine)
|
|
759
|
+
maps = maps_seen(engine, onto)
|
|
760
|
+
d = get_dialect_instance(store.dialect)
|
|
761
|
+
listed = sorted(members)
|
|
762
|
+
if SAME_AS in maps.by_predicate:
|
|
763
|
+
values = " ".join(f"<{m}>" for m in listed)
|
|
764
|
+
query = (f"SELECT (STR(?a) AS ?x) (STR(?b) AS ?y) WHERE {{ VALUES ?a {{ {values} }} ?a <{SAME_AS}> ?b "
|
|
765
|
+
f"FILTER(isIRI(?b)) }}")
|
|
766
|
+
t = sparql_front.translate(query, maps, _held(engine._schema, store.dialect, True), lambda m: (),
|
|
767
|
+
union_default=getattr(engine, "default_graph", "union") == "union", ontology=onto)
|
|
768
|
+
sql = lower.to_sql(t.plan, store.dialect, base_iri=maps.settings.base_iri)
|
|
769
|
+
givers = [g.map for g in maps.by_predicate[SAME_AS]]
|
|
770
|
+
for x, y in store.db.execute(f"SELECT {d.quote('x')}, {d.quote('y')} FROM ({sql}) AS _l").fetchall():
|
|
771
|
+
if y not in members or x == y:
|
|
772
|
+
continue
|
|
773
|
+
ids = sorted({m.id for m in givers if _gives(m, x, y)})
|
|
774
|
+
# One map that can give the link gave it, unless the ontology states it too; with several that may,
|
|
775
|
+
# the data does not say which.
|
|
776
|
+
layer = getattr(engine, "_ontology", None)
|
|
777
|
+
ontology = layer is not None and y in layer.get_same_as_group(x)
|
|
778
|
+
names = ids + (["the ontology"] if ontology else [])
|
|
779
|
+
out.append((x, y, "stated" + (f" by {ids[0]}" if len(ids) == 1 and not ontology else
|
|
780
|
+
f"; may be given by {', '.join(names)}" if names else "")))
|
|
781
|
+
meta = meta_table(store)
|
|
782
|
+
if store.db.has_table(meta):
|
|
783
|
+
rows = store.db.execute(f"SELECT a, b, note FROM {meta} WHERE kind = 'link' AND generation = (SELECT "
|
|
784
|
+
f"max(generation) FROM {meta} WHERE kind = 'generation')").fetchall()
|
|
785
|
+
out += [(x, y, src) for x, y, src in rows if x in members and y in members]
|
|
786
|
+
return out
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
def _gives(m: Any, x: str, y: str) -> bool:
|
|
790
|
+
"""Whether map `m` may give the triple (x owl:sameAs y) (its subject and object term maps)."""
|
|
791
|
+
from prograph.plan.ir.sparql_front import _may_give
|
|
792
|
+
from prograph.plan.ir.terms import IRITerm
|
|
793
|
+
subject = getattr(m, "subject", None) or getattr(getattr(m, "start", None), "term", None)
|
|
794
|
+
obj = getattr(getattr(m, "end", None), "term", None)
|
|
795
|
+
if subject is None:
|
|
796
|
+
return True
|
|
797
|
+
return _may_give(subject, IRITerm(x)) and (obj is None or _may_give(obj, IRITerm(y)))
|