prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/plan/ir/slot.py
ADDED
|
@@ -0,0 +1,509 @@
|
|
|
1
|
+
"""What a column of the one plan holds: its slot (one-plan.md 3, 6.4).
|
|
2
|
+
|
|
3
|
+
A slot is logical. It says what a present cell may be (its kinds, a
|
|
4
|
+
literal's datatypes and language), whether rows have one (`bound` for an RDF
|
|
5
|
+
term, `null` for a value), where its terms can come from (`provenance`), and
|
|
6
|
+
which datatype's SQL value the cell holds (`native`). It holds no
|
|
7
|
+
expression: how a cell is written in SQL is the lowering's
|
|
8
|
+
(`prograph.plan.ir.lower`), from the slot.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import functools
|
|
14
|
+
from dataclasses import dataclass, replace
|
|
15
|
+
from enum import Enum
|
|
16
|
+
from typing import FrozenSet, Iterable, Mapping, Optional, Tuple
|
|
17
|
+
|
|
18
|
+
from prograph.rules import home
|
|
19
|
+
from prograph.schema import xsd
|
|
20
|
+
from prograph.schema.source_map import BLANK, IRI, LITERAL, TermMap
|
|
21
|
+
|
|
22
|
+
# A term's kinds (RDF 1.1 Concepts 3), as source maps name them.
|
|
23
|
+
TERM_KINDS: FrozenSet[str] = frozenset({IRI, BLANK, LITERAL})
|
|
24
|
+
NODE_KINDS: FrozenSet[str] = frozenset({IRI, BLANK})
|
|
25
|
+
|
|
26
|
+
# What a column holds where it holds no RDF term: a database value as the
|
|
27
|
+
# source holds it (a raw column of a map scan, a Cypher property), or a
|
|
28
|
+
# Cypher list, map or path. A missing value of these kinds is a null; a
|
|
29
|
+
# missing RDF term is unbound.
|
|
30
|
+
VALUE = "value"
|
|
31
|
+
LIST = "list"
|
|
32
|
+
MAP = "map"
|
|
33
|
+
PATH = "path"
|
|
34
|
+
MIXED = "mixed" # a Cypher value that is an integer, float, string or boolean per row (rule cypher-mixed-values)
|
|
35
|
+
VALUE_KINDS: FrozenSet[str] = frozenset({VALUE, LIST, MAP, PATH, MIXED})
|
|
36
|
+
|
|
37
|
+
# Zones of a date-time slot.
|
|
38
|
+
ZONED = "zoned"
|
|
39
|
+
LOCAL = "local"
|
|
40
|
+
|
|
41
|
+
# A datatype the source did not record. Not an IRI, so it never equals one.
|
|
42
|
+
UNKNOWN = "\0unknown"
|
|
43
|
+
|
|
44
|
+
# A language tag that rows decide: the slot's tagged literals have their
|
|
45
|
+
# tag in a part column (`<c>__lang`), not one tag for every row.
|
|
46
|
+
ROW = "\0row"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class Bound(Enum):
|
|
50
|
+
"""Whether rows have something: in every row, in some, in none."""
|
|
51
|
+
ALWAYS = "always"
|
|
52
|
+
MAYBE = "maybe"
|
|
53
|
+
NEVER = "never"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _either(a: Bound, b: Bound) -> Bound:
|
|
57
|
+
"""What a column of rows of `a` and rows of `b` has."""
|
|
58
|
+
return a if a is b else Bound.MAYBE
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass(frozen=True)
|
|
62
|
+
class Slot:
|
|
63
|
+
"""One column's slot.
|
|
64
|
+
|
|
65
|
+
Attributes:
|
|
66
|
+
kinds: The kinds a present cell may have: some of `IRI`, `BLANK`,
|
|
67
|
+
`LITERAL` (terms) or `VALUE`, `LIST`, `MAP`, `PATH`, `MIXED` (values).
|
|
68
|
+
Empty only where no row has one.
|
|
69
|
+
datatypes: A literal's possible datatypes (a tagged literal's
|
|
70
|
+
rdf:langString), or a value's, or a list's elements' (Cypher's
|
|
71
|
+
lists hold one type); `UNKNOWN` with `reason` where the source
|
|
72
|
+
did not say.
|
|
73
|
+
language: The tag of the slot's tagged literals: one tag (lower
|
|
74
|
+
case), `ROW` where rows decide, None where it holds none.
|
|
75
|
+
bound: Whether rows bind the term (an RDF term's).
|
|
76
|
+
null: Whether rows hold a null (a value's).
|
|
77
|
+
provenance: The term maps its terms can come from (a constant
|
|
78
|
+
written in the query is its `constant_term`); None where
|
|
79
|
+
unknown, which counts as any term.
|
|
80
|
+
native: The datatype whose SQL value a present cell holds, where
|
|
81
|
+
the cell is not text: an integer column read as xsd:integer
|
|
82
|
+
holds the integer, in canonical lexical form when written. None
|
|
83
|
+
where the cell is the term's text (an IRI, a blank node's label,
|
|
84
|
+
a literal's lexical form as given).
|
|
85
|
+
zone: For date-times, `ZONED` or `LOCAL`; None where rows differ.
|
|
86
|
+
reason: Why a datatype is `UNKNOWN`.
|
|
87
|
+
digits: A native decimal's SQL precision and scale, where known: a
|
|
88
|
+
DECIMAL(p, s) column's, a constant's, an arithmetic result's
|
|
89
|
+
(`facts.arithmetic_slot`); None where unknown. A native integer
|
|
90
|
+
held wider than 64 bits (a SUM's DECIMAL(38, 0), D7) has its
|
|
91
|
+
digits too; None for one held as a 64-bit integer.
|
|
92
|
+
held_as: Where the cell is held as the values its one template map
|
|
93
|
+
makes the term from (`held_as_values`), their datatypes, one
|
|
94
|
+
per column the template reads, in order of first reading; None
|
|
95
|
+
where the cell is the term's text or native value.
|
|
96
|
+
bytewise: A string value whose SQL equality is the equality of its
|
|
97
|
+
bytes: its column's engine and collation compare so
|
|
98
|
+
(`collations.compares_bytes`). Kept by a union or a join only
|
|
99
|
+
where both sides have it.
|
|
100
|
+
uuid: A string value that is a uuid column's text, the name of its
|
|
101
|
+
reader (`source_map.UUID_READERS`; rule `uuid-text`); an IRI
|
|
102
|
+
built from it is held as the native column (`held_as_values`).
|
|
103
|
+
text_of: A string value that is the database's own text of a value
|
|
104
|
+
of this datatype: a DECIMAL column read under the schema's
|
|
105
|
+
`decimal_text: database` (`sparql_front.held_under`); its literal
|
|
106
|
+
is that datatype's, its text as read.
|
|
107
|
+
"""
|
|
108
|
+
kinds: FrozenSet[str]
|
|
109
|
+
datatypes: FrozenSet[str] = frozenset()
|
|
110
|
+
language: Optional[str] = None
|
|
111
|
+
bound: Bound = Bound.ALWAYS
|
|
112
|
+
null: Bound = Bound.NEVER
|
|
113
|
+
provenance: Optional[FrozenSet[TermMap]] = None
|
|
114
|
+
native: Optional[str] = None
|
|
115
|
+
zone: Optional[str] = None
|
|
116
|
+
reason: Optional[str] = None
|
|
117
|
+
held_as: Optional[Tuple[str, ...]] = None
|
|
118
|
+
digits: Optional[Tuple[int, int]] = None
|
|
119
|
+
bytewise: bool = False
|
|
120
|
+
text_of: Optional[str] = None
|
|
121
|
+
uuid: Optional[str] = None
|
|
122
|
+
|
|
123
|
+
def __post_init__(self) -> None:
|
|
124
|
+
if not self.kinds <= TERM_KINDS | VALUE_KINDS:
|
|
125
|
+
raise ValueError(f"no kinds {sorted(self.kinds - TERM_KINDS - VALUE_KINDS)}")
|
|
126
|
+
if self.kinds & TERM_KINDS and self.kinds & VALUE_KINDS:
|
|
127
|
+
raise ValueError("a slot holds terms or values, not both")
|
|
128
|
+
if not self.kinds and self.present is not Bound.NEVER:
|
|
129
|
+
raise ValueError("a slot that may have a cell has a kind")
|
|
130
|
+
if self.datatypes and not self.kinds & {LITERAL, VALUE, LIST, MAP, MIXED}:
|
|
131
|
+
raise ValueError("only a literal, a value or a list (its elements') has a datatype")
|
|
132
|
+
if UNKNOWN in self.datatypes and not self.reason:
|
|
133
|
+
raise ValueError("an unknown datatype needs its reason")
|
|
134
|
+
if (self.language is not None) != (xsd.RDF_LANGSTRING in self.datatypes):
|
|
135
|
+
raise ValueError("a language goes with rdf:langString, and only with it")
|
|
136
|
+
if self.native is not None and self.native not in self.datatypes:
|
|
137
|
+
raise ValueError("a native datatype is one of the slot's")
|
|
138
|
+
if self.native is not None and self.kinds & TERM_KINDS and \
|
|
139
|
+
(self.kinds != {LITERAL} or len(self.datatypes) != 1):
|
|
140
|
+
raise ValueError("a term is held natively only as a literal of one datatype")
|
|
141
|
+
if self.held_as is not None and (self.kinds != {IRI} or not self.provenance
|
|
142
|
+
or not disjoint_templates(self.provenance)):
|
|
143
|
+
raise ValueError("a term is held as its values only as an IRI of one template shape, "
|
|
144
|
+
"or of several no two of which may give one IRI")
|
|
145
|
+
|
|
146
|
+
@property
|
|
147
|
+
def terms(self) -> bool:
|
|
148
|
+
"""It holds RDF terms (a missing one is unbound)."""
|
|
149
|
+
return bool(self.kinds & TERM_KINDS)
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def present(self) -> Bound:
|
|
153
|
+
"""Whether rows have a cell at all: SQL's NOT NULL."""
|
|
154
|
+
if self.terms:
|
|
155
|
+
return self.bound
|
|
156
|
+
if not self.kinds:
|
|
157
|
+
return Bound.NEVER
|
|
158
|
+
return {Bound.NEVER: Bound.ALWAYS, Bound.ALWAYS: Bound.NEVER}.get(self.null, Bound.MAYBE)
|
|
159
|
+
|
|
160
|
+
@property
|
|
161
|
+
def datatype(self) -> Optional[str]:
|
|
162
|
+
"""The one datatype, None where there are several or none."""
|
|
163
|
+
return next(iter(self.datatypes)) if len(self.datatypes) == 1 else None
|
|
164
|
+
|
|
165
|
+
# -- the parts rows decide (one-plan.md 6.4) ---------------------------------
|
|
166
|
+
|
|
167
|
+
@property
|
|
168
|
+
def kind_varies(self) -> bool:
|
|
169
|
+
"""Rows differ in their term's kind: a `<c>__kind` part."""
|
|
170
|
+
return len(self.kinds & TERM_KINDS) > 1
|
|
171
|
+
|
|
172
|
+
@property
|
|
173
|
+
def datatype_varies(self) -> bool:
|
|
174
|
+
"""Literal rows differ in datatype: a `<c>__dt` part."""
|
|
175
|
+
return LITERAL in self.kinds and len(self.datatypes) > 1
|
|
176
|
+
|
|
177
|
+
@property
|
|
178
|
+
def language_varies(self) -> bool:
|
|
179
|
+
"""Tagged rows differ in tag: a `<c>__lang` part."""
|
|
180
|
+
return self.language == ROW
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
NOTHING = Slot(kinds=frozenset(), bound=Bound.NEVER)
|
|
184
|
+
"""A column no row has (SPARQL's projection of a variable not in scope)."""
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def value_slot(datatype: Optional[str], null: Bound, reason: Optional[str] = None,
|
|
188
|
+
digits: Optional[Tuple[int, int]] = None, bytewise: bool = False,
|
|
189
|
+
text_of: Optional[str] = None, uuid: Optional[str] = None) -> Slot:
|
|
190
|
+
"""A database value of `datatype` (None: unknown, with `reason`); `bytewise` a string column that
|
|
191
|
+
compares byte for byte (`Slot.bytewise`)."""
|
|
192
|
+
if datatype is None:
|
|
193
|
+
return Slot(kinds=frozenset({VALUE}), datatypes=frozenset({UNKNOWN}), bound=Bound.NEVER,
|
|
194
|
+
null=null, reason=reason or "the source recorded no type")
|
|
195
|
+
return Slot(kinds=frozenset({VALUE}), datatypes=frozenset({datatype}), bound=Bound.NEVER,
|
|
196
|
+
null=null, native=datatype,
|
|
197
|
+
digits=digits if datatype == xsd.DECIMAL or (text_of == xsd.DECIMAL and datatype == xsd.STRING) else None,
|
|
198
|
+
bytewise=bytewise and datatype == xsd.STRING,
|
|
199
|
+
text_of=text_of if datatype == xsd.STRING else None,
|
|
200
|
+
uuid=uuid if datatype == xsd.STRING else None)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def literal_slot(datatype: str, bound: Bound, *, native: bool = False,
|
|
204
|
+
language: Optional[str] = None,
|
|
205
|
+
provenance: Optional[FrozenSet[TermMap]] = None) -> Slot:
|
|
206
|
+
"""A literal of one datatype (rdf:langString with `language`)."""
|
|
207
|
+
return Slot(kinds=frozenset({LITERAL}), datatypes=frozenset({datatype}), language=language,
|
|
208
|
+
bound=bound, provenance=provenance, native=datatype if native else None)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
BOOLEAN = literal_slot(xsd.BOOLEAN, Bound.MAYBE, native=True)
|
|
212
|
+
"""An expression's boolean: unbound where it is an error."""
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def optional(slot: Slot) -> Slot:
|
|
216
|
+
"""`slot` in rows that may not have it (OPTIONAL's right side)."""
|
|
217
|
+
if slot.present is Bound.NEVER:
|
|
218
|
+
return slot
|
|
219
|
+
if slot.terms:
|
|
220
|
+
return replace(slot, bound=_either(slot.bound, Bound.NEVER))
|
|
221
|
+
return replace(slot, null=_either(slot.null, Bound.ALWAYS))
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def union(slots: Iterable[Slot]) -> Slot:
|
|
225
|
+
"""The slot of a column whose rows come from each of `slots` (a UNION's
|
|
226
|
+
branches; a compatible join's column where either side may be
|
|
227
|
+
unbound). A branch that never has the column only adds missing rows."""
|
|
228
|
+
slots = list(slots)
|
|
229
|
+
if not slots:
|
|
230
|
+
return NOTHING
|
|
231
|
+
out = slots[0]
|
|
232
|
+
for s in slots[1:]:
|
|
233
|
+
out = _union2(out, s)
|
|
234
|
+
return out
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _union2(a: Slot, b: Slot) -> Slot:
|
|
238
|
+
if a.present is Bound.NEVER and not a.kinds:
|
|
239
|
+
return optional(b) if b.present is not Bound.NEVER else b
|
|
240
|
+
if b.present is Bound.NEVER and not b.kinds:
|
|
241
|
+
return optional(a)
|
|
242
|
+
if a.terms != b.terms:
|
|
243
|
+
raise ValueError("a column holds terms in some rows and values in others")
|
|
244
|
+
datatypes = a.datatypes | b.datatypes
|
|
245
|
+
if a.language is None:
|
|
246
|
+
language = b.language
|
|
247
|
+
elif b.language is None or a.language == b.language:
|
|
248
|
+
language = a.language
|
|
249
|
+
else:
|
|
250
|
+
language = ROW
|
|
251
|
+
kinds = a.kinds | b.kinds
|
|
252
|
+
native = a.native if a.native == b.native else None
|
|
253
|
+
if native is not None and a.terms and (kinds != {LITERAL} or len(datatypes) != 1):
|
|
254
|
+
native = None
|
|
255
|
+
provenance = (None if a.provenance is None or b.provenance is None
|
|
256
|
+
else a.provenance | b.provenance)
|
|
257
|
+
reason = a.reason or b.reason if UNKNOWN in datatypes else None
|
|
258
|
+
return Slot(kinds=kinds, datatypes=datatypes, language=language,
|
|
259
|
+
bound=_either(a.bound, b.bound), null=_either(a.null, b.null),
|
|
260
|
+
provenance=provenance, native=native,
|
|
261
|
+
zone=a.zone if a.zone == b.zone else None, reason=reason,
|
|
262
|
+
held_as=_held_alike(a, b),
|
|
263
|
+
digits=_digits_alike(a, b, native),
|
|
264
|
+
bytewise=a.bytewise and b.bytewise and native == xsd.STRING,
|
|
265
|
+
text_of=a.text_of if a.text_of == b.text_of and native == xsd.STRING else None,
|
|
266
|
+
uuid=a.uuid if a.uuid == b.uuid and native == xsd.STRING else None)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _digits_alike(a: Slot, b: Slot, native) -> Optional[Tuple[int, int]]:
|
|
270
|
+
"""The digits of a cell either slot's cell may be: both sides' where alike; for integers, the
|
|
271
|
+
wider (one held as DECIMAL(38, 0) and one as 64 bits are both held as the DECIMAL)."""
|
|
272
|
+
if native is None:
|
|
273
|
+
return None
|
|
274
|
+
if a.digits == b.digits:
|
|
275
|
+
return a.digits
|
|
276
|
+
if native in xsd.INTEGER_DATATYPES | {xsd.INTEGER}:
|
|
277
|
+
return max((d for d in (a.digits, b.digits) if d is not None), default=None)
|
|
278
|
+
return None
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
# Datatypes whose native cells are always written in canonical form: a
|
|
282
|
+
# column's value or a computed one (xsd.canonical_lexical; D3, 2026-11-13: a decimal in XML Schema 1.1's form, `5`).
|
|
283
|
+
CANONICAL_NATIVE = frozenset({xsd.INTEGER, xsd.DECIMAL})
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def never_one_term(a: Slot, b: Slot) -> bool:
|
|
287
|
+
"""Whether no term of `a` can be a term of `b`, both bound: no kind,
|
|
288
|
+
datatype or map in common (`both`), or one a native integer or decimal,
|
|
289
|
+
always in canonical form, and the other a constant written otherwise
|
|
290
|
+
(`5.00`, `"05"^^xsd:integer`: another term, RDF 1.1 3.3). The front end
|
|
291
|
+
drops a pattern's map on it and says so; the lowering writes FALSE."""
|
|
292
|
+
if a.terms and b.terms and not both(a, b).kinds:
|
|
293
|
+
return True
|
|
294
|
+
for x, y in ((a, b), (b, a)):
|
|
295
|
+
if x.native in CANONICAL_NATIVE and x.kinds == {LITERAL} and \
|
|
296
|
+
y.kinds == {LITERAL} and y.datatypes == {x.native} and \
|
|
297
|
+
y.provenance is not None and len(y.provenance) == 1:
|
|
298
|
+
c = next(iter(y.provenance))
|
|
299
|
+
if c.constant is not None and not _canonical_text(c.constant, x.native):
|
|
300
|
+
return True
|
|
301
|
+
return False
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _canonical_text(lexical: str, datatype: str) -> bool:
|
|
305
|
+
"""Whether `lexical` is the canonical form of its value (an ill-typed
|
|
306
|
+
text is the canonical form of none)."""
|
|
307
|
+
try:
|
|
308
|
+
return xsd.canonical_lexical(xsd.parse(lexical, datatype), datatype) == lexical
|
|
309
|
+
except (ValueError, TypeError, ArithmeticError):
|
|
310
|
+
return False
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def both(a: Slot, b: Slot) -> Slot:
|
|
314
|
+
"""The slot of a column that a row has only where `a` and `b` hold one
|
|
315
|
+
term (a join's shared column, where both sides bind it in every row):
|
|
316
|
+
the kinds and datatypes they share. The physical form is `a`'s where
|
|
317
|
+
`b`'s is the same, else text."""
|
|
318
|
+
kinds = a.kinds & b.kinds
|
|
319
|
+
datatypes = a.datatypes & b.datatypes
|
|
320
|
+
if LITERAL in kinds and a.datatypes and b.datatypes and not datatypes \
|
|
321
|
+
and UNKNOWN not in a.datatypes and UNKNOWN not in b.datatypes:
|
|
322
|
+
# Two literals of different datatypes are never one term (RDF 1.1
|
|
323
|
+
# 3.3): no literal row joins.
|
|
324
|
+
kinds = kinds - {LITERAL}
|
|
325
|
+
if UNKNOWN in a.datatypes or UNKNOWN in b.datatypes:
|
|
326
|
+
datatypes = a.datatypes | b.datatypes if LITERAL in kinds else frozenset()
|
|
327
|
+
if LITERAL not in kinds:
|
|
328
|
+
datatypes = frozenset()
|
|
329
|
+
language = a.language if a.language == b.language else (
|
|
330
|
+
ROW if xsd.RDF_LANGSTRING in datatypes else None)
|
|
331
|
+
if xsd.RDF_LANGSTRING not in datatypes:
|
|
332
|
+
language = None
|
|
333
|
+
# A row's term is one term on both sides: a side that holds it natively
|
|
334
|
+
# holds it so for the row.
|
|
335
|
+
native = next((x.native for x in (a, b) if x.native is not None and x.native in datatypes),
|
|
336
|
+
None)
|
|
337
|
+
if native is not None and (kinds != {LITERAL} or len(datatypes) != 1):
|
|
338
|
+
native = None
|
|
339
|
+
# Each row's term is given by a map of each side: either side's maps
|
|
340
|
+
# describe it (two maps of one shape, an edge's end and the node's
|
|
341
|
+
# subject, are two maps of the same terms, so not an intersection).
|
|
342
|
+
# Only maps that may give a term the other side's maps give can hold a
|
|
343
|
+
# row's term; where none can, no row has one (as with no kind shared).
|
|
344
|
+
if a.provenance is not None and b.provenance is not None:
|
|
345
|
+
ka = frozenset(m for m in a.provenance if any(m.may_equal(n) for n in b.provenance))
|
|
346
|
+
kb = frozenset(m for m in b.provenance if any(m.may_equal(n) for n in a.provenance))
|
|
347
|
+
if not ka or not kb:
|
|
348
|
+
return NOTHING
|
|
349
|
+
# Either side's maps describe the terms; the fewer, and on a tie a
|
|
350
|
+
# constant (one term), whatever the join's order.
|
|
351
|
+
provenance = min(ka, kb, key=lambda k: (len(k), not any(m.constant is not None for m in k)))
|
|
352
|
+
else:
|
|
353
|
+
provenance = b.provenance if a.provenance is None else a.provenance
|
|
354
|
+
if not kinds:
|
|
355
|
+
return NOTHING
|
|
356
|
+
return Slot(kinds=kinds, datatypes=datatypes, language=language, bound=Bound.ALWAYS,
|
|
357
|
+
provenance=provenance, native=native,
|
|
358
|
+
zone=a.zone if a.zone == b.zone else None,
|
|
359
|
+
reason=(a.reason or b.reason) if UNKNOWN in datatypes else None,
|
|
360
|
+
held_as=_held_alike(a, b),
|
|
361
|
+
digits=_digits_alike(a, b, native),
|
|
362
|
+
bytewise=a.bytewise and b.bytewise and native == xsd.STRING,
|
|
363
|
+
text_of=a.text_of if a.text_of == b.text_of and native == xsd.STRING else None,
|
|
364
|
+
uuid=a.uuid if a.uuid == b.uuid and native == xsd.STRING else None)
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _held_alike(a: Slot, b: Slot) -> Optional[Tuple[str, ...]]:
|
|
368
|
+
"""The values both slots' cells are held as, where they are held alike: values of one type, of
|
|
369
|
+
one template shape, or of several no two of which may give one IRI (then with a tag naming the
|
|
370
|
+
template, `tagged`); else None, the term's text."""
|
|
371
|
+
if a.held_as is None or a.held_as != b.held_as:
|
|
372
|
+
return None
|
|
373
|
+
if {template_shape(m) for m in a.provenance} == {template_shape(m) for m in b.provenance}:
|
|
374
|
+
return a.held_as
|
|
375
|
+
return a.held_as if disjoint_templates(a.provenance | b.provenance) else None
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def template_tag(m: TermMap) -> int:
|
|
379
|
+
"""A template shape's tag: one integer for the shape wherever a cell holds it, in
|
|
380
|
+
every cell and on both sides of a join (never a position in one slot's templates), from
|
|
381
|
+
the shape itself. Two shapes of one tag are never tagged together (`disjoint_templates`)."""
|
|
382
|
+
return _tag_of(template_shape(m))
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
@functools.lru_cache(maxsize=8192)
|
|
386
|
+
def _tag_of(shape) -> int:
|
|
387
|
+
import zlib
|
|
388
|
+
return zlib.crc32(repr(shape).encode("utf-8")) & 0x3FFFFFFF
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def disjoint_templates(maps) -> bool:
|
|
392
|
+
"""Whether IRIs of these template maps may be held as (tag, values): one shape, or several
|
|
393
|
+
whose tags differ and no two of which may give one IRI (`TermMap.may_equal`), so one IRI has
|
|
394
|
+
exactly one tag and one set of values."""
|
|
395
|
+
shapes = {}
|
|
396
|
+
for m in maps:
|
|
397
|
+
if m.template is None:
|
|
398
|
+
return False
|
|
399
|
+
shapes.setdefault(template_shape(m), m)
|
|
400
|
+
if len(shapes) == 1:
|
|
401
|
+
return True
|
|
402
|
+
reps = list(shapes.values())
|
|
403
|
+
if len({template_tag(m) for m in reps}) != len(reps):
|
|
404
|
+
return False
|
|
405
|
+
return all(not x.may_equal(y) for i, x in enumerate(reps) for y in reps[i + 1:])
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def tagged(slot: Slot) -> bool:
|
|
409
|
+
"""A held IRI of several template shapes: its cell holds the template's tag before the values."""
|
|
410
|
+
return slot.held_as is not None and len({template_shape(m) for m in slot.provenance}) > 1
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def template_shape(m: TermMap):
|
|
414
|
+
"""A template map's shape: its term type, how it is written (R2RML or
|
|
415
|
+
ProGraph's), its fixed text and where each value goes, its column names
|
|
416
|
+
left out. Two maps of one shape over values of one type give one term
|
|
417
|
+
exactly where the values are equal (each is injective, `held_as_values`):
|
|
418
|
+
a node's subject `.../person/{id}` and an edge end's `.../person/{Person.id}`."""
|
|
419
|
+
return _shape_of(m.term_type, m.lexical, tuple(m.template or ()))
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
@functools.lru_cache(maxsize=8192)
|
|
423
|
+
def _shape_of(term_type, lexical, template):
|
|
424
|
+
order: list = []
|
|
425
|
+
runs = []
|
|
426
|
+
for run, is_column in template:
|
|
427
|
+
if is_column:
|
|
428
|
+
if run not in order:
|
|
429
|
+
order.append(run)
|
|
430
|
+
runs.append((order.index(run), True))
|
|
431
|
+
else:
|
|
432
|
+
runs.append((run, False))
|
|
433
|
+
return (term_type, lexical, tuple(runs))
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
# Datatypes whose SQL equality is the equality of the text a template
|
|
437
|
+
# writes for them (`lower.value_text`): an integer's digits; and a string's
|
|
438
|
+
# where its column compares bytes (`Slot.bytewise`).
|
|
439
|
+
_EQUAL_AS_TEXT = frozenset(xsd.INTEGER_DATATYPES | {xsd.INTEGER})
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def equal_as_text(slot: Slot) -> bool:
|
|
443
|
+
"""A value whose SQL equality is the equality of the text a template writes for it: an integer,
|
|
444
|
+
or a string whose column compares byte for byte (`collations.compares_bytes`)."""
|
|
445
|
+
if slot.terms:
|
|
446
|
+
return False
|
|
447
|
+
return slot.native in _EQUAL_AS_TEXT or (slot.native == xsd.STRING and slot.bytewise)
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
@home("term-held-as-values", "An IRI held in SQL as its template's values",
|
|
451
|
+
terms=("held_as", "late materialization", "IRI text", "template values", "injective"))
|
|
452
|
+
def held_as_values(m: TermMap, args: Mapping[str, Slot]) -> Optional[Tuple[str, ...]]:
|
|
453
|
+
"""The datatypes of the values a term of `m` is held as, in SQL, in
|
|
454
|
+
place of its text (m2-performance.md 1; one-plan.md 6.4); None where
|
|
455
|
+
it is held as text.
|
|
456
|
+
|
|
457
|
+
Held so where `m` is an IRI template that is injective
|
|
458
|
+
(`TermMap.one_term_per_values`), so that equal values are one term and
|
|
459
|
+
one term has one set of values, and every value it reads is of a type
|
|
460
|
+
whose SQL equality is the equality of its text (`equal_as_text`): an
|
|
461
|
+
integer, or a string whose column compares bytes (a decimal's
|
|
462
|
+
`1.0 = 1.00`, a double's `-0.0 = 0.0`, a string under a collation that
|
|
463
|
+
is not binary or with PAD SPACE, 'a' = 'a ', are equal in SQL as two
|
|
464
|
+
texts). A string's text is its value made IRI-safe, which is injective
|
|
465
|
+
('%' is encoded too: 'a b' and 'a%20b' are two IRIs). Joins, DISTINCT
|
|
466
|
+
and GROUP BY then compare the values, between cells of one template
|
|
467
|
+
shape and one tuple of datatypes only (`_held_alike`); the text is
|
|
468
|
+
built only where text is read (`lower.lexical`)."""
|
|
469
|
+
if m.term_type != IRI or m.template is None:
|
|
470
|
+
return None
|
|
471
|
+
columns = []
|
|
472
|
+
for run, is_column in m.template:
|
|
473
|
+
if is_column and run not in columns:
|
|
474
|
+
columns.append(run)
|
|
475
|
+
if not columns:
|
|
476
|
+
return None
|
|
477
|
+
dts = []
|
|
478
|
+
for c in columns:
|
|
479
|
+
slot = args.get(c)
|
|
480
|
+
if slot is None or not equal_as_text(slot):
|
|
481
|
+
return None
|
|
482
|
+
dts.append(slot.uuid or slot.native) # a uuid column: held as the column itself, named by its reader
|
|
483
|
+
if not injective_over(m, args):
|
|
484
|
+
return None
|
|
485
|
+
return tuple(dts)
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def injective_over(m: TermMap, args: Mapping[str, Slot]) -> bool:
|
|
489
|
+
"""`m.one_term_per_values()` for the values it reads here (`args`, its
|
|
490
|
+
template columns' slots): where every one is an integer, a separator
|
|
491
|
+
starting with a non-digit is enough to cut them apart
|
|
492
|
+
(`{l_orderkey}_{l_linenumber}`); a string value needs an IRI-unsafe
|
|
493
|
+
one. A key of the values is then a key of the terms (`facts`, R3)."""
|
|
494
|
+
if m.one_term_per_values():
|
|
495
|
+
return True
|
|
496
|
+
if m.template is None:
|
|
497
|
+
return False
|
|
498
|
+
columns = {run for run, is_column in m.template if is_column}
|
|
499
|
+
slots = [args.get(c) for c in columns]
|
|
500
|
+
return all(s is not None and not s.terms and s.native in _EQUAL_AS_TEXT and s.native != xsd.STRING
|
|
501
|
+
for s in slots) and m.one_term_per_values(integer_values=True)
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def list_slot(element: Optional[str], null: Bound) -> Slot:
|
|
505
|
+
"""A Cypher list of values of `element` (None: the empty list's, of no type), elements possibly null."""
|
|
506
|
+
if element is None:
|
|
507
|
+
return Slot(kinds=frozenset({LIST}), bound=Bound.NEVER, null=null)
|
|
508
|
+
# No `native`: a list is not its element type (a list of booleans is no boolean).
|
|
509
|
+
return Slot(kinds=frozenset({LIST}), datatypes=frozenset({element}), bound=Bound.NEVER, null=null)
|