prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/schema/xsd.py
ADDED
|
@@ -0,0 +1,569 @@
|
|
|
1
|
+
"""XSD datatypes: the one rule for literal values everywhere.
|
|
2
|
+
|
|
3
|
+
Every layer that meets an RDF literal asks the same questions: what value a
|
|
4
|
+
lexical form stands for under a datatype, whether it is in the datatype's
|
|
5
|
+
lexical space at all, which datatype a value has, how to write a value back
|
|
6
|
+
as text or as an rdflib Literal, and which column type holds a datatype.
|
|
7
|
+
They used to answer each on their own (the SPARQL parser, the Python and
|
|
8
|
+
SQL DATATYPE, STRDT, union markers, CONSTRUCT, Update,
|
|
9
|
+
load-ttl, infer-schema), and they disagreed: a float was xsd:decimal in
|
|
10
|
+
one and xsd:double in another, and a datatype none of them knew was text,
|
|
11
|
+
so "1"^^xsd:time equalled "1".
|
|
12
|
+
|
|
13
|
+
Values. A datatype ProGraph has a Python value for (the integers, decimal,
|
|
14
|
+
double, float, boolean, string, date, dateTime) parses to that value. When
|
|
15
|
+
the datatype is not the value's default (xsd:int for an int, xsd:decimal
|
|
16
|
+
for a float), the value is a `TypedInt` or `TypedFloat`: the same number,
|
|
17
|
+
carrying the datatype it was written with, so DATATYPE answers it and
|
|
18
|
+
arithmetic and comparisons are unchanged. Any other datatype (xsd:time,
|
|
19
|
+
gYear, duration, anyURI, a custom one) has no value but its text: a
|
|
20
|
+
`Lexical`, which equals only a Lexical of the same text and datatype, never
|
|
21
|
+
a plain string.
|
|
22
|
+
|
|
23
|
+
Default datatypes of plain values: bool is xsd:boolean, int xsd:integer,
|
|
24
|
+
float xsd:double (a DOUBLE column holds doubles), Decimal xsd:decimal, str
|
|
25
|
+
xsd:string, datetime xsd:dateTime, date xsd:date.
|
|
26
|
+
|
|
27
|
+
Lost on the way, as before: language tags (prograph.schema.literals) and a
|
|
28
|
+
datetime's offset where a column normalizes it.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
from prograph.rules import home, module_home
|
|
34
|
+
import datetime as _dt
|
|
35
|
+
import decimal as _decimal
|
|
36
|
+
import math
|
|
37
|
+
import re
|
|
38
|
+
from decimal import Decimal
|
|
39
|
+
from typing import Any, Callable, Dict, Optional
|
|
40
|
+
|
|
41
|
+
module_home("xsd-datatypes", "XSD datatypes",
|
|
42
|
+
terms=("datatype", "lexical form", "canonical", "XSD", "sql_type"))
|
|
43
|
+
|
|
44
|
+
XSD = "http://www.w3.org/2001/XMLSchema#"
|
|
45
|
+
RDF_LANGSTRING = "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString"
|
|
46
|
+
|
|
47
|
+
INTEGER = XSD + "integer"
|
|
48
|
+
DECIMAL = XSD + "decimal"
|
|
49
|
+
DOUBLE = XSD + "double"
|
|
50
|
+
FLOAT = XSD + "float"
|
|
51
|
+
BOOLEAN = XSD + "boolean"
|
|
52
|
+
HEX_BINARY = XSD + "hexBinary"
|
|
53
|
+
STRING = XSD + "string"
|
|
54
|
+
DATE = XSD + "date"
|
|
55
|
+
DATETIME = XSD + "dateTime"
|
|
56
|
+
TIME = XSD + "time"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class TypedInt(int):
|
|
60
|
+
"""An integer written with a datatype other than xsd:integer
|
|
61
|
+
("1"^^xsd:int): the number, and the datatype it was written with."""
|
|
62
|
+
|
|
63
|
+
def __new__(cls, value: int, datatype: str):
|
|
64
|
+
obj = int.__new__(cls, value)
|
|
65
|
+
obj.datatype = datatype
|
|
66
|
+
return obj
|
|
67
|
+
|
|
68
|
+
def __getnewargs__(self):
|
|
69
|
+
return (int(self), self.datatype)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class TypedFloat(float):
|
|
73
|
+
"""A number written as xsd:decimal or xsd:float (1.5, "1.5"^^xsd:float):
|
|
74
|
+
the float, and the datatype it was written with."""
|
|
75
|
+
|
|
76
|
+
def __new__(cls, value: float, datatype: str):
|
|
77
|
+
obj = float.__new__(cls, value)
|
|
78
|
+
obj.datatype = datatype
|
|
79
|
+
return obj
|
|
80
|
+
|
|
81
|
+
def __getnewargs__(self):
|
|
82
|
+
return (float(self), self.datatype)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class Lexical(str):
|
|
86
|
+
"""A literal of a datatype with no Python value ("13:00:00"^^xsd:time):
|
|
87
|
+
its text, and its datatype. Equal only to a Lexical with the same text
|
|
88
|
+
and datatype; a plain string with the same text is another term."""
|
|
89
|
+
|
|
90
|
+
def __new__(cls, text: str, datatype: str):
|
|
91
|
+
obj = str.__new__(cls, text)
|
|
92
|
+
obj.datatype = datatype
|
|
93
|
+
return obj
|
|
94
|
+
|
|
95
|
+
def __getnewargs__(self):
|
|
96
|
+
return (str(self), self.datatype)
|
|
97
|
+
|
|
98
|
+
def __eq__(self, other: Any) -> bool:
|
|
99
|
+
return (isinstance(other, Lexical) and str.__eq__(self, other)
|
|
100
|
+
and self.datatype == other.datatype)
|
|
101
|
+
|
|
102
|
+
def __ne__(self, other: Any) -> bool:
|
|
103
|
+
return not self.__eq__(other)
|
|
104
|
+
|
|
105
|
+
__hash__ = str.__hash__
|
|
106
|
+
|
|
107
|
+
def __repr__(self) -> str:
|
|
108
|
+
return f"{str(self)!r}^^<{self.datatype}>"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# --- lexical spaces ----------------------------------------------------------
|
|
112
|
+
|
|
113
|
+
_INT = re.compile(r"[+-]?[0-9]+")
|
|
114
|
+
_DECIMAL = re.compile(r"[+-]?([0-9]+(\.[0-9]*)?|\.[0-9]+)")
|
|
115
|
+
_DOUBLE = re.compile(r"([+-]?([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][+-]?[0-9]+)?|[+-]?INF|NaN)")
|
|
116
|
+
_TZ = r"(Z|[+-][0-9]{2}:[0-9]{2})?"
|
|
117
|
+
_DATE = re.compile(r"-?[0-9]{4,}-[0-9]{2}-[0-9]{2}" + _TZ)
|
|
118
|
+
_DATETIME = re.compile(r"-?[0-9]{4,}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?" + _TZ)
|
|
119
|
+
|
|
120
|
+
# Integer datatypes and their value ranges (None: unbounded).
|
|
121
|
+
_INTEGER_RANGES: Dict[str, tuple] = {
|
|
122
|
+
"integer": (None, None), "nonNegativeInteger": (0, None), "positiveInteger": (1, None),
|
|
123
|
+
"nonPositiveInteger": (None, 0), "negativeInteger": (None, -1),
|
|
124
|
+
"long": (-2**63, 2**63 - 1), "int": (-2**31, 2**31 - 1),
|
|
125
|
+
"short": (-2**15, 2**15 - 1), "byte": (-128, 127),
|
|
126
|
+
"unsignedLong": (0, 2**64 - 1), "unsignedInt": (0, 2**32 - 1),
|
|
127
|
+
"unsignedShort": (0, 2**16 - 1), "unsignedByte": (0, 255),
|
|
128
|
+
}
|
|
129
|
+
INTEGER_DATATYPES = frozenset(XSD + n for n in _INTEGER_RANGES)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def integer_range(datatype: str) -> tuple:
|
|
133
|
+
"""An integer datatype's value range (low, high), None for unbounded (XSD 1.1 3.4)."""
|
|
134
|
+
return _INTEGER_RANGES[datatype[len(XSD):]]
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def lexical_pattern(datatype: str) -> Optional[str]:
|
|
138
|
+
"""The regular expression an integer, decimal, double, float or boolean lexical form matches
|
|
139
|
+
whole (XSD 1.1 3.3, 3.4; `parse`'s), for SQL's regex match; None for another datatype."""
|
|
140
|
+
if datatype in INTEGER_DATATYPES:
|
|
141
|
+
return _INT.pattern
|
|
142
|
+
if datatype == DECIMAL:
|
|
143
|
+
return _DECIMAL.pattern
|
|
144
|
+
if datatype in (DOUBLE, FLOAT):
|
|
145
|
+
return _DOUBLE.pattern
|
|
146
|
+
if datatype == BOOLEAN:
|
|
147
|
+
return "true|false|1|0"
|
|
148
|
+
return None
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _parse_integer(text: str, datatype: str) -> int:
|
|
152
|
+
if not _INT.fullmatch(text):
|
|
153
|
+
raise ValueError(f"{text!r} is not an integer")
|
|
154
|
+
value = int(text)
|
|
155
|
+
low, high = _INTEGER_RANGES[datatype[len(XSD):]]
|
|
156
|
+
if (low is not None and value < low) or (high is not None and value > high):
|
|
157
|
+
raise ValueError(f"{value} is out of the range of {datatype}")
|
|
158
|
+
return value if datatype == INTEGER else TypedInt(value, datatype)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _parse_decimal(text: str, datatype: str) -> Decimal:
|
|
162
|
+
# An exact number: 0.06 - 0.01 is 0.05, not the doubles' 0.0499...
|
|
163
|
+
# (`numeric` below).
|
|
164
|
+
if not _DECIMAL.fullmatch(text):
|
|
165
|
+
raise ValueError(f"{text!r} is not a decimal")
|
|
166
|
+
return Decimal(text)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _parse_double(text: str, datatype: str) -> float:
|
|
170
|
+
if not _DOUBLE.fullmatch(text):
|
|
171
|
+
raise ValueError(f"{text!r} is not a {datatype[len(XSD):]}")
|
|
172
|
+
value = float(text.replace("INF", "inf"))
|
|
173
|
+
return value if datatype == DOUBLE else TypedFloat(value, datatype)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _parse_boolean(text: str, datatype: str) -> bool:
|
|
177
|
+
if text not in ("true", "false", "1", "0"):
|
|
178
|
+
raise ValueError(f"{text!r} is not a boolean")
|
|
179
|
+
return text in ("true", "1")
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _parse_date(text: str, datatype: str) -> _dt.date:
|
|
183
|
+
if not _DATE.fullmatch(text):
|
|
184
|
+
raise ValueError(f"{text!r} is not a date")
|
|
185
|
+
# A date's offset has no place in a Python date; it is not kept.
|
|
186
|
+
return _dt.date.fromisoformat(re.sub(_TZ + "$", "", text, count=1))
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _parse_datetime(text: str, datatype: str) -> _dt.datetime:
|
|
190
|
+
if not _DATETIME.fullmatch(text):
|
|
191
|
+
raise ValueError(f"{text!r} is not a dateTime")
|
|
192
|
+
# Python before 3.11 reads 3 or 6 places of a second only: the fraction as six (padded; past six, cut, as
|
|
193
|
+
# Python 3.11's reader does).
|
|
194
|
+
text = re.sub(r"\.(\d+)", lambda m: "." + (m.group(1) + "000000")[:6], text, count=1)
|
|
195
|
+
return _dt.datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
_PARSERS: Dict[str, Callable[[str, str], Any]] = {
|
|
199
|
+
**{iri: _parse_integer for iri in INTEGER_DATATYPES},
|
|
200
|
+
DECIMAL: _parse_decimal,
|
|
201
|
+
DOUBLE: _parse_double,
|
|
202
|
+
FLOAT: _parse_double,
|
|
203
|
+
BOOLEAN: _parse_boolean,
|
|
204
|
+
STRING: lambda text, datatype: text,
|
|
205
|
+
DATE: _parse_date,
|
|
206
|
+
DATETIME: _parse_datetime,
|
|
207
|
+
XSD + "dateTimeStamp": _parse_datetime,
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
# Datatypes with a Python value, each to its value kind
|
|
211
|
+
# (prograph.schema.types).
|
|
212
|
+
_KINDS: Dict[str, str] = {
|
|
213
|
+
**{iri: "integer" for iri in INTEGER_DATATYPES},
|
|
214
|
+
DECIMAL: "float", DOUBLE: "float", FLOAT: "float",
|
|
215
|
+
BOOLEAN: "boolean", STRING: "string", DATE: "date", DATETIME: "datetime",
|
|
216
|
+
XSD + "dateTimeStamp": "datetime",
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
# The datatype of each value kind, for a value that carries none (a column's).
|
|
220
|
+
_DATATYPE_OF_KIND: Dict[str, str] = {
|
|
221
|
+
"integer": INTEGER, "float": DOUBLE, "string": STRING, "boolean": BOOLEAN,
|
|
222
|
+
"date": DATE, "datetime": DATETIME, "binary": HEX_BINARY,
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def is_known(datatype: str) -> bool:
|
|
227
|
+
"""True when ProGraph has a Python value for the datatype."""
|
|
228
|
+
return datatype in _PARSERS
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def parse(text: str, datatype: Optional[str]) -> Any:
|
|
232
|
+
"""The value `text` stands for under `datatype`: an int, float, bool,
|
|
233
|
+
str, date or datetime (typed when the datatype is not the value's
|
|
234
|
+
default), or a Lexical for a datatype with no Python value. Raises
|
|
235
|
+
ValueError for text outside a known datatype's lexical space. No
|
|
236
|
+
datatype (or xsd:string, or rdf:langString) is the text itself."""
|
|
237
|
+
if datatype is None or datatype in (STRING, RDF_LANGSTRING):
|
|
238
|
+
return text
|
|
239
|
+
parser = _PARSERS.get(datatype)
|
|
240
|
+
if parser is None:
|
|
241
|
+
return Lexical(text, datatype)
|
|
242
|
+
return parser(text.strip() if datatype != STRING else text, datatype)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def parse_or_text(text: Any, datatype: Optional[str]) -> Any:
|
|
246
|
+
"""`parse`, but text outside the lexical space stays text (a value read
|
|
247
|
+
back from a column that held it as text)."""
|
|
248
|
+
if not isinstance(text, str):
|
|
249
|
+
return text
|
|
250
|
+
try:
|
|
251
|
+
return parse(text, datatype)
|
|
252
|
+
except ValueError:
|
|
253
|
+
return text
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
# --- numeric operations (SPARQL 1.1 17.3, XPath op:numeric-*) ---------------
|
|
257
|
+
|
|
258
|
+
# The scale of an exact quotient: op:numeric-divide of two xsd:decimal (or
|
|
259
|
+
# two xsd:integer) values is an xsd:decimal, which XPath asks to be accurate
|
|
260
|
+
# to at least 18 digits. ProGraph's quotient is the exact one rounded half
|
|
261
|
+
# away from zero at the 18th digit after the point, a DECIMAL(38, 18): 20
|
|
262
|
+
# digits are left before the point, and a quotient past them is an error on
|
|
263
|
+
# the databases, as a DECIMAL past 38 digits is (the Python path keeps it).
|
|
264
|
+
# The SQL is each dialect's EXACT_QUOTIENT (prograph/plan/interpreter/
|
|
265
|
+
# dialect.py; docs/architecture/rules.md, "SPARQL numeric type promotion").
|
|
266
|
+
QUOTIENT_SCALE = 18
|
|
267
|
+
QUOTIENT_PRECISION = 38
|
|
268
|
+
|
|
269
|
+
# The numeric datatypes by rank: an operation's datatype is its operands'
|
|
270
|
+
# higher one (xsd:integer, then xsd:decimal, then xsd:double), except that
|
|
271
|
+
# two integers divide to a decimal. xsd:float is held as a double, and so
|
|
272
|
+
# its operations are xsd:double's.
|
|
273
|
+
_NUMERIC_RANK = {INTEGER: 0, DECIMAL: 1, FLOAT: 2, DOUBLE: 2}
|
|
274
|
+
_RANKED = (INTEGER, DECIMAL, DOUBLE)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
@home("numeric-operator-mapping", "SPARQL's numeric operator mapping",
|
|
278
|
+
terms=("arithmetic", "type promotion", "+", "-", "*", "/", "SUM"))
|
|
279
|
+
def numeric_datatype(op: str, left: Optional[str], right: Optional[str]) -> Optional[str]:
|
|
280
|
+
"""The datatype of `left op right` for two numeric datatypes (`op` one
|
|
281
|
+
of + - * /), SPARQL 1.1 17.3's operator mapping: xsd:integer for two
|
|
282
|
+
integers (an integer datatype derived from it counts as one), but a
|
|
283
|
+
quotient of two integers is an xsd:decimal; xsd:decimal where a decimal
|
|
284
|
+
meets an integer or a decimal; xsd:double where either is a float or a
|
|
285
|
+
double. None where either is not known to be a number."""
|
|
286
|
+
ranks = []
|
|
287
|
+
for datatype in (left, right):
|
|
288
|
+
if datatype in INTEGER_DATATYPES:
|
|
289
|
+
datatype = INTEGER
|
|
290
|
+
if datatype not in _NUMERIC_RANK:
|
|
291
|
+
return None
|
|
292
|
+
ranks.append(_NUMERIC_RANK[datatype])
|
|
293
|
+
rank = max(ranks)
|
|
294
|
+
if op == "/" and rank == 0:
|
|
295
|
+
rank = 1
|
|
296
|
+
return _RANKED[rank]
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _exact(value: Any) -> Decimal:
|
|
302
|
+
if isinstance(value, Decimal):
|
|
303
|
+
return value
|
|
304
|
+
if isinstance(value, float):
|
|
305
|
+
return Decimal(repr(float(value)))
|
|
306
|
+
return Decimal(int(value))
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
# A decimal quotient is rounded with every digit kept before it (`quotient`).
|
|
312
|
+
_EXACT = _decimal.Context(prec=_decimal.MAX_PREC, Emax=_decimal.MAX_EMAX,
|
|
313
|
+
Emin=_decimal.MIN_EMIN)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def quotient(left: Any, right: Any) -> Optional[Decimal]:
|
|
317
|
+
"""op:numeric-divide of two integers or decimals: the exact quotient
|
|
318
|
+
rounded half away from zero to QUOTIENT_SCALE digits after the point
|
|
319
|
+
(`QUOTIENT_SCALE`). None for a zero divisor, an error."""
|
|
320
|
+
a, b = _exact(left), _exact(right)
|
|
321
|
+
if b == 0:
|
|
322
|
+
return None
|
|
323
|
+
digits = max(a.adjusted() - b.adjusted(), 0) + QUOTIENT_SCALE + 8
|
|
324
|
+
with _decimal.localcontext(_decimal.Context(prec=digits, rounding=_decimal.ROUND_DOWN)):
|
|
325
|
+
q = a / b
|
|
326
|
+
return q.quantize(_QUOTIENT_UNIT, rounding=_decimal.ROUND_HALF_UP, context=_EXACT)
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
_QUOTIENT_UNIT = Decimal(1).scaleb(-QUOTIENT_SCALE)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
# --- casts (SPARQL 1.1 17.5) -------------------------------------------------
|
|
333
|
+
|
|
334
|
+
# The XSD constructor functions SPARQL names, xsd:T(?v). xsd:string is STR's
|
|
335
|
+
# lexical form (the parser writes it as STR); the others are `plan.ir.casts`'s.
|
|
336
|
+
CAST_DATATYPES = frozenset((STRING, FLOAT, DOUBLE, DECIMAL, INTEGER, DATETIME, BOOLEAN))
|
|
337
|
+
|
|
338
|
+
# A cast's lexical space of each target, for the SQL that tests text before
|
|
339
|
+
# it casts it: the patterns `parse` matches, whitespace trimmed first.
|
|
340
|
+
CAST_LEXICAL = {
|
|
341
|
+
INTEGER: _INT.pattern, DECIMAL: _DECIMAL.pattern, DOUBLE: _DOUBLE.pattern,
|
|
342
|
+
FLOAT: _DOUBLE.pattern, DATETIME: _DATETIME.pattern, BOOLEAN: "true|false|1|0",
|
|
343
|
+
}
|
|
344
|
+
# The lexical space of each datatype STRDT reads text as in SQL (a mapping's
|
|
345
|
+
# declared datatype): a cast's, and a date's without its
|
|
346
|
+
# zone, which a SQL DATE cast does not read.
|
|
347
|
+
TEXT_LEXICAL = {**CAST_LEXICAL, DATE: r"-?[0-9]{4,}-[0-9]{2}-[0-9]{2}"}
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def datatype_of(value: Any) -> Optional[str]:
|
|
353
|
+
"""The datatype of a value: the one it was written with, else its
|
|
354
|
+
type's default. None for a value that is no literal (None, a node)."""
|
|
355
|
+
declared = getattr(value, "datatype", None)
|
|
356
|
+
if isinstance(declared, str) and isinstance(value, (int, float, str)):
|
|
357
|
+
return declared
|
|
358
|
+
if value is None:
|
|
359
|
+
return None
|
|
360
|
+
if isinstance(value, bool):
|
|
361
|
+
return BOOLEAN
|
|
362
|
+
if isinstance(value, int):
|
|
363
|
+
return INTEGER
|
|
364
|
+
if isinstance(value, float):
|
|
365
|
+
return DOUBLE
|
|
366
|
+
if isinstance(value, Decimal):
|
|
367
|
+
return DECIMAL
|
|
368
|
+
if isinstance(value, str):
|
|
369
|
+
return STRING
|
|
370
|
+
if isinstance(value, _dt.datetime) or hasattr(value, "to_pydatetime"):
|
|
371
|
+
return DATETIME
|
|
372
|
+
if isinstance(value, _dt.date):
|
|
373
|
+
return DATE
|
|
374
|
+
if hasattr(value, "item"): # a numpy scalar
|
|
375
|
+
return datatype_of(value.item())
|
|
376
|
+
return None
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def datatype_of_kind(kind: Optional[str]) -> Optional[str]:
|
|
380
|
+
"""The datatype a value of a kind has (prograph.schema.types kinds)."""
|
|
381
|
+
return _DATATYPE_OF_KIND.get(kind) if kind else None
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def datatype_of_type(column_type: Any) -> Optional[str]:
|
|
385
|
+
"""The datatype a column's own type (prograph.schema.types) gives its
|
|
386
|
+
values: a DECIMAL's is xsd:decimal, a DOUBLE's xsd:double."""
|
|
387
|
+
kind = getattr(column_type, "kind", None)
|
|
388
|
+
if kind == "float" and getattr(column_type, "exact", False):
|
|
389
|
+
return DECIMAL
|
|
390
|
+
if kind == "time":
|
|
391
|
+
# A TIME column is xsd:time (R2RML 10.2); a TIME WITH TIME ZONE has no text here yet.
|
|
392
|
+
return None if getattr(column_type, "zoned", False) else TIME
|
|
393
|
+
return datatype_of_kind(kind)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def canonical_lexical(value: Any, datatype: str) -> str:
|
|
397
|
+
"""A value's canonical lexical form under `datatype`, the form of the
|
|
398
|
+
literal a column holding the value holds (the R2RML natural mapping,
|
|
399
|
+
R2RML 10.2), of a template's text for it (R2RML 7.3) and of every value
|
|
400
|
+
ProGraph computes: an xsd:double's or xsd:float's one digit before the
|
|
401
|
+
point, at least one after and an exponent ("1.0E0", "1.5E-3", "INF",
|
|
402
|
+
"NaN"); an xsd:decimal's `canonical_decimal` ("30", "1200.5", "0";
|
|
403
|
+
sparql-choices.md D3, decided 2026-11-13); every other datatype's is
|
|
404
|
+
`lexical`'s."""
|
|
405
|
+
if datatype in (DOUBLE, FLOAT) and isinstance(value, (int, float)) \
|
|
406
|
+
and not isinstance(value, bool):
|
|
407
|
+
x = float(value)
|
|
408
|
+
if not math.isfinite(x):
|
|
409
|
+
return lexical(x) # INF, -INF, NaN
|
|
410
|
+
if x == 0:
|
|
411
|
+
return "-0.0E0" if math.copysign(1.0, x) < 0 else "0.0E0"
|
|
412
|
+
sign, digits, exponent = Decimal(repr(x)).normalize().as_tuple()
|
|
413
|
+
text = "".join(map(str, digits))
|
|
414
|
+
power = exponent + len(text) - 1
|
|
415
|
+
mantissa = text[0] + "." + (text[1:] or "0")
|
|
416
|
+
return ("-" if sign else "") + mantissa + "E" + str(power)
|
|
417
|
+
if datatype == DECIMAL and isinstance(value, Decimal):
|
|
418
|
+
return canonical_decimal(value)
|
|
419
|
+
return lexical(value)
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
class NotADecimal(ValueError):
|
|
423
|
+
"""A decimal value outside xsd:decimal's value space (a NUMERIC 'NaN' or an infinity), which
|
|
424
|
+
has no xsd:decimal text: refused, as the SQL refuses it (`XSD_DECIMAL_TEXT`)."""
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def canonical_decimal(value: Decimal) -> str:
|
|
428
|
+
"""An xsd:decimal's canonical form, XML Schema 1.1 Part 2 3.3.3.2 (the edition R2RML 10.2
|
|
429
|
+
cites; R2RML 10.5 writes 42.0 as 42): no exponent, no leading zero before a nonzero digit,
|
|
430
|
+
no trailing zero after the point and no point where no fraction is left, '0' for zero (-0
|
|
431
|
+
too), '-' only before a negative: "1", "1.5", "-0.25". Exact at any number of digits (no
|
|
432
|
+
context rounds it, as `Decimal.normalize` would past 28). NaN and the infinities are no
|
|
433
|
+
xsd:decimal: `NotADecimal`."""
|
|
434
|
+
if not value.is_finite():
|
|
435
|
+
raise NotADecimal(f"{value} is no xsd:decimal value")
|
|
436
|
+
text = format(value, "f")
|
|
437
|
+
if "." in text:
|
|
438
|
+
text = text.rstrip("0").rstrip(".")
|
|
439
|
+
return "0" if text in ("0", "-0") else text
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
@home("value-text", "A value's text",
|
|
443
|
+
terms=("STR", "toString", "lexical form", "text of a value", "canonical decimal"))
|
|
444
|
+
def lexical(value: Any) -> str:
|
|
445
|
+
"""A value's lexical form (STR of a literal; RDF output)."""
|
|
446
|
+
if isinstance(value, bool):
|
|
447
|
+
return "true" if value else "false"
|
|
448
|
+
if isinstance(value, (_dt.date, _dt.datetime)) or hasattr(value, "isoformat"):
|
|
449
|
+
text = value.isoformat()
|
|
450
|
+
if ("T" in text or isinstance(value, _dt.time)) and "." in text:
|
|
451
|
+
# xsd:dateTime's (and xsd:time's) canonical form: no trailing zero in the
|
|
452
|
+
# fraction of a second, no dot when none is left (pandas writes every digit, `...T10:00:00.500000`).
|
|
453
|
+
head, _, rest = text.partition(".")
|
|
454
|
+
digits = re.match(r"\d*", rest).group(0)
|
|
455
|
+
fraction = digits.rstrip("0")
|
|
456
|
+
text = head + ("." + fraction if fraction else "") + rest[len(digits):]
|
|
457
|
+
if "T" in text and text.endswith("+00:00"):
|
|
458
|
+
# A zero offset is `Z`: XSD 1.1's canonical form (Part 2.3.7) and openCypher's (TCK Temporal2 [5]), as the SQL
|
|
459
|
+
# writes it. Another offset stays +hh:mm.
|
|
460
|
+
text = text[:-len("+00:00")] + "Z"
|
|
461
|
+
return text
|
|
462
|
+
if isinstance(value, float) and not isinstance(value, TypedFloat):
|
|
463
|
+
if math.isinf(value):
|
|
464
|
+
return "INF" if value > 0 else "-INF"
|
|
465
|
+
if math.isnan(value):
|
|
466
|
+
return "NaN"
|
|
467
|
+
if isinstance(value, Decimal) and value.is_finite():
|
|
468
|
+
# xsd:decimal's canonical form, XML Schema 1.1's (D3, 2026-11-13): a DECIMAL(5,2)
|
|
469
|
+
# column's 9.50 is "9.5", its 9.00 "9".
|
|
470
|
+
return canonical_decimal(value)
|
|
471
|
+
if isinstance(value, (TypedInt, TypedFloat)):
|
|
472
|
+
return str(int(value) if isinstance(value, TypedInt) else float(value))
|
|
473
|
+
return str(value)
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def to_rdflib(value: Any, datatype: Optional[str] = None):
|
|
477
|
+
"""A value as an rdflib Literal of its datatype (`datatype` when given,
|
|
478
|
+
a declared one, else the value's own). An xsd:string is a simple
|
|
479
|
+
literal, as RDF 1.1 has it."""
|
|
480
|
+
from rdflib import Literal, URIRef
|
|
481
|
+
datatype = datatype or datatype_of(value)
|
|
482
|
+
if datatype in (None, STRING):
|
|
483
|
+
return Literal(lexical(value) if not isinstance(value, str) else str(value))
|
|
484
|
+
return Literal(lexical(value), datatype=URIRef(datatype))
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def from_rdflib(lit: Any) -> Any:
|
|
488
|
+
"""An rdflib Literal as a value (`parse`); text outside its datatype's
|
|
489
|
+
lexical space stays text. A language tag is not kept."""
|
|
490
|
+
try:
|
|
491
|
+
from rdflib import Literal
|
|
492
|
+
except ImportError: # pragma: no cover - rdflib is the RDF extra
|
|
493
|
+
return lit
|
|
494
|
+
if not isinstance(lit, Literal):
|
|
495
|
+
return lit
|
|
496
|
+
return parse_or_text(str(lit), str(lit.datatype) if lit.datatype else None)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
# --- column types ------------------------------------------------------------
|
|
500
|
+
|
|
501
|
+
# xsd:integer is unbounded, so it takes the widest integer column (the type of
|
|
502
|
+
# xsd:long); only xsd:int, 32 bits by definition, takes the narrow one.
|
|
503
|
+
_SQL_TYPES: Dict[str, Dict[str, str]] = {
|
|
504
|
+
"postgresql": {
|
|
505
|
+
"string": "VARCHAR(255)", "integer": "BIGINT", "int": "INTEGER", "long": "BIGINT",
|
|
506
|
+
"decimal": "DECIMAL(18,4)", "float": "REAL", "double": "DOUBLE PRECISION",
|
|
507
|
+
"boolean": "BOOLEAN", "date": "DATE", "dateTime": "TIMESTAMP", "time": "TIME",
|
|
508
|
+
},
|
|
509
|
+
"spark": {
|
|
510
|
+
"string": "STRING", "integer": "BIGINT", "int": "INT", "long": "BIGINT",
|
|
511
|
+
"decimal": "DECIMAL(18,4)", "float": "FLOAT", "double": "DOUBLE",
|
|
512
|
+
"boolean": "BOOLEAN", "date": "DATE", "dateTime": "TIMESTAMP",
|
|
513
|
+
"time": "STRING", # Spark has no TIME type
|
|
514
|
+
},
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
# An integer type whose range passes signed 64 bits (xsd:unsignedLong) needs an
|
|
519
|
+
# exact decimal wide enough for 2**64 - 1 in every dialect.
|
|
520
|
+
_BEYOND_64_BITS = "DECIMAL(20,0)"
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def _integer_sql_type(low: Optional[int], high: Optional[int], types: Dict[str, str]) -> str:
|
|
524
|
+
"""The column type for an integer datatype of range [low, high], derived
|
|
525
|
+
from the range: the 32-bit type when the whole range fits it, the 64-bit
|
|
526
|
+
type for a wider or unbounded range, an exact decimal past 64 bits."""
|
|
527
|
+
if low is None or high is None:
|
|
528
|
+
return types["long"]
|
|
529
|
+
if -2**31 <= low and high <= 2**31 - 1:
|
|
530
|
+
return types["int"]
|
|
531
|
+
if -2**63 <= low and high <= 2**63 - 1:
|
|
532
|
+
return types["long"]
|
|
533
|
+
return _BEYOND_64_BITS
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def sql_type(datatype: Optional[str], dialect: str = "postgresql") -> str:
|
|
537
|
+
"""The column type that holds a datatype's values; text for one with no
|
|
538
|
+
column type of its own. An integer datatype's width follows its declared
|
|
539
|
+
range (`_INTEGER_RANGES`); xsd:dateTimeStamp is a timestamp."""
|
|
540
|
+
types = _SQL_TYPES["postgresql" if dialect == "postgresql" else "spark"]
|
|
541
|
+
default = types["string"]
|
|
542
|
+
if not datatype:
|
|
543
|
+
return default
|
|
544
|
+
local = str(datatype).rsplit("#", 1)[-1].rsplit("/", 1)[-1]
|
|
545
|
+
if local in types:
|
|
546
|
+
return types[local]
|
|
547
|
+
if local in _INTEGER_RANGES:
|
|
548
|
+
return _integer_sql_type(*_INTEGER_RANGES[local], types)
|
|
549
|
+
if _KINDS.get(XSD + local) == "datetime":
|
|
550
|
+
return types["dateTime"]
|
|
551
|
+
return default
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def column_datatype(kind: Optional[str], declared: Optional[str] = None) -> Optional[str]:
|
|
555
|
+
"""The datatype of a mapped column's values: the PredicateMapping's
|
|
556
|
+
declared one, else its type kind's."""
|
|
557
|
+
return declared or datatype_of_kind(kind)
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
__all__ = [
|
|
561
|
+
"XSD", "RDF_LANGSTRING", "INTEGER", "DECIMAL", "DOUBLE", "FLOAT", "BOOLEAN", "STRING",
|
|
562
|
+
"DATE", "DATETIME", "INTEGER_DATATYPES", "TypedInt", "TypedFloat", "Lexical",
|
|
563
|
+
"is_known", "parse", "parse_or_text", "datatype_of", "datatype_of_kind", "datatype_of_type",
|
|
564
|
+
"CAST_DATATYPES", "CAST_LEXICAL", "TEXT_LEXICAL",
|
|
565
|
+
"QUOTIENT_SCALE", "QUOTIENT_PRECISION", "numeric_datatype",
|
|
566
|
+
"quotient",
|
|
567
|
+
"lexical", "to_rdflib", "from_rdflib", "sql_type", "column_datatype",
|
|
568
|
+
]
|
|
569
|
+
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""SHACL validation for ProGraph.
|
|
2
|
+
|
|
3
|
+
Validates the RDF graph SPARQL answers against W3C SHACL shapes and returns a typed
|
|
4
|
+
:class:`ValidationReport` describing any violations.
|
|
5
|
+
|
|
6
|
+
Implementation strategy: wrap ``pyshacl`` rather than reinventing SHACL. pyshacl already passes the
|
|
7
|
+
W3C SHACL Core conformance suite; our value-add is the typed report shape.
|
|
8
|
+
|
|
9
|
+
Quick usage::
|
|
10
|
+
|
|
11
|
+
from prograph import ProGraph
|
|
12
|
+
|
|
13
|
+
engine = ProGraph(...)
|
|
14
|
+
report = engine.validate_shapes("shapes.ttl")
|
|
15
|
+
|
|
16
|
+
if not report.conforms:
|
|
17
|
+
for v in report.violations:
|
|
18
|
+
print(v.focus_node, v.message)
|
|
19
|
+
|
|
20
|
+
Or directly, over any rdflib graph (for example a CONSTRUCT result)::
|
|
21
|
+
|
|
22
|
+
from prograph.shacl import validate
|
|
23
|
+
|
|
24
|
+
report = validate(data_graph, shapes="shapes.ttl")
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from prograph.shacl.report import ValidationReport, ViolationDetail
|
|
28
|
+
from prograph.shacl.validator import validate
|
|
29
|
+
|
|
30
|
+
__all__ = ["validate", "ValidationReport", "ViolationDetail"]
|
prograph/shacl/dqv.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""A SHACL validation report as Data Quality Vocabulary measurements (https://www.w3.org/TR/vocab-dqv/).
|
|
2
|
+
|
|
3
|
+
DQV records a value of a metric computed on a dataset: a `dqv:QualityMeasurement` (`dqv:computedOn` the
|
|
4
|
+
dataset, `dqv:isMeasurementOf` a `dqv:Metric`, `dqv:value`), reached from the dataset by
|
|
5
|
+
`dqv:hasQualityMeasurement` (the inverse of `computedOn`). Two metrics are written, `shaclConforms`
|
|
6
|
+
(xsd:boolean) and `shaclViolations` (xsd:integer, the results of severity sh:Violation), both in the
|
|
7
|
+
dimension `validity`. DQV has no term for the report itself: its results stay a `sh:ValidationReport`
|
|
8
|
+
(`ValidationReport.raw_rdf`), which `include_report` adds to the graph and ties to each measurement with
|
|
9
|
+
`dcterms:source`.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from prograph.rules import module_home
|
|
17
|
+
|
|
18
|
+
module_home("quality-measurements", "A SHACL validation report as DQV measurements of a dataset",
|
|
19
|
+
terms=("DQV", "dqv:QualityMeasurement", "hasQualityMeasurement", "quality measurement", "validation report"))
|
|
20
|
+
|
|
21
|
+
DQV = "http://www.w3.org/ns/dqv#"
|
|
22
|
+
METRIC_BASE = "urn:prograph:metric:"
|
|
23
|
+
DIMENSION = "urn:prograph:dimension:validity"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def add_measurements(g: Any, dataset: Any, report: Any, *, include_report: bool = False) -> None:
|
|
27
|
+
"""Add `report`'s measurements of `dataset` (a node of `g`) to the rdflib Graph `g`."""
|
|
28
|
+
from rdflib import BNode, Literal, Namespace, URIRef
|
|
29
|
+
from rdflib.namespace import DCTERMS, RDF, XSD
|
|
30
|
+
dqv = Namespace(DQV)
|
|
31
|
+
g.bind("dqv", dqv)
|
|
32
|
+
dimension = URIRef(DIMENSION)
|
|
33
|
+
g.add((dimension, RDF.type, dqv.Dimension))
|
|
34
|
+
source = None
|
|
35
|
+
if include_report and getattr(report, "raw_rdf", None):
|
|
36
|
+
from rdflib import Graph
|
|
37
|
+
reported = Graph().parse(data=report.raw_rdf, format="turtle")
|
|
38
|
+
for triple in reported:
|
|
39
|
+
g.add(triple)
|
|
40
|
+
source = next(reported.subjects(RDF.type, Namespace("http://www.w3.org/ns/shacl#").ValidationReport), None)
|
|
41
|
+
for name, datatype, value in (("shaclConforms", XSD.boolean, bool(report.conforms)),
|
|
42
|
+
("shaclViolations", XSD.integer, int(_violations(report)))):
|
|
43
|
+
metric = URIRef(METRIC_BASE + name)
|
|
44
|
+
g.add((metric, RDF.type, dqv.Metric))
|
|
45
|
+
g.add((metric, dqv.expectedDataType, datatype))
|
|
46
|
+
g.add((metric, dqv.inDimension, dimension))
|
|
47
|
+
m = BNode()
|
|
48
|
+
g.add((m, RDF.type, dqv.QualityMeasurement))
|
|
49
|
+
g.add((m, dqv.computedOn, dataset))
|
|
50
|
+
g.add((m, dqv.isMeasurementOf, metric))
|
|
51
|
+
g.add((m, dqv.value, Literal(value, datatype=datatype)))
|
|
52
|
+
g.add((dataset, dqv.hasQualityMeasurement, m))
|
|
53
|
+
if source is not None:
|
|
54
|
+
g.add((m, DCTERMS.source, source))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _violations(report: Any) -> int:
|
|
58
|
+
return sum(1 for v in report.violations if v.severity == "Violation")
|