prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
"""An R2RML mapping as source maps.
|
|
2
|
+
|
|
3
|
+
`compile_r2rml(mapping)` gives the `SourceMaps` of a parsed R2RML mapping
|
|
4
|
+
(`prograph.r2rml.model`) straight from its triples maps, with no
|
|
5
|
+
property-graph label: the engine's one model of an R2RML file (docs/architecture/r2rml-without-the-bridge.md; `r2rml.schema.R2RMLSchema`
|
|
6
|
+
holds it). An R2RML mapping has no property-graph view in 1.0.
|
|
7
|
+
|
|
8
|
+
What each construct becomes (R2RML 6 to 11):
|
|
9
|
+
|
|
10
|
+
- A triples map is one `NodeMap`, its id the map's IRI: its logical table
|
|
11
|
+
the `Source` (rr:tableName, or rr:sqlQuery as a query), its subject map
|
|
12
|
+
the subject `TermMap` with its term type (an rr:BlankNode subject stays a
|
|
13
|
+
blank node), its key the subject's columns with
|
|
14
|
+
`stated` False (a template states identity, not uniqueness), and every
|
|
15
|
+
rr:class its classes, with each constant rdf:type object
|
|
16
|
+
(rr:predicate rdf:type; rr:object C) of a predicate-object map with no
|
|
17
|
+
graph map of its own: the same triples. An rdf:type object made from
|
|
18
|
+
the row, or one with graph maps of its own, stays an edge map.
|
|
19
|
+
- A literal object (column, template or constant) is a `PredicateObject` of
|
|
20
|
+
the node map, keeping rr:datatype and rr:language, on a map with no class
|
|
21
|
+
too.
|
|
22
|
+
- An IRI or blank-node object is an `EdgeMap` from the map's subject to the
|
|
23
|
+
object's own term (`EndRef` with no map and `exists` False): no label is
|
|
24
|
+
invented for the objects (the bridge's `_object` labels).
|
|
25
|
+
- A referencing object map (rr:parentTriplesMap) is an `EdgeMap` from the
|
|
26
|
+
child's subject to the parent's over R2RML 8's joint query, held as a
|
|
27
|
+
`JointQuery` (child, parent, conditions), never as SQL text.
|
|
28
|
+
- Graph maps are IRI term maps on each triple (R2RML 11.1: a class triple
|
|
29
|
+
in the subject map's graphs, any other in the subject map's and its
|
|
30
|
+
predicate-object map's together; rr:defaultGraph the default graph).
|
|
31
|
+
|
|
32
|
+
Every term map is `LEXICAL_R2RML`: a value becomes text by its natural RDF
|
|
33
|
+
lexical form (R2RML 10.2), IRI-safe in an IRI template (R2RML 7.3), a
|
|
34
|
+
relative IRI resolved against the base IRI given here.
|
|
35
|
+
|
|
36
|
+
A mapping R2RML calls invalid raises :class:`R2RMLMappingError` here, as
|
|
37
|
+
far as the parsed document shows it (no subject map or two, a subject or
|
|
38
|
+
graph of literal term type, an unknown rr:sqlVersion, an ill-formed
|
|
39
|
+
language tag or template, a datatype beside a language, a parent map that
|
|
40
|
+
is not there, a referencing object map across two logical tables with no
|
|
41
|
+
join condition). What only the database shows (a table or column that is
|
|
42
|
+
not there, a query that does not run, a column named twice in a query's
|
|
43
|
+
result) is the database's to report when the maps are read. A data error
|
|
44
|
+
(R2RML 11: an IRI term map giving no valid IRI, a datatype override giving
|
|
45
|
+
an ill-typed literal) depends on the rows: whatever generates triples from
|
|
46
|
+
these maps raises :class:`R2RMLDataError` for it, never drops the triple
|
|
47
|
+
silently.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
from __future__ import annotations
|
|
51
|
+
|
|
52
|
+
import hashlib
|
|
53
|
+
import re
|
|
54
|
+
from typing import Dict, List, Optional, Tuple
|
|
55
|
+
|
|
56
|
+
from prograph.r2rml.model import (
|
|
57
|
+
GraphMap,
|
|
58
|
+
ObjectMap,
|
|
59
|
+
PredicateMap,
|
|
60
|
+
R2RMLMapping,
|
|
61
|
+
SubjectMap,
|
|
62
|
+
TriplesMap,
|
|
63
|
+
)
|
|
64
|
+
from prograph.schema.iri import DEFAULT_BASE_IRI, TemplateSyntaxError, template_parts
|
|
65
|
+
from prograph.schema.source_map import (
|
|
66
|
+
BLANK,
|
|
67
|
+
IRI,
|
|
68
|
+
LEXICAL_R2RML,
|
|
69
|
+
LITERAL,
|
|
70
|
+
EdgeMap,
|
|
71
|
+
EndRef,
|
|
72
|
+
JointQuery,
|
|
73
|
+
Key,
|
|
74
|
+
NodeMap,
|
|
75
|
+
PredicateObject,
|
|
76
|
+
Settings,
|
|
77
|
+
Source,
|
|
78
|
+
SourceMaps,
|
|
79
|
+
TermMap,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
RR = "http://www.w3.org/ns/r2rml#"
|
|
83
|
+
RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
|
|
84
|
+
|
|
85
|
+
# R2RML 5.2: rr:SQL2008, and the identifiers the RDB2RDF Working Group
|
|
86
|
+
# lists (https://www.w3.org/2001/sw/wiki/RDB2RDF/SQL_Version_IRIs).
|
|
87
|
+
SQL_VERSIONS = frozenset(RR + name for name in (
|
|
88
|
+
"SQL2008", "Oracle", "MySQL", "MSSQLServer", "HSQLDB", "PostgreSQL", "DB2",
|
|
89
|
+
"Informix", "Ingres", "Progress", "SybaseASE", "SybaseSQLAnywhere", "Virtuoso",
|
|
90
|
+
"Firebird"))
|
|
91
|
+
|
|
92
|
+
# A language tag (R2RML 7.6: rr:language is a BCP 47 tag): a language of two
|
|
93
|
+
# or three letters with up to three extended subtags, or one of four to
|
|
94
|
+
# eight letters is held to the registry, which has none: such a tag
|
|
95
|
+
# ("english") is refused. Then script, region, variants, extensions and a
|
|
96
|
+
# private use part; or a private use tag alone (x-...), or a grandfathered
|
|
97
|
+
# i- tag.
|
|
98
|
+
_LANGUAGE_TAG = re.compile(
|
|
99
|
+
r"^(?:(?:[a-z]{2,3}(?:-[a-z]{3}){0,3})"
|
|
100
|
+
r"(?:-[a-z]{4})?"
|
|
101
|
+
r"(?:-(?:[a-z]{2}|[0-9]{3}))?"
|
|
102
|
+
r"(?:-(?:[a-z0-9]{5,8}|[0-9][a-z0-9]{3}))*"
|
|
103
|
+
r"(?:-[0-9a-wy-z](?:-[a-z0-9]{2,8})+)*"
|
|
104
|
+
r"(?:-x(?:-[a-z0-9]{1,8})+)?"
|
|
105
|
+
r"|x(?:-[a-z0-9]{1,8})+"
|
|
106
|
+
r"|i-[a-z]{2,8})$",
|
|
107
|
+
re.IGNORECASE)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class R2RMLMappingError(ValueError):
|
|
111
|
+
"""An R2RML mapping that is not valid (R2RML: "an R2RML processor ...
|
|
112
|
+
MUST report" it), found by compiling it."""
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class R2RMLDataError(ValueError):
|
|
116
|
+
"""A data error (R2RML 11): a row whose term an R2RML processor cannot
|
|
117
|
+
make, an IRI term map giving no valid IRI or a datatype override giving
|
|
118
|
+
an ill-typed literal. Raised by what generates triples from source maps
|
|
119
|
+
compiled from R2RML; the triple is never dropped silently."""
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def compile_r2rml(mapping: R2RMLMapping, base_iri: Optional[str] = None) -> SourceMaps:
|
|
123
|
+
"""The source maps of an R2RML mapping.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
mapping: The parsed mapping (`R2RMLParser`).
|
|
127
|
+
base_iri: The base IRI a relative IRI is resolved against (R2RML
|
|
128
|
+
7.3, 11: the processor's to give); ProGraph's default instance
|
|
129
|
+
namespace where none is given.
|
|
130
|
+
|
|
131
|
+
Raises:
|
|
132
|
+
R2RMLMappingError: The mapping is not valid R2RML.
|
|
133
|
+
"""
|
|
134
|
+
by_uri: Dict[str, TriplesMap] = {}
|
|
135
|
+
for tm in mapping.triples_maps:
|
|
136
|
+
if tm.uri in by_uri:
|
|
137
|
+
raise R2RMLMappingError(f"two triples maps are named {tm.uri}")
|
|
138
|
+
by_uri[tm.uri] = tm
|
|
139
|
+
nodes: List[NodeMap] = []
|
|
140
|
+
edges: List[EdgeMap] = []
|
|
141
|
+
for tm in mapping.triples_maps:
|
|
142
|
+
node, its_edges = _triples_map(tm, by_uri, len(nodes), len(edges))
|
|
143
|
+
nodes.append(node)
|
|
144
|
+
edges.extend(its_edges)
|
|
145
|
+
settings = Settings(name=None, base_iri=base_iri or DEFAULT_BASE_IRI,
|
|
146
|
+
default_catalog=None, default_schema=None,
|
|
147
|
+
table_qualifications=(),
|
|
148
|
+
prefixes=tuple(sorted(mapping.prefixes.items())))
|
|
149
|
+
return SourceMaps(settings=settings, nodes=tuple(nodes), edges=tuple(edges))
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# -- the parts of one triples map ------------------------------------------
|
|
153
|
+
|
|
154
|
+
def _regular_columns(tm: TriplesMap, by_uri: Dict[str, TriplesMap]) -> frozenset:
|
|
155
|
+
"""The columns of `tm`'s logical table that some map of the mapping names undelimited (its
|
|
156
|
+
own term maps and rr:child, another map's rr:parent into it): the database folds them
|
|
157
|
+
(`Dialect.fold_identifier`). One set for every map of the table, so that they
|
|
158
|
+
read one source. A column named both ways in one table is read folded (on PostgreSQL the two
|
|
159
|
+
spellings are two columns, which one source cannot tell apart; not built)."""
|
|
160
|
+
regular = set()
|
|
161
|
+
for other in by_uri.values():
|
|
162
|
+
if _same_logical_table(other, tm):
|
|
163
|
+
regular |= other.regular_columns
|
|
164
|
+
for po in other.predicate_object_maps:
|
|
165
|
+
for om in po.object_maps:
|
|
166
|
+
parent = by_uri.get(om.parent_triples_map) if om.parent_triples_map else None
|
|
167
|
+
if parent is not None and _same_logical_table(parent, tm):
|
|
168
|
+
regular |= {jc.parent for jc in om.join_conditions if jc.parent_regular}
|
|
169
|
+
return frozenset(regular)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _source(tm: TriplesMap, by_uri: Dict[str, TriplesMap]) -> Source:
|
|
173
|
+
"""A triples map's logical table (R2RML 5): a table by its name, or a
|
|
174
|
+
query (an R2RML view) named by its text, so that two maps of one query
|
|
175
|
+
read one source and their subjects one identity. A table is read by its
|
|
176
|
+
name as written (a delimited part in its quotes, `Dialect.table_name`)."""
|
|
177
|
+
lt = tm.logical_table
|
|
178
|
+
if lt.table_name and lt.sql_query:
|
|
179
|
+
raise R2RMLMappingError(
|
|
180
|
+
f"{tm.uri}: a logical table has rr:tableName or rr:sqlQuery, not both")
|
|
181
|
+
for version in lt.sql_versions:
|
|
182
|
+
if version not in SQL_VERSIONS:
|
|
183
|
+
raise R2RMLMappingError(f"{tm.uri}: {version} is no SQL version identifier")
|
|
184
|
+
regular = _regular_columns(tm, by_uri)
|
|
185
|
+
if lt.table_name:
|
|
186
|
+
return Source(table=lt.table_name, qualified=lt.written or lt.table_name, regular_columns=regular)
|
|
187
|
+
name = "__r2rml_q_" + hashlib.sha1(lt.sql_query.encode("utf-8")).hexdigest()[:12]
|
|
188
|
+
return Source(table=name, qualified=name, query=lt.sql_query, regular_columns=regular)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _parts(where: str, template: str) -> Tuple[Tuple[str, bool], ...]:
|
|
192
|
+
try:
|
|
193
|
+
parts = tuple(template_parts(template, escapes=True))
|
|
194
|
+
except TemplateSyntaxError as error:
|
|
195
|
+
raise R2RMLMappingError(f"{where}: {error}") from None
|
|
196
|
+
if not parts:
|
|
197
|
+
raise R2RMLMappingError(f"{where}: an empty template")
|
|
198
|
+
return parts
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _term(where: str, term_type: str, template: Optional[str], column: Optional[str],
|
|
202
|
+
constant: Optional[str], datatype: Optional[str] = None,
|
|
203
|
+
language: Optional[str] = None) -> TermMap:
|
|
204
|
+
"""One term map (R2RML 7): exactly one of a template, a column and a
|
|
205
|
+
constant, of the term type given."""
|
|
206
|
+
given = [x is not None for x in (template, column, constant)]
|
|
207
|
+
if sum(given) != 1:
|
|
208
|
+
raise R2RMLMappingError(
|
|
209
|
+
f"{where}: a term map has exactly one of rr:template, rr:column and "
|
|
210
|
+
f"rr:constant ({sum(given)} given)")
|
|
211
|
+
if term_type == BLANK and constant is not None:
|
|
212
|
+
raise R2RMLMappingError(f"{where}: a constant term map is no blank node")
|
|
213
|
+
if term_type != LITERAL and (datatype or language):
|
|
214
|
+
raise R2RMLMappingError(f"{where}: rr:datatype or rr:language on a term map "
|
|
215
|
+
f"that gives no literal")
|
|
216
|
+
if datatype and language:
|
|
217
|
+
raise R2RMLMappingError(f"{where}: both rr:datatype and rr:language")
|
|
218
|
+
if language is not None and not _LANGUAGE_TAG.match(language):
|
|
219
|
+
raise R2RMLMappingError(f"{where}: {language!r} is no language tag")
|
|
220
|
+
return TermMap(term_type,
|
|
221
|
+
template=_parts(where, template) if template is not None else None,
|
|
222
|
+
column=column, constant=constant,
|
|
223
|
+
datatype=datatype, language=language, lexical=LEXICAL_R2RML)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
_TERM_TYPES = {"IRI": IRI, "BlankNode": BLANK, "Literal": LITERAL}
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _subject(tm: TriplesMap) -> TermMap:
|
|
230
|
+
if tm.subject_map_count != 1:
|
|
231
|
+
raise R2RMLMappingError(
|
|
232
|
+
f"{tm.uri}: a triples map has exactly one subject map "
|
|
233
|
+
f"({tm.subject_map_count} given)")
|
|
234
|
+
sm: SubjectMap = tm.subject_map
|
|
235
|
+
term_type = _TERM_TYPES[sm.term_type]
|
|
236
|
+
if term_type == LITERAL:
|
|
237
|
+
raise R2RMLMappingError(f"{tm.uri}: a subject is an IRI or a blank node, "
|
|
238
|
+
f"not a literal (R2RML 7.4)")
|
|
239
|
+
return _term(f"{tm.uri} subject map", term_type, sm.template, sm.column, sm.constant)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _graphs(where: str, maps: List[GraphMap]) -> Tuple[TermMap, ...]:
|
|
243
|
+
"""Graph maps as IRI term maps (R2RML 6.4, 7.4: a graph is an IRI),
|
|
244
|
+
each once, in order."""
|
|
245
|
+
out: List[TermMap] = []
|
|
246
|
+
for gm in maps:
|
|
247
|
+
if gm.term_type != "IRI":
|
|
248
|
+
raise R2RMLMappingError(f"{where}: a graph map gives IRIs, not "
|
|
249
|
+
f"{gm.term_type} terms (R2RML 7.4)")
|
|
250
|
+
term = _term(f"{where} graph map", IRI, gm.template, gm.column, gm.constant)
|
|
251
|
+
if term not in out:
|
|
252
|
+
out.append(term)
|
|
253
|
+
return tuple(out)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _predicate(where: str, pm: PredicateMap) -> Tuple[Optional[str], Optional[TermMap]]:
|
|
257
|
+
"""A predicate map: its IRI where constant, else its term map."""
|
|
258
|
+
term = _term(f"{where} predicate map", IRI, pm.template, pm.column, pm.constant)
|
|
259
|
+
if term.constant is not None:
|
|
260
|
+
return term.constant, None
|
|
261
|
+
return None, term
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _object(where: str, om: ObjectMap) -> TermMap:
|
|
265
|
+
return _term(f"{where} object map", _TERM_TYPES[om.term_type], om.template, om.column,
|
|
266
|
+
om.constant, om.datatype, om.language)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _same_logical_table(a: TriplesMap, b: TriplesMap) -> bool:
|
|
270
|
+
"""Whether two maps' effective SQL queries are one (R2RML 8): one table
|
|
271
|
+
name, or one query text."""
|
|
272
|
+
la, lb = a.logical_table, b.logical_table
|
|
273
|
+
return (la.table_name, la.sql_query) == (lb.table_name, lb.sql_query)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _triples_map(tm: TriplesMap, by_uri: Dict[str, TriplesMap], order: int,
|
|
277
|
+
edge_order: int) -> Tuple[NodeMap, List[EdgeMap]]:
|
|
278
|
+
source = _source(tm, by_uri)
|
|
279
|
+
subject = _subject(tm)
|
|
280
|
+
subject_graphs = _graphs(f"{tm.uri} subject map",
|
|
281
|
+
tm.subject_map.graph_maps + tm.graph_maps)
|
|
282
|
+
start = EndRef(map_id=tm.uri, label=None, columns=subject.columns(), listed=False,
|
|
283
|
+
term=subject, exists=True)
|
|
284
|
+
literals: List[PredicateObject] = []
|
|
285
|
+
edges: List[EdgeMap] = []
|
|
286
|
+
classes: List[str] = list(tm.subject_map.classes)
|
|
287
|
+
for i, pom in enumerate(tm.predicate_object_maps):
|
|
288
|
+
where = f"{tm.uri} predicate-object map {i}"
|
|
289
|
+
graphs = subject_graphs + tuple(g for g in _graphs(where, pom.graph_maps)
|
|
290
|
+
if g not in subject_graphs)
|
|
291
|
+
if not pom.predicate_maps:
|
|
292
|
+
raise R2RMLMappingError(f"{where}: no predicate map")
|
|
293
|
+
if not pom.object_maps:
|
|
294
|
+
raise R2RMLMappingError(f"{where}: no object map")
|
|
295
|
+
predicates = [_predicate(where, pm) for pm in pom.predicate_maps]
|
|
296
|
+
for k, om in enumerate(pom.object_maps):
|
|
297
|
+
if om.parent_triples_map is not None:
|
|
298
|
+
made = _referencing(tm, source, om, by_uri, f"{where} object map {k}")
|
|
299
|
+
for j, (iri, pmap) in enumerate(predicates):
|
|
300
|
+
edges.append(EdgeMap(
|
|
301
|
+
id=f"{tm.uri}#po{i}.{j}.{k}", order=edge_order + len(edges),
|
|
302
|
+
source=source, predicate=iri, rel_type=None, start=start,
|
|
303
|
+
end=made[0], join=made[1], predicate_map=pmap, graph_maps=graphs))
|
|
304
|
+
continue
|
|
305
|
+
if om.join_conditions:
|
|
306
|
+
raise R2RMLMappingError(f"{where}: rr:joinCondition with no "
|
|
307
|
+
f"rr:parentTriplesMap")
|
|
308
|
+
term = _object(where, om)
|
|
309
|
+
for j, (iri, pmap) in enumerate(predicates):
|
|
310
|
+
if term.term_type == LITERAL:
|
|
311
|
+
literals.append(PredicateObject(predicate=iri, object=term,
|
|
312
|
+
predicate_map=pmap, graph_maps=graphs))
|
|
313
|
+
continue
|
|
314
|
+
if (iri == RDF_TYPE and term.term_type == IRI and term.constant is not None
|
|
315
|
+
and not pom.graph_maps):
|
|
316
|
+
# rr:predicate rdf:type; rr:object C gives each subject
|
|
317
|
+
# the triple rr:class C gives it, in the subject map's
|
|
318
|
+
# graphs: a class (`SourceMaps.by_class`). With graph
|
|
319
|
+
# maps of its own its triples are in other graphs too,
|
|
320
|
+
# and it stays an edge, as does a type made from the
|
|
321
|
+
# row (a template or column object).
|
|
322
|
+
classes.append(term.constant)
|
|
323
|
+
continue
|
|
324
|
+
end = EndRef(map_id=None, label=None, columns=term.columns(), listed=False,
|
|
325
|
+
term=term, exists=False)
|
|
326
|
+
edges.append(EdgeMap(
|
|
327
|
+
id=f"{tm.uri}#po{i}.{j}.{k}", order=edge_order + len(edges),
|
|
328
|
+
source=source, predicate=iri, rel_type=None, start=start, end=end,
|
|
329
|
+
predicate_map=pmap, graph_maps=graphs))
|
|
330
|
+
node = NodeMap(
|
|
331
|
+
id=tm.uri, order=order, source=source,
|
|
332
|
+
key=Key(columns=subject.columns(), stated=False), subject=subject,
|
|
333
|
+
classes=tuple(dict.fromkeys(classes)),
|
|
334
|
+
predicate_objects=tuple(literals), label=None,
|
|
335
|
+
rdf_class=bool(classes), graph_maps=subject_graphs)
|
|
336
|
+
return node, edges
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _referencing(tm: TriplesMap, source: Source, om: ObjectMap,
|
|
340
|
+
by_uri: Dict[str, TriplesMap], where: str
|
|
341
|
+
) -> Tuple[EndRef, Optional[JointQuery]]:
|
|
342
|
+
"""A referencing object map's end and join (R2RML 8): the parent's
|
|
343
|
+
subject, from the same row where the two logical tables are one and
|
|
344
|
+
there is no join condition, else from the parent row the joint query
|
|
345
|
+
matches."""
|
|
346
|
+
parent = by_uri.get(om.parent_triples_map)
|
|
347
|
+
if parent is None:
|
|
348
|
+
raise R2RMLMappingError(f"{where}: rr:parentTriplesMap {om.parent_triples_map} "
|
|
349
|
+
f"is no triples map of the mapping")
|
|
350
|
+
if om.template is not None or om.column is not None or om.constant is not None:
|
|
351
|
+
raise R2RMLMappingError(f"{where}: a referencing object map has only "
|
|
352
|
+
f"rr:parentTriplesMap and rr:joinCondition")
|
|
353
|
+
parent_subject = _subject(parent)
|
|
354
|
+
conditions = tuple((jc.child, jc.parent) for jc in om.join_conditions)
|
|
355
|
+
same = _same_logical_table(tm, parent)
|
|
356
|
+
if not conditions:
|
|
357
|
+
if not same:
|
|
358
|
+
raise R2RMLMappingError(
|
|
359
|
+
f"{where}: a referencing object map whose parent's logical table is "
|
|
360
|
+
f"not the child's has a join condition (R2RML 8)")
|
|
361
|
+
return EndRef(map_id=parent.uri, label=None, columns=parent_subject.columns(),
|
|
362
|
+
listed=False, term=parent_subject, exists=True), None
|
|
363
|
+
join = JointQuery(child=source, parent=_source(parent, by_uri), conditions=conditions)
|
|
364
|
+
child_of = {}
|
|
365
|
+
for child, parent_column in conditions:
|
|
366
|
+
child_of.setdefault(parent_column, child)
|
|
367
|
+
wanted = parent_subject.columns()
|
|
368
|
+
columns = (tuple(child_of[c] for c in wanted)
|
|
369
|
+
if wanted and all(c in child_of for c in wanted) else ())
|
|
370
|
+
end = EndRef(map_id=parent.uri, label=None, columns=columns, listed=False,
|
|
371
|
+
term=parent_subject, exists=True, parent_columns=wanted)
|
|
372
|
+
return end, join
|
prograph/r2rml/direct.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""The W3C Direct Mapping (https://www.w3.org/TR/rdb-direct-mapping/) as an R2RML mapping.
|
|
2
|
+
|
|
3
|
+
`direct_mapping_r2rml` reads a database's catalog (`catalog.read_tables`) and writes the mapping the
|
|
4
|
+
standard's section 3 defines as one triples map per table, which the engine reads as it reads any R2RML
|
|
5
|
+
mapping (`ProGraph.from_direct_mapping`): one plan over source maps, no copy of the data.
|
|
6
|
+
|
|
7
|
+
- Row node of a table with a primary key: `<base>Table/pk1=v1;pk2=v2`, the names IRI-safe (R2RML 7.3,
|
|
8
|
+
`iri.iri_safe`) and each value IRI-safe by the template's own rule, in key order.
|
|
9
|
+
- Row node of a table with none: a blank node per row, duplicate rows too. The logical table is the table
|
|
10
|
+
with each row numbered (`ROW_NUMBER() OVER (ORDER BY the text of every column)`); the label is the table's
|
|
11
|
+
name and the number, so two tables never share a node. Equal rows tie in the order but are interchangeable,
|
|
12
|
+
so the graph is the same whichever is numbered first. Every read of the table sorts it. Only base tables are
|
|
13
|
+
mapped, not views.
|
|
14
|
+
- Class `<base>Table`; a literal triple per column, `<base>Table#Column`, its object the natural RDF literal
|
|
15
|
+
(R2RML 10.2; none for NULL); a reference triple per foreign key, `<base>Table#ref-c1;c2`, to the
|
|
16
|
+
referenced row's node, a referencing object map joined on the key's columns (a NULL joins nothing).
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import re
|
|
22
|
+
from typing import Any, List, Optional, Sequence
|
|
23
|
+
|
|
24
|
+
from prograph.r2rml.catalog import DIALECTS, DirectMappingError, Table, read_tables
|
|
25
|
+
from prograph.r2rml.model import (
|
|
26
|
+
JoinCondition, LogicalTable, ObjectMap, PredicateMap, PredicateObjectMap, R2RMLMapping,
|
|
27
|
+
SubjectMap, TriplesMap)
|
|
28
|
+
from prograph.rules import home
|
|
29
|
+
from prograph.schema.iri import iri_safe
|
|
30
|
+
|
|
31
|
+
_TEXT = {"duckdb": "VARCHAR", "postgresql": "TEXT", "databricks_sql": "STRING"}
|
|
32
|
+
RR = "http://www.w3.org/ns/r2rml#"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@home("direct-mapping", "The Direct Mapping of a database: its terms and a keyless row's blank node",
|
|
36
|
+
terms=("direct mapping", "Direct Mapping", "rdb-direct-mapping", "row node", "table IRI", "ref-",
|
|
37
|
+
"keyless table", "blank node per row", "from_direct_mapping"))
|
|
38
|
+
def direct_mapping_r2rml(connection: Any, base_iri: str, schema: Optional[str] = None,
|
|
39
|
+
tables: Optional[Sequence[str]] = None, *,
|
|
40
|
+
backend: Optional[str] = None) -> R2RMLMapping:
|
|
41
|
+
"""The Direct Mapping of `connection`'s tables (the module's doc) as an R2RML mapping.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
connection: A DuckDB connection, or a PostgreSQL or Databricks SQL one with `backend` named.
|
|
45
|
+
base_iri: The IRI every generated IRI is relative to (the standard fixes none); absolute.
|
|
46
|
+
schema: The schema to map (`catalog.schema` on Databricks); the current one by default.
|
|
47
|
+
tables: Only these tables of it, by name; a foreign key to another is refused.
|
|
48
|
+
backend: 'duckdb' (the default for a DuckDB connection), 'postgresql' or 'databricks_sql'.
|
|
49
|
+
|
|
50
|
+
Raises:
|
|
51
|
+
DirectMappingError: A database with no catalog reader, a relative base IRI, a missing table, or a
|
|
52
|
+
foreign key to a table not mapped.
|
|
53
|
+
"""
|
|
54
|
+
dialect = _dialect(connection, backend)
|
|
55
|
+
if isinstance(connection, str):
|
|
56
|
+
# A PostgreSQL URI, opened as the engine opens one and closed once the catalog is read.
|
|
57
|
+
from prograph.graph.postgresql_graph import open_connection
|
|
58
|
+
opened = open_connection(connection)
|
|
59
|
+
try:
|
|
60
|
+
return direct_mapping_r2rml(opened, base_iri, schema, tables, backend=backend)
|
|
61
|
+
finally:
|
|
62
|
+
opened.close()
|
|
63
|
+
if not re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", base_iri or ""):
|
|
64
|
+
raise DirectMappingError(f"base_iri must be an absolute IRI (http://example.com/base/); got {base_iri!r}")
|
|
65
|
+
found = read_tables(connection, dialect, schema, tables)
|
|
66
|
+
from prograph.plan.sql.dialect import get_dialect_instance
|
|
67
|
+
quote = get_dialect_instance(dialect).quote_exact
|
|
68
|
+
qualify = (lambda name: ".".join(quote(p) for p in (schema.split(".") if schema else []) + [name]))
|
|
69
|
+
# rr:tableName is written the standard way, each part delimited (R2RML 5): the dialect's quotes
|
|
70
|
+
# are written where SQL is.
|
|
71
|
+
delimit = (lambda name: ".".join('"' + p.replace('"', '""') + '"' for p in (schema.split(".") if schema else []) + [name]))
|
|
72
|
+
return R2RMLMapping(triples_maps=[_triples_map(t, base_iri, qualify, quote, _TEXT[dialect], delimit) for t in found],
|
|
73
|
+
prefixes={"rr": RR}, base=base_iri, dialect=dialect)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _dialect(connection: Any, backend: Optional[str]) -> str:
|
|
77
|
+
if isinstance(connection, str):
|
|
78
|
+
if backend != "postgresql":
|
|
79
|
+
raise DirectMappingError("a connection given as a URI is read only for backend='postgresql'; "
|
|
80
|
+
"open the connection and pass it for another database")
|
|
81
|
+
return "postgresql"
|
|
82
|
+
if backend is None:
|
|
83
|
+
import duckdb
|
|
84
|
+
if isinstance(connection, duckdb.DuckDBPyConnection):
|
|
85
|
+
return "duckdb"
|
|
86
|
+
raise DirectMappingError(
|
|
87
|
+
f"a connection of type {type(connection).__name__!r} needs its backend named "
|
|
88
|
+
f"(backend='postgresql' or 'databricks_sql')")
|
|
89
|
+
if backend not in DIALECTS:
|
|
90
|
+
raise DirectMappingError(f"the Direct Mapping reads DuckDB, PostgreSQL and Databricks; not {backend!r}")
|
|
91
|
+
return backend
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _uri(name: str) -> str:
|
|
95
|
+
return "urn:direct-mapping:" + iri_safe(name)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _placeholder(column: str) -> str:
|
|
99
|
+
return "{" + re.sub(r"([\\{}])", r"\\\1", column) + "}"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _label(name: str) -> str:
|
|
103
|
+
"""A table's name as a blank node label's text: letters and digits, any other character `_hex_`."""
|
|
104
|
+
return "".join(c if c.isascii() and c.isalnum() else f"_{ord(c):x}_" for c in name)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _subject(t: Table, base: str, row: Optional[str]) -> SubjectMap:
|
|
108
|
+
table_iri = base + iri_safe(t.name)
|
|
109
|
+
if not t.primary_key:
|
|
110
|
+
return SubjectMap(template=f"{_label(t.name)}-{{{row}}}", term_type="BlankNode", classes=[table_iri])
|
|
111
|
+
parts = ";".join(f"{iri_safe(c)}={_placeholder(c)}" for c in t.primary_key)
|
|
112
|
+
return SubjectMap(template=f"{_escaped_base(base)}{iri_safe(t.name)}/{parts}", classes=[table_iri])
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _escaped_base(base: str) -> str:
|
|
116
|
+
return re.sub(r"([\\{}])", r"\\\1", base)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _triples_map(t: Table, base: str, qualify: Any, quote: Any, text_type: str, delimit: Any) -> TriplesMap:
|
|
120
|
+
row = None
|
|
121
|
+
if t.primary_key:
|
|
122
|
+
table = LogicalTable(table_name=t.name, written=delimit(t.name))
|
|
123
|
+
else:
|
|
124
|
+
row = "__row"
|
|
125
|
+
while row in t.columns:
|
|
126
|
+
row += "_"
|
|
127
|
+
# By each column's text, so a type with no ordering (json, geometry) still sorts; equal rows tie.
|
|
128
|
+
order = ", ".join(f"CAST(t.{quote(c)} AS {text_type})" for c in t.columns)
|
|
129
|
+
table = LogicalTable(sql_query=(f"SELECT t.*, ROW_NUMBER() OVER (ORDER BY {order}) AS {quote(row)} "
|
|
130
|
+
f"FROM {qualify(t.name)} t"))
|
|
131
|
+
poms: List[PredicateObjectMap] = [
|
|
132
|
+
PredicateObjectMap([PredicateMap(constant=f"{base}{iri_safe(t.name)}#{iri_safe(c)}")],
|
|
133
|
+
[ObjectMap(column=c)]) for c in t.columns]
|
|
134
|
+
for fk in t.foreign_keys:
|
|
135
|
+
poms.append(PredicateObjectMap(
|
|
136
|
+
[PredicateMap(constant=f"{base}{iri_safe(t.name)}#ref-{';'.join(iri_safe(c) for c in fk.columns)}")],
|
|
137
|
+
[ObjectMap(parent_triples_map=_uri(fk.parent), term_type="IRI",
|
|
138
|
+
join_conditions=[JoinCondition(c, p) for c, p in zip(fk.columns, fk.parent_columns)])]))
|
|
139
|
+
return TriplesMap(uri=_uri(t.name), logical_table=table, subject_map=_subject(t, base, row),
|
|
140
|
+
predicate_object_maps=poms)
|