cognite-neat 0.86.0__py3-none-any.whl → 0.87.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/configuration.py +1 -10
- cognite/neat/app/api/routers/data_exploration.py +1 -1
- cognite/neat/config.py +84 -17
- cognite/neat/constants.py +11 -9
- cognite/neat/graph/extractors/_classic_cdf/_assets.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_events.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_files.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_labels.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_relationships.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_sequences.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_timeseries.py +1 -1
- cognite/neat/graph/extractors/_dexpi.py +1 -1
- cognite/neat/graph/extractors/_mock_graph_generator.py +8 -9
- cognite/neat/graph/loaders/__init__.py +5 -2
- cognite/neat/graph/loaders/_base.py +13 -5
- cognite/neat/graph/loaders/_rdf2asset.py +185 -55
- cognite/neat/graph/loaders/_rdf2dms.py +7 -7
- cognite/neat/graph/queries/_base.py +20 -11
- cognite/neat/graph/queries/_construct.py +5 -5
- cognite/neat/graph/queries/_shared.py +21 -7
- cognite/neat/graph/stores/_base.py +16 -4
- cognite/neat/graph/transformers/__init__.py +3 -0
- cognite/neat/graph/transformers/_rdfpath.py +42 -0
- cognite/neat/legacy/graph/extractors/_dexpi.py +0 -5
- cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +1 -1
- cognite/neat/legacy/graph/loaders/_asset_loader.py +2 -2
- cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +5 -2
- cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +4 -1
- cognite/neat/legacy/graph/loaders/rdf_to_dms.py +3 -1
- cognite/neat/legacy/graph/stores/_base.py +24 -8
- cognite/neat/legacy/graph/stores/_graphdb_store.py +3 -2
- cognite/neat/legacy/graph/stores/_memory_store.py +3 -3
- cognite/neat/legacy/graph/stores/_oxigraph_store.py +8 -4
- cognite/neat/legacy/graph/stores/_rdf_to_graph.py +5 -3
- cognite/neat/legacy/graph/transformations/query_generator/sparql.py +49 -16
- cognite/neat/legacy/graph/transformations/transformer.py +1 -1
- cognite/neat/legacy/rules/exporters/_rules2dms.py +8 -3
- cognite/neat/legacy/rules/exporters/_rules2graphql.py +1 -1
- cognite/neat/legacy/rules/exporters/_rules2ontology.py +2 -1
- cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +3 -4
- cognite/neat/legacy/rules/importers/_dms2rules.py +4 -1
- cognite/neat/legacy/rules/importers/_graph2rules.py +3 -3
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +1 -1
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +2 -1
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +1 -1
- cognite/neat/legacy/rules/models/raw_rules.py +19 -7
- cognite/neat/legacy/rules/models/rules.py +32 -12
- cognite/neat/rules/_shared.py +6 -1
- cognite/neat/rules/analysis/__init__.py +4 -4
- cognite/neat/rules/analysis/_asset.py +143 -0
- cognite/neat/rules/analysis/_base.py +385 -6
- cognite/neat/rules/analysis/_information.py +183 -0
- cognite/neat/rules/exporters/_rules2dms.py +1 -1
- cognite/neat/rules/exporters/_rules2ontology.py +6 -5
- cognite/neat/rules/importers/_dms2rules.py +3 -1
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +2 -8
- cognite/neat/rules/importers/_inference2rules.py +3 -7
- cognite/neat/rules/importers/_owl2rules/_owl2classes.py +1 -1
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +2 -1
- cognite/neat/rules/importers/_owl2rules/_owl2properties.py +1 -1
- cognite/neat/rules/issues/spreadsheet.py +35 -0
- cognite/neat/rules/models/_base.py +7 -7
- cognite/neat/rules/models/_rdfpath.py +17 -21
- cognite/neat/rules/models/asset/_rules.py +4 -5
- cognite/neat/rules/models/asset/_validation.py +38 -1
- cognite/neat/rules/models/dms/_converter.py +1 -2
- cognite/neat/rules/models/dms/_exporter.py +7 -3
- cognite/neat/rules/models/dms/_rules.py +3 -0
- cognite/neat/rules/models/dms/_schema.py +5 -4
- cognite/neat/rules/models/domain.py +5 -2
- cognite/neat/rules/models/entities.py +28 -17
- cognite/neat/rules/models/information/_rules.py +10 -8
- cognite/neat/rules/models/information/_rules_input.py +1 -2
- cognite/neat/rules/models/information/_validation.py +2 -2
- cognite/neat/utils/__init__.py +0 -3
- cognite/neat/utils/auth.py +47 -28
- cognite/neat/utils/auxiliary.py +141 -1
- cognite/neat/utils/cdf/__init__.py +0 -0
- cognite/neat/utils/{cdf_classes.py → cdf/data_classes.py} +122 -2
- cognite/neat/utils/{cdf_loaders → cdf/loaders}/_data_modeling.py +37 -0
- cognite/neat/utils/{cdf_loaders → cdf/loaders}/_ingestion.py +2 -1
- cognite/neat/utils/collection_.py +18 -0
- cognite/neat/utils/rdf_.py +165 -0
- cognite/neat/utils/text.py +4 -0
- cognite/neat/utils/time_.py +17 -0
- cognite/neat/utils/upload.py +13 -1
- cognite/neat/workflows/_exceptions.py +5 -5
- cognite/neat/workflows/base.py +1 -1
- cognite/neat/workflows/steps/lib/current/graph_store.py +28 -8
- cognite/neat/workflows/steps/lib/current/rules_validator.py +2 -2
- cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +130 -28
- cognite/neat/workflows/steps/lib/legacy/graph_loader.py +1 -1
- cognite/neat/workflows/steps/lib/legacy/graph_store.py +4 -4
- cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +1 -1
- cognite/neat/workflows/steps/lib/legacy/rules_importer.py +1 -1
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/METADATA +2 -2
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/RECORD +103 -102
- cognite/neat/rules/analysis/_information_rules.py +0 -476
- cognite/neat/utils/cdf.py +0 -59
- cognite/neat/utils/cdf_loaders/data_classes.py +0 -121
- cognite/neat/utils/exceptions.py +0 -41
- cognite/neat/utils/utils.py +0 -429
- /cognite/neat/utils/{cdf_loaders → cdf/loaders}/__init__.py +0 -0
- /cognite/neat/utils/{cdf_loaders → cdf/loaders}/_base.py +0 -0
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/LICENSE +0 -0
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/WHEEL +0 -0
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/entry_points.txt +0 -0
|
@@ -18,7 +18,10 @@ from rdflib.term import URIRef
|
|
|
18
18
|
from cognite.neat.legacy.graph.loaders.core.models import AssetTemplate
|
|
19
19
|
from cognite.neat.legacy.graph.stores import NeatGraphStoreBase
|
|
20
20
|
from cognite.neat.legacy.rules.models.rules import Property, Rules
|
|
21
|
-
from cognite.neat.utils.
|
|
21
|
+
from cognite.neat.utils.auxiliary import retry_decorator
|
|
22
|
+
from cognite.neat.utils.collection_ import chunker
|
|
23
|
+
from cognite.neat.utils.rdf_ import remove_namespace_from_uri
|
|
24
|
+
from cognite.neat.utils.time_ import datetime_utc_now
|
|
22
25
|
|
|
23
26
|
if sys.version_info >= (3, 11):
|
|
24
27
|
from datetime import UTC
|
|
@@ -899,7 +902,7 @@ def _micro_batch_push(
|
|
|
899
902
|
except CogniteDuplicatedError:
|
|
900
903
|
# this is handling of very rare case when some assets might be lost . Normally this should not happen.
|
|
901
904
|
# Last attempt to recover
|
|
902
|
-
client.assets.create_hierarchy(batch, upsert=True)
|
|
905
|
+
client.assets.create_hierarchy(batch, upsert=True) # type: ignore[arg-type]
|
|
903
906
|
|
|
904
907
|
delta_time = (datetime_utc_now() - start_time).seconds
|
|
905
908
|
|
|
@@ -14,7 +14,10 @@ from cognite.neat.legacy.graph.loaders.core.models import RelationshipDefinition
|
|
|
14
14
|
from cognite.neat.legacy.graph.loaders.core.rdf_to_assets import _categorize_cdf_assets
|
|
15
15
|
from cognite.neat.legacy.graph.stores import NeatGraphStoreBase
|
|
16
16
|
from cognite.neat.legacy.rules.models.rules import Rules
|
|
17
|
-
from cognite.neat.utils.
|
|
17
|
+
from cognite.neat.utils.auxiliary import retry_decorator
|
|
18
|
+
from cognite.neat.utils.collection_ import chunker
|
|
19
|
+
from cognite.neat.utils.rdf_ import remove_namespace_from_uri
|
|
20
|
+
from cognite.neat.utils.time_ import datetime_utc_now, epoch_now_ms
|
|
18
21
|
|
|
19
22
|
|
|
20
23
|
def define_relationships(rules: Rules, data_set_id: int, stop_on_exception: bool = False) -> RelationshipDefinitions:
|
|
@@ -13,7 +13,9 @@ from cognite.neat.legacy.graph.transformations.query_generator.sparql import tri
|
|
|
13
13
|
from cognite.neat.legacy.rules.exporters._rules2dms import DMSSchemaComponents
|
|
14
14
|
from cognite.neat.legacy.rules.exporters._rules2pydantic_models import add_class_prefix_to_xid, rules_to_pydantic_models
|
|
15
15
|
from cognite.neat.legacy.rules.models.rules import Rules
|
|
16
|
-
from cognite.neat.utils.
|
|
16
|
+
from cognite.neat.utils.auxiliary import retry_decorator
|
|
17
|
+
from cognite.neat.utils.collection_ import chunker
|
|
18
|
+
from cognite.neat.utils.time_ import datetime_utc_now
|
|
17
19
|
|
|
18
20
|
from ._base import CogniteLoader
|
|
19
21
|
|
|
@@ -11,7 +11,7 @@ from prometheus_client import Gauge, Summary
|
|
|
11
11
|
from rdflib import Graph, Namespace, URIRef
|
|
12
12
|
from rdflib.query import Result, ResultRow
|
|
13
13
|
|
|
14
|
-
from cognite.neat.constants import DEFAULT_NAMESPACE,
|
|
14
|
+
from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
|
|
15
15
|
from cognite.neat.legacy.graph.models import Triple
|
|
16
16
|
from cognite.neat.legacy.graph.stores._rdf_to_graph import rdf_file_to_graph
|
|
17
17
|
from cognite.neat.legacy.rules.models.rules import Rules
|
|
@@ -25,7 +25,11 @@ prom_qsm = Summary("store_query_time_summary_legacy", "Time spent processing que
|
|
|
25
25
|
prom_sq = Gauge("store_single_query_time_legacy", "Time spent processing a single query", ["query"])
|
|
26
26
|
|
|
27
27
|
MIMETypes: TypeAlias = Literal[
|
|
28
|
-
"application/rdf+xml",
|
|
28
|
+
"application/rdf+xml",
|
|
29
|
+
"text/turtle",
|
|
30
|
+
"application/n-triple",
|
|
31
|
+
"application/n-quads",
|
|
32
|
+
"application/trig",
|
|
29
33
|
]
|
|
30
34
|
|
|
31
35
|
|
|
@@ -35,7 +39,8 @@ class NeatGraphStoreBase(ABC):
|
|
|
35
39
|
|
|
36
40
|
Args:
|
|
37
41
|
graph : Instance of rdflib.Graph class for graph storage
|
|
38
|
-
base_prefix : Used as a base prefix for graph namespace,
|
|
42
|
+
base_prefix : Used as a base prefix for graph namespace,
|
|
43
|
+
allowing querying graph data using a short form of a URI
|
|
39
44
|
namespace : Namespace (aka URI) used to resolve any relative URI in the graph
|
|
40
45
|
prefixes : Dictionary of additional prefixes used and bounded to the graph
|
|
41
46
|
"""
|
|
@@ -47,12 +52,12 @@ class NeatGraphStoreBase(ABC):
|
|
|
47
52
|
graph: Graph | None = None,
|
|
48
53
|
base_prefix: str = "", # usually empty
|
|
49
54
|
namespace: Namespace = DEFAULT_NAMESPACE,
|
|
50
|
-
prefixes: dict =
|
|
55
|
+
prefixes: dict[str, Namespace] | None = None,
|
|
51
56
|
):
|
|
52
57
|
self.graph = graph or Graph()
|
|
53
58
|
self.base_prefix: str = base_prefix
|
|
54
59
|
self.namespace: Namespace = namespace
|
|
55
|
-
self.prefixes: dict[str, Namespace] = prefixes
|
|
60
|
+
self.prefixes: dict[str, Namespace] = prefixes or get_default_prefixes()
|
|
56
61
|
|
|
57
62
|
self.rdf_store_query_url: str | None = None
|
|
58
63
|
self.rdf_store_update_url: str | None = None
|
|
@@ -164,7 +169,10 @@ class NeatGraphStoreBase(ABC):
|
|
|
164
169
|
return None
|
|
165
170
|
|
|
166
171
|
def import_from_file(
|
|
167
|
-
self,
|
|
172
|
+
self,
|
|
173
|
+
graph_file: Path,
|
|
174
|
+
mime_type: MIMETypes = "application/rdf+xml",
|
|
175
|
+
add_base_iri: bool = True,
|
|
168
176
|
) -> None:
|
|
169
177
|
"""Imports graph data from file.
|
|
170
178
|
|
|
@@ -175,7 +183,10 @@ class NeatGraphStoreBase(ABC):
|
|
|
175
183
|
"""
|
|
176
184
|
if add_base_iri:
|
|
177
185
|
self.graph = rdf_file_to_graph(
|
|
178
|
-
self.graph,
|
|
186
|
+
self.graph,
|
|
187
|
+
graph_file,
|
|
188
|
+
base_namespace=self.namespace,
|
|
189
|
+
prefixes=self.prefixes,
|
|
179
190
|
)
|
|
180
191
|
else:
|
|
181
192
|
self.graph = rdf_file_to_graph(self.graph, graph_file, prefixes=self.prefixes)
|
|
@@ -290,7 +301,12 @@ class NeatGraphStoreBase(ABC):
|
|
|
290
301
|
"rdf_store_update_url": self.rdf_store_update_url,
|
|
291
302
|
}
|
|
292
303
|
|
|
293
|
-
def add_triples(
|
|
304
|
+
def add_triples(
|
|
305
|
+
self,
|
|
306
|
+
triples: list[Triple] | set[Triple],
|
|
307
|
+
batch_size: int = 10_000,
|
|
308
|
+
verbose: bool = False,
|
|
309
|
+
):
|
|
294
310
|
"""Adds triples to the graph store in batches.
|
|
295
311
|
|
|
296
312
|
Args:
|
|
@@ -4,7 +4,7 @@ import requests
|
|
|
4
4
|
from rdflib import Graph, Namespace
|
|
5
5
|
from rdflib.plugins.stores.sparqlstore import SPARQLUpdateStore
|
|
6
6
|
|
|
7
|
-
from cognite.neat.constants import DEFAULT_NAMESPACE,
|
|
7
|
+
from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
|
|
8
8
|
|
|
9
9
|
from ._base import NeatGraphStoreBase
|
|
10
10
|
|
|
@@ -28,8 +28,9 @@ class GraphDBStore(NeatGraphStoreBase):
|
|
|
28
28
|
graph: Graph | None = None,
|
|
29
29
|
base_prefix: str = "", # usually empty
|
|
30
30
|
namespace: Namespace = DEFAULT_NAMESPACE,
|
|
31
|
-
prefixes: dict =
|
|
31
|
+
prefixes: dict[str, Namespace] | None = None,
|
|
32
32
|
):
|
|
33
|
+
prefixes = prefixes if prefixes else get_default_prefixes()
|
|
33
34
|
super().__init__(graph, base_prefix, namespace, prefixes)
|
|
34
35
|
self.graph_db_rest_url: str = "http://localhost:7200"
|
|
35
36
|
|
|
@@ -2,7 +2,7 @@ import logging
|
|
|
2
2
|
|
|
3
3
|
from rdflib import Graph, Namespace
|
|
4
4
|
|
|
5
|
-
from cognite.neat.constants import DEFAULT_NAMESPACE,
|
|
5
|
+
from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
|
|
6
6
|
|
|
7
7
|
from ._base import NeatGraphStoreBase
|
|
8
8
|
|
|
@@ -26,9 +26,9 @@ class MemoryStore(NeatGraphStoreBase):
|
|
|
26
26
|
graph: Graph | None = None,
|
|
27
27
|
base_prefix: str = "", # usually empty
|
|
28
28
|
namespace: Namespace = DEFAULT_NAMESPACE,
|
|
29
|
-
prefixes: dict =
|
|
29
|
+
prefixes: dict[str, Namespace] | None = None,
|
|
30
30
|
):
|
|
31
|
-
# Init repeated to get nice docstring
|
|
31
|
+
prefixes = prefixes if prefixes else get_default_prefixes() # Init repeated to get nice docstring
|
|
32
32
|
super().__init__(graph, base_prefix, namespace, prefixes)
|
|
33
33
|
|
|
34
34
|
def _set_graph(self):
|
|
@@ -5,7 +5,7 @@ from pathlib import Path
|
|
|
5
5
|
|
|
6
6
|
from rdflib import Graph, Namespace
|
|
7
7
|
|
|
8
|
-
from cognite.neat.constants import DEFAULT_NAMESPACE,
|
|
8
|
+
from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
|
|
9
9
|
from cognite.neat.utils.auxiliary import local_import
|
|
10
10
|
|
|
11
11
|
from ._base import MIMETypes, NeatGraphStoreBase
|
|
@@ -29,8 +29,9 @@ class OxiGraphStore(NeatGraphStoreBase):
|
|
|
29
29
|
graph: Graph | None = None,
|
|
30
30
|
base_prefix: str = "", # usually empty
|
|
31
31
|
namespace: Namespace = DEFAULT_NAMESPACE,
|
|
32
|
-
prefixes: dict =
|
|
32
|
+
prefixes: dict[str, Namespace] | None = None,
|
|
33
33
|
):
|
|
34
|
+
prefixes = prefixes if prefixes else get_default_prefixes()
|
|
34
35
|
super().__init__(graph, base_prefix, namespace, prefixes)
|
|
35
36
|
|
|
36
37
|
def _set_graph(self) -> None:
|
|
@@ -44,7 +45,7 @@ class OxiGraphStore(NeatGraphStoreBase):
|
|
|
44
45
|
for i in range(4):
|
|
45
46
|
try:
|
|
46
47
|
oxstore = pyoxigraph.Store(
|
|
47
|
-
path=str(self.internal_storage_dir) if self.internal_storage_dir else None
|
|
48
|
+
path=(str(self.internal_storage_dir) if self.internal_storage_dir else None)
|
|
48
49
|
) # Store (Rust object) accepts only str as path and not Path.
|
|
49
50
|
break
|
|
50
51
|
except OSError as e:
|
|
@@ -76,7 +77,10 @@ class OxiGraphStore(NeatGraphStoreBase):
|
|
|
76
77
|
logging.info("GraphStore restarted")
|
|
77
78
|
|
|
78
79
|
def import_from_file(
|
|
79
|
-
self,
|
|
80
|
+
self,
|
|
81
|
+
graph_file: Path,
|
|
82
|
+
mime_type: MIMETypes = "application/rdf+xml",
|
|
83
|
+
add_base_iri: bool = True,
|
|
80
84
|
) -> None:
|
|
81
85
|
"""Imports graph data from file.
|
|
82
86
|
|
|
@@ -2,7 +2,7 @@ from pathlib import Path
|
|
|
2
2
|
|
|
3
3
|
from rdflib import Graph, Namespace
|
|
4
4
|
|
|
5
|
-
from cognite.neat.constants import
|
|
5
|
+
from cognite.neat.constants import get_default_prefixes
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
def rdf_file_to_graph(
|
|
@@ -10,7 +10,7 @@ def rdf_file_to_graph(
|
|
|
10
10
|
filepath: Path,
|
|
11
11
|
base_prefix: str | None = None,
|
|
12
12
|
base_namespace: Namespace | None = None,
|
|
13
|
-
prefixes: dict[str, Namespace] =
|
|
13
|
+
prefixes: dict[str, Namespace] | None = None,
|
|
14
14
|
) -> Graph:
|
|
15
15
|
"""Created rdflib Graph instance loaded with RDF triples from file
|
|
16
16
|
|
|
@@ -18,13 +18,15 @@ def rdf_file_to_graph(
|
|
|
18
18
|
filepath: Path to the RDF file
|
|
19
19
|
base_prefix: base prefix for URIs. Defaults to None.
|
|
20
20
|
base_namespace: base namespace for URIs . Defaults to None.
|
|
21
|
-
prefixes: Dictionary of prefixes to bind to graph. Defaults to
|
|
21
|
+
prefixes: Dictionary of prefixes to bind to graph. Defaults to internal set of prefixes.
|
|
22
22
|
graph: Graph instance to load RDF triples into. Defaults to None.
|
|
23
23
|
|
|
24
24
|
Returns:
|
|
25
25
|
Graph instance loaded with RDF triples from file
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
|
+
prefixes = prefixes if prefixes else get_default_prefixes()
|
|
29
|
+
|
|
28
30
|
if filepath.is_file():
|
|
29
31
|
graph.parse(filepath, publicID=base_namespace)
|
|
30
32
|
else:
|
|
@@ -6,7 +6,7 @@ from typing import cast
|
|
|
6
6
|
from rdflib import Graph, Namespace
|
|
7
7
|
from rdflib.term import URIRef
|
|
8
8
|
|
|
9
|
-
from cognite.neat.constants import
|
|
9
|
+
from cognite.neat.constants import get_default_prefixes
|
|
10
10
|
from cognite.neat.legacy.rules.analysis import get_classes_with_properties
|
|
11
11
|
from cognite.neat.legacy.rules.models._base import Triple
|
|
12
12
|
from cognite.neat.legacy.rules.models.rdfpath import (
|
|
@@ -21,10 +21,10 @@ from cognite.neat.legacy.rules.models.rdfpath import (
|
|
|
21
21
|
parse_traversal,
|
|
22
22
|
)
|
|
23
23
|
from cognite.neat.legacy.rules.models.rules import Rules
|
|
24
|
-
from cognite.neat.utils.
|
|
24
|
+
from cognite.neat.utils.rdf_ import remove_namespace_from_uri
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
def _generate_prefix_header(prefixes: dict[str, Namespace] =
|
|
27
|
+
def _generate_prefix_header(prefixes: dict[str, Namespace] | None = None) -> str:
|
|
28
28
|
"""Generate prefix header which is added to SPARQL query and allows for shorten query statements
|
|
29
29
|
|
|
30
30
|
Parameters
|
|
@@ -37,11 +37,15 @@ def _generate_prefix_header(prefixes: dict[str, Namespace] = PREFIXES) -> str:
|
|
|
37
37
|
str
|
|
38
38
|
Prefix header
|
|
39
39
|
"""
|
|
40
|
+
prefixes = prefixes if prefixes else get_default_prefixes()
|
|
40
41
|
return "".join(f"PREFIX {key}:<{value}>\n" for key, value in prefixes.items())
|
|
41
42
|
|
|
42
43
|
|
|
43
44
|
def _get_predicate_id(
|
|
44
|
-
graph: Graph,
|
|
45
|
+
graph: Graph,
|
|
46
|
+
subject_type_id: str,
|
|
47
|
+
object_type_id: str,
|
|
48
|
+
prefixes: dict[str, Namespace] | None = None,
|
|
45
49
|
) -> URIRef:
|
|
46
50
|
"""Returns predicate (aka property) URI (i.e., ID) that connects subject and object
|
|
47
51
|
|
|
@@ -61,6 +65,7 @@ def _get_predicate_id(
|
|
|
61
65
|
URIRef
|
|
62
66
|
ID of predicate (aka property) connecting subject and object
|
|
63
67
|
"""
|
|
68
|
+
prefixes = prefixes if prefixes else get_default_prefixes()
|
|
64
69
|
query = """
|
|
65
70
|
|
|
66
71
|
SELECT ?predicateTypeID
|
|
@@ -122,11 +127,21 @@ def _get_hop_triples(graph, path: Hop, prefixes) -> list[Triple]:
|
|
|
122
127
|
if previous_step.property:
|
|
123
128
|
triples.extend(
|
|
124
129
|
[
|
|
125
|
-
Triple(subject=f"?{previous_step.class_.suffix}ID", predicate="a", object=previous_step.class_.id),
|
|
126
130
|
Triple(
|
|
127
|
-
subject=f"?{previous_step.class_.suffix}ID",
|
|
131
|
+
subject=f"?{previous_step.class_.suffix}ID",
|
|
132
|
+
predicate="a",
|
|
133
|
+
object=previous_step.class_.id,
|
|
134
|
+
),
|
|
135
|
+
Triple(
|
|
136
|
+
subject=f"?{previous_step.class_.suffix}ID",
|
|
137
|
+
predicate=previous_step.property.id,
|
|
138
|
+
object="?object",
|
|
139
|
+
),
|
|
140
|
+
Triple(
|
|
141
|
+
subject="?predicate",
|
|
142
|
+
predicate="a",
|
|
143
|
+
object=previous_step.property.id,
|
|
128
144
|
),
|
|
129
|
-
Triple(subject="?predicate", predicate="a", object=previous_step.property.id),
|
|
130
145
|
]
|
|
131
146
|
)
|
|
132
147
|
else:
|
|
@@ -139,7 +154,7 @@ def _get_hop_triples(graph, path: Hop, prefixes) -> list[Triple]:
|
|
|
139
154
|
return triples
|
|
140
155
|
|
|
141
156
|
|
|
142
|
-
def _get_path_triples(graph: Graph, traversal: Traversal, prefixes: dict[str, Namespace] =
|
|
157
|
+
def _get_path_triples(graph: Graph, traversal: Traversal, prefixes: dict[str, Namespace] | None = None) -> list[Triple]:
|
|
143
158
|
"""Creates triples subject-predicate-object from declarative graph traversal path
|
|
144
159
|
|
|
145
160
|
Parameters
|
|
@@ -227,7 +242,9 @@ def _generate_all_references_query_statement(
|
|
|
227
242
|
|
|
228
243
|
|
|
229
244
|
def _generate_single_property_query_statement(
|
|
230
|
-
subject: str,
|
|
245
|
+
subject: str,
|
|
246
|
+
predicate: str,
|
|
247
|
+
query_template: str = SINGLE_PROPERTY_SPARQL_QUERY_TEMPLATE,
|
|
231
248
|
) -> str:
|
|
232
249
|
query_insertions = "\n".join([f"\t\t?subject a {subject} .", f"\t\t?subject {predicate} ?object ."])
|
|
233
250
|
|
|
@@ -257,7 +274,7 @@ def _generate_hop_query_statement(triples: list[Triple], query_template: str = B
|
|
|
257
274
|
def build_sparql_query(
|
|
258
275
|
graph: Graph,
|
|
259
276
|
traversal_path: str | Traversal,
|
|
260
|
-
prefixes: dict[str, Namespace] =
|
|
277
|
+
prefixes: dict[str, Namespace] | None = None,
|
|
261
278
|
insert_prefixes: bool = False,
|
|
262
279
|
) -> str:
|
|
263
280
|
"""Builds SPARQL query based on declarative traversal path
|
|
@@ -276,7 +293,7 @@ def build_sparql_query(
|
|
|
276
293
|
str
|
|
277
294
|
SPARQL query
|
|
278
295
|
"""
|
|
279
|
-
|
|
296
|
+
prefixes = prefixes if prefixes else get_default_prefixes()
|
|
280
297
|
traversal = parse_traversal(traversal_path) if isinstance(traversal_path, str) else traversal_path
|
|
281
298
|
triples = _get_path_triples(graph, traversal, prefixes)
|
|
282
299
|
|
|
@@ -297,7 +314,10 @@ def build_sparql_query(
|
|
|
297
314
|
for prefix, URI in prefixes.items():
|
|
298
315
|
query = query.replace(URI, f"{prefix}:")
|
|
299
316
|
|
|
300
|
-
return query.replace(
|
|
317
|
+
return query.replace(
|
|
318
|
+
"insertPrefixes\n\n",
|
|
319
|
+
_generate_prefix_header(prefixes) if insert_prefixes else "",
|
|
320
|
+
)
|
|
301
321
|
|
|
302
322
|
|
|
303
323
|
def compress_uri(uri: URIRef, prefixes: dict) -> str:
|
|
@@ -420,7 +440,8 @@ def _add_filter(class_instances, query_template):
|
|
|
420
440
|
if class_instances:
|
|
421
441
|
class_instances_formatted = [f"<{instance}>" for instance in class_instances]
|
|
422
442
|
query_template = query_template.replace(
|
|
423
|
-
"insert_filter",
|
|
443
|
+
"insert_filter",
|
|
444
|
+
f"\n\nFILTER (?subject IN ({', '.join(class_instances_formatted)}))",
|
|
424
445
|
)
|
|
425
446
|
else:
|
|
426
447
|
query_template = query_template.replace("insert_filter", "")
|
|
@@ -439,7 +460,10 @@ def _triples2sparql_statement(triples: list[Triple]):
|
|
|
439
460
|
|
|
440
461
|
|
|
441
462
|
def _to_construct_triples(
|
|
442
|
-
graph: Graph,
|
|
463
|
+
graph: Graph,
|
|
464
|
+
class_: str,
|
|
465
|
+
transformation_rules: Rules,
|
|
466
|
+
properties_optional: bool = True,
|
|
443
467
|
) -> tuple[list[Triple], list[Triple]]:
|
|
444
468
|
"""Converts class definition to CONSTRUCT triples which are used to generate CONSTRUCT query
|
|
445
469
|
|
|
@@ -485,7 +509,10 @@ def _to_construct_triples(
|
|
|
485
509
|
# by binding them to certain property
|
|
486
510
|
if isinstance(traversal, AllReferences):
|
|
487
511
|
graph_pattern_triple = Triple(
|
|
488
|
-
subject="BIND(?subject",
|
|
512
|
+
subject="BIND(?subject",
|
|
513
|
+
predicate="AS",
|
|
514
|
+
object=f"{graph_template_triple.object})",
|
|
515
|
+
optional=False,
|
|
489
516
|
)
|
|
490
517
|
|
|
491
518
|
elif isinstance(traversal, SingleProperty):
|
|
@@ -510,7 +537,13 @@ def _to_construct_triples(
|
|
|
510
537
|
|
|
511
538
|
# add first triple for graph pattern stating type of object
|
|
512
539
|
patterns.insert(
|
|
513
|
-
0,
|
|
540
|
+
0,
|
|
541
|
+
Triple(
|
|
542
|
+
subject="?subject",
|
|
543
|
+
predicate="a",
|
|
544
|
+
object=_most_occurring_element(class_ids),
|
|
545
|
+
optional=False,
|
|
546
|
+
),
|
|
514
547
|
)
|
|
515
548
|
|
|
516
549
|
return templates, patterns
|
|
@@ -24,7 +24,7 @@ from cognite.neat.legacy.rules.models.rdfpath import (
|
|
|
24
24
|
parse_rule,
|
|
25
25
|
)
|
|
26
26
|
from cognite.neat.legacy.rules.models.rules import Rules
|
|
27
|
-
from cognite.neat.utils.
|
|
27
|
+
from cognite.neat.utils.rdf_ import remove_namespace_from_uri
|
|
28
28
|
|
|
29
29
|
prom_total_proc_rules_g = Gauge("neat_total_processed_rules", "Number of processed rules", ["state"])
|
|
30
30
|
rules_processing_timing_metric = Gauge(
|
|
@@ -8,6 +8,7 @@ from pathlib import Path
|
|
|
8
8
|
from typing import ClassVar, Literal, cast, no_type_check
|
|
9
9
|
|
|
10
10
|
import yaml
|
|
11
|
+
from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
|
|
11
12
|
|
|
12
13
|
from cognite.neat.legacy.rules.models.value_types import ValueTypeMapping
|
|
13
14
|
|
|
@@ -45,7 +46,7 @@ from cognite.neat.legacy.rules.exporters._base import BaseExporter
|
|
|
45
46
|
from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant
|
|
46
47
|
from cognite.neat.legacy.rules.models._base import ContainerEntity, EntityTypes, ParentClass
|
|
47
48
|
from cognite.neat.legacy.rules.models.rules import Class, Property, Rules
|
|
48
|
-
from cognite.neat.utils.
|
|
49
|
+
from cognite.neat.utils.auxiliary import generate_exception_report
|
|
49
50
|
|
|
50
51
|
if sys.version_info < (3, 11):
|
|
51
52
|
from exceptiongroup import ExceptionGroup
|
|
@@ -278,13 +279,17 @@ class DMSSchemaComponents(BaseModel):
|
|
|
278
279
|
existing_property, default_value=None
|
|
279
280
|
)
|
|
280
281
|
|
|
281
|
-
# scenario: property
|
|
282
|
+
# scenario: property holds multiple values -> set is_list to True
|
|
282
283
|
if (
|
|
283
284
|
not isinstance(existing_property.type, DirectRelation)
|
|
284
285
|
and not isinstance(api_container_property.type, DirectRelation)
|
|
286
|
+
and isinstance(existing_property.type, ListablePropertyType)
|
|
287
|
+
and isinstance(api_container_property.type, ListablePropertyType)
|
|
285
288
|
and existing_property.type.is_list != api_container_property.type.is_list
|
|
286
289
|
):
|
|
287
|
-
containers[container_id].properties[container_property_id].type
|
|
290
|
+
type_ = containers[container_id].properties[container_property_id].type
|
|
291
|
+
if isinstance(type_, ListablePropertyType):
|
|
292
|
+
type_.is_list = True
|
|
288
293
|
|
|
289
294
|
if errors:
|
|
290
295
|
raise ExceptionGroup("Properties value types have been redefined! This is prohibited! Aborting!", errors)
|
|
@@ -14,7 +14,7 @@ from cognite.neat.legacy.rules.analysis import to_class_property_pairs
|
|
|
14
14
|
from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant, are_properties_redefined
|
|
15
15
|
from cognite.neat.legacy.rules.models.rules import Rules
|
|
16
16
|
from cognite.neat.legacy.rules.models.value_types import XSD_VALUE_TYPE_MAPPINGS
|
|
17
|
-
from cognite.neat.utils.
|
|
17
|
+
from cognite.neat.utils.auxiliary import generate_exception_report
|
|
18
18
|
|
|
19
19
|
from ._base import BaseExporter
|
|
20
20
|
|
|
@@ -14,7 +14,8 @@ from cognite.neat.legacy.rules.exporters._base import BaseExporter
|
|
|
14
14
|
from cognite.neat.legacy.rules.exporters._validation import are_properties_redefined
|
|
15
15
|
from cognite.neat.legacy.rules.models.rules import Class, Metadata, Property, Rules
|
|
16
16
|
from cognite.neat.legacy.rules.models.value_types import XSD_VALUE_TYPE_MAPPINGS
|
|
17
|
-
from cognite.neat.utils.
|
|
17
|
+
from cognite.neat.utils.auxiliary import generate_exception_report
|
|
18
|
+
from cognite.neat.utils.rdf_ import remove_namespace_from_uri
|
|
18
19
|
|
|
19
20
|
if sys.version_info >= (3, 11):
|
|
20
21
|
from typing import Self
|
|
@@ -11,7 +11,6 @@ from cognite.client.data_classes.data_modeling.views import SingleHopConnectionD
|
|
|
11
11
|
from pydantic import BaseModel, ConfigDict, Field, create_model
|
|
12
12
|
from pydantic._internal._model_construction import ModelMetaclass
|
|
13
13
|
from rdflib import Graph, URIRef
|
|
14
|
-
from typing_extensions import TypeAliasType
|
|
15
14
|
|
|
16
15
|
from cognite.neat.legacy.graph.loaders.core.rdf_to_assets import NeatMetadataKeys
|
|
17
16
|
from cognite.neat.legacy.graph.transformations.query_generator.sparql import build_construct_query, triples2dictionary
|
|
@@ -21,7 +20,7 @@ from cognite.neat.legacy.rules.exporters._rules2dms import DMSSchemaComponents
|
|
|
21
20
|
from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant
|
|
22
21
|
from cognite.neat.legacy.rules.models.rules import Property, Rules
|
|
23
22
|
from cognite.neat.legacy.rules.models.value_types import ValueTypeMapping
|
|
24
|
-
from cognite.neat.utils.
|
|
23
|
+
from cognite.neat.utils.auxiliary import create_sha256_hash, generate_exception_report
|
|
25
24
|
|
|
26
25
|
if sys.version_info >= (3, 11):
|
|
27
26
|
from datetime import UTC
|
|
@@ -30,8 +29,8 @@ else:
|
|
|
30
29
|
|
|
31
30
|
UTC = timezone.utc
|
|
32
31
|
|
|
33
|
-
EdgeOneToOne: TypeAlias =
|
|
34
|
-
EdgeOneToMany: TypeAlias =
|
|
32
|
+
EdgeOneToOne: TypeAlias = str # type: ignore[valid-type]
|
|
33
|
+
EdgeOneToMany: TypeAlias = list[str] # type: ignore[valid-type]
|
|
35
34
|
|
|
36
35
|
|
|
37
36
|
def default_model_configuration(
|
|
@@ -13,6 +13,7 @@ from cognite.client.data_classes.data_modeling import (
|
|
|
13
13
|
SingleHopConnectionDefinition,
|
|
14
14
|
View,
|
|
15
15
|
)
|
|
16
|
+
from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
|
|
16
17
|
from cognite.client.data_classes.data_modeling.ids import DataModelIdentifier, ViewId
|
|
17
18
|
|
|
18
19
|
from cognite.neat.legacy.rules.models.tables import Tables
|
|
@@ -129,7 +130,9 @@ class DMSImporter(BaseImporter):
|
|
|
129
130
|
|
|
130
131
|
max_count: str | float = "1"
|
|
131
132
|
if isinstance(prop, SingleHopConnectionDefinition) or (
|
|
132
|
-
isinstance(prop, MappedProperty)
|
|
133
|
+
isinstance(prop, MappedProperty)
|
|
134
|
+
and isinstance(prop.type, ListablePropertyType)
|
|
135
|
+
and prop.type.is_list
|
|
133
136
|
):
|
|
134
137
|
max_count = float("nan")
|
|
135
138
|
|
|
@@ -10,11 +10,11 @@ from typing import cast
|
|
|
10
10
|
import pandas as pd
|
|
11
11
|
from rdflib import Graph, Literal, Namespace, URIRef
|
|
12
12
|
|
|
13
|
-
from cognite.neat.constants import
|
|
13
|
+
from cognite.neat.constants import get_default_prefixes
|
|
14
14
|
from cognite.neat.legacy.rules import exceptions
|
|
15
15
|
from cognite.neat.legacy.rules.exporters._rules2rules import to_dms_name
|
|
16
16
|
from cognite.neat.legacy.rules.models.tables import Tables
|
|
17
|
-
from cognite.neat.utils.
|
|
17
|
+
from cognite.neat.utils.rdf_ import get_namespace, remove_namespace_from_uri, uri_to_short_form
|
|
18
18
|
|
|
19
19
|
from ._base import BaseImporter
|
|
20
20
|
|
|
@@ -169,7 +169,7 @@ def _graph_to_data_model_dict(graph: Graph, max_number_of_instance: int = -1) ->
|
|
|
169
169
|
"""
|
|
170
170
|
data_model: dict[str, dict] = {}
|
|
171
171
|
|
|
172
|
-
prefixes: dict[str, Namespace] =
|
|
172
|
+
prefixes: dict[str, Namespace] = get_default_prefixes()
|
|
173
173
|
|
|
174
174
|
for class_ in _get_class_ids(graph):
|
|
175
175
|
_add_uri_namespace_to_prefixes(class_, prefixes)
|
|
@@ -4,7 +4,7 @@ import numpy as np
|
|
|
4
4
|
import pandas as pd
|
|
5
5
|
from rdflib import OWL, Graph
|
|
6
6
|
|
|
7
|
-
from cognite.neat.utils.
|
|
7
|
+
from cognite.neat.utils.rdf_ import remove_namespace_from_uri
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
def parse_owl_classes(graph: Graph, make_compliant: bool = False, language: str = "en") -> pd.DataFrame:
|
|
@@ -10,7 +10,8 @@ from cognite.neat.legacy.rules.models.rules import (
|
|
|
10
10
|
prefix_compliance_regex,
|
|
11
11
|
version_compliance_regex,
|
|
12
12
|
)
|
|
13
|
-
from cognite.neat.utils.
|
|
13
|
+
from cognite.neat.utils.collection_ import remove_none_elements_from_set
|
|
14
|
+
from cognite.neat.utils.rdf_ import convert_rdflib_content
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
def parse_owl_metadata(graph: Graph, make_compliant: bool = False) -> pd.DataFrame:
|
|
@@ -4,7 +4,7 @@ import numpy as np
|
|
|
4
4
|
import pandas as pd
|
|
5
5
|
from rdflib import Graph
|
|
6
6
|
|
|
7
|
-
from cognite.neat.utils.
|
|
7
|
+
from cognite.neat.utils.rdf_ import remove_namespace_from_uri
|
|
8
8
|
|
|
9
9
|
from ._owl2classes import _data_type_property_class, _object_property_class, _thing_class
|
|
10
10
|
|
|
@@ -9,16 +9,22 @@ from pydantic import field_validator
|
|
|
9
9
|
from pydantic_core import ErrorDetails, ValidationError
|
|
10
10
|
from rdflib import Namespace
|
|
11
11
|
|
|
12
|
-
from cognite.neat.constants import
|
|
12
|
+
from cognite.neat.constants import get_default_prefixes
|
|
13
13
|
from cognite.neat.exceptions import wrangle_warnings
|
|
14
14
|
|
|
15
15
|
# rules model and model components:
|
|
16
|
-
from cognite.neat.legacy.rules.models.rules import
|
|
16
|
+
from cognite.neat.legacy.rules.models.rules import (
|
|
17
|
+
Class,
|
|
18
|
+
Metadata,
|
|
19
|
+
Property,
|
|
20
|
+
RuleModel,
|
|
21
|
+
Rules,
|
|
22
|
+
)
|
|
17
23
|
from cognite.neat.legacy.rules.models.tables import Tables
|
|
18
24
|
|
|
19
25
|
# importers:
|
|
20
26
|
from cognite.neat.rules import exceptions
|
|
21
|
-
from cognite.neat.utils.
|
|
27
|
+
from cognite.neat.utils.auxiliary import generate_exception_report
|
|
22
28
|
|
|
23
29
|
__all__ = ["RawRules"]
|
|
24
30
|
|
|
@@ -36,7 +42,7 @@ class RawRules(RuleModel):
|
|
|
36
42
|
classes: Classes defined in the data model
|
|
37
43
|
properties: Class properties defined in the data model with accompanying transformation rules
|
|
38
44
|
to transform data from source to target representation
|
|
39
|
-
prefixes: Prefixes used in the data model. Defaults to
|
|
45
|
+
prefixes: Prefixes used in the data model. Defaults to internal set of prefixes
|
|
40
46
|
instances: Instances defined in the data model. Defaults to None
|
|
41
47
|
"""
|
|
42
48
|
|
|
@@ -242,7 +248,9 @@ def _raw_tables_to_rules_dict(raw_tables: RawRules, validators_to_skip: set | No
|
|
|
242
248
|
"metadata": _metadata_table2dict(raw_tables.Metadata),
|
|
243
249
|
"classes": _classes_table2dict(raw_tables.Classes),
|
|
244
250
|
"properties": _properties_table2dict(raw_tables.Properties),
|
|
245
|
-
"prefixes":
|
|
251
|
+
"prefixes": (
|
|
252
|
+
get_default_prefixes() if raw_tables.Prefixes.empty else _prefixes_table2dict(raw_tables.Prefixes)
|
|
253
|
+
),
|
|
246
254
|
}
|
|
247
255
|
|
|
248
256
|
rules_dict["instances"] = (
|
|
@@ -272,11 +280,15 @@ def _metadata_table2dict(meta_df: pd.DataFrame) -> dict[str, Any]:
|
|
|
272
280
|
return metadata_dict
|
|
273
281
|
|
|
274
282
|
|
|
275
|
-
def _classes_table2dict(
|
|
283
|
+
def _classes_table2dict(
|
|
284
|
+
classes_df: pd.DataFrame,
|
|
285
|
+
) -> dict[Any | None, dict[Hashable, Any]]:
|
|
276
286
|
return {class_.get("Class"): class_ for class_ in classes_df.to_dict(orient="records")}
|
|
277
287
|
|
|
278
288
|
|
|
279
|
-
def _properties_table2dict(
|
|
289
|
+
def _properties_table2dict(
|
|
290
|
+
properties_df: pd.DataFrame,
|
|
291
|
+
) -> dict[str, dict[Hashable, Any]]:
|
|
280
292
|
return {f"row {i+3}": property_ for i, property_ in enumerate(properties_df.to_dict(orient="records"))}
|
|
281
293
|
|
|
282
294
|
|