cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/__init__.py +4 -0
- cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
- cognite/neat/_client/_api/schema.py +50 -0
- cognite/neat/_client/_api_client.py +17 -0
- cognite/neat/_client/data_classes/__init__.py +0 -0
- cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
- cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
- cognite/neat/_graph/_shared.py +14 -15
- cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
- cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
- cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
- cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
- cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
- cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
- cognite/neat/_graph/extractors/_rdf_file.py +6 -7
- cognite/neat/_graph/loaders/__init__.py +1 -2
- cognite/neat/_graph/queries/_base.py +17 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
- cognite/neat/_graph/transformers/_prune_graph.py +1 -1
- cognite/neat/_graph/transformers/_rdfpath.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +6 -0
- cognite/neat/_issues/warnings/_external.py +8 -0
- cognite/neat/_issues/warnings/_models.py +9 -0
- cognite/neat/_issues/warnings/_properties.py +16 -0
- cognite/neat/_rules/_constants.py +7 -6
- cognite/neat/_rules/_shared.py +3 -8
- cognite/neat/_rules/analysis/__init__.py +1 -2
- cognite/neat/_rules/analysis/_base.py +10 -27
- cognite/neat/_rules/analysis/_dms.py +4 -10
- cognite/neat/_rules/analysis/_information.py +2 -10
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_base.py +3 -4
- cognite/neat/_rules/exporters/_rules2dms.py +29 -40
- cognite/neat/_rules/exporters/_rules2excel.py +15 -72
- cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
- cognite/neat/_rules/importers/_base.py +3 -4
- cognite/neat/_rules/importers/_dms2rules.py +21 -45
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
- cognite/neat/_rules/importers/_rdf/_base.py +17 -29
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
- cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
- cognite/neat/_rules/models/__init__.py +3 -17
- cognite/neat/_rules/models/_base_rules.py +118 -62
- cognite/neat/_rules/models/dms/__init__.py +2 -2
- cognite/neat/_rules/models/dms/_exporter.py +20 -178
- cognite/neat/_rules/models/dms/_rules.py +65 -128
- cognite/neat/_rules/models/dms/_rules_input.py +72 -56
- cognite/neat/_rules/models/dms/_validation.py +16 -109
- cognite/neat/_rules/models/entities/_single_value.py +32 -4
- cognite/neat/_rules/models/information/_rules.py +19 -122
- cognite/neat/_rules/models/information/_rules_input.py +32 -41
- cognite/neat/_rules/models/information/_validation.py +34 -102
- cognite/neat/_rules/models/mapping/__init__.py +2 -3
- cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
- cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
- cognite/neat/_rules/transformers/__init__.py +3 -6
- cognite/neat/_rules/transformers/_converters.py +128 -206
- cognite/neat/_rules/transformers/_mapping.py +105 -34
- cognite/neat/_rules/transformers/_verification.py +5 -16
- cognite/neat/_session/_base.py +83 -21
- cognite/neat/_session/_collector.py +126 -0
- cognite/neat/_session/_drop.py +35 -0
- cognite/neat/_session/_inspect.py +22 -10
- cognite/neat/_session/_mapping.py +39 -0
- cognite/neat/_session/_prepare.py +222 -27
- cognite/neat/_session/_read.py +109 -19
- cognite/neat/_session/_set.py +2 -2
- cognite/neat/_session/_show.py +11 -11
- cognite/neat/_session/_to.py +27 -14
- cognite/neat/_session/exceptions.py +20 -3
- cognite/neat/_store/_base.py +27 -24
- cognite/neat/_store/_provenance.py +2 -2
- cognite/neat/_utils/auxiliary.py +19 -0
- cognite/neat/_utils/rdf_.py +28 -1
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/steps/data_contracts.py +2 -10
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
- cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
- cognite/neat/_rules/analysis/_asset.py +0 -173
- cognite/neat/_rules/models/asset/__init__.py +0 -13
- cognite/neat/_rules/models/asset/_rules.py +0 -109
- cognite/neat/_rules/models/asset/_rules_input.py +0 -101
- cognite/neat/_rules/models/asset/_validation.py +0 -45
- cognite/neat/_rules/models/domain.py +0 -136
- cognite/neat/_rules/models/mapping/_base.py +0 -131
- cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
- cognite/neat/_utils/cdf/loaders/_base.py +0 -54
- cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
- cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
- /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
|
@@ -12,7 +12,7 @@ from cognite.neat._rules._constants import EntityTypes
|
|
|
12
12
|
from cognite.neat._rules.models.entities import ClassEntity
|
|
13
13
|
from cognite.neat._rules.models.information import InformationRules
|
|
14
14
|
from cognite.neat._shared import InstanceType
|
|
15
|
-
from cognite.neat._utils.rdf_ import remove_namespace_from_uri
|
|
15
|
+
from cognite.neat._utils.rdf_ import remove_instance_ids_in_batch, remove_namespace_from_uri
|
|
16
16
|
|
|
17
17
|
from ._construct import build_construct_query
|
|
18
18
|
|
|
@@ -342,3 +342,19 @@ class Queries:
|
|
|
342
342
|
self.graph.query(query.format(unknownType=str(UNKNOWN_TYPE))),
|
|
343
343
|
):
|
|
344
344
|
yield cast(URIRef, source_type), cast(URIRef, property_), [URIRef(uri) for uri in value_types.split(",")]
|
|
345
|
+
|
|
346
|
+
def drop_types(self, type_: list[URIRef]) -> dict[URIRef, int]:
|
|
347
|
+
"""Drop types from the graph store
|
|
348
|
+
|
|
349
|
+
Args:
|
|
350
|
+
type_: List of types to drop
|
|
351
|
+
|
|
352
|
+
Returns:
|
|
353
|
+
Dictionary of dropped types
|
|
354
|
+
"""
|
|
355
|
+
dropped_types: dict[URIRef, int] = {}
|
|
356
|
+
for t in type_:
|
|
357
|
+
instance_ids = self.list_instances_ids_of_class(t)
|
|
358
|
+
dropped_types[t] = len(instance_ids)
|
|
359
|
+
remove_instance_ids_in_batch(self.graph, instance_ids)
|
|
360
|
+
return dropped_types
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import textwrap
|
|
1
2
|
import warnings
|
|
3
|
+
from abc import ABC
|
|
2
4
|
from typing import cast
|
|
3
5
|
|
|
4
6
|
from rdflib import RDF, Graph, Literal, Namespace, URIRef
|
|
@@ -7,7 +9,7 @@ from rdflib.query import ResultRow
|
|
|
7
9
|
from cognite.neat._constants import CLASSIC_CDF_NAMESPACE, DEFAULT_NAMESPACE
|
|
8
10
|
from cognite.neat._graph import extractors
|
|
9
11
|
from cognite.neat._issues.warnings import ResourceNotFoundWarning
|
|
10
|
-
from cognite.neat._utils.rdf_ import remove_namespace_from_uri
|
|
12
|
+
from cognite.neat._utils.rdf_ import Triple, add_triples_in_batch, remove_namespace_from_uri
|
|
11
13
|
|
|
12
14
|
from ._base import BaseTransformer
|
|
13
15
|
|
|
@@ -32,8 +34,8 @@ class AddAssetDepth(BaseTransformer):
|
|
|
32
34
|
depth_typing: dict[int, str] | None = None,
|
|
33
35
|
):
|
|
34
36
|
self.asset_type = asset_type or DEFAULT_NAMESPACE.Asset
|
|
35
|
-
self.root_prop = root_prop or DEFAULT_NAMESPACE.
|
|
36
|
-
self.parent_prop = parent_prop or DEFAULT_NAMESPACE.
|
|
37
|
+
self.root_prop = root_prop or DEFAULT_NAMESPACE.rootId
|
|
38
|
+
self.parent_prop = parent_prop or DEFAULT_NAMESPACE.parentId
|
|
37
39
|
self.depth_typing = depth_typing
|
|
38
40
|
|
|
39
41
|
def transform(self, graph: Graph) -> None:
|
|
@@ -75,7 +77,33 @@ class AddAssetDepth(BaseTransformer):
|
|
|
75
77
|
return None
|
|
76
78
|
|
|
77
79
|
|
|
78
|
-
class
|
|
80
|
+
class BaseAssetConnector(BaseTransformer, ABC):
|
|
81
|
+
_asset_type: URIRef = DEFAULT_NAMESPACE.Asset
|
|
82
|
+
_item_type: URIRef
|
|
83
|
+
_default_attribute: URIRef
|
|
84
|
+
_connection_type: URIRef
|
|
85
|
+
|
|
86
|
+
_select_item_ids = "SELECT DISTINCT ?item_id WHERE {{?item_id a <{item_type}>}}"
|
|
87
|
+
_select_connected_assets: str = textwrap.dedent("""SELECT ?asset_id WHERE {{
|
|
88
|
+
<{item_id}> <{attribute}> ?asset_id .
|
|
89
|
+
?asset_id a <{asset_type}>}}""")
|
|
90
|
+
|
|
91
|
+
def __init__(self, attribute: URIRef | None = None) -> None:
|
|
92
|
+
self._attribute = attribute or self._default_attribute
|
|
93
|
+
|
|
94
|
+
def transform(self, graph: Graph) -> None:
|
|
95
|
+
for item_id, *_ in graph.query(self._select_item_ids.format(item_type=self._item_type)): # type: ignore[misc]
|
|
96
|
+
triples: list[Triple] = []
|
|
97
|
+
for asset_id, *_ in graph.query( # type: ignore[misc]
|
|
98
|
+
self._select_connected_assets.format(
|
|
99
|
+
item_id=item_id, attribute=self._attribute, asset_type=self._asset_type
|
|
100
|
+
)
|
|
101
|
+
):
|
|
102
|
+
triples.append((asset_id, self._connection_type, item_id)) # type: ignore[arg-type]
|
|
103
|
+
add_triples_in_batch(graph, triples)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class AssetTimeSeriesConnector(BaseAssetConnector):
|
|
79
107
|
description: str = "Connects assets to timeseries, thus forming bi-directional connection"
|
|
80
108
|
_use_only_once: bool = True
|
|
81
109
|
_need_changes = frozenset(
|
|
@@ -84,41 +112,12 @@ class AssetTimeSeriesConnector(BaseTransformer):
|
|
|
84
112
|
str(extractors.TimeSeriesExtractor.__name__),
|
|
85
113
|
}
|
|
86
114
|
)
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
115
|
+
_item_type = DEFAULT_NAMESPACE.TimeSeries
|
|
116
|
+
_default_attribute = DEFAULT_NAMESPACE.assetId
|
|
117
|
+
_connection_type = DEFAULT_NAMESPACE.timeSeries
|
|
90
118
|
|
|
91
|
-
def __init__(
|
|
92
|
-
self,
|
|
93
|
-
asset_type: URIRef | None = None,
|
|
94
|
-
timeseries_type: URIRef | None = None,
|
|
95
|
-
asset_prop: URIRef | None = None,
|
|
96
|
-
):
|
|
97
|
-
self.asset_type = asset_type or DEFAULT_NAMESPACE.Asset
|
|
98
|
-
self.timeseries_type = timeseries_type or DEFAULT_NAMESPACE.TimeSeries
|
|
99
|
-
self.asset_prop = asset_prop or DEFAULT_NAMESPACE.asset
|
|
100
119
|
|
|
101
|
-
|
|
102
|
-
for ts_id_result in graph.query(
|
|
103
|
-
f"SELECT DISTINCT ?timeseries_id WHERE {{?timeseries_id a <{self.timeseries_type}>}}"
|
|
104
|
-
):
|
|
105
|
-
timeseries_id: URIRef = cast(tuple, ts_id_result)[0]
|
|
106
|
-
|
|
107
|
-
if asset_id_res := list(
|
|
108
|
-
graph.query(
|
|
109
|
-
self._asset_template.format(
|
|
110
|
-
timeseries_id=timeseries_id,
|
|
111
|
-
asset_prop=self.asset_prop,
|
|
112
|
-
asset_type=self.asset_type,
|
|
113
|
-
)
|
|
114
|
-
)
|
|
115
|
-
):
|
|
116
|
-
# timeseries can be connected to only one asset in the graph
|
|
117
|
-
asset_id = cast(list[tuple], asset_id_res)[0][0]
|
|
118
|
-
graph.add((asset_id, DEFAULT_NAMESPACE.timeSeries, timeseries_id))
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
class AssetSequenceConnector(BaseTransformer):
|
|
120
|
+
class AssetSequenceConnector(BaseAssetConnector):
|
|
122
121
|
description: str = "Connects assets to sequences, thus forming bi-directional connection"
|
|
123
122
|
_use_only_once: bool = True
|
|
124
123
|
_need_changes = frozenset(
|
|
@@ -127,41 +126,12 @@ class AssetSequenceConnector(BaseTransformer):
|
|
|
127
126
|
str(extractors.SequencesExtractor.__name__),
|
|
128
127
|
}
|
|
129
128
|
)
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
def __init__(
|
|
135
|
-
self,
|
|
136
|
-
asset_type: URIRef | None = None,
|
|
137
|
-
sequence_type: URIRef | None = None,
|
|
138
|
-
asset_prop: URIRef | None = None,
|
|
139
|
-
):
|
|
140
|
-
self.asset_type = asset_type or DEFAULT_NAMESPACE.Asset
|
|
141
|
-
self.sequence_type = sequence_type or DEFAULT_NAMESPACE.Sequence
|
|
142
|
-
self.asset_prop = asset_prop or DEFAULT_NAMESPACE.asset
|
|
143
|
-
|
|
144
|
-
def transform(self, graph: Graph) -> None:
|
|
145
|
-
for sequency_id_result in graph.query(
|
|
146
|
-
f"SELECT DISTINCT ?sequence_id WHERE {{?sequence_id a <{self.sequence_type}>}}"
|
|
147
|
-
):
|
|
148
|
-
sequence_id: URIRef = cast(tuple, sequency_id_result)[0]
|
|
149
|
-
|
|
150
|
-
if asset_id_res := list(
|
|
151
|
-
graph.query(
|
|
152
|
-
self._asset_template.format(
|
|
153
|
-
sequence_id=sequence_id,
|
|
154
|
-
asset_prop=self.asset_prop,
|
|
155
|
-
asset_type=self.asset_type,
|
|
156
|
-
)
|
|
157
|
-
)
|
|
158
|
-
):
|
|
159
|
-
# sequence can be connected to only one asset in the graph
|
|
160
|
-
asset_id = cast(list[tuple], asset_id_res)[0][0]
|
|
161
|
-
graph.add((asset_id, DEFAULT_NAMESPACE.sequence, sequence_id))
|
|
129
|
+
_item_type = DEFAULT_NAMESPACE.Sequence
|
|
130
|
+
_default_attribute = DEFAULT_NAMESPACE.assetId
|
|
131
|
+
_connection_type = DEFAULT_NAMESPACE.sequence
|
|
162
132
|
|
|
163
133
|
|
|
164
|
-
class AssetFileConnector(
|
|
134
|
+
class AssetFileConnector(BaseAssetConnector):
|
|
165
135
|
description: str = "Connects assets to files, thus forming bi-directional connection"
|
|
166
136
|
_use_only_once: bool = True
|
|
167
137
|
_need_changes = frozenset(
|
|
@@ -170,39 +140,12 @@ class AssetFileConnector(BaseTransformer):
|
|
|
170
140
|
str(extractors.FilesExtractor.__name__),
|
|
171
141
|
}
|
|
172
142
|
)
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
def __init__(
|
|
178
|
-
self,
|
|
179
|
-
asset_type: URIRef | None = None,
|
|
180
|
-
file_type: URIRef | None = None,
|
|
181
|
-
asset_prop: URIRef | None = None,
|
|
182
|
-
):
|
|
183
|
-
self.asset_type = asset_type or DEFAULT_NAMESPACE.Asset
|
|
184
|
-
self.file_type = file_type or DEFAULT_NAMESPACE.File
|
|
185
|
-
self.asset_prop = asset_prop or DEFAULT_NAMESPACE.asset
|
|
143
|
+
_item_type = DEFAULT_NAMESPACE.File
|
|
144
|
+
_default_attribute = DEFAULT_NAMESPACE.assetIds
|
|
145
|
+
_connection_type = DEFAULT_NAMESPACE.file
|
|
186
146
|
|
|
187
|
-
def transform(self, graph: Graph) -> None:
|
|
188
|
-
for sequency_id_result in graph.query(f"SELECT DISTINCT ?file_id WHERE {{?file_id a <{self.file_type}>}}"):
|
|
189
|
-
file_id: URIRef = cast(tuple, sequency_id_result)[0]
|
|
190
|
-
|
|
191
|
-
if assets_id_res := list(
|
|
192
|
-
graph.query(
|
|
193
|
-
self._asset_template.format(
|
|
194
|
-
file_id=file_id,
|
|
195
|
-
asset_prop=self.asset_prop,
|
|
196
|
-
asset_type=self.asset_type,
|
|
197
|
-
)
|
|
198
|
-
)
|
|
199
|
-
):
|
|
200
|
-
# files can be connected to multiple assets in the graph
|
|
201
|
-
for (asset_id,) in cast(list[tuple], assets_id_res):
|
|
202
|
-
graph.add((asset_id, DEFAULT_NAMESPACE.file, file_id))
|
|
203
147
|
|
|
204
|
-
|
|
205
|
-
class AssetEventConnector(BaseTransformer):
|
|
148
|
+
class AssetEventConnector(BaseAssetConnector):
|
|
206
149
|
description: str = "Connects assets to events, thus forming bi-directional connection"
|
|
207
150
|
_use_only_once: bool = True
|
|
208
151
|
_need_changes = frozenset(
|
|
@@ -211,36 +154,9 @@ class AssetEventConnector(BaseTransformer):
|
|
|
211
154
|
str(extractors.EventsExtractor.__name__),
|
|
212
155
|
}
|
|
213
156
|
)
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
def __init__(
|
|
219
|
-
self,
|
|
220
|
-
asset_type: URIRef | None = None,
|
|
221
|
-
event_type: URIRef | None = None,
|
|
222
|
-
asset_prop: URIRef | None = None,
|
|
223
|
-
):
|
|
224
|
-
self.asset_type = asset_type or DEFAULT_NAMESPACE.Asset
|
|
225
|
-
self.event_type = event_type or DEFAULT_NAMESPACE.Event
|
|
226
|
-
self.asset_prop = asset_prop or DEFAULT_NAMESPACE.asset
|
|
227
|
-
|
|
228
|
-
def transform(self, graph: Graph) -> None:
|
|
229
|
-
for event_id_result in graph.query(f"SELECT DISTINCT ?event_id WHERE {{?event_id a <{self.event_type}>}}"):
|
|
230
|
-
event_id: URIRef = cast(tuple, event_id_result)[0]
|
|
231
|
-
|
|
232
|
-
if assets_id_res := list(
|
|
233
|
-
graph.query(
|
|
234
|
-
self._asset_template.format(
|
|
235
|
-
event_id=event_id,
|
|
236
|
-
asset_prop=self.asset_prop,
|
|
237
|
-
asset_type=self.asset_type,
|
|
238
|
-
)
|
|
239
|
-
)
|
|
240
|
-
):
|
|
241
|
-
# files can be connected to multiple assets in the graph
|
|
242
|
-
for (asset_id,) in cast(list[tuple], assets_id_res):
|
|
243
|
-
graph.add((asset_id, DEFAULT_NAMESPACE.event, event_id))
|
|
157
|
+
_item_type = DEFAULT_NAMESPACE.Event
|
|
158
|
+
_default_attribute = DEFAULT_NAMESPACE.assetIds
|
|
159
|
+
_connection_type = DEFAULT_NAMESPACE.event
|
|
244
160
|
|
|
245
161
|
|
|
246
162
|
class AssetRelationshipConnector(BaseTransformer):
|
|
@@ -271,9 +187,9 @@ class AssetRelationshipConnector(BaseTransformer):
|
|
|
271
187
|
):
|
|
272
188
|
self.asset_type = asset_type or DEFAULT_NAMESPACE.Asset
|
|
273
189
|
self.relationship_type = relationship_type or DEFAULT_NAMESPACE.Relationship
|
|
274
|
-
self.relationship_source_xid_prop = relationship_source_xid_prop or DEFAULT_NAMESPACE.
|
|
275
|
-
self.relationship_target_xid_prop = relationship_target_xid_prop or DEFAULT_NAMESPACE.
|
|
276
|
-
self.asset_xid_property = asset_xid_property or DEFAULT_NAMESPACE.
|
|
190
|
+
self.relationship_source_xid_prop = relationship_source_xid_prop or DEFAULT_NAMESPACE.sourceExternalId
|
|
191
|
+
self.relationship_target_xid_prop = relationship_target_xid_prop or DEFAULT_NAMESPACE.targetExternalId
|
|
192
|
+
self.asset_xid_property = asset_xid_property or DEFAULT_NAMESPACE.externalId
|
|
277
193
|
|
|
278
194
|
def transform(self, graph: Graph) -> None:
|
|
279
195
|
for relationship_id_result in graph.query(
|
|
@@ -123,4 +123,4 @@ class PruneDanglingNodes(BaseTransformer):
|
|
|
123
123
|
for node in nodes_without_neighbours:
|
|
124
124
|
# Remove node and its property triples in the graph
|
|
125
125
|
if isinstance(node, ResultRow):
|
|
126
|
-
graph.remove(
|
|
126
|
+
graph.remove((node["subject"], None, None))
|
|
@@ -6,6 +6,7 @@ from cognite.neat._issues._base import DefaultWarning, NeatWarning, _get_subclas
|
|
|
6
6
|
|
|
7
7
|
from . import user_modeling
|
|
8
8
|
from ._external import (
|
|
9
|
+
AuthWarning,
|
|
9
10
|
FileItemNotSupportedWarning,
|
|
10
11
|
FileMissingRequiredFieldWarning,
|
|
11
12
|
FileReadWarning,
|
|
@@ -26,6 +27,8 @@ from ._models import (
|
|
|
26
27
|
from ._properties import (
|
|
27
28
|
PropertyDefinitionDuplicatedWarning,
|
|
28
29
|
PropertyNotFoundWarning,
|
|
30
|
+
PropertyOverwritingWarning,
|
|
31
|
+
PropertySkippedWarning,
|
|
29
32
|
PropertyTypeNotSupportedWarning,
|
|
30
33
|
PropertyValueTypeUndefinedWarning,
|
|
31
34
|
)
|
|
@@ -52,6 +55,8 @@ __all__ = [
|
|
|
52
55
|
"PropertyTypeNotSupportedWarning",
|
|
53
56
|
"PropertyNotFoundWarning",
|
|
54
57
|
"PropertyValueTypeUndefinedWarning",
|
|
58
|
+
"PropertyOverwritingWarning",
|
|
59
|
+
"PropertySkippedWarning",
|
|
55
60
|
"ResourceNeatWarning",
|
|
56
61
|
"ResourcesDuplicatedWarning",
|
|
57
62
|
"RegexViolationWarning",
|
|
@@ -64,6 +69,7 @@ __all__ = [
|
|
|
64
69
|
"NotSupportedViewContainerLimitWarning",
|
|
65
70
|
"NotSupportedHasDataFilterLimitWarning",
|
|
66
71
|
"UndefinedViewWarning",
|
|
72
|
+
"AuthWarning",
|
|
67
73
|
"user_modeling",
|
|
68
74
|
]
|
|
69
75
|
|
|
@@ -92,6 +92,15 @@ class NotSupportedHasDataFilterLimitWarning(CDFNotSupportedWarning):
|
|
|
92
92
|
limit: int = DMS_VIEW_CONTAINER_SIZE_LIMIT
|
|
93
93
|
|
|
94
94
|
|
|
95
|
+
@dataclass(unsafe_hash=True)
|
|
96
|
+
class UndefinedClassWarning(UserModelingWarning):
|
|
97
|
+
"""Class {class_id} has no explicit properties defined neither implements other class"""
|
|
98
|
+
|
|
99
|
+
fix = "Define properties for class or inherit properties by implementing another class."
|
|
100
|
+
|
|
101
|
+
class_id: str
|
|
102
|
+
|
|
103
|
+
|
|
95
104
|
@dataclass(unsafe_hash=True)
|
|
96
105
|
class UndefinedViewWarning(UserModelingWarning):
|
|
97
106
|
"""Undefined view {value_type} has been referred as value type for property <{view_property}> of view {view_id}."""
|
|
@@ -54,3 +54,19 @@ class PropertyValueTypeUndefinedWarning(PropertyWarning[T_Identifier]):
|
|
|
54
54
|
|
|
55
55
|
default_action: str
|
|
56
56
|
recommended_action: str | None = None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(unsafe_hash=True)
|
|
60
|
+
class PropertyOverwritingWarning(PropertyWarning[T_Identifier]):
|
|
61
|
+
"""Overwriting the {overwriting} for {property_name} in the {resource_type}
|
|
62
|
+
with identifier {identifier}."""
|
|
63
|
+
|
|
64
|
+
overwriting: tuple[str, ...]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(unsafe_hash=True)
|
|
68
|
+
class PropertySkippedWarning(PropertyWarning[T_Identifier]):
|
|
69
|
+
"""The {resource_type} with identifier {identifier} has a property {property_name}
|
|
70
|
+
which is skipped. {reason}."""
|
|
71
|
+
|
|
72
|
+
reason: str
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import re
|
|
2
2
|
import sys
|
|
3
3
|
from functools import cached_property
|
|
4
|
+
from typing import Literal
|
|
4
5
|
|
|
5
6
|
if sys.version_info >= (3, 11):
|
|
6
7
|
from enum import StrEnum
|
|
@@ -42,7 +43,7 @@ class EntityTypes(StrEnum):
|
|
|
42
43
|
space = "space"
|
|
43
44
|
|
|
44
45
|
|
|
45
|
-
def get_reserved_words() ->
|
|
46
|
+
def get_reserved_words(key: Literal["class", "view", "property", "space"]) -> list[str]:
|
|
46
47
|
return {
|
|
47
48
|
"class": ["Class", "class"],
|
|
48
49
|
"view": [
|
|
@@ -82,7 +83,7 @@ def get_reserved_words() -> dict[str, list[str]]:
|
|
|
82
83
|
"extensions",
|
|
83
84
|
],
|
|
84
85
|
"space": ["space", "cdf", "dms", "pg3", "shared", "system", "node", "edge"],
|
|
85
|
-
}
|
|
86
|
+
}[key]
|
|
86
87
|
|
|
87
88
|
|
|
88
89
|
ENTITY_PATTERN = re.compile(r"^(?P<prefix>.*?):?(?P<suffix>[^(:]*)(\((?P<content>.+)\))?$")
|
|
@@ -93,20 +94,20 @@ MORE_THAN_ONE_NONE_ALPHANUMERIC_REGEX = r"([_-]{2,})"
|
|
|
93
94
|
PREFIX_COMPLIANCE_REGEX = r"^([a-zA-Z]+)([a-zA-Z0-9]*[_-]{0,1}[a-zA-Z0-9_-]*)([a-zA-Z0-9]*)$"
|
|
94
95
|
|
|
95
96
|
SPACE_COMPLIANCE_REGEX = (
|
|
96
|
-
rf"(?!^({'|'.join(get_reserved_words(
|
|
97
|
+
rf"(?!^({'|'.join(get_reserved_words('space'))})$)" r"(^[a-zA-Z][a-zA-Z0-9_-]{0,41}[a-zA-Z0-9]?$)"
|
|
97
98
|
)
|
|
98
99
|
|
|
99
100
|
|
|
100
101
|
DATA_MODEL_COMPLIANCE_REGEX = r"^[a-zA-Z]([a-zA-Z0-9_]{0,253}[a-zA-Z0-9])?$"
|
|
101
102
|
|
|
102
103
|
VIEW_ID_COMPLIANCE_REGEX = (
|
|
103
|
-
rf"(?!^({'|'.join(get_reserved_words(
|
|
104
|
+
rf"(?!^({'|'.join(get_reserved_words('view'))})$)" r"(^[a-zA-Z][a-zA-Z0-9_]{0,253}[a-zA-Z0-9]?$)"
|
|
104
105
|
)
|
|
105
106
|
DMS_PROPERTY_ID_COMPLIANCE_REGEX = (
|
|
106
|
-
rf"(?!^({'|'.join(get_reserved_words(
|
|
107
|
+
rf"(?!^({'|'.join(get_reserved_words('property'))})$)" r"(^[a-zA-Z][a-zA-Z0-9_]{0,253}[a-zA-Z0-9]?$)"
|
|
107
108
|
)
|
|
108
109
|
CLASS_ID_COMPLIANCE_REGEX = (
|
|
109
|
-
rf"(?!^({'|'.join(get_reserved_words(
|
|
110
|
+
rf"(?!^({'|'.join(get_reserved_words('class'))})$)" r"(^[a-zA-Z][a-zA-Z0-9._-]{0,253}[a-zA-Z0-9]?$)"
|
|
110
111
|
)
|
|
111
112
|
|
|
112
113
|
INFORMATION_PROPERTY_ID_COMPLIANCE_REGEX = (
|
cognite/neat/_rules/_shared.py
CHANGED
|
@@ -4,20 +4,15 @@ from typing import Any, Generic, TypeAlias, TypeVar
|
|
|
4
4
|
|
|
5
5
|
from cognite.neat._issues import IssueList
|
|
6
6
|
from cognite.neat._rules.models import (
|
|
7
|
-
AssetRules,
|
|
8
7
|
DMSRules,
|
|
9
|
-
DomainRules,
|
|
10
8
|
InformationRules,
|
|
11
9
|
)
|
|
12
|
-
from cognite.neat._rules.models.asset._rules_input import AssetInputRules
|
|
13
10
|
from cognite.neat._rules.models.dms._rules_input import DMSInputRules
|
|
14
11
|
from cognite.neat._rules.models.information._rules_input import InformationInputRules
|
|
15
12
|
|
|
16
|
-
VerifiedRules: TypeAlias =
|
|
17
|
-
InputRules: TypeAlias =
|
|
18
|
-
Rules: TypeAlias =
|
|
19
|
-
AssetInputRules | DMSInputRules | InformationInputRules | DomainRules | InformationRules | DMSRules | AssetRules
|
|
20
|
-
)
|
|
13
|
+
VerifiedRules: TypeAlias = InformationRules | DMSRules
|
|
14
|
+
InputRules: TypeAlias = DMSInputRules | InformationInputRules
|
|
15
|
+
Rules: TypeAlias = DMSInputRules | InformationInputRules | InformationRules | DMSRules
|
|
21
16
|
T_Rules = TypeVar("T_Rules", bound=Rules)
|
|
22
17
|
T_VerifiedRules = TypeVar("T_VerifiedRules", bound=VerifiedRules)
|
|
23
18
|
T_InputRules = TypeVar("T_InputRules", bound=InputRules)
|
|
@@ -14,7 +14,6 @@ from cognite.neat._rules.models._rdfpath import RDFPath
|
|
|
14
14
|
from cognite.neat._rules.models.entities import (
|
|
15
15
|
ClassEntity,
|
|
16
16
|
Entity,
|
|
17
|
-
ReferenceEntity,
|
|
18
17
|
)
|
|
19
18
|
from cognite.neat._rules.models.information import InformationProperty
|
|
20
19
|
from cognite.neat._utils.rdf_ import get_inheritance_path
|
|
@@ -76,10 +75,6 @@ class BaseAnalysis(ABC, Generic[T_Rules, T_Class, T_Property, T_ClassEntity, T_P
|
|
|
76
75
|
def _get_properties(self) -> list[T_Property]:
|
|
77
76
|
raise NotImplementedError
|
|
78
77
|
|
|
79
|
-
@abstractmethod
|
|
80
|
-
def _get_reference(self, class_or_property: T_Class | T_Property) -> ReferenceEntity | None:
|
|
81
|
-
raise NotImplementedError
|
|
82
|
-
|
|
83
78
|
@abstractmethod
|
|
84
79
|
def _get_cls_entity(self, class_: T_Class | T_Property) -> T_ClassEntity:
|
|
85
80
|
raise NotImplementedError
|
|
@@ -92,10 +87,6 @@ class BaseAnalysis(ABC, Generic[T_Rules, T_Class, T_Property, T_ClassEntity, T_P
|
|
|
92
87
|
def _get_cls_parents(self, class_: T_Class) -> list[T_ClassEntity] | None:
|
|
93
88
|
raise NotImplementedError
|
|
94
89
|
|
|
95
|
-
@abstractmethod
|
|
96
|
-
def _get_reference_rules(self) -> T_Rules | None:
|
|
97
|
-
raise NotImplementedError
|
|
98
|
-
|
|
99
90
|
@classmethod
|
|
100
91
|
@abstractmethod
|
|
101
92
|
def _set_cls_entity(cls, property_: T_Property, class_: T_ClassEntity) -> None:
|
|
@@ -111,33 +102,21 @@ class BaseAnalysis(ABC, Generic[T_Rules, T_Class, T_Property, T_ClassEntity, T_P
|
|
|
111
102
|
|
|
112
103
|
@property
|
|
113
104
|
def directly_referred_classes(self) -> set[ClassEntity]:
|
|
114
|
-
|
|
115
|
-
if ref_rules is None:
|
|
116
|
-
return set()
|
|
117
|
-
prefix = ref_rules.metadata.get_prefix()
|
|
118
|
-
return {
|
|
119
|
-
ref.as_class_entity()
|
|
120
|
-
for class_ in self._get_classes()
|
|
121
|
-
if isinstance((ref := self._get_reference(class_)), ReferenceEntity) and ref.prefix == prefix
|
|
122
|
-
}
|
|
105
|
+
raise NotImplementedError
|
|
123
106
|
|
|
124
107
|
@property
|
|
125
108
|
def inherited_referred_classes(self) -> set[ClassEntity]:
|
|
126
|
-
|
|
127
|
-
inherited_referred_classes = []
|
|
128
|
-
for class_ in dir_referred_classes:
|
|
129
|
-
inherited_referred_classes.extend(self.class_inheritance_path(class_))
|
|
130
|
-
return set(inherited_referred_classes)
|
|
109
|
+
raise NotImplementedError
|
|
131
110
|
|
|
132
111
|
# Todo Lru cache this method.
|
|
133
|
-
def class_parent_pairs(self) -> dict[T_ClassEntity, list[T_ClassEntity]]:
|
|
112
|
+
def class_parent_pairs(self, allow_different_space: bool = False) -> dict[T_ClassEntity, list[T_ClassEntity]]:
|
|
134
113
|
"""This only returns class - parent pairs only if parent is in the same data model"""
|
|
135
114
|
class_subclass_pairs: dict[T_ClassEntity, list[T_ClassEntity]] = {}
|
|
136
115
|
for cls_ in self._get_classes():
|
|
137
116
|
entity = self._get_cls_entity(cls_)
|
|
138
117
|
class_subclass_pairs[entity] = []
|
|
139
118
|
for parent in self._get_cls_parents(cls_) or []:
|
|
140
|
-
if parent.prefix == entity.prefix:
|
|
119
|
+
if parent.prefix == entity.prefix or allow_different_space:
|
|
141
120
|
class_subclass_pairs[entity].append(parent)
|
|
142
121
|
else:
|
|
143
122
|
warnings.warn(
|
|
@@ -147,11 +126,15 @@ class BaseAnalysis(ABC, Generic[T_Rules, T_Class, T_Property, T_ClassEntity, T_P
|
|
|
147
126
|
|
|
148
127
|
return class_subclass_pairs
|
|
149
128
|
|
|
150
|
-
def classes_with_properties(
|
|
129
|
+
def classes_with_properties(
|
|
130
|
+
self, consider_inheritance: bool = False, allow_different_namespace: bool = False
|
|
131
|
+
) -> dict[T_ClassEntity, list[T_Property]]:
|
|
151
132
|
"""Returns classes that have been defined in the data model.
|
|
152
133
|
|
|
153
134
|
Args:
|
|
154
135
|
consider_inheritance: Whether to consider inheritance or not. Defaults False
|
|
136
|
+
allow_different_namespace: When considering inheritance, whether to allow parents from
|
|
137
|
+
different namespaces or not. Defaults False
|
|
155
138
|
|
|
156
139
|
Returns:
|
|
157
140
|
Dictionary of classes with a list of properties defined for them
|
|
@@ -171,7 +154,7 @@ class BaseAnalysis(ABC, Generic[T_Rules, T_Class, T_Property, T_ClassEntity, T_P
|
|
|
171
154
|
class_property_pairs[self._get_cls_entity(property_)].append(property_) # type: ignore
|
|
172
155
|
|
|
173
156
|
if consider_inheritance:
|
|
174
|
-
class_parent_pairs = self.class_parent_pairs()
|
|
157
|
+
class_parent_pairs = self.class_parent_pairs(allow_different_namespace)
|
|
175
158
|
for class_ in class_parent_pairs:
|
|
176
159
|
self._add_inherited_properties(class_, class_property_pairs, class_parent_pairs)
|
|
177
160
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from cognite.neat._constants import DMS_LISTABLE_PROPERTY_LIMIT
|
|
2
2
|
from cognite.neat._rules.models.dms import DMSProperty, DMSRules, DMSView
|
|
3
|
-
from cognite.neat._rules.models.entities import
|
|
3
|
+
from cognite.neat._rules.models.entities import ViewEntity
|
|
4
4
|
|
|
5
5
|
from ._base import BaseAnalysis
|
|
6
6
|
|
|
@@ -14,21 +14,12 @@ class DMSAnalysis(BaseAnalysis[DMSRules, DMSView, DMSProperty, ViewEntity, str])
|
|
|
14
14
|
def _get_properties(self) -> list[DMSProperty]:
|
|
15
15
|
return list(self.rules.properties)
|
|
16
16
|
|
|
17
|
-
def _get_reference(self, class_or_property: DMSView | DMSProperty) -> ReferenceEntity | None:
|
|
18
|
-
return class_or_property.reference if isinstance(class_or_property.reference, ReferenceEntity) else None
|
|
19
|
-
|
|
20
17
|
def _get_cls_entity(self, class_: DMSView | DMSProperty) -> ViewEntity:
|
|
21
18
|
return class_.view
|
|
22
19
|
|
|
23
|
-
def _get_prop_entity(self, property_: DMSProperty) -> str:
|
|
24
|
-
return property_.property_
|
|
25
|
-
|
|
26
20
|
def _get_cls_parents(self, class_: DMSView) -> list[ViewEntity] | None:
|
|
27
21
|
return list(class_.implements) if class_.implements else None
|
|
28
22
|
|
|
29
|
-
def _get_reference_rules(self) -> DMSRules | None:
|
|
30
|
-
return self.rules.reference
|
|
31
|
-
|
|
32
23
|
@classmethod
|
|
33
24
|
def _set_cls_entity(cls, property_: DMSProperty, class_: ViewEntity) -> None:
|
|
34
25
|
property_.view = class_
|
|
@@ -41,3 +32,6 @@ class DMSAnalysis(BaseAnalysis[DMSRules, DMSView, DMSProperty, ViewEntity, str])
|
|
|
41
32
|
|
|
42
33
|
def subset_rules(self, desired_classes: set[ViewEntity]) -> DMSRules:
|
|
43
34
|
raise NotImplementedError()
|
|
35
|
+
|
|
36
|
+
def _get_prop_entity(self, property_: DMSProperty) -> str:
|
|
37
|
+
return property_.view_property
|
|
@@ -12,7 +12,7 @@ from cognite.neat._rules.models._rdfpath import (
|
|
|
12
12
|
SelfReferenceProperty,
|
|
13
13
|
SingleProperty,
|
|
14
14
|
)
|
|
15
|
-
from cognite.neat._rules.models.entities import ClassEntity
|
|
15
|
+
from cognite.neat._rules.models.entities import ClassEntity
|
|
16
16
|
from cognite.neat._rules.models.entities._multi_value import MultiValueTypeInfo
|
|
17
17
|
from cognite.neat._rules.models.information import (
|
|
18
18
|
InformationClass,
|
|
@@ -34,9 +34,6 @@ class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, Infor
|
|
|
34
34
|
def _get_max_occurrence(self, property_: InformationProperty) -> int | float | None:
|
|
35
35
|
return property_.max_count
|
|
36
36
|
|
|
37
|
-
def _get_reference(self, class_or_property: InformationClass | InformationProperty) -> ReferenceEntity | None:
|
|
38
|
-
return class_or_property.reference if isinstance(class_or_property.reference, ReferenceEntity) else None
|
|
39
|
-
|
|
40
37
|
def _get_cls_entity(self, class_: InformationClass | InformationProperty) -> ClassEntity:
|
|
41
38
|
return class_.class_
|
|
42
39
|
|
|
@@ -48,10 +45,7 @@ class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, Infor
|
|
|
48
45
|
return property_.property_
|
|
49
46
|
|
|
50
47
|
def _get_cls_parents(self, class_: InformationClass) -> list[ClassEntity] | None:
|
|
51
|
-
return list(class_.
|
|
52
|
-
|
|
53
|
-
def _get_reference_rules(self) -> InformationRules | None:
|
|
54
|
-
return self.rules.reference
|
|
48
|
+
return list(class_.implements or []) or None
|
|
55
49
|
|
|
56
50
|
def _get_properties(self) -> list[InformationProperty]:
|
|
57
51
|
return list(self.rules.properties)
|
|
@@ -154,8 +148,6 @@ class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, Infor
|
|
|
154
148
|
If it fails, it will return a partial rules with a warning message, validated
|
|
155
149
|
only with base Pydantic validators.
|
|
156
150
|
"""
|
|
157
|
-
if self.rules.metadata.schema_ is not SchemaCompleteness.complete:
|
|
158
|
-
raise ValueError("Rules are not complete cannot perform reduction!")
|
|
159
151
|
class_as_dict = self.as_class_dict()
|
|
160
152
|
class_parents_pairs = self.class_parent_pairs()
|
|
161
153
|
defined_classes = self.defined_classes(consider_inheritance=True)
|
|
Binary file
|
|
@@ -3,8 +3,7 @@ from collections.abc import Iterable
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
from typing import Generic, TypeVar
|
|
5
5
|
|
|
6
|
-
from cognite.
|
|
7
|
-
|
|
6
|
+
from cognite.neat._client import NeatClient
|
|
8
7
|
from cognite.neat._rules._shared import T_VerifiedRules
|
|
9
8
|
from cognite.neat._utils.auxiliary import class_html_doc
|
|
10
9
|
from cognite.neat._utils.upload import UploadResult, UploadResultList
|
|
@@ -32,11 +31,11 @@ class BaseExporter(ABC, Generic[T_VerifiedRules, T_Export]):
|
|
|
32
31
|
class CDFExporter(BaseExporter[T_VerifiedRules, T_Export]):
|
|
33
32
|
@abstractmethod
|
|
34
33
|
def export_to_cdf_iterable(
|
|
35
|
-
self, rules: T_VerifiedRules, client:
|
|
34
|
+
self, rules: T_VerifiedRules, client: NeatClient, dry_run: bool = False, fallback_one_by_one: bool = False
|
|
36
35
|
) -> Iterable[UploadResult]:
|
|
37
36
|
raise NotImplementedError
|
|
38
37
|
|
|
39
38
|
def export_to_cdf(
|
|
40
|
-
self, rules: T_VerifiedRules, client:
|
|
39
|
+
self, rules: T_VerifiedRules, client: NeatClient, dry_run: bool = False, fallback_one_by_one: bool = False
|
|
41
40
|
) -> UploadResultList:
|
|
42
41
|
return UploadResultList(self.export_to_cdf_iterable(rules, client, dry_run, fallback_one_by_one))
|