cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/__init__.py +4 -0
- cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
- cognite/neat/_client/_api/schema.py +50 -0
- cognite/neat/_client/_api_client.py +17 -0
- cognite/neat/_client/data_classes/__init__.py +0 -0
- cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
- cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
- cognite/neat/_graph/_shared.py +14 -15
- cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
- cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
- cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
- cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
- cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
- cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
- cognite/neat/_graph/extractors/_rdf_file.py +6 -7
- cognite/neat/_graph/loaders/__init__.py +1 -2
- cognite/neat/_graph/queries/_base.py +17 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
- cognite/neat/_graph/transformers/_prune_graph.py +1 -1
- cognite/neat/_graph/transformers/_rdfpath.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +6 -0
- cognite/neat/_issues/warnings/_external.py +8 -0
- cognite/neat/_issues/warnings/_models.py +9 -0
- cognite/neat/_issues/warnings/_properties.py +16 -0
- cognite/neat/_rules/_constants.py +7 -6
- cognite/neat/_rules/_shared.py +3 -8
- cognite/neat/_rules/analysis/__init__.py +1 -2
- cognite/neat/_rules/analysis/_base.py +10 -27
- cognite/neat/_rules/analysis/_dms.py +4 -10
- cognite/neat/_rules/analysis/_information.py +2 -10
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_base.py +3 -4
- cognite/neat/_rules/exporters/_rules2dms.py +29 -40
- cognite/neat/_rules/exporters/_rules2excel.py +15 -72
- cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
- cognite/neat/_rules/importers/_base.py +3 -4
- cognite/neat/_rules/importers/_dms2rules.py +21 -45
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
- cognite/neat/_rules/importers/_rdf/_base.py +17 -29
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
- cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
- cognite/neat/_rules/models/__init__.py +3 -17
- cognite/neat/_rules/models/_base_rules.py +118 -62
- cognite/neat/_rules/models/dms/__init__.py +2 -2
- cognite/neat/_rules/models/dms/_exporter.py +20 -178
- cognite/neat/_rules/models/dms/_rules.py +65 -128
- cognite/neat/_rules/models/dms/_rules_input.py +72 -56
- cognite/neat/_rules/models/dms/_validation.py +16 -109
- cognite/neat/_rules/models/entities/_single_value.py +32 -4
- cognite/neat/_rules/models/information/_rules.py +19 -122
- cognite/neat/_rules/models/information/_rules_input.py +32 -41
- cognite/neat/_rules/models/information/_validation.py +34 -102
- cognite/neat/_rules/models/mapping/__init__.py +2 -3
- cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
- cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
- cognite/neat/_rules/transformers/__init__.py +3 -6
- cognite/neat/_rules/transformers/_converters.py +128 -206
- cognite/neat/_rules/transformers/_mapping.py +105 -34
- cognite/neat/_rules/transformers/_verification.py +5 -16
- cognite/neat/_session/_base.py +83 -21
- cognite/neat/_session/_collector.py +126 -0
- cognite/neat/_session/_drop.py +35 -0
- cognite/neat/_session/_inspect.py +22 -10
- cognite/neat/_session/_mapping.py +39 -0
- cognite/neat/_session/_prepare.py +222 -27
- cognite/neat/_session/_read.py +109 -19
- cognite/neat/_session/_set.py +2 -2
- cognite/neat/_session/_show.py +11 -11
- cognite/neat/_session/_to.py +27 -14
- cognite/neat/_session/exceptions.py +20 -3
- cognite/neat/_store/_base.py +27 -24
- cognite/neat/_store/_provenance.py +2 -2
- cognite/neat/_utils/auxiliary.py +19 -0
- cognite/neat/_utils/rdf_.py +28 -1
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/steps/data_contracts.py +2 -10
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
- cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
- cognite/neat/_rules/analysis/_asset.py +0 -173
- cognite/neat/_rules/models/asset/__init__.py +0 -13
- cognite/neat/_rules/models/asset/_rules.py +0 -109
- cognite/neat/_rules/models/asset/_rules_input.py +0 -101
- cognite/neat/_rules/models/asset/_validation.py +0 -45
- cognite/neat/_rules/models/domain.py +0 -136
- cognite/neat/_rules/models/mapping/_base.py +0 -131
- cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
- cognite/neat/_utils/cdf/loaders/_base.py +0 -54
- cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
- cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
- /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
from datetime import datetime, timezone
|
|
2
|
+
|
|
3
|
+
from cognite.neat._rules.models.mapping import load_classic_to_core_mapping
|
|
4
|
+
from cognite.neat._rules.transformers import RuleMapper
|
|
5
|
+
from cognite.neat._store._provenance import Change
|
|
6
|
+
|
|
7
|
+
from ._state import SessionState
|
|
8
|
+
from .exceptions import session_class_wrapper
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@session_class_wrapper
|
|
12
|
+
class MappingAPI:
|
|
13
|
+
def __init__(self, state: SessionState):
|
|
14
|
+
self._state = state
|
|
15
|
+
|
|
16
|
+
def classic_to_core(self, org_name: str) -> None:
|
|
17
|
+
"""Map classic types to core types.
|
|
18
|
+
|
|
19
|
+
Note this automatically creates an extended CogniteCore model.
|
|
20
|
+
|
|
21
|
+
"""
|
|
22
|
+
source_id, rules = self._state.data_model.last_verified_dms_rules
|
|
23
|
+
|
|
24
|
+
start = datetime.now(timezone.utc)
|
|
25
|
+
transformer = RuleMapper(load_classic_to_core_mapping(org_name, rules.metadata.space, rules.metadata.version))
|
|
26
|
+
output = transformer.transform(rules)
|
|
27
|
+
end = datetime.now(timezone.utc)
|
|
28
|
+
|
|
29
|
+
change = Change.from_rules_activity(
|
|
30
|
+
output,
|
|
31
|
+
transformer.agent,
|
|
32
|
+
start,
|
|
33
|
+
end,
|
|
34
|
+
"Mapping classic to core",
|
|
35
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
36
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
self._state.data_model.write(output.rules, change)
|
|
@@ -1,30 +1,48 @@
|
|
|
1
|
+
import copy
|
|
1
2
|
from collections.abc import Collection
|
|
2
3
|
from datetime import datetime, timezone
|
|
3
|
-
from typing import Literal
|
|
4
|
+
from typing import Literal, cast
|
|
4
5
|
|
|
5
6
|
from cognite.client.data_classes.data_modeling import DataModelIdentifier
|
|
6
7
|
from rdflib import URIRef
|
|
7
8
|
|
|
9
|
+
from cognite.neat._client import NeatClient
|
|
10
|
+
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
11
|
+
from cognite.neat._graph.transformers import RelationshipToSchemaTransformer
|
|
8
12
|
from cognite.neat._graph.transformers._rdfpath import MakeConnectionOnExactMatch
|
|
9
|
-
from cognite.neat._rules._shared import ReadRules
|
|
13
|
+
from cognite.neat._rules._shared import InputRules, ReadRules
|
|
14
|
+
from cognite.neat._rules.importers import DMSImporter
|
|
15
|
+
from cognite.neat._rules.models import DMSRules
|
|
10
16
|
from cognite.neat._rules.models.information._rules_input import InformationInputRules
|
|
11
|
-
from cognite.neat._rules.transformers import
|
|
17
|
+
from cognite.neat._rules.transformers import (
|
|
18
|
+
PrefixEntities,
|
|
19
|
+
ReduceCogniteModel,
|
|
20
|
+
ToCompliantEntities,
|
|
21
|
+
ToExtension,
|
|
22
|
+
VerifyDMSRules,
|
|
23
|
+
)
|
|
24
|
+
from cognite.neat._store._provenance import Agent as ProvenanceAgent
|
|
12
25
|
from cognite.neat._store._provenance import Change
|
|
13
26
|
|
|
14
27
|
from ._state import SessionState
|
|
15
|
-
from .exceptions import NeatSessionError,
|
|
28
|
+
from .exceptions import NeatSessionError, session_class_wrapper
|
|
16
29
|
|
|
30
|
+
try:
|
|
31
|
+
from rich import print
|
|
32
|
+
except ImportError:
|
|
33
|
+
...
|
|
17
34
|
|
|
18
|
-
|
|
35
|
+
|
|
36
|
+
@session_class_wrapper
|
|
19
37
|
class PrepareAPI:
|
|
20
|
-
def __init__(self, state: SessionState, verbose: bool) -> None:
|
|
38
|
+
def __init__(self, client: NeatClient | None, state: SessionState, verbose: bool) -> None:
|
|
21
39
|
self._state = state
|
|
22
40
|
self._verbose = verbose
|
|
23
|
-
self.data_model = DataModelPrepareAPI(state, verbose)
|
|
41
|
+
self.data_model = DataModelPrepareAPI(client, state, verbose)
|
|
24
42
|
self.instances = InstancePrepareAPI(state, verbose)
|
|
25
43
|
|
|
26
44
|
|
|
27
|
-
@
|
|
45
|
+
@session_class_wrapper
|
|
28
46
|
class InstancePrepareAPI:
|
|
29
47
|
def __init__(self, state: SessionState, verbose: bool) -> None:
|
|
30
48
|
self._state = state
|
|
@@ -94,34 +112,77 @@ class InstancePrepareAPI:
|
|
|
94
112
|
raise NeatSessionError(f"Property {property_} is not defined for type {type_}. Cannot make connection")
|
|
95
113
|
return type_uri[0], property_uri[0]
|
|
96
114
|
|
|
115
|
+
def relationships_as_connections(self, limit: int = 1) -> None:
|
|
116
|
+
"""This assumes that you have read a classic CDF knowledge graph including relationships.
|
|
97
117
|
|
|
98
|
-
|
|
118
|
+
This transformer analyzes the relationships in the graph and modifies them to be part of the schema
|
|
119
|
+
for Assets, Events, Files, Sequences, and TimeSeries. Relationships without any properties
|
|
120
|
+
are replaced by a simple relationship between the source and target nodes. Relationships with
|
|
121
|
+
properties are replaced by a schema that contains the properties as attributes.
|
|
122
|
+
|
|
123
|
+
Args:
|
|
124
|
+
limit: The minimum number of relationships that need to be present for it
|
|
125
|
+
to be converted into a schema. Default is 1.
|
|
126
|
+
|
|
127
|
+
"""
|
|
128
|
+
transformer = RelationshipToSchemaTransformer(limit=limit)
|
|
129
|
+
self._state.instances.store.transform(transformer)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@session_class_wrapper
|
|
99
133
|
class DataModelPrepareAPI:
|
|
100
|
-
def __init__(self, state: SessionState, verbose: bool) -> None:
|
|
134
|
+
def __init__(self, client: NeatClient | None, state: SessionState, verbose: bool) -> None:
|
|
135
|
+
self._client = client
|
|
101
136
|
self._state = state
|
|
102
137
|
self._verbose = verbose
|
|
103
138
|
|
|
104
139
|
def cdf_compliant_external_ids(self) -> None:
|
|
105
140
|
"""Convert data model component external ids to CDF compliant entities."""
|
|
106
|
-
|
|
107
|
-
|
|
141
|
+
source_id, rules = self._state.data_model.last_info_unverified_rule
|
|
142
|
+
|
|
143
|
+
start = datetime.now(timezone.utc)
|
|
144
|
+
transformer = ToCompliantEntities()
|
|
145
|
+
output: ReadRules[InformationInputRules] = transformer.transform(rules)
|
|
146
|
+
end = datetime.now(timezone.utc)
|
|
147
|
+
|
|
148
|
+
change = Change.from_rules_activity(
|
|
149
|
+
output,
|
|
150
|
+
transformer.agent,
|
|
151
|
+
start,
|
|
152
|
+
end,
|
|
153
|
+
"Converted external ids to CDF compliant entities",
|
|
154
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
155
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
156
|
+
)
|
|
108
157
|
|
|
109
|
-
|
|
110
|
-
transformer = ToCompliantEntities()
|
|
111
|
-
output: ReadRules[InformationInputRules] = transformer.transform(rules)
|
|
112
|
-
end = datetime.now(timezone.utc)
|
|
158
|
+
self._state.data_model.write(output, change)
|
|
113
159
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
transformer.agent,
|
|
117
|
-
start,
|
|
118
|
-
end,
|
|
119
|
-
"Converted external ids to CDF compliant entities",
|
|
120
|
-
self._state.data_model.provenance.source_entity(source_id)
|
|
121
|
-
or self._state.data_model.provenance.target_entity(source_id),
|
|
122
|
-
)
|
|
160
|
+
def prefix(self, prefix: str) -> None:
|
|
161
|
+
"""Prefix all views in the data model with the given prefix.
|
|
123
162
|
|
|
124
|
-
|
|
163
|
+
Args:
|
|
164
|
+
prefix: The prefix to add to the views in the data model.
|
|
165
|
+
|
|
166
|
+
"""
|
|
167
|
+
source_id, rules = self._state.data_model.last_unverified_rule
|
|
168
|
+
|
|
169
|
+
start = datetime.now(timezone.utc)
|
|
170
|
+
transformer = PrefixEntities(prefix)
|
|
171
|
+
new_rules = cast(InputRules, copy.deepcopy(rules.get_rules()))
|
|
172
|
+
output = transformer.transform(new_rules)
|
|
173
|
+
end = datetime.now(timezone.utc)
|
|
174
|
+
|
|
175
|
+
change = Change.from_rules_activity(
|
|
176
|
+
output,
|
|
177
|
+
transformer.agent,
|
|
178
|
+
start,
|
|
179
|
+
end,
|
|
180
|
+
"Added prefix to the data model views",
|
|
181
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
182
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
self._state.data_model.write(output, change)
|
|
125
186
|
|
|
126
187
|
def to_enterprise(
|
|
127
188
|
self,
|
|
@@ -185,7 +246,7 @@ class DataModelPrepareAPI:
|
|
|
185
246
|
data_model_id: DataModelIdentifier,
|
|
186
247
|
org_name: str = "My",
|
|
187
248
|
mode: Literal["read", "write"] = "read",
|
|
188
|
-
dummy_property: str = "
|
|
249
|
+
dummy_property: str = "GUID",
|
|
189
250
|
) -> None:
|
|
190
251
|
"""Uses the current data model as a basis to create solution data model
|
|
191
252
|
|
|
@@ -235,6 +296,81 @@ class DataModelPrepareAPI:
|
|
|
235
296
|
|
|
236
297
|
self._state.data_model.write(output.rules, change)
|
|
237
298
|
|
|
299
|
+
def to_data_product(
|
|
300
|
+
self,
|
|
301
|
+
data_model_id: DataModelIdentifier,
|
|
302
|
+
org_name: str = "",
|
|
303
|
+
include: Literal["same-space", "all"] = "same-space",
|
|
304
|
+
) -> None:
|
|
305
|
+
"""Uses the current data model as a basis to create data product data model.
|
|
306
|
+
|
|
307
|
+
A data product model is a data model that ONLY maps to containers and do not use implements. This is
|
|
308
|
+
typically used for defining the data in a data product.
|
|
309
|
+
|
|
310
|
+
Args:
|
|
311
|
+
data_model_id: The data product data model id that is being created.
|
|
312
|
+
org_name: Organization name to use for the views in the new data model.
|
|
313
|
+
include: The views to include in the data product data model. Can be either "same-space" or "all".
|
|
314
|
+
If you set same-space, only the views in the same space as the data model will be included.
|
|
315
|
+
"""
|
|
316
|
+
source_id, rules = self._state.data_model.last_verified_dms_rules
|
|
317
|
+
|
|
318
|
+
dms_ref: DMSRules | None = None
|
|
319
|
+
view_ids, container_ids = rules.imported_views_and_containers_ids(include_model_views_with_no_properties=True)
|
|
320
|
+
if view_ids or container_ids:
|
|
321
|
+
if self._client is None:
|
|
322
|
+
raise NeatSessionError(
|
|
323
|
+
"No client provided. You are referencing unknown views and containers in your data model, "
|
|
324
|
+
"NEAT needs a client to lookup the definitions. "
|
|
325
|
+
"Please set the client in the session, NeatSession(client=client)."
|
|
326
|
+
)
|
|
327
|
+
schema = self._client.schema.retrieve(list(view_ids), list(container_ids))
|
|
328
|
+
|
|
329
|
+
importer = DMSImporter(schema)
|
|
330
|
+
reference_rules = importer.to_rules().rules
|
|
331
|
+
if reference_rules is not None:
|
|
332
|
+
imported = VerifyDMSRules("continue").transform(reference_rules)
|
|
333
|
+
if dms_ref := imported.rules:
|
|
334
|
+
rules = rules.model_copy(deep=True)
|
|
335
|
+
if rules.containers is None:
|
|
336
|
+
rules.containers = dms_ref.containers
|
|
337
|
+
else:
|
|
338
|
+
existing_containers = {c.container for c in rules.containers}
|
|
339
|
+
rules.containers.extend(
|
|
340
|
+
[c for c in dms_ref.containers or [] if c.container not in existing_containers]
|
|
341
|
+
)
|
|
342
|
+
existing_views = {v.view for v in rules.views}
|
|
343
|
+
rules.views.extend([v for v in dms_ref.views if v.view not in existing_views])
|
|
344
|
+
existing_properties = {(p.view, p.view_property) for p in rules.properties}
|
|
345
|
+
rules.properties.extend(
|
|
346
|
+
[p for p in dms_ref.properties if (p.view, p.view_property) not in existing_properties]
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
start = datetime.now(timezone.utc)
|
|
350
|
+
transformer = ToExtension(
|
|
351
|
+
new_model_id=data_model_id,
|
|
352
|
+
org_name=org_name,
|
|
353
|
+
type_="data_product",
|
|
354
|
+
include=include,
|
|
355
|
+
)
|
|
356
|
+
output = transformer.transform(rules)
|
|
357
|
+
end = datetime.now(timezone.utc)
|
|
358
|
+
|
|
359
|
+
change = Change.from_rules_activity(
|
|
360
|
+
output,
|
|
361
|
+
transformer.agent,
|
|
362
|
+
start,
|
|
363
|
+
end,
|
|
364
|
+
(
|
|
365
|
+
f"Prepared data model {data_model_id} to be data product model "
|
|
366
|
+
f"on top of {rules.metadata.as_data_model_id()}"
|
|
367
|
+
),
|
|
368
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
369
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
self._state.data_model.write(output.rules, change)
|
|
373
|
+
|
|
238
374
|
def reduce(self, drop: Collection[Literal["3D", "Annotation", "BaseViews"] | str]) -> None:
|
|
239
375
|
"""This is a special method that allow you to drop parts of the data model.
|
|
240
376
|
This only applies to Cognite Data Models.
|
|
@@ -267,3 +403,62 @@ class DataModelPrepareAPI:
|
|
|
267
403
|
)
|
|
268
404
|
|
|
269
405
|
self._state.data_model.write(output.rules, change)
|
|
406
|
+
|
|
407
|
+
def include_referenced(self) -> None:
|
|
408
|
+
"""Include referenced views and containers in the data model."""
|
|
409
|
+
start = datetime.now(timezone.utc)
|
|
410
|
+
|
|
411
|
+
source_id, rules = self._state.data_model.last_verified_dms_rules
|
|
412
|
+
view_ids, container_ids = rules.imported_views_and_containers_ids(include_model_views_with_no_properties=True)
|
|
413
|
+
if not (view_ids or container_ids):
|
|
414
|
+
print(
|
|
415
|
+
f"Data model {rules.metadata.as_data_model_id()} does not have any referenced views or containers."
|
|
416
|
+
f"that is not already included in the data model."
|
|
417
|
+
)
|
|
418
|
+
return
|
|
419
|
+
if self._client is None:
|
|
420
|
+
raise NeatSessionError(
|
|
421
|
+
"No client provided. You are referencing unknown views and containers in your data model, "
|
|
422
|
+
"NEAT needs a client to lookup the definitions. "
|
|
423
|
+
"Please set the client in the session, NeatSession(client=client)."
|
|
424
|
+
)
|
|
425
|
+
schema = self._client.schema.retrieve(list(view_ids), list(container_ids))
|
|
426
|
+
copy_ = rules.model_copy(deep=True)
|
|
427
|
+
copy_.metadata.version = f"{rules.metadata.version}_completed"
|
|
428
|
+
importer = DMSImporter(schema)
|
|
429
|
+
imported = importer.to_rules()
|
|
430
|
+
if imported.rules is None:
|
|
431
|
+
self._state.data_model.issue_lists.append(imported.issues)
|
|
432
|
+
raise NeatSessionError(
|
|
433
|
+
"Could not import the referenced views and containers. "
|
|
434
|
+
"See `neat.inspect.issues()` for more information."
|
|
435
|
+
)
|
|
436
|
+
verified = VerifyDMSRules("continue", post_validate=False).transform(imported.rules)
|
|
437
|
+
if verified.rules is None:
|
|
438
|
+
self._state.data_model.issue_lists.append(verified.issues)
|
|
439
|
+
raise NeatSessionError(
|
|
440
|
+
"Could not verify the referenced views and containers. "
|
|
441
|
+
"See `neat.inspect.issues()` for more information."
|
|
442
|
+
)
|
|
443
|
+
if copy_.containers is None:
|
|
444
|
+
copy_.containers = verified.rules.containers
|
|
445
|
+
else:
|
|
446
|
+
existing_containers = {c.container for c in copy_.containers}
|
|
447
|
+
copy_.containers.extend(
|
|
448
|
+
[c for c in verified.rules.containers or [] if c.container not in existing_containers]
|
|
449
|
+
)
|
|
450
|
+
existing_views = {v.view for v in copy_.views}
|
|
451
|
+
copy_.views.extend([v for v in verified.rules.views if v.view not in existing_views])
|
|
452
|
+
end = datetime.now(timezone.utc)
|
|
453
|
+
|
|
454
|
+
change = Change.from_rules_activity(
|
|
455
|
+
copy_,
|
|
456
|
+
ProvenanceAgent(id_=DEFAULT_NAMESPACE["agent/"]),
|
|
457
|
+
start,
|
|
458
|
+
end,
|
|
459
|
+
(f"Included referenced views and containers in the data model {rules.metadata.as_data_model_id()}"),
|
|
460
|
+
self._state.data_model.provenance.source_entity(source_id)
|
|
461
|
+
or self._state.data_model.provenance.target_entity(source_id),
|
|
462
|
+
)
|
|
463
|
+
|
|
464
|
+
self._state.data_model.write(copy_, change)
|
cognite/neat/_session/_read.py
CHANGED
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
import tempfile
|
|
2
2
|
from datetime import datetime, timezone
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
from typing import Any
|
|
4
|
+
from typing import Any, Literal
|
|
5
5
|
|
|
6
|
-
from cognite.client import CogniteClient
|
|
7
6
|
from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
|
|
8
7
|
|
|
8
|
+
from cognite.neat._client import NeatClient
|
|
9
|
+
from cognite.neat._constants import COGNITE_SPACES
|
|
9
10
|
from cognite.neat._graph import examples as instances_examples
|
|
10
11
|
from cognite.neat._graph import extractors
|
|
11
12
|
from cognite.neat._issues import IssueList
|
|
12
13
|
from cognite.neat._issues.errors import NeatValueError
|
|
13
14
|
from cognite.neat._rules import importers
|
|
14
15
|
from cognite.neat._rules._shared import ReadRules
|
|
16
|
+
from cognite.neat._rules.importers import BaseImporter
|
|
15
17
|
from cognite.neat._store._provenance import Activity as ProvenanceActivity
|
|
16
18
|
from cognite.neat._store._provenance import Change
|
|
17
19
|
from cognite.neat._store._provenance import Entity as ProvenanceEntity
|
|
@@ -20,23 +22,24 @@ from cognite.neat._utils.reader import GitHubReader, NeatReader, PathReader
|
|
|
20
22
|
from ._state import SessionState
|
|
21
23
|
from ._wizard import NeatObjectType, RDFFileType, object_wizard, rdf_dm_wizard
|
|
22
24
|
from .engine import import_engine
|
|
23
|
-
from .exceptions import NeatSessionError,
|
|
25
|
+
from .exceptions import NeatSessionError, session_class_wrapper
|
|
24
26
|
|
|
25
27
|
|
|
26
|
-
@
|
|
28
|
+
@session_class_wrapper
|
|
27
29
|
class ReadAPI:
|
|
28
|
-
def __init__(self, state: SessionState, client:
|
|
30
|
+
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|
|
29
31
|
self._state = state
|
|
30
32
|
self._verbose = verbose
|
|
31
33
|
self.cdf = CDFReadAPI(state, client, verbose)
|
|
32
34
|
self.rdf = RDFReadAPI(state, client, verbose)
|
|
33
35
|
self.excel = ExcelReadAPI(state, client, verbose)
|
|
34
36
|
self.csv = CSVReadAPI(state, client, verbose)
|
|
37
|
+
self.yaml = YamlReadAPI(state, client, verbose)
|
|
35
38
|
|
|
36
39
|
|
|
37
|
-
@
|
|
40
|
+
@session_class_wrapper
|
|
38
41
|
class BaseReadAPI:
|
|
39
|
-
def __init__(self, state: SessionState, client:
|
|
42
|
+
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|
|
40
43
|
self._state = state
|
|
41
44
|
self._verbose = verbose
|
|
42
45
|
self._client = client
|
|
@@ -62,14 +65,14 @@ class BaseReadAPI:
|
|
|
62
65
|
raise NeatValueError(f"Expected str or Path, got {type(io)}")
|
|
63
66
|
|
|
64
67
|
|
|
65
|
-
@
|
|
68
|
+
@session_class_wrapper
|
|
66
69
|
class CDFReadAPI(BaseReadAPI):
|
|
67
|
-
def __init__(self, state: SessionState, client:
|
|
70
|
+
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|
|
68
71
|
super().__init__(state, client, verbose)
|
|
69
72
|
self.classic = CDFClassicAPI(state, client, verbose)
|
|
70
73
|
|
|
71
74
|
@property
|
|
72
|
-
def _get_client(self) ->
|
|
75
|
+
def _get_client(self) -> NeatClient:
|
|
73
76
|
if self._client is None:
|
|
74
77
|
raise NeatValueError("No client provided. Please provide a client to read a data model.")
|
|
75
78
|
return self._client
|
|
@@ -107,22 +110,59 @@ class CDFReadAPI(BaseReadAPI):
|
|
|
107
110
|
return self._store_rules(rules, change)
|
|
108
111
|
|
|
109
112
|
|
|
110
|
-
@
|
|
113
|
+
@session_class_wrapper
|
|
111
114
|
class CDFClassicAPI(BaseReadAPI):
|
|
112
115
|
@property
|
|
113
|
-
def _get_client(self) ->
|
|
116
|
+
def _get_client(self) -> NeatClient:
|
|
114
117
|
if self._client is None:
|
|
115
118
|
raise ValueError("No client provided. Please provide a client to read a data model.")
|
|
116
119
|
return self._client
|
|
117
120
|
|
|
118
|
-
def
|
|
119
|
-
|
|
121
|
+
def graph(self, root_asset_external_id: str) -> None:
|
|
122
|
+
"""Reads the classic knowledge graph from CDF.
|
|
123
|
+
|
|
124
|
+
The Classic Graph consists of the following core resource type.
|
|
125
|
+
|
|
126
|
+
Classic Node CDF Resources:
|
|
127
|
+
- Assets
|
|
128
|
+
- TimeSeries
|
|
129
|
+
- Sequences
|
|
130
|
+
- Events
|
|
131
|
+
- Files
|
|
132
|
+
|
|
133
|
+
All the classic node CDF resources can have one or more connections to one or more assets. This
|
|
134
|
+
will match a direct relationship in the data modeling of CDF.
|
|
135
|
+
|
|
136
|
+
In addition, you have relationships between the classic node CDF resources. This matches an edge
|
|
137
|
+
in the data modeling of CDF.
|
|
138
|
+
|
|
139
|
+
Finally, you have labels and data sets that to organize the graph. In which data sets have a similar,
|
|
140
|
+
but different, role as a space in data modeling. While labels can be compared to node types in data modeling,
|
|
141
|
+
used to quickly filter and find nodes/edges.
|
|
142
|
+
|
|
143
|
+
This extractor will extract the classic CDF graph into Neat starting from either a data set or a root asset.
|
|
144
|
+
|
|
145
|
+
It works as follows:
|
|
146
|
+
|
|
147
|
+
1. Extract all core nodes (assets, time series, sequences, events, files) filtered by the given data set or
|
|
148
|
+
root asset.
|
|
149
|
+
2. Extract all relationships starting from any of the extracted core nodes.
|
|
150
|
+
3. Extract all core nodes that are targets of the relationships that are not already extracted.
|
|
151
|
+
4. Extract all labels that are connected to the extracted core nodes/relationships.
|
|
152
|
+
5. Extract all data sets that are connected to the extracted core nodes/relationships.
|
|
153
|
+
|
|
154
|
+
Args:
|
|
155
|
+
root_asset_external_id: The external id of the root asset
|
|
156
|
+
|
|
157
|
+
"""
|
|
158
|
+
extractor = extractors.ClassicGraphExtractor(self._get_client, root_asset_external_id=root_asset_external_id)
|
|
159
|
+
|
|
120
160
|
self._state.instances.store.write(extractor)
|
|
121
161
|
if self._verbose:
|
|
122
|
-
print(f"
|
|
162
|
+
print(f"Classic Graph {root_asset_external_id} read successfully")
|
|
123
163
|
|
|
124
164
|
|
|
125
|
-
@
|
|
165
|
+
@session_class_wrapper
|
|
126
166
|
class ExcelReadAPI(BaseReadAPI):
|
|
127
167
|
def __call__(self, io: Any) -> IssueList:
|
|
128
168
|
reader = NeatReader.create(io)
|
|
@@ -142,11 +182,61 @@ class ExcelReadAPI(BaseReadAPI):
|
|
|
142
182
|
description=f"Excel file {reader!s} read as unverified data model",
|
|
143
183
|
)
|
|
144
184
|
self._store_rules(input_rules, change)
|
|
185
|
+
self._state.data_model.issue_lists.append(input_rules.issues)
|
|
186
|
+
return input_rules.issues
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@session_class_wrapper
|
|
190
|
+
class YamlReadAPI(BaseReadAPI):
|
|
191
|
+
def __call__(self, io: Any, format: Literal["neat", "toolkit"] = "neat") -> IssueList:
|
|
192
|
+
reader = NeatReader.create(io)
|
|
193
|
+
if not isinstance(reader, PathReader):
|
|
194
|
+
raise NeatValueError("Only file paths are supported for YAML files")
|
|
195
|
+
start = datetime.now(timezone.utc)
|
|
196
|
+
importer: BaseImporter
|
|
197
|
+
if format == "neat":
|
|
198
|
+
importer = importers.YAMLImporter.from_file(reader.path)
|
|
199
|
+
elif format == "toolkit":
|
|
200
|
+
if reader.path.is_file():
|
|
201
|
+
dms_importer = importers.DMSImporter.from_zip_file(reader.path)
|
|
202
|
+
elif reader.path.is_dir():
|
|
203
|
+
dms_importer = importers.DMSImporter.from_directory(reader.path)
|
|
204
|
+
else:
|
|
205
|
+
raise NeatValueError(f"Unsupported YAML format: {format}")
|
|
206
|
+
ref_containers = dms_importer.root_schema.referenced_container()
|
|
207
|
+
if system_container_ids := [
|
|
208
|
+
container_id for container_id in ref_containers if container_id.space in COGNITE_SPACES
|
|
209
|
+
]:
|
|
210
|
+
if self._client is None:
|
|
211
|
+
raise NeatSessionError(
|
|
212
|
+
"No client provided. You are referencing Cognite containers in your data model, "
|
|
213
|
+
"NEAT needs a client to lookup the container definitions. "
|
|
214
|
+
"Please set the client in the session, NeatSession(client=client)."
|
|
215
|
+
)
|
|
216
|
+
system_containers = self._client.loaders.containers.retrieve(system_container_ids)
|
|
217
|
+
dms_importer.update_referenced_containers(system_containers)
|
|
218
|
+
|
|
219
|
+
importer = dms_importer
|
|
220
|
+
else:
|
|
221
|
+
raise NeatValueError(f"Unsupported YAML format: {format}")
|
|
222
|
+
input_rules: ReadRules = importer.to_rules()
|
|
223
|
+
|
|
224
|
+
end = datetime.now(timezone.utc)
|
|
225
|
+
|
|
226
|
+
if input_rules.rules:
|
|
227
|
+
change = Change.from_rules_activity(
|
|
228
|
+
input_rules,
|
|
229
|
+
importer.agent,
|
|
230
|
+
start,
|
|
231
|
+
end,
|
|
232
|
+
description=f"YAML file {reader!s} read as unverified data model",
|
|
233
|
+
)
|
|
234
|
+
self._store_rules(input_rules, change)
|
|
145
235
|
|
|
146
236
|
return input_rules.issues
|
|
147
237
|
|
|
148
238
|
|
|
149
|
-
@
|
|
239
|
+
@session_class_wrapper
|
|
150
240
|
class CSVReadAPI(BaseReadAPI):
|
|
151
241
|
def __call__(self, io: Any, type: str, primary_key: str) -> None:
|
|
152
242
|
reader = NeatReader.create(io)
|
|
@@ -167,9 +257,9 @@ class CSVReadAPI(BaseReadAPI):
|
|
|
167
257
|
self._state.instances.store.write(extractor)
|
|
168
258
|
|
|
169
259
|
|
|
170
|
-
@
|
|
260
|
+
@session_class_wrapper
|
|
171
261
|
class RDFReadAPI(BaseReadAPI):
|
|
172
|
-
def __init__(self, state: SessionState, client:
|
|
262
|
+
def __init__(self, state: SessionState, client: NeatClient | None, verbose: bool) -> None:
|
|
173
263
|
super().__init__(state, client, verbose)
|
|
174
264
|
self.examples = RDFExamples(state)
|
|
175
265
|
|
cognite/neat/_session/_set.py
CHANGED
|
@@ -7,10 +7,10 @@ from cognite.neat._rules.transformers import SetIDDMSModel
|
|
|
7
7
|
from cognite.neat._store._provenance import Change
|
|
8
8
|
|
|
9
9
|
from ._state import SessionState
|
|
10
|
-
from .exceptions import NeatSessionError,
|
|
10
|
+
from .exceptions import NeatSessionError, session_class_wrapper
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
@
|
|
13
|
+
@session_class_wrapper
|
|
14
14
|
class SetAPI:
|
|
15
15
|
def __init__(self, state: SessionState, verbose: bool) -> None:
|
|
16
16
|
self._state = state
|
cognite/neat/_session/_show.py
CHANGED
|
@@ -16,10 +16,10 @@ from cognite.neat._session.exceptions import NeatSessionError
|
|
|
16
16
|
from cognite.neat._utils.rdf_ import remove_namespace_from_uri
|
|
17
17
|
|
|
18
18
|
from ._state import SessionState
|
|
19
|
-
from .exceptions import
|
|
19
|
+
from .exceptions import session_class_wrapper
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
@
|
|
22
|
+
@session_class_wrapper
|
|
23
23
|
class ShowAPI:
|
|
24
24
|
def __init__(self, state: SessionState) -> None:
|
|
25
25
|
self._state = state
|
|
@@ -27,7 +27,7 @@ class ShowAPI:
|
|
|
27
27
|
self.instances = ShowInstanceAPI(self._state)
|
|
28
28
|
|
|
29
29
|
|
|
30
|
-
@
|
|
30
|
+
@session_class_wrapper
|
|
31
31
|
class ShowBaseAPI:
|
|
32
32
|
def __init__(self, state: SessionState) -> None:
|
|
33
33
|
self._state = state
|
|
@@ -63,7 +63,7 @@ class ShowBaseAPI:
|
|
|
63
63
|
return net.show(name)
|
|
64
64
|
|
|
65
65
|
|
|
66
|
-
@
|
|
66
|
+
@session_class_wrapper
|
|
67
67
|
class ShowDataModelAPI(ShowBaseAPI):
|
|
68
68
|
def __init__(self, state: SessionState) -> None:
|
|
69
69
|
super().__init__(state)
|
|
@@ -111,7 +111,7 @@ class ShowDataModelAPI(ShowBaseAPI):
|
|
|
111
111
|
di_graph.add_edge(
|
|
112
112
|
prop_.view.suffix,
|
|
113
113
|
prop_.value_type.suffix,
|
|
114
|
-
label=prop_.name or prop_.
|
|
114
|
+
label=prop_.name or prop_.view_property,
|
|
115
115
|
)
|
|
116
116
|
|
|
117
117
|
return di_graph
|
|
@@ -145,7 +145,7 @@ class ShowDataModelAPI(ShowBaseAPI):
|
|
|
145
145
|
return di_graph
|
|
146
146
|
|
|
147
147
|
|
|
148
|
-
@
|
|
148
|
+
@session_class_wrapper
|
|
149
149
|
class ShowDataModelImplementsAPI(ShowBaseAPI):
|
|
150
150
|
def __init__(self, state: SessionState) -> None:
|
|
151
151
|
super().__init__(state)
|
|
@@ -206,27 +206,27 @@ class ShowDataModelImplementsAPI(ShowBaseAPI):
|
|
|
206
206
|
# if possible use human readable label coming from the view name
|
|
207
207
|
|
|
208
208
|
# add subClassOff as edges
|
|
209
|
-
if class_.
|
|
209
|
+
if class_.implements:
|
|
210
210
|
if not di_graph.has_node(class_.class_.suffix):
|
|
211
211
|
di_graph.add_node(
|
|
212
212
|
class_.class_.suffix,
|
|
213
213
|
label=class_.name or class_.class_.suffix,
|
|
214
214
|
)
|
|
215
215
|
|
|
216
|
-
for parent in class_.
|
|
216
|
+
for parent in class_.implements:
|
|
217
217
|
if not di_graph.has_node(parent.suffix):
|
|
218
218
|
di_graph.add_node(parent.suffix, label=parent.suffix)
|
|
219
219
|
di_graph.add_edge(
|
|
220
220
|
class_.class_.suffix,
|
|
221
221
|
parent.suffix,
|
|
222
|
-
label="
|
|
222
|
+
label="implements",
|
|
223
223
|
dashes=True,
|
|
224
224
|
)
|
|
225
225
|
|
|
226
226
|
return di_graph
|
|
227
227
|
|
|
228
228
|
|
|
229
|
-
@
|
|
229
|
+
@session_class_wrapper
|
|
230
230
|
class ShowDataModelProvenanceAPI(ShowBaseAPI):
|
|
231
231
|
def __init__(self, state: SessionState) -> None:
|
|
232
232
|
super().__init__(state)
|
|
@@ -286,7 +286,7 @@ class ShowDataModelProvenanceAPI(ShowBaseAPI):
|
|
|
286
286
|
return remove_namespace_from_uri(thing)
|
|
287
287
|
|
|
288
288
|
|
|
289
|
-
@
|
|
289
|
+
@session_class_wrapper
|
|
290
290
|
class ShowInstanceAPI(ShowBaseAPI):
|
|
291
291
|
def __init__(self, state: SessionState) -> None:
|
|
292
292
|
super().__init__(state)
|