cognite-neat 0.75.7__py3-none-any.whl → 0.75.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/configuration.py +4 -9
- cognite/neat/app/api/routers/configuration.py +2 -1
- cognite/neat/app/api/routers/crud.py +5 -5
- cognite/neat/app/api/routers/data_exploration.py +3 -1
- cognite/neat/app/api/routers/rules.py +3 -3
- cognite/neat/app/api/routers/workflows.py +3 -3
- cognite/neat/app/ui/neat-app/build/asset-manifest.json +3 -3
- cognite/neat/app/ui/neat-app/build/index.html +1 -1
- cognite/neat/app/ui/neat-app/build/static/js/{main.4345d42f.js → main.ec7f72e2.js} +3 -3
- cognite/neat/app/ui/neat-app/build/static/js/{main.4345d42f.js.map → main.ec7f72e2.js.map} +1 -1
- cognite/neat/config.py +147 -12
- cognite/neat/constants.py +1 -0
- cognite/neat/graph/exceptions.py +1 -2
- cognite/neat/graph/extractors/_mock_graph_generator.py +6 -5
- cognite/neat/legacy/graph/exceptions.py +1 -2
- cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +1 -2
- cognite/neat/legacy/graph/loaders/_asset_loader.py +8 -13
- cognite/neat/legacy/graph/loaders/_base.py +2 -4
- cognite/neat/legacy/graph/loaders/_exceptions.py +1 -3
- cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +4 -8
- cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +2 -4
- cognite/neat/legacy/graph/loaders/rdf_to_dms.py +2 -4
- cognite/neat/legacy/graph/loaders/validator.py +1 -1
- cognite/neat/legacy/graph/transformations/transformer.py +1 -2
- cognite/neat/legacy/rules/exporters/_rules2dms.py +1 -2
- cognite/neat/legacy/rules/exporters/_validation.py +4 -8
- cognite/neat/legacy/rules/importers/_base.py +0 -4
- cognite/neat/legacy/rules/importers/_dms2rules.py +0 -2
- cognite/neat/legacy/rules/models/rdfpath.py +1 -2
- cognite/neat/legacy/workflows/examples/Export_DMS/workflow.yaml +89 -0
- cognite/neat/legacy/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +152 -0
- cognite/neat/legacy/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +139 -0
- cognite/neat/legacy/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml +270 -0
- cognite/neat/legacy/workflows/examples/Import_DMS/workflow.yaml +65 -0
- cognite/neat/legacy/workflows/examples/Ontology_to_Data_Model/workflow.yaml +116 -0
- cognite/neat/legacy/workflows/examples/Validate_Rules/workflow.yaml +67 -0
- cognite/neat/legacy/workflows/examples/Validate_Solution_Model/workflow.yaml +64 -0
- cognite/neat/legacy/workflows/examples/Visualize_Data_Model_Using_Mock_Graph/workflow.yaml +95 -0
- cognite/neat/legacy/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml +111 -0
- cognite/neat/rules/analysis/_base.py +1 -1
- cognite/neat/rules/analysis/_information_rules.py +4 -4
- cognite/neat/rules/exporters/_rules2excel.py +2 -2
- cognite/neat/rules/exporters/_rules2ontology.py +20 -17
- cognite/neat/rules/exporters/_validation.py +8 -10
- cognite/neat/rules/importers/_base.py +2 -4
- cognite/neat/rules/importers/_dms2rules.py +16 -19
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +21 -19
- cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +2 -4
- cognite/neat/rules/importers/_dtdl2rules/spec.py +3 -5
- cognite/neat/rules/importers/_owl2rules/_owl2rules.py +4 -6
- cognite/neat/rules/importers/_spreadsheet2rules.py +10 -9
- cognite/neat/rules/importers/_yaml2rules.py +10 -6
- cognite/neat/rules/issues/dms.py +3 -5
- cognite/neat/rules/issues/formatters.py +3 -1
- cognite/neat/rules/models/data_types.py +54 -31
- cognite/neat/rules/models/entities.py +184 -42
- cognite/neat/rules/models/rdfpath.py +112 -13
- cognite/neat/rules/models/rules/_base.py +2 -2
- cognite/neat/rules/models/rules/_dms_architect_rules.py +119 -189
- cognite/neat/rules/models/rules/_dms_rules_write.py +344 -0
- cognite/neat/rules/models/rules/_dms_schema.py +3 -3
- cognite/neat/rules/models/rules/_domain_rules.py +6 -3
- cognite/neat/rules/models/rules/_information_rules.py +68 -61
- cognite/neat/rules/models/rules/_types/__init__.py +0 -47
- cognite/neat/rules/models/rules/_types/_base.py +1 -309
- cognite/neat/rules/models/rules/_types/_field.py +0 -225
- cognite/neat/utils/cdf_loaders/_data_modeling.py +3 -1
- cognite/neat/utils/cdf_loaders/_ingestion.py +2 -4
- cognite/neat/utils/spreadsheet.py +2 -4
- cognite/neat/utils/utils.py +2 -4
- cognite/neat/workflows/base.py +5 -5
- cognite/neat/workflows/manager.py +32 -22
- cognite/neat/workflows/model.py +3 -3
- cognite/neat/workflows/steps/lib/__init__.py +0 -7
- cognite/neat/workflows/steps/lib/current/__init__.py +6 -0
- cognite/neat/workflows/steps/lib/{rules_exporter.py → current/rules_exporter.py} +8 -8
- cognite/neat/workflows/steps/lib/{rules_importer.py → current/rules_importer.py} +7 -7
- cognite/neat/workflows/steps/lib/io/__init__.py +1 -0
- cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_contextualization.py +2 -2
- cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_extractor.py +9 -9
- cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_loader.py +9 -9
- cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_store.py +4 -4
- cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_transformer.py +2 -2
- cognite/neat/workflows/steps/lib/{v1 → legacy}/rules_exporter.py +15 -17
- cognite/neat/workflows/steps/lib/{v1 → legacy}/rules_importer.py +7 -7
- cognite/neat/workflows/steps/step_model.py +5 -9
- cognite/neat/workflows/steps_registry.py +20 -11
- {cognite_neat-0.75.7.dist-info → cognite_neat-0.75.9.dist-info}/METADATA +1 -1
- {cognite_neat-0.75.7.dist-info → cognite_neat-0.75.9.dist-info}/RECORD +100 -90
- cognite/neat/app/api/data_classes/configuration.py +0 -121
- cognite/neat/rules/models/_entity.py +0 -142
- cognite/neat/rules/models/rules/_types/_value.py +0 -159
- /cognite/neat/app/ui/neat-app/build/static/js/{main.4345d42f.js.LICENSE.txt → main.ec7f72e2.js.LICENSE.txt} +0 -0
- /cognite/neat/workflows/steps/lib/{graph_extractor.py → current/graph_extractor.py} +0 -0
- /cognite/neat/workflows/steps/lib/{graph_loader.py → current/graph_loader.py} +0 -0
- /cognite/neat/workflows/steps/lib/{graph_store.py → current/graph_store.py} +0 -0
- /cognite/neat/workflows/steps/lib/{rules_validator.py → current/rules_validator.py} +0 -0
- /cognite/neat/workflows/steps/lib/{io_steps.py → io/io_steps.py} +0 -0
- /cognite/neat/workflows/steps/lib/{v1 → legacy}/__init__.py +0 -0
- {cognite_neat-0.75.7.dist-info → cognite_neat-0.75.9.dist-info}/LICENSE +0 -0
- {cognite_neat-0.75.7.dist-info → cognite_neat-0.75.9.dist-info}/WHEEL +0 -0
- {cognite_neat-0.75.7.dist-info → cognite_neat-0.75.9.dist-info}/entry_points.txt +0 -0
|
@@ -10,6 +10,14 @@ from cognite.client.utils import ms_to_datetime
|
|
|
10
10
|
from cognite.neat.rules import issues
|
|
11
11
|
from cognite.neat.rules.importers._base import BaseImporter, Rules
|
|
12
12
|
from cognite.neat.rules.issues import IssueList
|
|
13
|
+
from cognite.neat.rules.models.data_types import DataType
|
|
14
|
+
from cognite.neat.rules.models.entities import (
|
|
15
|
+
ClassEntity,
|
|
16
|
+
ContainerEntity,
|
|
17
|
+
DMSUnknownEntity,
|
|
18
|
+
ViewEntity,
|
|
19
|
+
ViewPropertyEntity,
|
|
20
|
+
)
|
|
13
21
|
from cognite.neat.rules.models.rules import DMSRules, DMSSchema, RoleTypes
|
|
14
22
|
from cognite.neat.rules.models.rules._base import ExtensionCategory, SchemaCompleteness
|
|
15
23
|
from cognite.neat.rules.models.rules._dms_architect_rules import (
|
|
@@ -19,15 +27,6 @@ from cognite.neat.rules.models.rules._dms_architect_rules import (
|
|
|
19
27
|
DMSView,
|
|
20
28
|
SheetList,
|
|
21
29
|
)
|
|
22
|
-
from cognite.neat.rules.models.rules._types import (
|
|
23
|
-
ClassEntity,
|
|
24
|
-
ContainerEntity,
|
|
25
|
-
DMSValueType,
|
|
26
|
-
Undefined,
|
|
27
|
-
Unknown,
|
|
28
|
-
ViewEntity,
|
|
29
|
-
ViewPropEntity,
|
|
30
|
-
)
|
|
31
30
|
|
|
32
31
|
|
|
33
32
|
class DMSImporter(BaseImporter):
|
|
@@ -82,14 +81,12 @@ class DMSImporter(BaseImporter):
|
|
|
82
81
|
return cls(DMSSchema.from_zip(zip_file))
|
|
83
82
|
|
|
84
83
|
@overload
|
|
85
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
|
|
86
|
-
...
|
|
84
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
|
|
87
85
|
|
|
88
86
|
@overload
|
|
89
87
|
def to_rules(
|
|
90
88
|
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
91
|
-
) -> tuple[Rules | None, IssueList]:
|
|
92
|
-
...
|
|
89
|
+
) -> tuple[Rules | None, IssueList]: ...
|
|
93
90
|
|
|
94
91
|
def to_rules(
|
|
95
92
|
self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
|
|
@@ -136,21 +133,21 @@ class DMSImporter(BaseImporter):
|
|
|
136
133
|
raise NotImplementedError(f"Constraint type {type(constraint_obj)} not implemented")
|
|
137
134
|
|
|
138
135
|
if isinstance(container_prop.type, dm.DirectRelation):
|
|
139
|
-
direct_value_type: str | ViewEntity |
|
|
136
|
+
direct_value_type: str | ViewEntity | DataType | DMSUnknownEntity
|
|
140
137
|
if prop.source is None:
|
|
141
138
|
issue_list.append(
|
|
142
139
|
issues.importing.UnknownValueTypeWarning(class_entity.versioned_id, prop_id)
|
|
143
140
|
)
|
|
144
|
-
direct_value_type =
|
|
141
|
+
direct_value_type = DMSUnknownEntity()
|
|
145
142
|
else:
|
|
146
|
-
direct_value_type =
|
|
143
|
+
direct_value_type = ViewEntity.from_id(prop.source)
|
|
147
144
|
|
|
148
145
|
dms_property = DMSProperty(
|
|
149
146
|
class_=class_entity,
|
|
150
147
|
property_=prop_id,
|
|
151
148
|
description=prop.description,
|
|
152
149
|
name=prop.name,
|
|
153
|
-
value_type=
|
|
150
|
+
value_type=direct_value_type,
|
|
154
151
|
relation="direct",
|
|
155
152
|
nullable=container_prop.nullable,
|
|
156
153
|
default=container_prop.default_value,
|
|
@@ -168,7 +165,7 @@ class DMSImporter(BaseImporter):
|
|
|
168
165
|
property_=prop_id,
|
|
169
166
|
description=prop.description,
|
|
170
167
|
name=prop.name,
|
|
171
|
-
value_type=cast(
|
|
168
|
+
value_type=cast(ViewPropertyEntity | DataType, container_prop.type._type),
|
|
172
169
|
nullable=container_prop.nullable,
|
|
173
170
|
is_list=container_prop.type.is_list,
|
|
174
171
|
default=container_prop.default_value,
|
|
@@ -180,7 +177,7 @@ class DMSImporter(BaseImporter):
|
|
|
180
177
|
constraint=unique_constraints or None,
|
|
181
178
|
)
|
|
182
179
|
elif isinstance(prop, dm.MultiEdgeConnectionApply):
|
|
183
|
-
view_entity =
|
|
180
|
+
view_entity = ViewEntity.from_id(prop.source)
|
|
184
181
|
dms_property = DMSProperty(
|
|
185
182
|
class_=ClassEntity(prefix=view.space, suffix=view.external_id, version=view.version),
|
|
186
183
|
property_=prop_id,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from collections import Counter
|
|
2
2
|
from collections.abc import Callable, Sequence
|
|
3
|
+
from typing import cast
|
|
3
4
|
|
|
4
5
|
import cognite.neat.rules.issues.importing
|
|
5
6
|
from cognite.neat.rules import issues
|
|
@@ -20,13 +21,9 @@ from cognite.neat.rules.importers._dtdl2rules.spec import (
|
|
|
20
21
|
TelemetryV2,
|
|
21
22
|
)
|
|
22
23
|
from cognite.neat.rules.issues import IssueList, ValidationIssue
|
|
24
|
+
from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_NAME, DataType, Json, String
|
|
25
|
+
from cognite.neat.rules.models.entities import ClassEntity, ParentClassEntity
|
|
23
26
|
from cognite.neat.rules.models.rules._information_rules import InformationClass, InformationProperty
|
|
24
|
-
from cognite.neat.rules.models.rules._types import (
|
|
25
|
-
XSD_VALUE_TYPE_MAPPINGS,
|
|
26
|
-
ClassEntity,
|
|
27
|
-
ParentClassEntity,
|
|
28
|
-
XSDValueType,
|
|
29
|
-
)
|
|
30
27
|
|
|
31
28
|
|
|
32
29
|
class _DTDLConverter:
|
|
@@ -92,7 +89,12 @@ class _DTDLConverter:
|
|
|
92
89
|
name=item.display_name,
|
|
93
90
|
description=item.description,
|
|
94
91
|
comment=item.comment,
|
|
95
|
-
parent=[
|
|
92
|
+
parent=[
|
|
93
|
+
cast(ParentClassEntity, parent_entity)
|
|
94
|
+
for parent in item.extends or []
|
|
95
|
+
if isinstance(parent_entity := ParentClassEntity.load(parent.as_class_id()), ParentClassEntity)
|
|
96
|
+
]
|
|
97
|
+
or None,
|
|
96
98
|
)
|
|
97
99
|
self.classes.append(class_)
|
|
98
100
|
for sub_item_or_id in item.contents or []:
|
|
@@ -123,7 +125,7 @@ class _DTDLConverter:
|
|
|
123
125
|
return None
|
|
124
126
|
|
|
125
127
|
prop = InformationProperty(
|
|
126
|
-
class_=ClassEntity.
|
|
128
|
+
class_=ClassEntity.load(parent),
|
|
127
129
|
property_=item.name,
|
|
128
130
|
name=item.display_name,
|
|
129
131
|
description=item.description,
|
|
@@ -175,7 +177,7 @@ class _DTDLConverter:
|
|
|
175
177
|
if value_type is None:
|
|
176
178
|
return
|
|
177
179
|
prop = InformationProperty(
|
|
178
|
-
class_=ClassEntity.
|
|
180
|
+
class_=ClassEntity.load(parent),
|
|
179
181
|
property_=item.name,
|
|
180
182
|
name=item.display_name,
|
|
181
183
|
description=item.description,
|
|
@@ -195,7 +197,7 @@ class _DTDLConverter:
|
|
|
195
197
|
if value_type is None:
|
|
196
198
|
return
|
|
197
199
|
prop = InformationProperty(
|
|
198
|
-
class_=ClassEntity.
|
|
200
|
+
class_=ClassEntity.load(parent),
|
|
199
201
|
property_=item.name,
|
|
200
202
|
name=item.display_name,
|
|
201
203
|
description=item.description,
|
|
@@ -211,7 +213,7 @@ class _DTDLConverter:
|
|
|
211
213
|
self._missing_parent_warning(item)
|
|
212
214
|
return None
|
|
213
215
|
if item.target is not None:
|
|
214
|
-
value_type:
|
|
216
|
+
value_type: DataType | ClassEntity
|
|
215
217
|
if item.target in self._item_by_id:
|
|
216
218
|
value_type = item.target.as_class_id()
|
|
217
219
|
else:
|
|
@@ -223,10 +225,10 @@ class _DTDLConverter:
|
|
|
223
225
|
instance_id=item.target.model_dump(),
|
|
224
226
|
)
|
|
225
227
|
)
|
|
226
|
-
value_type =
|
|
228
|
+
value_type = Json()
|
|
227
229
|
|
|
228
230
|
prop = InformationProperty(
|
|
229
|
-
class_=ClassEntity.
|
|
231
|
+
class_=ClassEntity.load(parent),
|
|
230
232
|
property_=item.name,
|
|
231
233
|
name=item.display_name,
|
|
232
234
|
description=item.description,
|
|
@@ -275,13 +277,13 @@ class _DTDLConverter:
|
|
|
275
277
|
|
|
276
278
|
def schema_to_value_type(
|
|
277
279
|
self, schema: Schema | Interface | DTMI | None, item: DTDLBase
|
|
278
|
-
) ->
|
|
280
|
+
) -> DataType | ClassEntity | None:
|
|
279
281
|
input_type = self._item_by_id.get(schema) if isinstance(schema, DTMI) else schema
|
|
280
282
|
|
|
281
283
|
if isinstance(input_type, Enum):
|
|
282
|
-
return
|
|
283
|
-
elif isinstance(input_type, str) and input_type in
|
|
284
|
-
return
|
|
284
|
+
return String()
|
|
285
|
+
elif isinstance(input_type, str) and input_type.casefold() in _DATA_TYPE_BY_NAME:
|
|
286
|
+
return _DATA_TYPE_BY_NAME[input_type.casefold()]()
|
|
285
287
|
elif isinstance(input_type, str):
|
|
286
288
|
self.issues.append(
|
|
287
289
|
cognite.neat.rules.issues.importing.UnsupportedPropertyTypeError(
|
|
@@ -301,11 +303,11 @@ class _DTDLConverter:
|
|
|
301
303
|
instance_name=input_type.display_name,
|
|
302
304
|
)
|
|
303
305
|
)
|
|
304
|
-
return
|
|
306
|
+
return Json()
|
|
305
307
|
else:
|
|
306
308
|
if isinstance(input_type, Object):
|
|
307
309
|
self.convert_object(input_type, None)
|
|
308
|
-
return ClassEntity.
|
|
310
|
+
return ClassEntity.load(input_type.id_.as_class_id())
|
|
309
311
|
else:
|
|
310
312
|
self.issues.append(
|
|
311
313
|
issues.importing.UnknownPropertyWarning(
|
|
@@ -119,14 +119,12 @@ class DTDLImporter(BaseImporter):
|
|
|
119
119
|
return cls(items, zip_file.stem, read_issues=issues)
|
|
120
120
|
|
|
121
121
|
@overload
|
|
122
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
|
|
123
|
-
...
|
|
122
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
|
|
124
123
|
|
|
125
124
|
@overload
|
|
126
125
|
def to_rules(
|
|
127
126
|
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
128
|
-
) -> tuple[Rules | None, IssueList]:
|
|
129
|
-
...
|
|
127
|
+
) -> tuple[Rules | None, IssueList]: ...
|
|
130
128
|
|
|
131
129
|
def to_rules(
|
|
132
130
|
self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
|
|
@@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeAlias
|
|
|
14
14
|
|
|
15
15
|
from pydantic import BaseModel, Field, field_validator, model_serializer, model_validator
|
|
16
16
|
|
|
17
|
-
from cognite.neat.rules.models.
|
|
17
|
+
from cognite.neat.rules.models.entities import ClassEntity
|
|
18
18
|
|
|
19
19
|
if TYPE_CHECKING:
|
|
20
20
|
from pydantic.type_adapter import IncEx
|
|
@@ -72,8 +72,7 @@ class DTMI(BaseModel):
|
|
|
72
72
|
exclude_none: bool = False,
|
|
73
73
|
round_trip: bool = False,
|
|
74
74
|
warnings: bool = True,
|
|
75
|
-
) -> str:
|
|
76
|
-
...
|
|
75
|
+
) -> str: ...
|
|
77
76
|
|
|
78
77
|
|
|
79
78
|
IRI: TypeAlias = str
|
|
@@ -124,8 +123,7 @@ class Unit(BaseModel, ABC):
|
|
|
124
123
|
exclude_none: bool = False,
|
|
125
124
|
round_trip: bool = False,
|
|
126
125
|
warnings: bool = True,
|
|
127
|
-
) -> str:
|
|
128
|
-
...
|
|
126
|
+
) -> str: ...
|
|
129
127
|
|
|
130
128
|
|
|
131
129
|
class DTDLBase(BaseModel, ABC):
|
|
@@ -10,8 +10,8 @@ from rdflib import DC, DCTERMS, OWL, RDF, RDFS, SKOS, Graph
|
|
|
10
10
|
|
|
11
11
|
from cognite.neat.rules.importers._base import BaseImporter, Rules
|
|
12
12
|
from cognite.neat.rules.issues import IssueList
|
|
13
|
+
from cognite.neat.rules.models.data_types import _XSD_TYPES
|
|
13
14
|
from cognite.neat.rules.models.rules import InformationRules, RoleTypes
|
|
14
|
-
from cognite.neat.rules.models.rules._types import XSD_VALUE_TYPE_MAPPINGS
|
|
15
15
|
|
|
16
16
|
from ._owl2classes import parse_owl_classes
|
|
17
17
|
from ._owl2metadata import parse_owl_metadata
|
|
@@ -43,14 +43,12 @@ class OWLImporter(BaseImporter):
|
|
|
43
43
|
self.make_compliant = make_compliant
|
|
44
44
|
|
|
45
45
|
@overload
|
|
46
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
|
|
47
|
-
...
|
|
46
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
|
|
48
47
|
|
|
49
48
|
@overload
|
|
50
49
|
def to_rules(
|
|
51
50
|
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
52
|
-
) -> tuple[Rules | None, IssueList]:
|
|
53
|
-
...
|
|
51
|
+
) -> tuple[Rules | None, IssueList]: ...
|
|
54
52
|
|
|
55
53
|
def to_rules(
|
|
56
54
|
self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
|
|
@@ -126,7 +124,7 @@ def _add_missing_value_types(components: dict) -> dict:
|
|
|
126
124
|
Updated tables with missing properties added to containers
|
|
127
125
|
"""
|
|
128
126
|
|
|
129
|
-
xsd_types =
|
|
127
|
+
xsd_types = _XSD_TYPES
|
|
130
128
|
candidate_value_types = {definition["Value Type"] for definition in components["Properties"]} - {
|
|
131
129
|
definition["Class"] for definition in components["Classes"]
|
|
132
130
|
}
|
|
@@ -15,6 +15,7 @@ from cognite.neat.rules import issues
|
|
|
15
15
|
from cognite.neat.rules.issues import IssueList
|
|
16
16
|
from cognite.neat.rules.models.rules import RULES_PER_ROLE, DMSRules, DomainRules, InformationRules
|
|
17
17
|
from cognite.neat.rules.models.rules._base import RoleTypes, SchemaCompleteness
|
|
18
|
+
from cognite.neat.rules.models.rules._dms_rules_write import DMSRulesWrite
|
|
18
19
|
from cognite.neat.utils.auxiliary import local_import
|
|
19
20
|
from cognite.neat.utils.spreadsheet import SpreadsheetRead, read_individual_sheet
|
|
20
21
|
|
|
@@ -152,16 +153,14 @@ class ExcelImporter(BaseImporter):
|
|
|
152
153
|
self.filepath = filepath
|
|
153
154
|
|
|
154
155
|
@overload
|
|
155
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
|
|
156
|
-
...
|
|
156
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
|
|
157
157
|
|
|
158
158
|
@overload
|
|
159
159
|
def to_rules(
|
|
160
160
|
self,
|
|
161
161
|
errors: Literal["continue"] = "continue",
|
|
162
162
|
role: RoleTypes | None = None,
|
|
163
|
-
) -> tuple[Rules | None, IssueList]:
|
|
164
|
-
...
|
|
163
|
+
) -> tuple[Rules | None, IssueList]: ...
|
|
165
164
|
|
|
166
165
|
def to_rules(
|
|
167
166
|
self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
|
|
@@ -214,7 +213,11 @@ class ExcelImporter(BaseImporter):
|
|
|
214
213
|
error_cls=issues.spreadsheet.InvalidSheetError,
|
|
215
214
|
error_args={"read_info_by_sheet": read_info_by_sheet},
|
|
216
215
|
) as future:
|
|
217
|
-
rules
|
|
216
|
+
rules: Rules
|
|
217
|
+
if rules_cls is DMSRules:
|
|
218
|
+
rules = DMSRulesWrite.load(sheets).as_read()
|
|
219
|
+
else:
|
|
220
|
+
rules = rules_cls.model_validate(sheets) # type: ignore[attr-defined]
|
|
218
221
|
|
|
219
222
|
if future.result == "failure" or issue_list.has_errors:
|
|
220
223
|
return self._return_or_raise(issue_list, errors)
|
|
@@ -239,14 +242,12 @@ class GoogleSheetImporter(BaseImporter):
|
|
|
239
242
|
self.skiprows = skiprows
|
|
240
243
|
|
|
241
244
|
@overload
|
|
242
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
|
|
243
|
-
...
|
|
245
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
|
|
244
246
|
|
|
245
247
|
@overload
|
|
246
248
|
def to_rules(
|
|
247
249
|
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
248
|
-
) -> tuple[Rules | None, IssueList]:
|
|
249
|
-
...
|
|
250
|
+
) -> tuple[Rules | None, IssueList]: ...
|
|
250
251
|
|
|
251
252
|
def to_rules(
|
|
252
253
|
self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
|
|
@@ -5,7 +5,8 @@ import yaml
|
|
|
5
5
|
|
|
6
6
|
from cognite.neat.rules import issues
|
|
7
7
|
from cognite.neat.rules.issues import IssueList, NeatValidationError, ValidationIssue
|
|
8
|
-
from cognite.neat.rules.models.rules import RULES_PER_ROLE, RoleTypes
|
|
8
|
+
from cognite.neat.rules.models.rules import RULES_PER_ROLE, DMSRules, RoleTypes
|
|
9
|
+
from cognite.neat.rules.models.rules._dms_rules_write import DMSRulesWrite
|
|
9
10
|
|
|
10
11
|
from ._base import BaseImporter, Rules, _handle_issues
|
|
11
12
|
|
|
@@ -45,14 +46,12 @@ class YAMLImporter(BaseImporter):
|
|
|
45
46
|
return cls(yaml.safe_load(filepath.read_text()), filepaths=[filepath])
|
|
46
47
|
|
|
47
48
|
@overload
|
|
48
|
-
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
|
|
49
|
-
...
|
|
49
|
+
def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
|
|
50
50
|
|
|
51
51
|
@overload
|
|
52
52
|
def to_rules(
|
|
53
53
|
self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
|
|
54
|
-
) -> tuple[Rules | None, IssueList]:
|
|
55
|
-
...
|
|
54
|
+
) -> tuple[Rules | None, IssueList]: ...
|
|
56
55
|
|
|
57
56
|
def to_rules(
|
|
58
57
|
self, errors: Literal["raise", "continue"] = "continue", role: RoleTypes | None = None
|
|
@@ -97,7 +96,12 @@ class YAMLImporter(BaseImporter):
|
|
|
97
96
|
rules_model = RULES_PER_ROLE[role_enum]
|
|
98
97
|
|
|
99
98
|
with _handle_issues(issue_list) as future:
|
|
100
|
-
rules
|
|
99
|
+
rules: Rules
|
|
100
|
+
if rules_model is DMSRules:
|
|
101
|
+
rules = DMSRulesWrite.load(self.raw_data).as_read()
|
|
102
|
+
else:
|
|
103
|
+
rules = rules_model.model_validate(self.raw_data)
|
|
104
|
+
|
|
101
105
|
if future.result == "failure":
|
|
102
106
|
if errors == "continue":
|
|
103
107
|
return None, issue_list
|
cognite/neat/rules/issues/dms.py
CHANGED
|
@@ -32,13 +32,11 @@ __all__ = [
|
|
|
32
32
|
|
|
33
33
|
|
|
34
34
|
@dataclass(frozen=True)
|
|
35
|
-
class DMSSchemaError(NeatValidationError, ABC):
|
|
36
|
-
...
|
|
35
|
+
class DMSSchemaError(NeatValidationError, ABC): ...
|
|
37
36
|
|
|
38
37
|
|
|
39
38
|
@dataclass(frozen=True)
|
|
40
|
-
class DMSSchemaWarning(ValidationWarning, ABC):
|
|
41
|
-
...
|
|
39
|
+
class DMSSchemaWarning(ValidationWarning, ABC): ...
|
|
42
40
|
|
|
43
41
|
|
|
44
42
|
@dataclass(frozen=True)
|
|
@@ -195,7 +193,7 @@ class DirectRelationMissingSourceWarning(DMSSchemaWarning):
|
|
|
195
193
|
property: str
|
|
196
194
|
|
|
197
195
|
def message(self) -> str:
|
|
198
|
-
return f"The source view referred to by {self.view_id}.{self.property} does not exist"
|
|
196
|
+
return f"The source view referred to by '{self.view_id.external_id}.{self.property}' does not exist."
|
|
199
197
|
|
|
200
198
|
def dump(self) -> dict[str, Any]:
|
|
201
199
|
output = super().dump()
|
|
@@ -79,5 +79,7 @@ class BasicHTML(Formatter):
|
|
|
79
79
|
|
|
80
80
|
|
|
81
81
|
FORMATTER_BY_NAME: dict[str, type[Formatter]] = {
|
|
82
|
-
subclass.__name__: subclass
|
|
82
|
+
subclass.__name__: subclass # type: ignore[type-abstract]
|
|
83
|
+
for subclass in Formatter.__subclasses__()
|
|
84
|
+
if ABC not in subclass.__bases__ # type: ignore[type-abstract]
|
|
83
85
|
}
|
|
@@ -13,6 +13,12 @@ else:
|
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
class DataType(BaseModel):
|
|
16
|
+
# These are necessary for Pydantic to work
|
|
17
|
+
# pydantic gets confused as we have no fields.
|
|
18
|
+
__pydantic_extra__ = None
|
|
19
|
+
__pydantic_fields_set__ = set()
|
|
20
|
+
__pydantic_private__ = {}
|
|
21
|
+
|
|
16
22
|
name: ClassVar[str]
|
|
17
23
|
python: ClassVar[type]
|
|
18
24
|
dms: ClassVar[type[dms.PropertyType]]
|
|
@@ -32,10 +38,12 @@ class DataType(BaseModel):
|
|
|
32
38
|
if isinstance(value, cls | dict):
|
|
33
39
|
return value
|
|
34
40
|
elif isinstance(value, str):
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
41
|
+
value_standardized = value.casefold()
|
|
42
|
+
if cls_ := _DATA_TYPE_BY_DMS_TYPE.get(value_standardized):
|
|
43
|
+
return cls_()
|
|
44
|
+
elif cls_ := _DATA_TYPE_BY_NAME.get(value_standardized):
|
|
45
|
+
return cls_()
|
|
46
|
+
raise ValueError(f"Unknown literal type: {value}") from None
|
|
39
47
|
raise ValueError(f"Cannot load {cls.__name__} from {value}")
|
|
40
48
|
|
|
41
49
|
@model_serializer(when_used="unless-none", return_type=str)
|
|
@@ -48,13 +56,20 @@ class DataType(BaseModel):
|
|
|
48
56
|
def __eq__(self, other: Any) -> bool:
|
|
49
57
|
return isinstance(other, type(self))
|
|
50
58
|
|
|
59
|
+
def __hash__(self) -> int:
|
|
60
|
+
return hash(str(self))
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def is_data_type(cls, value: str) -> bool:
|
|
64
|
+
return value.casefold() in _DATA_TYPE_BY_NAME or value.casefold() in _DATA_TYPE_BY_DMS_TYPE
|
|
65
|
+
|
|
51
66
|
|
|
52
67
|
class Boolean(DataType):
|
|
53
68
|
name = "boolean"
|
|
54
69
|
python = bool
|
|
55
70
|
dms = dms.Boolean
|
|
56
71
|
graphql = "Boolean"
|
|
57
|
-
xsd = "
|
|
72
|
+
xsd = "boolean"
|
|
58
73
|
sql = "BOOLEAN"
|
|
59
74
|
|
|
60
75
|
|
|
@@ -63,7 +78,7 @@ class Float(DataType):
|
|
|
63
78
|
python = float
|
|
64
79
|
dms = dms.Float32
|
|
65
80
|
graphql = "Float"
|
|
66
|
-
xsd = "
|
|
81
|
+
xsd = "float"
|
|
67
82
|
sql = "FLOAT"
|
|
68
83
|
|
|
69
84
|
|
|
@@ -72,7 +87,7 @@ class Double(DataType):
|
|
|
72
87
|
python = float
|
|
73
88
|
dms = dms.Float64
|
|
74
89
|
graphql = "Float"
|
|
75
|
-
xsd = "
|
|
90
|
+
xsd = "double"
|
|
76
91
|
sql = "FLOAT"
|
|
77
92
|
|
|
78
93
|
|
|
@@ -81,7 +96,7 @@ class Integer(DataType):
|
|
|
81
96
|
python = int
|
|
82
97
|
dms = dms.Int32
|
|
83
98
|
graphql = "Int"
|
|
84
|
-
xsd = "
|
|
99
|
+
xsd = "integer"
|
|
85
100
|
sql = "INTEGER"
|
|
86
101
|
|
|
87
102
|
|
|
@@ -90,7 +105,7 @@ class NonPositiveInteger(DataType):
|
|
|
90
105
|
python = int
|
|
91
106
|
dms = dms.Int32
|
|
92
107
|
graphql = "Int"
|
|
93
|
-
xsd = "
|
|
108
|
+
xsd = "nonPositiveInteger"
|
|
94
109
|
sql = "INTEGER"
|
|
95
110
|
|
|
96
111
|
|
|
@@ -99,7 +114,7 @@ class NonNegativeInteger(DataType):
|
|
|
99
114
|
python = int
|
|
100
115
|
dms = dms.Int32
|
|
101
116
|
graphql = "Int"
|
|
102
|
-
xsd = "
|
|
117
|
+
xsd = "nonNegativeInteger"
|
|
103
118
|
sql = "INTEGER"
|
|
104
119
|
|
|
105
120
|
|
|
@@ -108,7 +123,7 @@ class NegativeInteger(DataType):
|
|
|
108
123
|
python = int
|
|
109
124
|
dms = dms.Int32
|
|
110
125
|
graphql = "Int"
|
|
111
|
-
xsd = "
|
|
126
|
+
xsd = "negativeInteger"
|
|
112
127
|
sql = "INTEGER"
|
|
113
128
|
|
|
114
129
|
|
|
@@ -117,7 +132,7 @@ class Long(DataType):
|
|
|
117
132
|
python = int
|
|
118
133
|
dms = dms.Int64
|
|
119
134
|
graphql = "Int"
|
|
120
|
-
xsd = "
|
|
135
|
+
xsd = "long"
|
|
121
136
|
sql = "BIGINT"
|
|
122
137
|
|
|
123
138
|
|
|
@@ -126,7 +141,7 @@ class String(DataType):
|
|
|
126
141
|
python = str
|
|
127
142
|
dms = dms.Text
|
|
128
143
|
graphql = "String"
|
|
129
|
-
xsd = "
|
|
144
|
+
xsd = "string"
|
|
130
145
|
sql = "STRING"
|
|
131
146
|
|
|
132
147
|
|
|
@@ -135,7 +150,7 @@ class LangString(DataType):
|
|
|
135
150
|
python = str
|
|
136
151
|
dms = dms.Text
|
|
137
152
|
graphql = "String"
|
|
138
|
-
xsd = "
|
|
153
|
+
xsd = "string"
|
|
139
154
|
sql = "STRING"
|
|
140
155
|
|
|
141
156
|
|
|
@@ -144,7 +159,7 @@ class AnyURI(DataType):
|
|
|
144
159
|
python = str
|
|
145
160
|
dms = dms.Text
|
|
146
161
|
graphql = "String"
|
|
147
|
-
xsd = "
|
|
162
|
+
xsd = "anyURI"
|
|
148
163
|
sql = "STRING"
|
|
149
164
|
|
|
150
165
|
|
|
@@ -153,7 +168,7 @@ class NormalizedString(DataType):
|
|
|
153
168
|
python = str
|
|
154
169
|
dms = dms.Text
|
|
155
170
|
graphql = "String"
|
|
156
|
-
xsd = "
|
|
171
|
+
xsd = "normalizedString"
|
|
157
172
|
sql = "STRING"
|
|
158
173
|
|
|
159
174
|
|
|
@@ -162,7 +177,7 @@ class Token(DataType):
|
|
|
162
177
|
python = str
|
|
163
178
|
dms = dms.Text
|
|
164
179
|
graphql = "String"
|
|
165
|
-
xsd = "
|
|
180
|
+
xsd = "string"
|
|
166
181
|
sql = "STRING"
|
|
167
182
|
|
|
168
183
|
|
|
@@ -171,25 +186,25 @@ class DateTime(DataType):
|
|
|
171
186
|
python = datetime
|
|
172
187
|
dms = dms.Timestamp
|
|
173
188
|
graphql = "Timestamp"
|
|
174
|
-
xsd = "
|
|
189
|
+
xsd = "dateTimeStamp"
|
|
175
190
|
sql = "TIMESTAMP"
|
|
176
191
|
|
|
177
192
|
|
|
178
|
-
class
|
|
179
|
-
name = "
|
|
193
|
+
class Timestamp(DataType):
|
|
194
|
+
name = "timestamp"
|
|
180
195
|
python = datetime
|
|
181
196
|
dms = dms.Timestamp
|
|
182
197
|
graphql = "Timestamp"
|
|
183
|
-
xsd = "
|
|
198
|
+
xsd = "dateTimeStamp"
|
|
184
199
|
sql = "TIMESTAMP"
|
|
185
200
|
|
|
186
201
|
|
|
187
|
-
class
|
|
188
|
-
name = "
|
|
202
|
+
class DateTimeStamp(DataType):
|
|
203
|
+
name = "dateTimeStamp"
|
|
189
204
|
python = datetime
|
|
190
205
|
dms = dms.Timestamp
|
|
191
206
|
graphql = "Timestamp"
|
|
192
|
-
xsd = "
|
|
207
|
+
xsd = "dateTimeStamp"
|
|
193
208
|
sql = "TIMESTAMP"
|
|
194
209
|
|
|
195
210
|
|
|
@@ -198,7 +213,7 @@ class Date(DataType):
|
|
|
198
213
|
python = date
|
|
199
214
|
dms = dms.Date
|
|
200
215
|
graphql = "String"
|
|
201
|
-
xsd = "
|
|
216
|
+
xsd = "date"
|
|
202
217
|
sql = "DATE"
|
|
203
218
|
|
|
204
219
|
|
|
@@ -207,7 +222,7 @@ class PlainLiteral(DataType):
|
|
|
207
222
|
python = str
|
|
208
223
|
dms = dms.Text
|
|
209
224
|
graphql = "String"
|
|
210
|
-
xsd = "
|
|
225
|
+
xsd = "plainLiteral"
|
|
211
226
|
sql = "STRING"
|
|
212
227
|
|
|
213
228
|
|
|
@@ -216,7 +231,7 @@ class Literal(DataType):
|
|
|
216
231
|
python = str
|
|
217
232
|
dms = dms.Text
|
|
218
233
|
graphql = "String"
|
|
219
|
-
xsd = "
|
|
234
|
+
xsd = "string"
|
|
220
235
|
sql = "STRING"
|
|
221
236
|
|
|
222
237
|
|
|
@@ -225,7 +240,7 @@ class Timeseries(DataType):
|
|
|
225
240
|
python = dms.TimeSeriesReference
|
|
226
241
|
dms = dms.TimeSeriesReference
|
|
227
242
|
graphql = "TimeSeries"
|
|
228
|
-
xsd = "
|
|
243
|
+
xsd = "string"
|
|
229
244
|
sql = "STRING"
|
|
230
245
|
|
|
231
246
|
|
|
@@ -234,7 +249,7 @@ class File(DataType):
|
|
|
234
249
|
python = dms.FileReference
|
|
235
250
|
dms = dms.FileReference
|
|
236
251
|
graphql = "File"
|
|
237
|
-
xsd = "
|
|
252
|
+
xsd = "string"
|
|
238
253
|
sql = "STRING"
|
|
239
254
|
|
|
240
255
|
|
|
@@ -243,7 +258,7 @@ class Sequence(DataType):
|
|
|
243
258
|
python = dms.SequenceReference
|
|
244
259
|
dms = dms.SequenceReference
|
|
245
260
|
graphql = "Sequence"
|
|
246
|
-
xsd = "
|
|
261
|
+
xsd = "string"
|
|
247
262
|
sql = "STRING"
|
|
248
263
|
|
|
249
264
|
|
|
@@ -252,8 +267,16 @@ class Json(DataType):
|
|
|
252
267
|
python = dms.Json
|
|
253
268
|
dms = dms.Json
|
|
254
269
|
graphql = "Json"
|
|
255
|
-
xsd = "
|
|
270
|
+
xsd = "string"
|
|
256
271
|
sql = "STRING"
|
|
257
272
|
|
|
258
273
|
|
|
259
274
|
_DATA_TYPE_BY_NAME = {cls.name.casefold(): cls for cls in DataType.__subclasses__()}
|
|
275
|
+
_seen = set()
|
|
276
|
+
_DATA_TYPE_BY_DMS_TYPE = {
|
|
277
|
+
cls.dms._type.casefold(): cls
|
|
278
|
+
for cls in DataType.__subclasses__()
|
|
279
|
+
if cls.dms._type not in _seen and not _seen.add(cls.dms._type) # type: ignore[func-returns-value]
|
|
280
|
+
}
|
|
281
|
+
del _seen
|
|
282
|
+
_XSD_TYPES = {cls_.xsd for cls_ in _DATA_TYPE_BY_NAME.values()}
|