cognite-neat 0.109.3__py3-none-any.whl → 0.110.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_alpha.py +2 -0
- cognite/neat/_client/_api/schema.py +17 -1
- cognite/neat/_client/data_classes/schema.py +3 -3
- cognite/neat/_constants.py +11 -0
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +9 -10
- cognite/neat/_graph/extractors/_iodd.py +3 -3
- cognite/neat/_graph/extractors/_mock_graph_generator.py +9 -7
- cognite/neat/_graph/loaders/_rdf2dms.py +285 -346
- cognite/neat/_graph/queries/_base.py +28 -92
- cognite/neat/_graph/transformers/__init__.py +1 -3
- cognite/neat/_graph/transformers/_rdfpath.py +2 -49
- cognite/neat/_issues/__init__.py +1 -6
- cognite/neat/_issues/_base.py +21 -252
- cognite/neat/_issues/_contextmanagers.py +46 -0
- cognite/neat/_issues/_factory.py +61 -0
- cognite/neat/_issues/errors/__init__.py +18 -4
- cognite/neat/_issues/errors/_wrapper.py +81 -3
- cognite/neat/_issues/formatters.py +4 -4
- cognite/neat/_issues/warnings/__init__.py +3 -2
- cognite/neat/_issues/warnings/_properties.py +8 -0
- cognite/neat/_rules/_constants.py +9 -0
- cognite/neat/_rules/_shared.py +3 -2
- cognite/neat/_rules/analysis/__init__.py +2 -3
- cognite/neat/_rules/analysis/_base.py +450 -258
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_rules2excel.py +2 -8
- cognite/neat/_rules/exporters/_rules2instance_template.py +2 -2
- cognite/neat/_rules/exporters/_rules2ontology.py +5 -4
- cognite/neat/_rules/importers/_base.py +2 -47
- cognite/neat/_rules/importers/_dms2rules.py +7 -10
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +2 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +59 -25
- cognite/neat/_rules/importers/_rdf/_shared.py +1 -1
- cognite/neat/_rules/importers/_spreadsheet2rules.py +12 -9
- cognite/neat/_rules/models/dms/_rules.py +3 -1
- cognite/neat/_rules/models/dms/_rules_input.py +4 -0
- cognite/neat/_rules/models/dms/_validation.py +14 -4
- cognite/neat/_rules/models/entities/_loaders.py +1 -1
- cognite/neat/_rules/models/entities/_multi_value.py +2 -2
- cognite/neat/_rules/models/information/_rules.py +18 -17
- cognite/neat/_rules/models/information/_rules_input.py +2 -1
- cognite/neat/_rules/models/information/_validation.py +3 -1
- cognite/neat/_rules/transformers/__init__.py +8 -2
- cognite/neat/_rules/transformers/_converters.py +242 -43
- cognite/neat/_rules/transformers/_verification.py +5 -10
- cognite/neat/_session/_base.py +4 -4
- cognite/neat/_session/_prepare.py +12 -0
- cognite/neat/_session/_read.py +21 -17
- cognite/neat/_session/_show.py +11 -123
- cognite/neat/_session/_state.py +0 -2
- cognite/neat/_session/_subset.py +64 -0
- cognite/neat/_session/_to.py +63 -12
- cognite/neat/_store/_graph_store.py +5 -246
- cognite/neat/_utils/rdf_.py +2 -2
- cognite/neat/_utils/spreadsheet.py +44 -1
- cognite/neat/_utils/text.py +51 -32
- cognite/neat/_version.py +1 -1
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/METADATA +1 -1
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/RECORD +62 -64
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/WHEEL +1 -1
- cognite/neat/_graph/queries/_construct.py +0 -187
- cognite/neat/_graph/queries/_shared.py +0 -173
- cognite/neat/_rules/analysis/_dms.py +0 -57
- cognite/neat/_rules/analysis/_information.py +0 -249
- cognite/neat/_rules/models/_rdfpath.py +0 -372
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/entry_points.txt +0 -0
|
Binary file
|
|
@@ -13,6 +13,7 @@ from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
|
|
|
13
13
|
from openpyxl.worksheet.worksheet import Worksheet
|
|
14
14
|
from rdflib import Namespace
|
|
15
15
|
|
|
16
|
+
from cognite.neat._rules._constants import get_internal_properties
|
|
16
17
|
from cognite.neat._rules._shared import VerifiedRules
|
|
17
18
|
from cognite.neat._rules.models import (
|
|
18
19
|
ExtensionCategory,
|
|
@@ -64,13 +65,6 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
64
65
|
style_options = get_args(Style)
|
|
65
66
|
dump_options = get_args(DumpOptions)
|
|
66
67
|
|
|
67
|
-
_internal_columns: ClassVar[list[str]] = [
|
|
68
|
-
"physical",
|
|
69
|
-
"logical",
|
|
70
|
-
"conceptual",
|
|
71
|
-
"Neat ID",
|
|
72
|
-
]
|
|
73
|
-
|
|
74
68
|
def __init__(
|
|
75
69
|
self,
|
|
76
70
|
styling: Style = "default",
|
|
@@ -131,7 +125,7 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
|
|
|
131
125
|
if sheet.lower() == "metadata":
|
|
132
126
|
continue
|
|
133
127
|
ws = workbook[sheet]
|
|
134
|
-
for col in
|
|
128
|
+
for col in get_internal_properties():
|
|
135
129
|
column_letter = find_column_with_value(ws, col)
|
|
136
130
|
if column_letter:
|
|
137
131
|
ws.column_dimensions[column_letter].hidden = True
|
|
@@ -9,7 +9,7 @@ from openpyxl.utils import get_column_letter
|
|
|
9
9
|
from openpyxl.worksheet.datavalidation import DataValidation
|
|
10
10
|
|
|
11
11
|
from cognite.neat._rules._constants import EntityTypes
|
|
12
|
-
from cognite.neat._rules.analysis import
|
|
12
|
+
from cognite.neat._rules.analysis import RulesAnalysis
|
|
13
13
|
from cognite.neat._rules.models.entities._single_value import ClassEntity
|
|
14
14
|
from cognite.neat._rules.models.information._rules import InformationRules
|
|
15
15
|
|
|
@@ -55,7 +55,7 @@ class InstanceTemplateExporter(BaseExporter[InformationRules, Workbook]):
|
|
|
55
55
|
# Remove default sheet named "Sheet"
|
|
56
56
|
workbook.remove(workbook["Sheet"])
|
|
57
57
|
|
|
58
|
-
for class_, properties in
|
|
58
|
+
for class_, properties in RulesAnalysis(rules).properties_by_id_by_class().items():
|
|
59
59
|
workbook.create_sheet(title=class_.suffix)
|
|
60
60
|
|
|
61
61
|
# Add header rows
|
|
@@ -15,7 +15,7 @@ from cognite.neat._issues.errors import (
|
|
|
15
15
|
)
|
|
16
16
|
from cognite.neat._issues.warnings import PropertyDefinitionDuplicatedWarning
|
|
17
17
|
from cognite.neat._rules._constants import EntityTypes
|
|
18
|
-
from cognite.neat._rules.analysis import
|
|
18
|
+
from cognite.neat._rules.analysis import RulesAnalysis
|
|
19
19
|
from cognite.neat._rules.models.data_types import DataType
|
|
20
20
|
from cognite.neat._rules.models.entities import ClassEntity
|
|
21
21
|
from cognite.neat._rules.models.information import (
|
|
@@ -121,11 +121,12 @@ class Ontology(OntologyModel):
|
|
|
121
121
|
)
|
|
122
122
|
raise MultiValueError(errors)
|
|
123
123
|
|
|
124
|
-
|
|
124
|
+
analysis = RulesAnalysis(rules)
|
|
125
|
+
class_dict = analysis.class_by_suffix()
|
|
125
126
|
return cls(
|
|
126
127
|
properties=[
|
|
127
128
|
OWLProperty.from_list_of_properties(definition, rules.metadata.namespace)
|
|
128
|
-
for definition in
|
|
129
|
+
for definition in analysis.property_by_id().values()
|
|
129
130
|
],
|
|
130
131
|
classes=[
|
|
131
132
|
OWLClass.from_class(definition, rules.metadata.namespace, rules.prefixes)
|
|
@@ -137,7 +138,7 @@ class Ontology(OntologyModel):
|
|
|
137
138
|
list(properties.values()),
|
|
138
139
|
rules.metadata.namespace,
|
|
139
140
|
)
|
|
140
|
-
for class_, properties in
|
|
141
|
+
for class_, properties in analysis.properties_by_id_by_class().items()
|
|
141
142
|
]
|
|
142
143
|
+ [
|
|
143
144
|
SHACLNodeShape.from_rules(
|
|
@@ -1,15 +1,11 @@
|
|
|
1
|
-
import warnings
|
|
2
1
|
from abc import ABC, abstractmethod
|
|
3
|
-
from
|
|
4
|
-
from contextlib import contextmanager, suppress
|
|
2
|
+
from contextlib import suppress
|
|
5
3
|
from datetime import datetime
|
|
6
|
-
from typing import TYPE_CHECKING, Any, Generic
|
|
4
|
+
from typing import TYPE_CHECKING, Any, Generic
|
|
7
5
|
|
|
8
|
-
from pydantic import ValidationError
|
|
9
6
|
from rdflib import URIRef
|
|
10
7
|
|
|
11
8
|
from cognite.neat._constants import DEFAULT_NAMESPACE
|
|
12
|
-
from cognite.neat._issues import IssueList, NeatError, NeatWarning
|
|
13
9
|
from cognite.neat._rules._shared import ReadRules, T_InputRules
|
|
14
10
|
from cognite.neat._utils.auxiliary import class_html_doc
|
|
15
11
|
|
|
@@ -64,44 +60,3 @@ class BaseImporter(ABC, Generic[T_InputRules]):
|
|
|
64
60
|
@property
|
|
65
61
|
def source_uri(self) -> URIRef:
|
|
66
62
|
return DEFAULT_NAMESPACE["UNKNOWN"]
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
class _FutureResult:
|
|
70
|
-
def __init__(self) -> None:
|
|
71
|
-
self._result: Literal["success", "failure", "pending"] = "pending"
|
|
72
|
-
|
|
73
|
-
@property
|
|
74
|
-
def result(self) -> Literal["success", "failure", "pending"]:
|
|
75
|
-
return self._result
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
@contextmanager
|
|
79
|
-
def _handle_issues(
|
|
80
|
-
issues: IssueList,
|
|
81
|
-
error_cls: type[NeatError] = NeatError,
|
|
82
|
-
warning_cls: type[NeatWarning] = NeatWarning,
|
|
83
|
-
error_args: dict[str, Any] | None = None,
|
|
84
|
-
) -> Iterator[_FutureResult]:
|
|
85
|
-
"""This is an internal help function to handle issues and warnings.
|
|
86
|
-
|
|
87
|
-
Args:
|
|
88
|
-
issues: The issues list to append to.
|
|
89
|
-
error_cls: The class used to convert errors to issues.
|
|
90
|
-
warning_cls: The class used to convert warnings to issues.
|
|
91
|
-
|
|
92
|
-
Returns:
|
|
93
|
-
FutureResult: A future result object that can be used to check the result of the context manager.
|
|
94
|
-
"""
|
|
95
|
-
with warnings.catch_warnings(record=True) as warning_logger:
|
|
96
|
-
warnings.simplefilter("always")
|
|
97
|
-
future_result = _FutureResult()
|
|
98
|
-
try:
|
|
99
|
-
yield future_result
|
|
100
|
-
except ValidationError as e:
|
|
101
|
-
issues.extend(error_cls.from_errors(e.errors(), **(error_args or {}))) # type: ignore[arg-type]
|
|
102
|
-
future_result._result = "failure"
|
|
103
|
-
else:
|
|
104
|
-
future_result._result = "success"
|
|
105
|
-
finally:
|
|
106
|
-
if warning_logger:
|
|
107
|
-
issues.extend([warning_cls.from_warning(warning) for warning in warning_logger]) # type: ignore[misc]
|
|
@@ -19,7 +19,7 @@ from cognite.client.data_classes.data_modeling.views import (
|
|
|
19
19
|
from cognite.client.utils import ms_to_datetime
|
|
20
20
|
|
|
21
21
|
from cognite.neat._client import NeatClient
|
|
22
|
-
from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
|
|
22
|
+
from cognite.neat._issues import IssueList, MultiValueError, NeatIssue, catch_issues
|
|
23
23
|
from cognite.neat._issues.errors import (
|
|
24
24
|
FileTypeUnexpectedError,
|
|
25
25
|
NeatValueError,
|
|
@@ -35,7 +35,7 @@ from cognite.neat._issues.warnings import (
|
|
|
35
35
|
ResourceUnknownWarning,
|
|
36
36
|
)
|
|
37
37
|
from cognite.neat._rules._shared import ReadRules
|
|
38
|
-
from cognite.neat._rules.importers._base import BaseImporter
|
|
38
|
+
from cognite.neat._rules.importers._base import BaseImporter
|
|
39
39
|
from cognite.neat._rules.models import (
|
|
40
40
|
DMSInputRules,
|
|
41
41
|
DMSSchema,
|
|
@@ -131,11 +131,10 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
131
131
|
|
|
132
132
|
@classmethod
|
|
133
133
|
def from_data_model(cls, client: NeatClient, model: dm.DataModel[dm.View]) -> "DMSImporter":
|
|
134
|
-
|
|
135
|
-
with _handle_issues(issue_list) as result:
|
|
134
|
+
with catch_issues() as issue_list:
|
|
136
135
|
schema = client.schema.retrieve_data_model(model)
|
|
137
136
|
|
|
138
|
-
if
|
|
137
|
+
if issue_list.has_errors:
|
|
139
138
|
return cls(DMSSchema(), issue_list)
|
|
140
139
|
|
|
141
140
|
metadata = cls._create_metadata_from_model(model)
|
|
@@ -187,10 +186,9 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
187
186
|
|
|
188
187
|
@classmethod
|
|
189
188
|
def from_directory(cls, directory: str | Path, client: NeatClient | None = None) -> "DMSImporter":
|
|
190
|
-
|
|
191
|
-
with _handle_issues(issue_list) as _:
|
|
189
|
+
with catch_issues() as issue_list:
|
|
192
190
|
schema = DMSSchema.from_directory(directory)
|
|
193
|
-
# If there were errors during the import, the to_rules
|
|
191
|
+
# If there were errors during the import, the to_rules will raise them.
|
|
194
192
|
return cls(
|
|
195
193
|
schema, issue_list, referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client)
|
|
196
194
|
)
|
|
@@ -202,8 +200,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
|
|
|
202
200
|
DMSSchema(),
|
|
203
201
|
[FileTypeUnexpectedError(Path(zip_file), frozenset([".zip"]))],
|
|
204
202
|
)
|
|
205
|
-
|
|
206
|
-
with _handle_issues(issue_list) as _:
|
|
203
|
+
with catch_issues() as issue_list:
|
|
207
204
|
schema = DMSSchema.from_zip(zip_file)
|
|
208
205
|
return cls(
|
|
209
206
|
schema, issue_list, referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client)
|
|
@@ -19,7 +19,7 @@ from cognite.neat._rules.importers._dtdl2rules.dtdl_converter import _DTDLConver
|
|
|
19
19
|
from cognite.neat._rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
|
|
20
20
|
from cognite.neat._rules.models import InformationInputRules
|
|
21
21
|
from cognite.neat._rules.models.information import InformationInputMetadata
|
|
22
|
-
from cognite.neat._utils.text import humanize_collection,
|
|
22
|
+
from cognite.neat._utils.text import humanize_collection, to_pascal_case
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
class DTDLImporter(BaseImporter[InformationInputRules]):
|
|
@@ -130,7 +130,7 @@ class DTDLImporter(BaseImporter[InformationInputRules]):
|
|
|
130
130
|
metadata = self._default_metadata()
|
|
131
131
|
|
|
132
132
|
if self.name:
|
|
133
|
-
metadata["name"] =
|
|
133
|
+
metadata["name"] = to_pascal_case(self.name)
|
|
134
134
|
try:
|
|
135
135
|
most_common_prefix = converter.get_most_common_prefix()
|
|
136
136
|
except ValueError:
|
|
@@ -13,11 +13,12 @@ from rdflib import Literal as RdfLiteral
|
|
|
13
13
|
from cognite.neat._constants import NEAT, get_default_prefixes_and_namespaces
|
|
14
14
|
from cognite.neat._issues import IssueList
|
|
15
15
|
from cognite.neat._issues.warnings import PropertyValueTypeUndefinedWarning
|
|
16
|
-
from cognite.neat._rules.analysis import
|
|
16
|
+
from cognite.neat._rules.analysis import RulesAnalysis
|
|
17
17
|
from cognite.neat._rules.models import InformationRules, data_types
|
|
18
18
|
from cognite.neat._rules.models.data_types import AnyURI
|
|
19
19
|
from cognite.neat._rules.models.entities._single_value import UnknownEntity
|
|
20
20
|
from cognite.neat._rules.models.information import (
|
|
21
|
+
InformationClass,
|
|
21
22
|
InformationInputClass,
|
|
22
23
|
InformationInputProperty,
|
|
23
24
|
InformationMetadata,
|
|
@@ -256,7 +257,7 @@ class InferenceImporter(BaseRDFImporter):
|
|
|
256
257
|
property_["value_type"].remove(str(self.non_existing_node_type))
|
|
257
258
|
|
|
258
259
|
if len(property_["value_type"]) > 1:
|
|
259
|
-
property_["value_type"] = "
|
|
260
|
+
property_["value_type"] = ", ".join([str(t) for t in property_["value_type"]])
|
|
260
261
|
else:
|
|
261
262
|
property_["value_type"] = next(iter(property_["value_type"]))
|
|
262
263
|
|
|
@@ -402,7 +403,8 @@ class SubclassInferenceImporter(BaseRDFImporter):
|
|
|
402
403
|
else:
|
|
403
404
|
existing_classes = {}
|
|
404
405
|
classes: list[InformationInputClass] = []
|
|
405
|
-
|
|
406
|
+
properties_by_class_suffix_by_property_id_lowered: dict[str, dict[str, InformationInputProperty]] = {}
|
|
407
|
+
|
|
406
408
|
# Help for IDE
|
|
407
409
|
type_uri: URIRef
|
|
408
410
|
parent_uri: URIRef
|
|
@@ -440,26 +442,51 @@ class SubclassInferenceImporter(BaseRDFImporter):
|
|
|
440
442
|
InformationInputClass(
|
|
441
443
|
class_=class_suffix,
|
|
442
444
|
implements=parent_suffix,
|
|
445
|
+
instance_source=type_uri,
|
|
443
446
|
)
|
|
444
447
|
)
|
|
445
448
|
else:
|
|
446
449
|
classes.append(InformationInputClass.load(existing_classes[class_suffix].model_dump()))
|
|
450
|
+
|
|
451
|
+
properties_by_id: dict[str, InformationInputProperty] = {}
|
|
447
452
|
for property_uri, read_properties in properties_by_property_uri.items():
|
|
448
453
|
if property_uri in shared_property_uris:
|
|
449
454
|
shared_properties[property_uri].extend(read_properties)
|
|
450
455
|
continue
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
)
|
|
454
|
-
|
|
456
|
+
property_id = remove_namespace_from_uri(property_uri)
|
|
457
|
+
self._add_uri_namespace_to_prefixes(property_uri, prefixes)
|
|
458
|
+
if existing_prop := properties_by_id.get(property_id.casefold()):
|
|
459
|
+
if not isinstance(existing_prop.instance_source, list):
|
|
460
|
+
existing_prop.instance_source = (
|
|
461
|
+
[existing_prop.instance_source] if existing_prop.instance_source else []
|
|
462
|
+
)
|
|
463
|
+
existing_prop.instance_source.append(property_uri)
|
|
464
|
+
continue
|
|
465
|
+
else:
|
|
466
|
+
properties_by_id[property_id.casefold()] = self._create_property(
|
|
467
|
+
read_properties, class_suffix, property_uri, property_id, prefixes
|
|
468
|
+
)
|
|
469
|
+
properties_by_class_suffix_by_property_id_lowered[class_suffix] = properties_by_id
|
|
455
470
|
if parent_suffix:
|
|
471
|
+
properties_by_id = {}
|
|
456
472
|
for property_uri, read_properties in shared_properties.items():
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
473
|
+
property_id = remove_namespace_from_uri(property_uri)
|
|
474
|
+
self._add_uri_namespace_to_prefixes(property_uri, prefixes)
|
|
475
|
+
if existing_prop := properties_by_id.get(property_id.casefold()):
|
|
476
|
+
if not isinstance(existing_prop.instance_source, list):
|
|
477
|
+
existing_prop.instance_source = (
|
|
478
|
+
[existing_prop.instance_source] if existing_prop.instance_source else []
|
|
479
|
+
)
|
|
480
|
+
existing_prop.instance_source.append(property_uri)
|
|
481
|
+
else:
|
|
482
|
+
properties_by_id[property_uri.casefold()] = self._create_property(
|
|
483
|
+
read_properties, parent_suffix, property_uri, property_id, prefixes
|
|
460
484
|
)
|
|
461
|
-
|
|
462
|
-
|
|
485
|
+
return classes, [
|
|
486
|
+
prop
|
|
487
|
+
for properties in properties_by_class_suffix_by_property_id_lowered.values()
|
|
488
|
+
for prop in properties.values()
|
|
489
|
+
]
|
|
463
490
|
|
|
464
491
|
@staticmethod
|
|
465
492
|
def _get_properties_by_class_by_property(
|
|
@@ -481,26 +508,36 @@ class SubclassInferenceImporter(BaseRDFImporter):
|
|
|
481
508
|
type_uri, instance_count_literal = cast(tuple[URIRef, RdfLiteral], result_row)
|
|
482
509
|
count_by_type[type_uri] = instance_count_literal.toPython()
|
|
483
510
|
if self._rules:
|
|
484
|
-
analysis =
|
|
511
|
+
analysis = RulesAnalysis(self._rules)
|
|
485
512
|
existing_class_properties = {
|
|
486
|
-
(class_entity.suffix, prop.property_)
|
|
487
|
-
for class_entity, properties in analysis.
|
|
488
|
-
|
|
513
|
+
(class_entity.suffix, prop.property_): prop
|
|
514
|
+
for class_entity, properties in analysis.properties_by_class(
|
|
515
|
+
include_ancestors=True, include_different_space=True
|
|
489
516
|
).items()
|
|
490
517
|
for prop in properties
|
|
491
518
|
}
|
|
519
|
+
existing_classes = {cls_.class_.suffix: cls_ for cls_ in self._rules.classes}
|
|
492
520
|
else:
|
|
493
|
-
existing_class_properties =
|
|
521
|
+
existing_class_properties = {}
|
|
522
|
+
existing_classes = {}
|
|
494
523
|
properties_by_class_by_subclass: list[_ReadProperties] = []
|
|
524
|
+
existing_class: InformationClass | None
|
|
495
525
|
for type_uri, instance_count in count_by_type.items():
|
|
496
526
|
property_query = self._properties_query.format(type=type_uri, unknown_type=NEAT.UnknownType)
|
|
497
527
|
class_suffix = remove_namespace_from_uri(type_uri)
|
|
528
|
+
if (existing_class := existing_classes.get(class_suffix)) and existing_class.instance_source is None:
|
|
529
|
+
existing_class.instance_source = type_uri
|
|
530
|
+
|
|
498
531
|
for result_row in self.graph.query(property_query):
|
|
499
532
|
property_uri, value_type_uri = cast(tuple[URIRef, URIRef], result_row)
|
|
500
533
|
if property_uri == RDF.type:
|
|
501
534
|
continue
|
|
502
535
|
property_str = remove_namespace_from_uri(property_uri)
|
|
503
|
-
if (class_suffix, property_str)
|
|
536
|
+
if existing_property := existing_class_properties.get((class_suffix, property_str)):
|
|
537
|
+
if existing_property.instance_source is None:
|
|
538
|
+
existing_property.instance_source = [property_uri]
|
|
539
|
+
elif existing_property.instance_source and property_uri not in existing_property.instance_source:
|
|
540
|
+
existing_property.instance_source.append(property_uri)
|
|
504
541
|
continue
|
|
505
542
|
occurrence_query = self._max_occurrence_query.format(type=type_uri, property=property_uri)
|
|
506
543
|
max_occurrence = 1 # default value
|
|
@@ -531,21 +568,18 @@ class SubclassInferenceImporter(BaseRDFImporter):
|
|
|
531
568
|
self,
|
|
532
569
|
read_properties: list[_ReadProperties],
|
|
533
570
|
class_suffix: str,
|
|
534
|
-
type_uri: URIRef,
|
|
535
571
|
property_uri: URIRef,
|
|
572
|
+
property_id: str,
|
|
536
573
|
prefixes: dict[str, Namespace],
|
|
537
574
|
) -> InformationInputProperty:
|
|
538
575
|
first = read_properties[0]
|
|
539
576
|
value_type = self._get_value_type(read_properties, prefixes)
|
|
540
|
-
property_name = remove_namespace_from_uri(property_uri)
|
|
541
|
-
self._add_uri_namespace_to_prefixes(property_uri, prefixes)
|
|
542
|
-
|
|
543
577
|
return InformationInputProperty(
|
|
544
578
|
class_=class_suffix,
|
|
545
|
-
property_=
|
|
579
|
+
property_=property_id,
|
|
546
580
|
max_count=first.max_occurrence,
|
|
547
581
|
value_type=value_type,
|
|
548
|
-
instance_source=
|
|
582
|
+
instance_source=[property_uri],
|
|
549
583
|
)
|
|
550
584
|
|
|
551
585
|
def _get_value_type(
|
|
@@ -562,7 +596,7 @@ class SubclassInferenceImporter(BaseRDFImporter):
|
|
|
562
596
|
return UnknownEntity()
|
|
563
597
|
for uri_ref in value_types:
|
|
564
598
|
self._add_uri_namespace_to_prefixes(uri_ref, prefixes)
|
|
565
|
-
return "
|
|
599
|
+
return ", ".join(remove_namespace_from_uri(uri_ref) for uri_ref in value_types)
|
|
566
600
|
|
|
567
601
|
def _default_metadata(self) -> dict[str, Any]:
|
|
568
602
|
now = datetime.now(timezone.utc)
|
|
@@ -136,7 +136,7 @@ def parse_properties(graph: Graph, query: str, language: str, issue_list: IssueL
|
|
|
136
136
|
properties[id_]["value_type"].append(res["value_type"])
|
|
137
137
|
|
|
138
138
|
for prop in properties.values():
|
|
139
|
-
prop["value_type"] = "
|
|
139
|
+
prop["value_type"] = ", ".join(prop["value_type"])
|
|
140
140
|
|
|
141
141
|
if not properties:
|
|
142
142
|
issue_list.append(NeatValueError("Unable to parse properties"))
|
|
@@ -37,15 +37,15 @@ SOURCE_SHEET__TARGET_FIELD__HEADERS = [
|
|
|
37
37
|
"Properties",
|
|
38
38
|
"Properties",
|
|
39
39
|
{
|
|
40
|
-
RoleTypes.information: "Property",
|
|
41
|
-
RoleTypes.dms: "View Property",
|
|
40
|
+
RoleTypes.information: ["Class", "Property"],
|
|
41
|
+
RoleTypes.dms: ["View", "View Property"],
|
|
42
42
|
},
|
|
43
43
|
),
|
|
44
|
-
("Classes", "Classes", "Class"),
|
|
45
|
-
("Containers", "Containers", "Container"),
|
|
46
|
-
("Views", "Views", "View"),
|
|
47
|
-
("Enum", "Enum", "Collection"),
|
|
48
|
-
("Nodes", "Nodes", "Node"),
|
|
44
|
+
("Classes", "Classes", ["Class"]),
|
|
45
|
+
("Containers", "Containers", ["Container"]),
|
|
46
|
+
("Views", "Views", ["View"]),
|
|
47
|
+
("Enum", "Enum", ["Collection"]),
|
|
48
|
+
("Nodes", "Nodes", ["Node"]),
|
|
49
49
|
]
|
|
50
50
|
|
|
51
51
|
|
|
@@ -231,7 +231,10 @@ class SpreadsheetReader:
|
|
|
231
231
|
|
|
232
232
|
try:
|
|
233
233
|
sheets[target_sheet_name], read_info_by_sheet[source_sheet_name] = read_individual_sheet(
|
|
234
|
-
excel_file,
|
|
234
|
+
excel_file,
|
|
235
|
+
source_sheet_name,
|
|
236
|
+
return_read_info=True,
|
|
237
|
+
expected_headers=headers,
|
|
235
238
|
)
|
|
236
239
|
except Exception as e:
|
|
237
240
|
self.issue_list.append(FileReadError(cast(Path, excel_file.io), str(e)))
|
|
@@ -273,7 +276,7 @@ class ExcelImporter(BaseImporter[T_InputRules]):
|
|
|
273
276
|
|
|
274
277
|
rules_cls = INPUT_RULES_BY_ROLE[original_role]
|
|
275
278
|
rules = cast(T_InputRules, rules_cls.load(sheets))
|
|
276
|
-
return ReadRules(rules,
|
|
279
|
+
return ReadRules(rules, read_info_by_sheet)
|
|
277
280
|
|
|
278
281
|
@property
|
|
279
282
|
def description(self) -> str:
|
|
@@ -295,7 +295,9 @@ class DMSView(SheetRow):
|
|
|
295
295
|
None, alias="Filter", description="Explicitly define the filter for the view."
|
|
296
296
|
)
|
|
297
297
|
in_model: bool = Field(
|
|
298
|
-
True,
|
|
298
|
+
True,
|
|
299
|
+
alias="In Model",
|
|
300
|
+
description="Indicates whether the view being defined is a part of the data model.",
|
|
299
301
|
)
|
|
300
302
|
logical: URIRefType | None = Field(
|
|
301
303
|
None,
|
|
@@ -193,6 +193,10 @@ class DMSInputView(InputComponent[DMSView]):
|
|
|
193
193
|
neatId: str | URIRef | None = None
|
|
194
194
|
logical: str | URIRef | None = None
|
|
195
195
|
|
|
196
|
+
def __post_init__(self):
|
|
197
|
+
if self.in_model is None:
|
|
198
|
+
self.in_model = True
|
|
199
|
+
|
|
196
200
|
@classmethod
|
|
197
201
|
def _get_verified_cls(cls) -> type[DMSView]:
|
|
198
202
|
return DMSView
|
|
@@ -17,7 +17,7 @@ from cognite.neat._client import NeatClient
|
|
|
17
17
|
from cognite.neat._client.data_classes.data_modeling import ViewApplyDict
|
|
18
18
|
from cognite.neat._client.data_classes.schema import DMSSchema
|
|
19
19
|
from cognite.neat._constants import COGNITE_MODELS, DMS_CONTAINER_PROPERTY_SIZE_LIMIT, DMS_VIEW_CONTAINER_SIZE_LIMIT
|
|
20
|
-
from cognite.neat._issues import IssueList, NeatError
|
|
20
|
+
from cognite.neat._issues import IssueList, NeatError
|
|
21
21
|
from cognite.neat._issues.errors import (
|
|
22
22
|
CDFMissingClientError,
|
|
23
23
|
PropertyDefinitionDuplicatedError,
|
|
@@ -42,6 +42,7 @@ from cognite.neat._rules.models.entities import ContainerEntity, RawFilter
|
|
|
42
42
|
from cognite.neat._rules.models.entities._single_value import (
|
|
43
43
|
ViewEntity,
|
|
44
44
|
)
|
|
45
|
+
from cognite.neat._utils.spreadsheet import SpreadsheetRead
|
|
45
46
|
|
|
46
47
|
from ._rules import DMSProperty, DMSRules
|
|
47
48
|
|
|
@@ -54,13 +55,19 @@ class DMSValidation:
|
|
|
54
55
|
# For example, changing the filter is allowed, but changing the properties is not.
|
|
55
56
|
changeable_view_attributes: ClassVar[set[str]] = {"filter"}
|
|
56
57
|
|
|
57
|
-
def __init__(
|
|
58
|
+
def __init__(
|
|
59
|
+
self,
|
|
60
|
+
rules: DMSRules,
|
|
61
|
+
client: NeatClient | None = None,
|
|
62
|
+
read_info_by_spreadsheet: dict[str, SpreadsheetRead] | None = None,
|
|
63
|
+
) -> None:
|
|
58
64
|
self._rules = rules
|
|
59
65
|
self._client = client
|
|
60
66
|
self._metadata = rules.metadata
|
|
61
67
|
self._properties = rules.properties
|
|
62
68
|
self._containers = rules.containers
|
|
63
69
|
self._views = rules.views
|
|
70
|
+
self._read_info_by_spreadsheet = read_info_by_spreadsheet or {}
|
|
64
71
|
|
|
65
72
|
def imported_views_and_containers_ids(
|
|
66
73
|
self, include_views_with_no_properties: bool = True
|
|
@@ -87,7 +94,7 @@ class DMSValidation:
|
|
|
87
94
|
|
|
88
95
|
return imported_views, imported_containers
|
|
89
96
|
|
|
90
|
-
def validate(self) ->
|
|
97
|
+
def validate(self) -> IssueList:
|
|
91
98
|
imported_views, imported_containers = self.imported_views_and_containers_ids(
|
|
92
99
|
include_views_with_no_properties=False
|
|
93
100
|
)
|
|
@@ -212,13 +219,16 @@ class DMSValidation:
|
|
|
212
219
|
for prop_no, prop in enumerate(self._properties):
|
|
213
220
|
if prop.container and prop.container_property:
|
|
214
221
|
container_properties_by_id[(prop.container, prop.container_property)].append((prop_no, prop))
|
|
215
|
-
|
|
222
|
+
properties_sheet = self._read_info_by_spreadsheet.get("Properties")
|
|
216
223
|
errors = IssueList()
|
|
217
224
|
for (container, prop_name), properties in container_properties_by_id.items():
|
|
218
225
|
if len(properties) == 1:
|
|
219
226
|
continue
|
|
220
227
|
container_id = container.as_id()
|
|
228
|
+
|
|
221
229
|
row_numbers = {prop_no for prop_no, _ in properties}
|
|
230
|
+
if properties_sheet:
|
|
231
|
+
row_numbers = {properties_sheet.adjusted_row_number(row_no) for row_no in row_numbers}
|
|
222
232
|
value_types = {prop.value_type for _, prop in properties if prop.value_type}
|
|
223
233
|
# The container type 'direct' is an exception. On a container the type direct can point to any
|
|
224
234
|
# node. The value type is typically set on the view.
|
|
@@ -23,7 +23,7 @@ def load_value_type(
|
|
|
23
23
|
elif isinstance(raw, str):
|
|
24
24
|
# property holding xsd data type
|
|
25
25
|
# check if it is multi value type
|
|
26
|
-
if "
|
|
26
|
+
if "," in raw:
|
|
27
27
|
value_type = MultiValueTypeInfo.load(raw)
|
|
28
28
|
value_type.set_default_prefix(default_prefix)
|
|
29
29
|
return value_type
|
|
@@ -18,7 +18,7 @@ class MultiValueTypeInfo(BaseModel):
|
|
|
18
18
|
types: list[DataType | ClassEntity]
|
|
19
19
|
|
|
20
20
|
def __str__(self) -> str:
|
|
21
|
-
return "
|
|
21
|
+
return ", ".join([str(t) for t in self.types])
|
|
22
22
|
|
|
23
23
|
@model_serializer(when_used="unless-none", return_type=str)
|
|
24
24
|
def as_str(self) -> str:
|
|
@@ -52,7 +52,7 @@ class MultiValueTypeInfo(BaseModel):
|
|
|
52
52
|
|
|
53
53
|
@classmethod
|
|
54
54
|
def _parse(cls, raw: str) -> dict:
|
|
55
|
-
if not (types := [type_.strip() for type_ in raw.split("
|
|
55
|
+
if not (types := [type_.strip() for type_ in raw.split(",")]):
|
|
56
56
|
return {"types": [UnknownEntity()]}
|
|
57
57
|
else:
|
|
58
58
|
return {
|
|
@@ -8,7 +8,7 @@ from pydantic_core.core_schema import SerializationInfo
|
|
|
8
8
|
from rdflib import Namespace, URIRef
|
|
9
9
|
|
|
10
10
|
from cognite.neat._constants import get_default_prefixes_and_namespaces
|
|
11
|
-
from cognite.neat._issues.errors import
|
|
11
|
+
from cognite.neat._issues.errors import PropertyDefinitionError
|
|
12
12
|
from cognite.neat._rules._constants import EntityTypes
|
|
13
13
|
from cognite.neat._rules.models._base_rules import (
|
|
14
14
|
BaseMetadata,
|
|
@@ -18,11 +18,6 @@ from cognite.neat._rules.models._base_rules import (
|
|
|
18
18
|
SheetList,
|
|
19
19
|
SheetRow,
|
|
20
20
|
)
|
|
21
|
-
from cognite.neat._rules.models._rdfpath import (
|
|
22
|
-
RDFPath,
|
|
23
|
-
TransformationRuleType,
|
|
24
|
-
parse_rule,
|
|
25
|
-
)
|
|
26
21
|
from cognite.neat._rules.models._types import (
|
|
27
22
|
ClassEntityType,
|
|
28
23
|
InformationPropertyType,
|
|
@@ -78,7 +73,11 @@ class InformationClass(SheetRow):
|
|
|
78
73
|
default=None,
|
|
79
74
|
description="List of classes (comma separated) that the current class implements (parents).",
|
|
80
75
|
)
|
|
81
|
-
|
|
76
|
+
instance_source: URIRefType | None = Field(
|
|
77
|
+
alias="Instance Source",
|
|
78
|
+
default=None,
|
|
79
|
+
description="The link to to the rdf.type that have the instances for this class.",
|
|
80
|
+
)
|
|
82
81
|
physical: URIRefType | None = Field(
|
|
83
82
|
None,
|
|
84
83
|
description="Link to the class representation in the physical data model aspect",
|
|
@@ -153,11 +152,10 @@ class InformationProperty(SheetRow):
|
|
|
153
152
|
"which means that the property can hold any number of values (listable).",
|
|
154
153
|
)
|
|
155
154
|
default: Any | None = Field(alias="Default", default=None, description="Default value of the property.")
|
|
156
|
-
instance_source:
|
|
155
|
+
instance_source: list[URIRefType] | None = Field(
|
|
157
156
|
alias="Instance Source",
|
|
158
157
|
default=None,
|
|
159
|
-
description="The
|
|
160
|
-
"The rule is provided in a RDFPath query syntax which is converted to downstream solution query (e.g. SPARQL).",
|
|
158
|
+
description="The URIRef(s) in the graph to get the value of the property.",
|
|
161
159
|
)
|
|
162
160
|
inherited: bool = Field(
|
|
163
161
|
default=False,
|
|
@@ -182,13 +180,10 @@ class InformationProperty(SheetRow):
|
|
|
182
180
|
return value
|
|
183
181
|
|
|
184
182
|
@field_validator("instance_source", mode="before")
|
|
185
|
-
def
|
|
186
|
-
if
|
|
187
|
-
return value
|
|
188
|
-
|
|
189
|
-
return parse_rule(value, TransformationRuleType.rdfpath)
|
|
190
|
-
else:
|
|
191
|
-
raise NeatValueError(f"Invalid RDF Path: {value!s}")
|
|
183
|
+
def split_on_comma(cls, value: Any) -> Any:
|
|
184
|
+
if isinstance(value, str):
|
|
185
|
+
return [v.strip() for v in value.split(",")]
|
|
186
|
+
return value
|
|
192
187
|
|
|
193
188
|
@model_validator(mode="after")
|
|
194
189
|
def set_type_for_default(self):
|
|
@@ -214,6 +209,12 @@ class InformationProperty(SheetRow):
|
|
|
214
209
|
) from None
|
|
215
210
|
return self
|
|
216
211
|
|
|
212
|
+
@field_serializer("instance_source", when_used="unless-none")
|
|
213
|
+
def serialize_instance_source(self, value: list[URIRefType] | None) -> str | None:
|
|
214
|
+
if value is None:
|
|
215
|
+
return None
|
|
216
|
+
return ",".join(str(v) for v in value)
|
|
217
|
+
|
|
217
218
|
@field_serializer("max_count", when_used="json-unless-none")
|
|
218
219
|
def serialize_max_count(self, value: int | float | None) -> int | float | None | str:
|
|
219
220
|
if isinstance(value, float) and math.isinf(value):
|
|
@@ -84,7 +84,7 @@ class InformationInputProperty(InputComponent[InformationProperty]):
|
|
|
84
84
|
min_count: int | None = None
|
|
85
85
|
max_count: int | float | None = None
|
|
86
86
|
default: Any | None = None
|
|
87
|
-
instance_source: str | None = None
|
|
87
|
+
instance_source: str | list[str] | None = None
|
|
88
88
|
# Only used internally
|
|
89
89
|
inherited: bool = False
|
|
90
90
|
neatId: str | URIRef | None = None
|
|
@@ -110,6 +110,7 @@ class InformationInputClass(InputComponent[InformationClass]):
|
|
|
110
110
|
name: str | None = None
|
|
111
111
|
description: str | None = None
|
|
112
112
|
implements: str | list[ClassEntity] | None = None
|
|
113
|
+
instance_source: str | None = None
|
|
113
114
|
neatId: str | URIRef | None = None
|
|
114
115
|
# linking
|
|
115
116
|
physical: str | URIRef | None = None
|