cognite-neat 0.109.3__py3-none-any.whl → 0.110.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (67) hide show
  1. cognite/neat/_alpha.py +2 -0
  2. cognite/neat/_client/_api/schema.py +17 -1
  3. cognite/neat/_client/data_classes/schema.py +3 -3
  4. cognite/neat/_constants.py +11 -0
  5. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +9 -10
  6. cognite/neat/_graph/extractors/_iodd.py +3 -3
  7. cognite/neat/_graph/extractors/_mock_graph_generator.py +9 -7
  8. cognite/neat/_graph/loaders/_rdf2dms.py +285 -346
  9. cognite/neat/_graph/queries/_base.py +28 -92
  10. cognite/neat/_graph/transformers/__init__.py +1 -3
  11. cognite/neat/_graph/transformers/_rdfpath.py +2 -49
  12. cognite/neat/_issues/__init__.py +1 -6
  13. cognite/neat/_issues/_base.py +21 -252
  14. cognite/neat/_issues/_contextmanagers.py +46 -0
  15. cognite/neat/_issues/_factory.py +61 -0
  16. cognite/neat/_issues/errors/__init__.py +18 -4
  17. cognite/neat/_issues/errors/_wrapper.py +81 -3
  18. cognite/neat/_issues/formatters.py +4 -4
  19. cognite/neat/_issues/warnings/__init__.py +3 -2
  20. cognite/neat/_issues/warnings/_properties.py +8 -0
  21. cognite/neat/_rules/_constants.py +9 -0
  22. cognite/neat/_rules/_shared.py +3 -2
  23. cognite/neat/_rules/analysis/__init__.py +2 -3
  24. cognite/neat/_rules/analysis/_base.py +450 -258
  25. cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
  26. cognite/neat/_rules/exporters/_rules2excel.py +2 -8
  27. cognite/neat/_rules/exporters/_rules2instance_template.py +2 -2
  28. cognite/neat/_rules/exporters/_rules2ontology.py +5 -4
  29. cognite/neat/_rules/importers/_base.py +2 -47
  30. cognite/neat/_rules/importers/_dms2rules.py +7 -10
  31. cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +2 -2
  32. cognite/neat/_rules/importers/_rdf/_inference2rules.py +59 -25
  33. cognite/neat/_rules/importers/_rdf/_shared.py +1 -1
  34. cognite/neat/_rules/importers/_spreadsheet2rules.py +12 -9
  35. cognite/neat/_rules/models/dms/_rules.py +3 -1
  36. cognite/neat/_rules/models/dms/_rules_input.py +4 -0
  37. cognite/neat/_rules/models/dms/_validation.py +14 -4
  38. cognite/neat/_rules/models/entities/_loaders.py +1 -1
  39. cognite/neat/_rules/models/entities/_multi_value.py +2 -2
  40. cognite/neat/_rules/models/information/_rules.py +18 -17
  41. cognite/neat/_rules/models/information/_rules_input.py +2 -1
  42. cognite/neat/_rules/models/information/_validation.py +3 -1
  43. cognite/neat/_rules/transformers/__init__.py +8 -2
  44. cognite/neat/_rules/transformers/_converters.py +242 -43
  45. cognite/neat/_rules/transformers/_verification.py +5 -10
  46. cognite/neat/_session/_base.py +4 -4
  47. cognite/neat/_session/_prepare.py +12 -0
  48. cognite/neat/_session/_read.py +21 -17
  49. cognite/neat/_session/_show.py +11 -123
  50. cognite/neat/_session/_state.py +0 -2
  51. cognite/neat/_session/_subset.py +64 -0
  52. cognite/neat/_session/_to.py +63 -12
  53. cognite/neat/_store/_graph_store.py +5 -246
  54. cognite/neat/_utils/rdf_.py +2 -2
  55. cognite/neat/_utils/spreadsheet.py +44 -1
  56. cognite/neat/_utils/text.py +51 -32
  57. cognite/neat/_version.py +1 -1
  58. {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/METADATA +1 -1
  59. {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/RECORD +62 -64
  60. {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/WHEEL +1 -1
  61. cognite/neat/_graph/queries/_construct.py +0 -187
  62. cognite/neat/_graph/queries/_shared.py +0 -173
  63. cognite/neat/_rules/analysis/_dms.py +0 -57
  64. cognite/neat/_rules/analysis/_information.py +0 -249
  65. cognite/neat/_rules/models/_rdfpath.py +0 -372
  66. {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/LICENSE +0 -0
  67. {cognite_neat-0.109.3.dist-info → cognite_neat-0.110.0.dist-info}/entry_points.txt +0 -0
@@ -13,6 +13,7 @@ from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
13
13
  from openpyxl.worksheet.worksheet import Worksheet
14
14
  from rdflib import Namespace
15
15
 
16
+ from cognite.neat._rules._constants import get_internal_properties
16
17
  from cognite.neat._rules._shared import VerifiedRules
17
18
  from cognite.neat._rules.models import (
18
19
  ExtensionCategory,
@@ -64,13 +65,6 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
64
65
  style_options = get_args(Style)
65
66
  dump_options = get_args(DumpOptions)
66
67
 
67
- _internal_columns: ClassVar[list[str]] = [
68
- "physical",
69
- "logical",
70
- "conceptual",
71
- "Neat ID",
72
- ]
73
-
74
68
  def __init__(
75
69
  self,
76
70
  styling: Style = "default",
@@ -131,7 +125,7 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
131
125
  if sheet.lower() == "metadata":
132
126
  continue
133
127
  ws = workbook[sheet]
134
- for col in self._internal_columns:
128
+ for col in get_internal_properties():
135
129
  column_letter = find_column_with_value(ws, col)
136
130
  if column_letter:
137
131
  ws.column_dimensions[column_letter].hidden = True
@@ -9,7 +9,7 @@ from openpyxl.utils import get_column_letter
9
9
  from openpyxl.worksheet.datavalidation import DataValidation
10
10
 
11
11
  from cognite.neat._rules._constants import EntityTypes
12
- from cognite.neat._rules.analysis import InformationAnalysis
12
+ from cognite.neat._rules.analysis import RulesAnalysis
13
13
  from cognite.neat._rules.models.entities._single_value import ClassEntity
14
14
  from cognite.neat._rules.models.information._rules import InformationRules
15
15
 
@@ -55,7 +55,7 @@ class InstanceTemplateExporter(BaseExporter[InformationRules, Workbook]):
55
55
  # Remove default sheet named "Sheet"
56
56
  workbook.remove(workbook["Sheet"])
57
57
 
58
- for class_, properties in InformationAnalysis(rules).class_property_pairs().items():
58
+ for class_, properties in RulesAnalysis(rules).properties_by_id_by_class().items():
59
59
  workbook.create_sheet(title=class_.suffix)
60
60
 
61
61
  # Add header rows
@@ -15,7 +15,7 @@ from cognite.neat._issues.errors import (
15
15
  )
16
16
  from cognite.neat._issues.warnings import PropertyDefinitionDuplicatedWarning
17
17
  from cognite.neat._rules._constants import EntityTypes
18
- from cognite.neat._rules.analysis import InformationAnalysis
18
+ from cognite.neat._rules.analysis import RulesAnalysis
19
19
  from cognite.neat._rules.models.data_types import DataType
20
20
  from cognite.neat._rules.models.entities import ClassEntity
21
21
  from cognite.neat._rules.models.information import (
@@ -121,11 +121,12 @@ class Ontology(OntologyModel):
121
121
  )
122
122
  raise MultiValueError(errors)
123
123
 
124
- class_dict = InformationAnalysis(rules).as_class_dict()
124
+ analysis = RulesAnalysis(rules)
125
+ class_dict = analysis.class_by_suffix()
125
126
  return cls(
126
127
  properties=[
127
128
  OWLProperty.from_list_of_properties(definition, rules.metadata.namespace)
128
- for definition in InformationAnalysis(rules).as_property_dict().values()
129
+ for definition in analysis.property_by_id().values()
129
130
  ],
130
131
  classes=[
131
132
  OWLClass.from_class(definition, rules.metadata.namespace, rules.prefixes)
@@ -137,7 +138,7 @@ class Ontology(OntologyModel):
137
138
  list(properties.values()),
138
139
  rules.metadata.namespace,
139
140
  )
140
- for class_, properties in InformationAnalysis(rules).class_property_pairs().items()
141
+ for class_, properties in analysis.properties_by_id_by_class().items()
141
142
  ]
142
143
  + [
143
144
  SHACLNodeShape.from_rules(
@@ -1,15 +1,11 @@
1
- import warnings
2
1
  from abc import ABC, abstractmethod
3
- from collections.abc import Iterator
4
- from contextlib import contextmanager, suppress
2
+ from contextlib import suppress
5
3
  from datetime import datetime
6
- from typing import TYPE_CHECKING, Any, Generic, Literal
4
+ from typing import TYPE_CHECKING, Any, Generic
7
5
 
8
- from pydantic import ValidationError
9
6
  from rdflib import URIRef
10
7
 
11
8
  from cognite.neat._constants import DEFAULT_NAMESPACE
12
- from cognite.neat._issues import IssueList, NeatError, NeatWarning
13
9
  from cognite.neat._rules._shared import ReadRules, T_InputRules
14
10
  from cognite.neat._utils.auxiliary import class_html_doc
15
11
 
@@ -64,44 +60,3 @@ class BaseImporter(ABC, Generic[T_InputRules]):
64
60
  @property
65
61
  def source_uri(self) -> URIRef:
66
62
  return DEFAULT_NAMESPACE["UNKNOWN"]
67
-
68
-
69
- class _FutureResult:
70
- def __init__(self) -> None:
71
- self._result: Literal["success", "failure", "pending"] = "pending"
72
-
73
- @property
74
- def result(self) -> Literal["success", "failure", "pending"]:
75
- return self._result
76
-
77
-
78
- @contextmanager
79
- def _handle_issues(
80
- issues: IssueList,
81
- error_cls: type[NeatError] = NeatError,
82
- warning_cls: type[NeatWarning] = NeatWarning,
83
- error_args: dict[str, Any] | None = None,
84
- ) -> Iterator[_FutureResult]:
85
- """This is an internal help function to handle issues and warnings.
86
-
87
- Args:
88
- issues: The issues list to append to.
89
- error_cls: The class used to convert errors to issues.
90
- warning_cls: The class used to convert warnings to issues.
91
-
92
- Returns:
93
- FutureResult: A future result object that can be used to check the result of the context manager.
94
- """
95
- with warnings.catch_warnings(record=True) as warning_logger:
96
- warnings.simplefilter("always")
97
- future_result = _FutureResult()
98
- try:
99
- yield future_result
100
- except ValidationError as e:
101
- issues.extend(error_cls.from_errors(e.errors(), **(error_args or {}))) # type: ignore[arg-type]
102
- future_result._result = "failure"
103
- else:
104
- future_result._result = "success"
105
- finally:
106
- if warning_logger:
107
- issues.extend([warning_cls.from_warning(warning) for warning in warning_logger]) # type: ignore[misc]
@@ -19,7 +19,7 @@ from cognite.client.data_classes.data_modeling.views import (
19
19
  from cognite.client.utils import ms_to_datetime
20
20
 
21
21
  from cognite.neat._client import NeatClient
22
- from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
22
+ from cognite.neat._issues import IssueList, MultiValueError, NeatIssue, catch_issues
23
23
  from cognite.neat._issues.errors import (
24
24
  FileTypeUnexpectedError,
25
25
  NeatValueError,
@@ -35,7 +35,7 @@ from cognite.neat._issues.warnings import (
35
35
  ResourceUnknownWarning,
36
36
  )
37
37
  from cognite.neat._rules._shared import ReadRules
38
- from cognite.neat._rules.importers._base import BaseImporter, _handle_issues
38
+ from cognite.neat._rules.importers._base import BaseImporter
39
39
  from cognite.neat._rules.models import (
40
40
  DMSInputRules,
41
41
  DMSSchema,
@@ -131,11 +131,10 @@ class DMSImporter(BaseImporter[DMSInputRules]):
131
131
 
132
132
  @classmethod
133
133
  def from_data_model(cls, client: NeatClient, model: dm.DataModel[dm.View]) -> "DMSImporter":
134
- issue_list = IssueList()
135
- with _handle_issues(issue_list) as result:
134
+ with catch_issues() as issue_list:
136
135
  schema = client.schema.retrieve_data_model(model)
137
136
 
138
- if result.result == "failure" or issue_list.has_errors:
137
+ if issue_list.has_errors:
139
138
  return cls(DMSSchema(), issue_list)
140
139
 
141
140
  metadata = cls._create_metadata_from_model(model)
@@ -187,10 +186,9 @@ class DMSImporter(BaseImporter[DMSInputRules]):
187
186
 
188
187
  @classmethod
189
188
  def from_directory(cls, directory: str | Path, client: NeatClient | None = None) -> "DMSImporter":
190
- issue_list = IssueList()
191
- with _handle_issues(issue_list) as _:
189
+ with catch_issues() as issue_list:
192
190
  schema = DMSSchema.from_directory(directory)
193
- # If there were errors during the import, the to_rules
191
+ # If there were errors during the import, the to_rules will raise them.
194
192
  return cls(
195
193
  schema, issue_list, referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client)
196
194
  )
@@ -202,8 +200,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
202
200
  DMSSchema(),
203
201
  [FileTypeUnexpectedError(Path(zip_file), frozenset([".zip"]))],
204
202
  )
205
- issue_list = IssueList()
206
- with _handle_issues(issue_list) as _:
203
+ with catch_issues() as issue_list:
207
204
  schema = DMSSchema.from_zip(zip_file)
208
205
  return cls(
209
206
  schema, issue_list, referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client)
@@ -19,7 +19,7 @@ from cognite.neat._rules.importers._dtdl2rules.dtdl_converter import _DTDLConver
19
19
  from cognite.neat._rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
20
20
  from cognite.neat._rules.models import InformationInputRules
21
21
  from cognite.neat._rules.models.information import InformationInputMetadata
22
- from cognite.neat._utils.text import humanize_collection, to_pascal
22
+ from cognite.neat._utils.text import humanize_collection, to_pascal_case
23
23
 
24
24
 
25
25
  class DTDLImporter(BaseImporter[InformationInputRules]):
@@ -130,7 +130,7 @@ class DTDLImporter(BaseImporter[InformationInputRules]):
130
130
  metadata = self._default_metadata()
131
131
 
132
132
  if self.name:
133
- metadata["name"] = to_pascal(self.name)
133
+ metadata["name"] = to_pascal_case(self.name)
134
134
  try:
135
135
  most_common_prefix = converter.get_most_common_prefix()
136
136
  except ValueError:
@@ -13,11 +13,12 @@ from rdflib import Literal as RdfLiteral
13
13
  from cognite.neat._constants import NEAT, get_default_prefixes_and_namespaces
14
14
  from cognite.neat._issues import IssueList
15
15
  from cognite.neat._issues.warnings import PropertyValueTypeUndefinedWarning
16
- from cognite.neat._rules.analysis import InformationAnalysis
16
+ from cognite.neat._rules.analysis import RulesAnalysis
17
17
  from cognite.neat._rules.models import InformationRules, data_types
18
18
  from cognite.neat._rules.models.data_types import AnyURI
19
19
  from cognite.neat._rules.models.entities._single_value import UnknownEntity
20
20
  from cognite.neat._rules.models.information import (
21
+ InformationClass,
21
22
  InformationInputClass,
22
23
  InformationInputProperty,
23
24
  InformationMetadata,
@@ -256,7 +257,7 @@ class InferenceImporter(BaseRDFImporter):
256
257
  property_["value_type"].remove(str(self.non_existing_node_type))
257
258
 
258
259
  if len(property_["value_type"]) > 1:
259
- property_["value_type"] = " | ".join([str(t) for t in property_["value_type"]])
260
+ property_["value_type"] = ", ".join([str(t) for t in property_["value_type"]])
260
261
  else:
261
262
  property_["value_type"] = next(iter(property_["value_type"]))
262
263
 
@@ -402,7 +403,8 @@ class SubclassInferenceImporter(BaseRDFImporter):
402
403
  else:
403
404
  existing_classes = {}
404
405
  classes: list[InformationInputClass] = []
405
- properties: list[InformationInputProperty] = []
406
+ properties_by_class_suffix_by_property_id_lowered: dict[str, dict[str, InformationInputProperty]] = {}
407
+
406
408
  # Help for IDE
407
409
  type_uri: URIRef
408
410
  parent_uri: URIRef
@@ -440,26 +442,51 @@ class SubclassInferenceImporter(BaseRDFImporter):
440
442
  InformationInputClass(
441
443
  class_=class_suffix,
442
444
  implements=parent_suffix,
445
+ instance_source=type_uri,
443
446
  )
444
447
  )
445
448
  else:
446
449
  classes.append(InformationInputClass.load(existing_classes[class_suffix].model_dump()))
450
+
451
+ properties_by_id: dict[str, InformationInputProperty] = {}
447
452
  for property_uri, read_properties in properties_by_property_uri.items():
448
453
  if property_uri in shared_property_uris:
449
454
  shared_properties[property_uri].extend(read_properties)
450
455
  continue
451
- properties.append(
452
- self._create_property(read_properties, class_suffix, type_uri, property_uri, prefixes)
453
- )
454
-
456
+ property_id = remove_namespace_from_uri(property_uri)
457
+ self._add_uri_namespace_to_prefixes(property_uri, prefixes)
458
+ if existing_prop := properties_by_id.get(property_id.casefold()):
459
+ if not isinstance(existing_prop.instance_source, list):
460
+ existing_prop.instance_source = (
461
+ [existing_prop.instance_source] if existing_prop.instance_source else []
462
+ )
463
+ existing_prop.instance_source.append(property_uri)
464
+ continue
465
+ else:
466
+ properties_by_id[property_id.casefold()] = self._create_property(
467
+ read_properties, class_suffix, property_uri, property_id, prefixes
468
+ )
469
+ properties_by_class_suffix_by_property_id_lowered[class_suffix] = properties_by_id
455
470
  if parent_suffix:
471
+ properties_by_id = {}
456
472
  for property_uri, read_properties in shared_properties.items():
457
- properties.append(
458
- self._create_property(
459
- read_properties, parent_suffix, read_properties[0].type_uri, property_uri, prefixes
473
+ property_id = remove_namespace_from_uri(property_uri)
474
+ self._add_uri_namespace_to_prefixes(property_uri, prefixes)
475
+ if existing_prop := properties_by_id.get(property_id.casefold()):
476
+ if not isinstance(existing_prop.instance_source, list):
477
+ existing_prop.instance_source = (
478
+ [existing_prop.instance_source] if existing_prop.instance_source else []
479
+ )
480
+ existing_prop.instance_source.append(property_uri)
481
+ else:
482
+ properties_by_id[property_uri.casefold()] = self._create_property(
483
+ read_properties, parent_suffix, property_uri, property_id, prefixes
460
484
  )
461
- )
462
- return classes, properties
485
+ return classes, [
486
+ prop
487
+ for properties in properties_by_class_suffix_by_property_id_lowered.values()
488
+ for prop in properties.values()
489
+ ]
463
490
 
464
491
  @staticmethod
465
492
  def _get_properties_by_class_by_property(
@@ -481,26 +508,36 @@ class SubclassInferenceImporter(BaseRDFImporter):
481
508
  type_uri, instance_count_literal = cast(tuple[URIRef, RdfLiteral], result_row)
482
509
  count_by_type[type_uri] = instance_count_literal.toPython()
483
510
  if self._rules:
484
- analysis = InformationAnalysis(self._rules)
511
+ analysis = RulesAnalysis(self._rules)
485
512
  existing_class_properties = {
486
- (class_entity.suffix, prop.property_)
487
- for class_entity, properties in analysis.classes_with_properties(
488
- consider_inheritance=True, allow_different_namespace=True
513
+ (class_entity.suffix, prop.property_): prop
514
+ for class_entity, properties in analysis.properties_by_class(
515
+ include_ancestors=True, include_different_space=True
489
516
  ).items()
490
517
  for prop in properties
491
518
  }
519
+ existing_classes = {cls_.class_.suffix: cls_ for cls_ in self._rules.classes}
492
520
  else:
493
- existing_class_properties = set()
521
+ existing_class_properties = {}
522
+ existing_classes = {}
494
523
  properties_by_class_by_subclass: list[_ReadProperties] = []
524
+ existing_class: InformationClass | None
495
525
  for type_uri, instance_count in count_by_type.items():
496
526
  property_query = self._properties_query.format(type=type_uri, unknown_type=NEAT.UnknownType)
497
527
  class_suffix = remove_namespace_from_uri(type_uri)
528
+ if (existing_class := existing_classes.get(class_suffix)) and existing_class.instance_source is None:
529
+ existing_class.instance_source = type_uri
530
+
498
531
  for result_row in self.graph.query(property_query):
499
532
  property_uri, value_type_uri = cast(tuple[URIRef, URIRef], result_row)
500
533
  if property_uri == RDF.type:
501
534
  continue
502
535
  property_str = remove_namespace_from_uri(property_uri)
503
- if (class_suffix, property_str) in existing_class_properties:
536
+ if existing_property := existing_class_properties.get((class_suffix, property_str)):
537
+ if existing_property.instance_source is None:
538
+ existing_property.instance_source = [property_uri]
539
+ elif existing_property.instance_source and property_uri not in existing_property.instance_source:
540
+ existing_property.instance_source.append(property_uri)
504
541
  continue
505
542
  occurrence_query = self._max_occurrence_query.format(type=type_uri, property=property_uri)
506
543
  max_occurrence = 1 # default value
@@ -531,21 +568,18 @@ class SubclassInferenceImporter(BaseRDFImporter):
531
568
  self,
532
569
  read_properties: list[_ReadProperties],
533
570
  class_suffix: str,
534
- type_uri: URIRef,
535
571
  property_uri: URIRef,
572
+ property_id: str,
536
573
  prefixes: dict[str, Namespace],
537
574
  ) -> InformationInputProperty:
538
575
  first = read_properties[0]
539
576
  value_type = self._get_value_type(read_properties, prefixes)
540
- property_name = remove_namespace_from_uri(property_uri)
541
- self._add_uri_namespace_to_prefixes(property_uri, prefixes)
542
-
543
577
  return InformationInputProperty(
544
578
  class_=class_suffix,
545
- property_=property_name,
579
+ property_=property_id,
546
580
  max_count=first.max_occurrence,
547
581
  value_type=value_type,
548
- instance_source=(f"{uri_to_short_form(type_uri, prefixes)}({uri_to_short_form(property_uri, prefixes)})"),
582
+ instance_source=[property_uri],
549
583
  )
550
584
 
551
585
  def _get_value_type(
@@ -562,7 +596,7 @@ class SubclassInferenceImporter(BaseRDFImporter):
562
596
  return UnknownEntity()
563
597
  for uri_ref in value_types:
564
598
  self._add_uri_namespace_to_prefixes(uri_ref, prefixes)
565
- return " | ".join(remove_namespace_from_uri(uri_ref) for uri_ref in value_types)
599
+ return ", ".join(remove_namespace_from_uri(uri_ref) for uri_ref in value_types)
566
600
 
567
601
  def _default_metadata(self) -> dict[str, Any]:
568
602
  now = datetime.now(timezone.utc)
@@ -136,7 +136,7 @@ def parse_properties(graph: Graph, query: str, language: str, issue_list: IssueL
136
136
  properties[id_]["value_type"].append(res["value_type"])
137
137
 
138
138
  for prop in properties.values():
139
- prop["value_type"] = "|".join(prop["value_type"])
139
+ prop["value_type"] = ", ".join(prop["value_type"])
140
140
 
141
141
  if not properties:
142
142
  issue_list.append(NeatValueError("Unable to parse properties"))
@@ -37,15 +37,15 @@ SOURCE_SHEET__TARGET_FIELD__HEADERS = [
37
37
  "Properties",
38
38
  "Properties",
39
39
  {
40
- RoleTypes.information: "Property",
41
- RoleTypes.dms: "View Property",
40
+ RoleTypes.information: ["Class", "Property"],
41
+ RoleTypes.dms: ["View", "View Property"],
42
42
  },
43
43
  ),
44
- ("Classes", "Classes", "Class"),
45
- ("Containers", "Containers", "Container"),
46
- ("Views", "Views", "View"),
47
- ("Enum", "Enum", "Collection"),
48
- ("Nodes", "Nodes", "Node"),
44
+ ("Classes", "Classes", ["Class"]),
45
+ ("Containers", "Containers", ["Container"]),
46
+ ("Views", "Views", ["View"]),
47
+ ("Enum", "Enum", ["Collection"]),
48
+ ("Nodes", "Nodes", ["Node"]),
49
49
  ]
50
50
 
51
51
 
@@ -231,7 +231,10 @@ class SpreadsheetReader:
231
231
 
232
232
  try:
233
233
  sheets[target_sheet_name], read_info_by_sheet[source_sheet_name] = read_individual_sheet(
234
- excel_file, source_sheet_name, return_read_info=True, expected_headers=[headers]
234
+ excel_file,
235
+ source_sheet_name,
236
+ return_read_info=True,
237
+ expected_headers=headers,
235
238
  )
236
239
  except Exception as e:
237
240
  self.issue_list.append(FileReadError(cast(Path, excel_file.io), str(e)))
@@ -273,7 +276,7 @@ class ExcelImporter(BaseImporter[T_InputRules]):
273
276
 
274
277
  rules_cls = INPUT_RULES_BY_ROLE[original_role]
275
278
  rules = cast(T_InputRules, rules_cls.load(sheets))
276
- return ReadRules(rules, {"read_info_by_sheet": read_info_by_sheet})
279
+ return ReadRules(rules, read_info_by_sheet)
277
280
 
278
281
  @property
279
282
  def description(self) -> str:
@@ -295,7 +295,9 @@ class DMSView(SheetRow):
295
295
  None, alias="Filter", description="Explicitly define the filter for the view."
296
296
  )
297
297
  in_model: bool = Field(
298
- True, alias="In Model", description="Indicates whether the view being defined is a part of the data model."
298
+ True,
299
+ alias="In Model",
300
+ description="Indicates whether the view being defined is a part of the data model.",
299
301
  )
300
302
  logical: URIRefType | None = Field(
301
303
  None,
@@ -193,6 +193,10 @@ class DMSInputView(InputComponent[DMSView]):
193
193
  neatId: str | URIRef | None = None
194
194
  logical: str | URIRef | None = None
195
195
 
196
+ def __post_init__(self):
197
+ if self.in_model is None:
198
+ self.in_model = True
199
+
196
200
  @classmethod
197
201
  def _get_verified_cls(cls) -> type[DMSView]:
198
202
  return DMSView
@@ -17,7 +17,7 @@ from cognite.neat._client import NeatClient
17
17
  from cognite.neat._client.data_classes.data_modeling import ViewApplyDict
18
18
  from cognite.neat._client.data_classes.schema import DMSSchema
19
19
  from cognite.neat._constants import COGNITE_MODELS, DMS_CONTAINER_PROPERTY_SIZE_LIMIT, DMS_VIEW_CONTAINER_SIZE_LIMIT
20
- from cognite.neat._issues import IssueList, NeatError, NeatIssueList
20
+ from cognite.neat._issues import IssueList, NeatError
21
21
  from cognite.neat._issues.errors import (
22
22
  CDFMissingClientError,
23
23
  PropertyDefinitionDuplicatedError,
@@ -42,6 +42,7 @@ from cognite.neat._rules.models.entities import ContainerEntity, RawFilter
42
42
  from cognite.neat._rules.models.entities._single_value import (
43
43
  ViewEntity,
44
44
  )
45
+ from cognite.neat._utils.spreadsheet import SpreadsheetRead
45
46
 
46
47
  from ._rules import DMSProperty, DMSRules
47
48
 
@@ -54,13 +55,19 @@ class DMSValidation:
54
55
  # For example, changing the filter is allowed, but changing the properties is not.
55
56
  changeable_view_attributes: ClassVar[set[str]] = {"filter"}
56
57
 
57
- def __init__(self, rules: DMSRules, client: NeatClient | None = None) -> None:
58
+ def __init__(
59
+ self,
60
+ rules: DMSRules,
61
+ client: NeatClient | None = None,
62
+ read_info_by_spreadsheet: dict[str, SpreadsheetRead] | None = None,
63
+ ) -> None:
58
64
  self._rules = rules
59
65
  self._client = client
60
66
  self._metadata = rules.metadata
61
67
  self._properties = rules.properties
62
68
  self._containers = rules.containers
63
69
  self._views = rules.views
70
+ self._read_info_by_spreadsheet = read_info_by_spreadsheet or {}
64
71
 
65
72
  def imported_views_and_containers_ids(
66
73
  self, include_views_with_no_properties: bool = True
@@ -87,7 +94,7 @@ class DMSValidation:
87
94
 
88
95
  return imported_views, imported_containers
89
96
 
90
- def validate(self) -> NeatIssueList:
97
+ def validate(self) -> IssueList:
91
98
  imported_views, imported_containers = self.imported_views_and_containers_ids(
92
99
  include_views_with_no_properties=False
93
100
  )
@@ -212,13 +219,16 @@ class DMSValidation:
212
219
  for prop_no, prop in enumerate(self._properties):
213
220
  if prop.container and prop.container_property:
214
221
  container_properties_by_id[(prop.container, prop.container_property)].append((prop_no, prop))
215
-
222
+ properties_sheet = self._read_info_by_spreadsheet.get("Properties")
216
223
  errors = IssueList()
217
224
  for (container, prop_name), properties in container_properties_by_id.items():
218
225
  if len(properties) == 1:
219
226
  continue
220
227
  container_id = container.as_id()
228
+
221
229
  row_numbers = {prop_no for prop_no, _ in properties}
230
+ if properties_sheet:
231
+ row_numbers = {properties_sheet.adjusted_row_number(row_no) for row_no in row_numbers}
222
232
  value_types = {prop.value_type for _, prop in properties if prop.value_type}
223
233
  # The container type 'direct' is an exception. On a container the type direct can point to any
224
234
  # node. The value type is typically set on the view.
@@ -23,7 +23,7 @@ def load_value_type(
23
23
  elif isinstance(raw, str):
24
24
  # property holding xsd data type
25
25
  # check if it is multi value type
26
- if "|" in raw:
26
+ if "," in raw:
27
27
  value_type = MultiValueTypeInfo.load(raw)
28
28
  value_type.set_default_prefix(default_prefix)
29
29
  return value_type
@@ -18,7 +18,7 @@ class MultiValueTypeInfo(BaseModel):
18
18
  types: list[DataType | ClassEntity]
19
19
 
20
20
  def __str__(self) -> str:
21
- return " | ".join([str(t) for t in self.types])
21
+ return ", ".join([str(t) for t in self.types])
22
22
 
23
23
  @model_serializer(when_used="unless-none", return_type=str)
24
24
  def as_str(self) -> str:
@@ -52,7 +52,7 @@ class MultiValueTypeInfo(BaseModel):
52
52
 
53
53
  @classmethod
54
54
  def _parse(cls, raw: str) -> dict:
55
- if not (types := [type_.strip() for type_ in raw.split("|")]):
55
+ if not (types := [type_.strip() for type_ in raw.split(",")]):
56
56
  return {"types": [UnknownEntity()]}
57
57
  else:
58
58
  return {
@@ -8,7 +8,7 @@ from pydantic_core.core_schema import SerializationInfo
8
8
  from rdflib import Namespace, URIRef
9
9
 
10
10
  from cognite.neat._constants import get_default_prefixes_and_namespaces
11
- from cognite.neat._issues.errors import NeatValueError, PropertyDefinitionError
11
+ from cognite.neat._issues.errors import PropertyDefinitionError
12
12
  from cognite.neat._rules._constants import EntityTypes
13
13
  from cognite.neat._rules.models._base_rules import (
14
14
  BaseMetadata,
@@ -18,11 +18,6 @@ from cognite.neat._rules.models._base_rules import (
18
18
  SheetList,
19
19
  SheetRow,
20
20
  )
21
- from cognite.neat._rules.models._rdfpath import (
22
- RDFPath,
23
- TransformationRuleType,
24
- parse_rule,
25
- )
26
21
  from cognite.neat._rules.models._types import (
27
22
  ClassEntityType,
28
23
  InformationPropertyType,
@@ -78,7 +73,11 @@ class InformationClass(SheetRow):
78
73
  default=None,
79
74
  description="List of classes (comma separated) that the current class implements (parents).",
80
75
  )
81
-
76
+ instance_source: URIRefType | None = Field(
77
+ alias="Instance Source",
78
+ default=None,
79
+ description="The link to to the rdf.type that have the instances for this class.",
80
+ )
82
81
  physical: URIRefType | None = Field(
83
82
  None,
84
83
  description="Link to the class representation in the physical data model aspect",
@@ -153,11 +152,10 @@ class InformationProperty(SheetRow):
153
152
  "which means that the property can hold any number of values (listable).",
154
153
  )
155
154
  default: Any | None = Field(alias="Default", default=None, description="Default value of the property.")
156
- instance_source: RDFPath | None = Field(
155
+ instance_source: list[URIRefType] | None = Field(
157
156
  alias="Instance Source",
158
157
  default=None,
159
- description="The link to to the instance property for the model. "
160
- "The rule is provided in a RDFPath query syntax which is converted to downstream solution query (e.g. SPARQL).",
158
+ description="The URIRef(s) in the graph to get the value of the property.",
161
159
  )
162
160
  inherited: bool = Field(
163
161
  default=False,
@@ -182,13 +180,10 @@ class InformationProperty(SheetRow):
182
180
  return value
183
181
 
184
182
  @field_validator("instance_source", mode="before")
185
- def generate_rdfpath(cls, value: str | RDFPath | None) -> RDFPath | None:
186
- if value is None or isinstance(value, RDFPath):
187
- return value
188
- elif isinstance(value, str):
189
- return parse_rule(value, TransformationRuleType.rdfpath)
190
- else:
191
- raise NeatValueError(f"Invalid RDF Path: {value!s}")
183
+ def split_on_comma(cls, value: Any) -> Any:
184
+ if isinstance(value, str):
185
+ return [v.strip() for v in value.split(",")]
186
+ return value
192
187
 
193
188
  @model_validator(mode="after")
194
189
  def set_type_for_default(self):
@@ -214,6 +209,12 @@ class InformationProperty(SheetRow):
214
209
  ) from None
215
210
  return self
216
211
 
212
+ @field_serializer("instance_source", when_used="unless-none")
213
+ def serialize_instance_source(self, value: list[URIRefType] | None) -> str | None:
214
+ if value is None:
215
+ return None
216
+ return ",".join(str(v) for v in value)
217
+
217
218
  @field_serializer("max_count", when_used="json-unless-none")
218
219
  def serialize_max_count(self, value: int | float | None) -> int | float | None | str:
219
220
  if isinstance(value, float) and math.isinf(value):
@@ -84,7 +84,7 @@ class InformationInputProperty(InputComponent[InformationProperty]):
84
84
  min_count: int | None = None
85
85
  max_count: int | float | None = None
86
86
  default: Any | None = None
87
- instance_source: str | None = None
87
+ instance_source: str | list[str] | None = None
88
88
  # Only used internally
89
89
  inherited: bool = False
90
90
  neatId: str | URIRef | None = None
@@ -110,6 +110,7 @@ class InformationInputClass(InputComponent[InformationClass]):
110
110
  name: str | None = None
111
111
  description: str | None = None
112
112
  implements: str | list[ClassEntity] | None = None
113
+ instance_source: str | None = None
113
114
  neatId: str | URIRef | None = None
114
115
  # linking
115
116
  physical: str | URIRef | None = None