cognite-neat 0.109.4__py3-none-any.whl → 0.111.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (88) hide show
  1. cognite/neat/_alpha.py +8 -0
  2. cognite/neat/_client/_api/schema.py +43 -1
  3. cognite/neat/_client/data_classes/schema.py +4 -4
  4. cognite/neat/_constants.py +15 -1
  5. cognite/neat/_graph/extractors/__init__.py +4 -0
  6. cognite/neat/_graph/extractors/_classic_cdf/_base.py +8 -16
  7. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +48 -19
  8. cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +23 -17
  9. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +15 -17
  10. cognite/neat/_graph/extractors/_dict.py +102 -0
  11. cognite/neat/_graph/extractors/_dms.py +27 -40
  12. cognite/neat/_graph/extractors/_dms_graph.py +30 -3
  13. cognite/neat/_graph/extractors/_iodd.py +3 -3
  14. cognite/neat/_graph/extractors/_mock_graph_generator.py +9 -7
  15. cognite/neat/_graph/extractors/_raw.py +67 -0
  16. cognite/neat/_graph/loaders/_base.py +20 -4
  17. cognite/neat/_graph/loaders/_rdf2dms.py +476 -383
  18. cognite/neat/_graph/queries/_base.py +163 -133
  19. cognite/neat/_graph/transformers/__init__.py +1 -3
  20. cognite/neat/_graph/transformers/_classic_cdf.py +6 -22
  21. cognite/neat/_graph/transformers/_rdfpath.py +2 -49
  22. cognite/neat/_issues/__init__.py +1 -6
  23. cognite/neat/_issues/_base.py +21 -252
  24. cognite/neat/_issues/_contextmanagers.py +46 -0
  25. cognite/neat/_issues/_factory.py +69 -0
  26. cognite/neat/_issues/errors/__init__.py +20 -4
  27. cognite/neat/_issues/errors/_external.py +7 -0
  28. cognite/neat/_issues/errors/_wrapper.py +81 -3
  29. cognite/neat/_issues/formatters.py +4 -4
  30. cognite/neat/_issues/warnings/__init__.py +3 -2
  31. cognite/neat/_issues/warnings/_properties.py +8 -0
  32. cognite/neat/_issues/warnings/user_modeling.py +12 -0
  33. cognite/neat/_rules/_constants.py +12 -0
  34. cognite/neat/_rules/_shared.py +3 -2
  35. cognite/neat/_rules/analysis/__init__.py +2 -3
  36. cognite/neat/_rules/analysis/_base.py +430 -259
  37. cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
  38. cognite/neat/_rules/exporters/_rules2excel.py +3 -9
  39. cognite/neat/_rules/exporters/_rules2instance_template.py +2 -2
  40. cognite/neat/_rules/exporters/_rules2ontology.py +5 -4
  41. cognite/neat/_rules/importers/_base.py +2 -47
  42. cognite/neat/_rules/importers/_dms2rules.py +7 -10
  43. cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +2 -2
  44. cognite/neat/_rules/importers/_rdf/_inference2rules.py +66 -26
  45. cognite/neat/_rules/importers/_rdf/_shared.py +1 -1
  46. cognite/neat/_rules/importers/_spreadsheet2rules.py +12 -9
  47. cognite/neat/_rules/models/_base_rules.py +0 -2
  48. cognite/neat/_rules/models/data_types.py +7 -0
  49. cognite/neat/_rules/models/dms/_exporter.py +9 -8
  50. cognite/neat/_rules/models/dms/_rules.py +29 -2
  51. cognite/neat/_rules/models/dms/_rules_input.py +9 -1
  52. cognite/neat/_rules/models/dms/_validation.py +115 -5
  53. cognite/neat/_rules/models/entities/_loaders.py +1 -1
  54. cognite/neat/_rules/models/entities/_multi_value.py +2 -2
  55. cognite/neat/_rules/models/entities/_single_value.py +8 -3
  56. cognite/neat/_rules/models/entities/_wrapped.py +2 -2
  57. cognite/neat/_rules/models/information/_rules.py +18 -17
  58. cognite/neat/_rules/models/information/_rules_input.py +3 -1
  59. cognite/neat/_rules/models/information/_validation.py +66 -17
  60. cognite/neat/_rules/transformers/__init__.py +8 -2
  61. cognite/neat/_rules/transformers/_converters.py +234 -44
  62. cognite/neat/_rules/transformers/_verification.py +5 -10
  63. cognite/neat/_session/_base.py +6 -4
  64. cognite/neat/_session/_explore.py +39 -0
  65. cognite/neat/_session/_inspect.py +25 -6
  66. cognite/neat/_session/_prepare.py +12 -0
  67. cognite/neat/_session/_read.py +88 -20
  68. cognite/neat/_session/_set.py +7 -1
  69. cognite/neat/_session/_show.py +11 -123
  70. cognite/neat/_session/_state.py +6 -2
  71. cognite/neat/_session/_subset.py +64 -0
  72. cognite/neat/_session/_to.py +177 -19
  73. cognite/neat/_store/_graph_store.py +9 -246
  74. cognite/neat/_utils/rdf_.py +36 -5
  75. cognite/neat/_utils/spreadsheet.py +44 -1
  76. cognite/neat/_utils/text.py +124 -37
  77. cognite/neat/_utils/upload.py +2 -0
  78. cognite/neat/_version.py +2 -2
  79. {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/METADATA +1 -1
  80. {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/RECORD +83 -82
  81. {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/WHEEL +1 -1
  82. cognite/neat/_graph/queries/_construct.py +0 -187
  83. cognite/neat/_graph/queries/_shared.py +0 -173
  84. cognite/neat/_rules/analysis/_dms.py +0 -57
  85. cognite/neat/_rules/analysis/_information.py +0 -249
  86. cognite/neat/_rules/models/_rdfpath.py +0 -372
  87. {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/LICENSE +0 -0
  88. {cognite_neat-0.109.4.dist-info → cognite_neat-0.111.0.dist-info}/entry_points.txt +0 -0
@@ -13,6 +13,7 @@ from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
13
13
  from openpyxl.worksheet.worksheet import Worksheet
14
14
  from rdflib import Namespace
15
15
 
16
+ from cognite.neat._rules._constants import get_internal_properties
16
17
  from cognite.neat._rules._shared import VerifiedRules
17
18
  from cognite.neat._rules.models import (
18
19
  ExtensionCategory,
@@ -54,7 +55,7 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
54
55
  Style = Literal["none", "minimal", "default", "maximal"]
55
56
  DumpOptions = Literal["user", "last", "reference"]
56
57
  _main_header_by_sheet_name: ClassVar[dict[str, str]] = {
57
- "Properties": "Definition of Properties per Class",
58
+ "Properties": "Definition of Properties",
58
59
  "Classes": "Definition of Classes",
59
60
  "Views": "Definition of Views",
60
61
  "Containers": "Definition of Containers",
@@ -64,13 +65,6 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
64
65
  style_options = get_args(Style)
65
66
  dump_options = get_args(DumpOptions)
66
67
 
67
- _internal_columns: ClassVar[list[str]] = [
68
- "physical",
69
- "logical",
70
- "conceptual",
71
- "Neat ID",
72
- ]
73
-
74
68
  def __init__(
75
69
  self,
76
70
  styling: Style = "default",
@@ -131,7 +125,7 @@ class ExcelExporter(BaseExporter[VerifiedRules, Workbook]):
131
125
  if sheet.lower() == "metadata":
132
126
  continue
133
127
  ws = workbook[sheet]
134
- for col in self._internal_columns:
128
+ for col in get_internal_properties():
135
129
  column_letter = find_column_with_value(ws, col)
136
130
  if column_letter:
137
131
  ws.column_dimensions[column_letter].hidden = True
@@ -9,7 +9,7 @@ from openpyxl.utils import get_column_letter
9
9
  from openpyxl.worksheet.datavalidation import DataValidation
10
10
 
11
11
  from cognite.neat._rules._constants import EntityTypes
12
- from cognite.neat._rules.analysis import InformationAnalysis
12
+ from cognite.neat._rules.analysis import RulesAnalysis
13
13
  from cognite.neat._rules.models.entities._single_value import ClassEntity
14
14
  from cognite.neat._rules.models.information._rules import InformationRules
15
15
 
@@ -55,7 +55,7 @@ class InstanceTemplateExporter(BaseExporter[InformationRules, Workbook]):
55
55
  # Remove default sheet named "Sheet"
56
56
  workbook.remove(workbook["Sheet"])
57
57
 
58
- for class_, properties in InformationAnalysis(rules).class_property_pairs().items():
58
+ for class_, properties in RulesAnalysis(rules).properties_by_id_by_class().items():
59
59
  workbook.create_sheet(title=class_.suffix)
60
60
 
61
61
  # Add header rows
@@ -15,7 +15,7 @@ from cognite.neat._issues.errors import (
15
15
  )
16
16
  from cognite.neat._issues.warnings import PropertyDefinitionDuplicatedWarning
17
17
  from cognite.neat._rules._constants import EntityTypes
18
- from cognite.neat._rules.analysis import InformationAnalysis
18
+ from cognite.neat._rules.analysis import RulesAnalysis
19
19
  from cognite.neat._rules.models.data_types import DataType
20
20
  from cognite.neat._rules.models.entities import ClassEntity
21
21
  from cognite.neat._rules.models.information import (
@@ -121,11 +121,12 @@ class Ontology(OntologyModel):
121
121
  )
122
122
  raise MultiValueError(errors)
123
123
 
124
- class_dict = InformationAnalysis(rules).as_class_dict()
124
+ analysis = RulesAnalysis(rules)
125
+ class_dict = analysis.class_by_suffix()
125
126
  return cls(
126
127
  properties=[
127
128
  OWLProperty.from_list_of_properties(definition, rules.metadata.namespace)
128
- for definition in InformationAnalysis(rules).as_property_dict().values()
129
+ for definition in analysis.property_by_id().values()
129
130
  ],
130
131
  classes=[
131
132
  OWLClass.from_class(definition, rules.metadata.namespace, rules.prefixes)
@@ -137,7 +138,7 @@ class Ontology(OntologyModel):
137
138
  list(properties.values()),
138
139
  rules.metadata.namespace,
139
140
  )
140
- for class_, properties in InformationAnalysis(rules).class_property_pairs().items()
141
+ for class_, properties in analysis.properties_by_id_by_class().items()
141
142
  ]
142
143
  + [
143
144
  SHACLNodeShape.from_rules(
@@ -1,15 +1,11 @@
1
- import warnings
2
1
  from abc import ABC, abstractmethod
3
- from collections.abc import Iterator
4
- from contextlib import contextmanager, suppress
2
+ from contextlib import suppress
5
3
  from datetime import datetime
6
- from typing import TYPE_CHECKING, Any, Generic, Literal
4
+ from typing import TYPE_CHECKING, Any, Generic
7
5
 
8
- from pydantic import ValidationError
9
6
  from rdflib import URIRef
10
7
 
11
8
  from cognite.neat._constants import DEFAULT_NAMESPACE
12
- from cognite.neat._issues import IssueList, NeatError, NeatWarning
13
9
  from cognite.neat._rules._shared import ReadRules, T_InputRules
14
10
  from cognite.neat._utils.auxiliary import class_html_doc
15
11
 
@@ -64,44 +60,3 @@ class BaseImporter(ABC, Generic[T_InputRules]):
64
60
  @property
65
61
  def source_uri(self) -> URIRef:
66
62
  return DEFAULT_NAMESPACE["UNKNOWN"]
67
-
68
-
69
- class _FutureResult:
70
- def __init__(self) -> None:
71
- self._result: Literal["success", "failure", "pending"] = "pending"
72
-
73
- @property
74
- def result(self) -> Literal["success", "failure", "pending"]:
75
- return self._result
76
-
77
-
78
- @contextmanager
79
- def _handle_issues(
80
- issues: IssueList,
81
- error_cls: type[NeatError] = NeatError,
82
- warning_cls: type[NeatWarning] = NeatWarning,
83
- error_args: dict[str, Any] | None = None,
84
- ) -> Iterator[_FutureResult]:
85
- """This is an internal help function to handle issues and warnings.
86
-
87
- Args:
88
- issues: The issues list to append to.
89
- error_cls: The class used to convert errors to issues.
90
- warning_cls: The class used to convert warnings to issues.
91
-
92
- Returns:
93
- FutureResult: A future result object that can be used to check the result of the context manager.
94
- """
95
- with warnings.catch_warnings(record=True) as warning_logger:
96
- warnings.simplefilter("always")
97
- future_result = _FutureResult()
98
- try:
99
- yield future_result
100
- except ValidationError as e:
101
- issues.extend(error_cls.from_errors(e.errors(), **(error_args or {}))) # type: ignore[arg-type]
102
- future_result._result = "failure"
103
- else:
104
- future_result._result = "success"
105
- finally:
106
- if warning_logger:
107
- issues.extend([warning_cls.from_warning(warning) for warning in warning_logger]) # type: ignore[misc]
@@ -19,7 +19,7 @@ from cognite.client.data_classes.data_modeling.views import (
19
19
  from cognite.client.utils import ms_to_datetime
20
20
 
21
21
  from cognite.neat._client import NeatClient
22
- from cognite.neat._issues import IssueList, MultiValueError, NeatIssue
22
+ from cognite.neat._issues import IssueList, MultiValueError, NeatIssue, catch_issues
23
23
  from cognite.neat._issues.errors import (
24
24
  FileTypeUnexpectedError,
25
25
  NeatValueError,
@@ -35,7 +35,7 @@ from cognite.neat._issues.warnings import (
35
35
  ResourceUnknownWarning,
36
36
  )
37
37
  from cognite.neat._rules._shared import ReadRules
38
- from cognite.neat._rules.importers._base import BaseImporter, _handle_issues
38
+ from cognite.neat._rules.importers._base import BaseImporter
39
39
  from cognite.neat._rules.models import (
40
40
  DMSInputRules,
41
41
  DMSSchema,
@@ -131,11 +131,10 @@ class DMSImporter(BaseImporter[DMSInputRules]):
131
131
 
132
132
  @classmethod
133
133
  def from_data_model(cls, client: NeatClient, model: dm.DataModel[dm.View]) -> "DMSImporter":
134
- issue_list = IssueList()
135
- with _handle_issues(issue_list) as result:
134
+ with catch_issues() as issue_list:
136
135
  schema = client.schema.retrieve_data_model(model)
137
136
 
138
- if result.result == "failure" or issue_list.has_errors:
137
+ if issue_list.has_errors:
139
138
  return cls(DMSSchema(), issue_list)
140
139
 
141
140
  metadata = cls._create_metadata_from_model(model)
@@ -187,10 +186,9 @@ class DMSImporter(BaseImporter[DMSInputRules]):
187
186
 
188
187
  @classmethod
189
188
  def from_directory(cls, directory: str | Path, client: NeatClient | None = None) -> "DMSImporter":
190
- issue_list = IssueList()
191
- with _handle_issues(issue_list) as _:
189
+ with catch_issues() as issue_list:
192
190
  schema = DMSSchema.from_directory(directory)
193
- # If there were errors during the import, the to_rules
191
+ # If there were errors during the import, the to_rules will raise them.
194
192
  return cls(
195
193
  schema, issue_list, referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client)
196
194
  )
@@ -202,8 +200,7 @@ class DMSImporter(BaseImporter[DMSInputRules]):
202
200
  DMSSchema(),
203
201
  [FileTypeUnexpectedError(Path(zip_file), frozenset([".zip"]))],
204
202
  )
205
- issue_list = IssueList()
206
- with _handle_issues(issue_list) as _:
203
+ with catch_issues() as issue_list:
207
204
  schema = DMSSchema.from_zip(zip_file)
208
205
  return cls(
209
206
  schema, issue_list, referenced_containers=cls._lookup_referenced_containers(schema, issue_list, client)
@@ -19,7 +19,7 @@ from cognite.neat._rules.importers._dtdl2rules.dtdl_converter import _DTDLConver
19
19
  from cognite.neat._rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
20
20
  from cognite.neat._rules.models import InformationInputRules
21
21
  from cognite.neat._rules.models.information import InformationInputMetadata
22
- from cognite.neat._utils.text import humanize_collection, to_pascal
22
+ from cognite.neat._utils.text import humanize_collection, to_pascal_case
23
23
 
24
24
 
25
25
  class DTDLImporter(BaseImporter[InformationInputRules]):
@@ -130,7 +130,7 @@ class DTDLImporter(BaseImporter[InformationInputRules]):
130
130
  metadata = self._default_metadata()
131
131
 
132
132
  if self.name:
133
- metadata["name"] = to_pascal(self.name)
133
+ metadata["name"] = to_pascal_case(self.name)
134
134
  try:
135
135
  most_common_prefix = converter.get_most_common_prefix()
136
136
  except ValueError:
@@ -10,14 +10,16 @@ from cognite.client import data_modeling as dm
10
10
  from rdflib import RDF, RDFS, Graph, Namespace, URIRef
11
11
  from rdflib import Literal as RdfLiteral
12
12
 
13
+ from cognite.neat._config import GLOBAL_CONFIG
13
14
  from cognite.neat._constants import NEAT, get_default_prefixes_and_namespaces
14
15
  from cognite.neat._issues import IssueList
15
16
  from cognite.neat._issues.warnings import PropertyValueTypeUndefinedWarning
16
- from cognite.neat._rules.analysis import InformationAnalysis
17
+ from cognite.neat._rules.analysis import RulesAnalysis
17
18
  from cognite.neat._rules.models import InformationRules, data_types
18
19
  from cognite.neat._rules.models.data_types import AnyURI
19
20
  from cognite.neat._rules.models.entities._single_value import UnknownEntity
20
21
  from cognite.neat._rules.models.information import (
22
+ InformationClass,
21
23
  InformationInputClass,
22
24
  InformationInputProperty,
23
25
  InformationMetadata,
@@ -26,6 +28,7 @@ from cognite.neat._store import NeatGraphStore
26
28
  from cognite.neat._store._provenance import INSTANCES_ENTITY
27
29
  from cognite.neat._utils.collection_ import iterate_progress_bar
28
30
  from cognite.neat._utils.rdf_ import remove_namespace_from_uri, uri_to_short_form
31
+ from cognite.neat._utils.text import NamingStandardization
29
32
 
30
33
  from ._base import DEFAULT_NON_EXISTING_NODE_TYPE, BaseRDFImporter
31
34
 
@@ -256,7 +259,7 @@ class InferenceImporter(BaseRDFImporter):
256
259
  property_["value_type"].remove(str(self.non_existing_node_type))
257
260
 
258
261
  if len(property_["value_type"]) > 1:
259
- property_["value_type"] = " | ".join([str(t) for t in property_["value_type"]])
262
+ property_["value_type"] = ", ".join([str(t) for t in property_["value_type"]])
260
263
  else:
261
264
  property_["value_type"] = next(iter(property_["value_type"]))
262
265
 
@@ -402,7 +405,8 @@ class SubclassInferenceImporter(BaseRDFImporter):
402
405
  else:
403
406
  existing_classes = {}
404
407
  classes: list[InformationInputClass] = []
405
- properties: list[InformationInputProperty] = []
408
+ properties_by_class_suffix_by_property_id: dict[str, dict[str, InformationInputProperty]] = {}
409
+
406
410
  # Help for IDE
407
411
  type_uri: URIRef
408
412
  parent_uri: URIRef
@@ -440,26 +444,51 @@ class SubclassInferenceImporter(BaseRDFImporter):
440
444
  InformationInputClass(
441
445
  class_=class_suffix,
442
446
  implements=parent_suffix,
447
+ instance_source=type_uri,
443
448
  )
444
449
  )
445
450
  else:
446
451
  classes.append(InformationInputClass.load(existing_classes[class_suffix].model_dump()))
452
+
453
+ properties_by_id: dict[str, InformationInputProperty] = {}
447
454
  for property_uri, read_properties in properties_by_property_uri.items():
448
455
  if property_uri in shared_property_uris:
449
456
  shared_properties[property_uri].extend(read_properties)
450
457
  continue
451
- properties.append(
452
- self._create_property(read_properties, class_suffix, type_uri, property_uri, prefixes)
453
- )
454
-
458
+ property_id = remove_namespace_from_uri(property_uri)
459
+ self._add_uri_namespace_to_prefixes(property_uri, prefixes)
460
+ property_id_standardized = NamingStandardization.standardize_property_str(property_uri)
461
+ if existing_prop := properties_by_id.get(property_id_standardized):
462
+ if not isinstance(existing_prop.instance_source, list):
463
+ existing_prop.instance_source = (
464
+ [existing_prop.instance_source] if existing_prop.instance_source else []
465
+ )
466
+ existing_prop.instance_source.append(property_uri)
467
+ continue
468
+ else:
469
+ properties_by_id[property_id_standardized] = self._create_property(
470
+ read_properties, class_suffix, property_uri, property_id, prefixes
471
+ )
472
+ properties_by_class_suffix_by_property_id[class_suffix] = properties_by_id
455
473
  if parent_suffix:
474
+ properties_by_id = {}
456
475
  for property_uri, read_properties in shared_properties.items():
457
- properties.append(
458
- self._create_property(
459
- read_properties, parent_suffix, read_properties[0].type_uri, property_uri, prefixes
476
+ property_id = remove_namespace_from_uri(property_uri)
477
+ self._add_uri_namespace_to_prefixes(property_uri, prefixes)
478
+ property_id_standardized = NamingStandardization.standardize_property_str(property_uri)
479
+ if existing_prop := properties_by_id.get(property_id_standardized):
480
+ if not isinstance(existing_prop.instance_source, list):
481
+ existing_prop.instance_source = (
482
+ [existing_prop.instance_source] if existing_prop.instance_source else []
483
+ )
484
+ existing_prop.instance_source.append(property_uri)
485
+ else:
486
+ properties_by_id[property_id_standardized] = self._create_property(
487
+ read_properties, parent_suffix, property_uri, property_id, prefixes
460
488
  )
461
- )
462
- return classes, properties
489
+ return classes, [
490
+ prop for properties in properties_by_class_suffix_by_property_id.values() for prop in properties.values()
491
+ ]
463
492
 
464
493
  @staticmethod
465
494
  def _get_properties_by_class_by_property(
@@ -481,26 +510,40 @@ class SubclassInferenceImporter(BaseRDFImporter):
481
510
  type_uri, instance_count_literal = cast(tuple[URIRef, RdfLiteral], result_row)
482
511
  count_by_type[type_uri] = instance_count_literal.toPython()
483
512
  if self._rules:
484
- analysis = InformationAnalysis(self._rules)
513
+ analysis = RulesAnalysis(self._rules)
485
514
  existing_class_properties = {
486
- (class_entity.suffix, prop.property_)
487
- for class_entity, properties in analysis.classes_with_properties(
488
- consider_inheritance=True, allow_different_namespace=True
515
+ (class_entity.suffix, prop.property_): prop
516
+ for class_entity, properties in analysis.properties_by_class(
517
+ include_ancestors=True, include_different_space=True
489
518
  ).items()
490
519
  for prop in properties
491
520
  }
521
+ existing_classes = {cls_.class_.suffix: cls_ for cls_ in self._rules.classes}
492
522
  else:
493
- existing_class_properties = set()
523
+ existing_class_properties = {}
524
+ existing_classes = {}
494
525
  properties_by_class_by_subclass: list[_ReadProperties] = []
495
- for type_uri, instance_count in count_by_type.items():
526
+ existing_class: InformationClass | None
527
+ total_instance_count = sum(count_by_type.values())
528
+ iterable = count_by_type.items()
529
+ if GLOBAL_CONFIG.use_iterate_bar_threshold and total_instance_count > GLOBAL_CONFIG.use_iterate_bar_threshold:
530
+ iterable = iterate_progress_bar(iterable, len(count_by_type), "Inferring types...") # type: ignore[assignment]
531
+ for type_uri, instance_count in iterable:
496
532
  property_query = self._properties_query.format(type=type_uri, unknown_type=NEAT.UnknownType)
497
533
  class_suffix = remove_namespace_from_uri(type_uri)
534
+ if (existing_class := existing_classes.get(class_suffix)) and existing_class.instance_source is None:
535
+ existing_class.instance_source = type_uri
536
+
498
537
  for result_row in self.graph.query(property_query):
499
538
  property_uri, value_type_uri = cast(tuple[URIRef, URIRef], result_row)
500
539
  if property_uri == RDF.type:
501
540
  continue
502
541
  property_str = remove_namespace_from_uri(property_uri)
503
- if (class_suffix, property_str) in existing_class_properties:
542
+ if existing_property := existing_class_properties.get((class_suffix, property_str)):
543
+ if existing_property.instance_source is None:
544
+ existing_property.instance_source = [property_uri]
545
+ elif existing_property.instance_source and property_uri not in existing_property.instance_source:
546
+ existing_property.instance_source.append(property_uri)
504
547
  continue
505
548
  occurrence_query = self._max_occurrence_query.format(type=type_uri, property=property_uri)
506
549
  max_occurrence = 1 # default value
@@ -531,21 +574,18 @@ class SubclassInferenceImporter(BaseRDFImporter):
531
574
  self,
532
575
  read_properties: list[_ReadProperties],
533
576
  class_suffix: str,
534
- type_uri: URIRef,
535
577
  property_uri: URIRef,
578
+ property_id: str,
536
579
  prefixes: dict[str, Namespace],
537
580
  ) -> InformationInputProperty:
538
581
  first = read_properties[0]
539
582
  value_type = self._get_value_type(read_properties, prefixes)
540
- property_name = remove_namespace_from_uri(property_uri)
541
- self._add_uri_namespace_to_prefixes(property_uri, prefixes)
542
-
543
583
  return InformationInputProperty(
544
584
  class_=class_suffix,
545
- property_=property_name,
585
+ property_=property_id,
546
586
  max_count=first.max_occurrence,
547
587
  value_type=value_type,
548
- instance_source=(f"{uri_to_short_form(type_uri, prefixes)}({uri_to_short_form(property_uri, prefixes)})"),
588
+ instance_source=[property_uri],
549
589
  )
550
590
 
551
591
  def _get_value_type(
@@ -562,7 +602,7 @@ class SubclassInferenceImporter(BaseRDFImporter):
562
602
  return UnknownEntity()
563
603
  for uri_ref in value_types:
564
604
  self._add_uri_namespace_to_prefixes(uri_ref, prefixes)
565
- return " | ".join(remove_namespace_from_uri(uri_ref) for uri_ref in value_types)
605
+ return ", ".join(remove_namespace_from_uri(uri_ref) for uri_ref in value_types)
566
606
 
567
607
  def _default_metadata(self) -> dict[str, Any]:
568
608
  now = datetime.now(timezone.utc)
@@ -136,7 +136,7 @@ def parse_properties(graph: Graph, query: str, language: str, issue_list: IssueL
136
136
  properties[id_]["value_type"].append(res["value_type"])
137
137
 
138
138
  for prop in properties.values():
139
- prop["value_type"] = "|".join(prop["value_type"])
139
+ prop["value_type"] = ", ".join(prop["value_type"])
140
140
 
141
141
  if not properties:
142
142
  issue_list.append(NeatValueError("Unable to parse properties"))
@@ -37,15 +37,15 @@ SOURCE_SHEET__TARGET_FIELD__HEADERS = [
37
37
  "Properties",
38
38
  "Properties",
39
39
  {
40
- RoleTypes.information: "Property",
41
- RoleTypes.dms: "View Property",
40
+ RoleTypes.information: ["Class", "Property"],
41
+ RoleTypes.dms: ["View", "View Property"],
42
42
  },
43
43
  ),
44
- ("Classes", "Classes", "Class"),
45
- ("Containers", "Containers", "Container"),
46
- ("Views", "Views", "View"),
47
- ("Enum", "Enum", "Collection"),
48
- ("Nodes", "Nodes", "Node"),
44
+ ("Classes", "Classes", ["Class"]),
45
+ ("Containers", "Containers", ["Container"]),
46
+ ("Views", "Views", ["View"]),
47
+ ("Enum", "Enum", ["Collection"]),
48
+ ("Nodes", "Nodes", ["Node"]),
49
49
  ]
50
50
 
51
51
 
@@ -231,7 +231,10 @@ class SpreadsheetReader:
231
231
 
232
232
  try:
233
233
  sheets[target_sheet_name], read_info_by_sheet[source_sheet_name] = read_individual_sheet(
234
- excel_file, source_sheet_name, return_read_info=True, expected_headers=[headers]
234
+ excel_file,
235
+ source_sheet_name,
236
+ return_read_info=True,
237
+ expected_headers=headers,
235
238
  )
236
239
  except Exception as e:
237
240
  self.issue_list.append(FileReadError(cast(Path, excel_file.io), str(e)))
@@ -273,7 +276,7 @@ class ExcelImporter(BaseImporter[T_InputRules]):
273
276
 
274
277
  rules_cls = INPUT_RULES_BY_ROLE[original_role]
275
278
  rules = cast(T_InputRules, rules_cls.load(sheets))
276
- return ReadRules(rules, {"read_info_by_sheet": read_info_by_sheet})
279
+ return ReadRules(rules, read_info_by_sheet)
277
280
 
278
281
  @property
279
282
  def description(self) -> str:
@@ -126,7 +126,6 @@ class SchemaModel(BaseModel):
126
126
  extra="ignore",
127
127
  use_enum_values=True,
128
128
  )
129
- validators_to_skip: set[str] = Field(default_factory=set, exclude=True)
130
129
 
131
130
  @classmethod
132
131
  def mandatory_fields(cls, use_alias=False) -> set[str]:
@@ -256,7 +255,6 @@ class BaseRules(SchemaModel, ABC):
256
255
 
257
256
  Args:
258
257
  metadata: Data model metadata
259
- validators_to_skip: List of validators to skip. Defaults to []
260
258
  """
261
259
 
262
260
  metadata: BaseMetadata
@@ -130,6 +130,13 @@ class DataType(BaseModel):
130
130
  def as_xml_uri_ref(cls) -> URIRef:
131
131
  return XML_SCHEMA_NAMESPACE[cls.xsd]
132
132
 
133
+ @classmethod
134
+ def convert_value(cls, value: Any) -> Any:
135
+ if cls != Boolean:
136
+ return cls.python(value)
137
+ else:
138
+ return value.strip().lower() in {"true", "1", "yes"}
139
+
133
140
 
134
141
  class Boolean(DataType):
135
142
  python = bool
@@ -99,6 +99,7 @@ class _DMSExporter:
99
99
  key=lambda x: x.as_tuple(), # type: ignore[union-attr]
100
100
  )
101
101
  spaces = self._create_spaces(rules.metadata, containers, views, data_model)
102
+
102
103
  return DMSSchema(
103
104
  spaces=spaces,
104
105
  data_model=data_model,
@@ -114,14 +115,14 @@ class _DMSExporter:
114
115
  views: ViewApplyDict,
115
116
  data_model: dm.DataModelApply,
116
117
  ) -> SpaceApplyDict:
117
- used_spaces = {container.space for container in containers.values()} | {view.space for view in views.values()}
118
- if len(used_spaces) == 1:
119
- # We skip the default space and only use this space for the data model
120
- data_model.space = used_spaces.pop()
121
- spaces = SpaceApplyDict([dm.SpaceApply(space=data_model.space)])
122
- else:
123
- used_spaces.add(metadata.space)
124
- spaces = SpaceApplyDict([dm.SpaceApply(space=space) for space in used_spaces])
118
+ used_spaces = (
119
+ {container.space for container in containers.values()}
120
+ | {view.space for view in views.values()}
121
+ | {data_model.space}
122
+ | {metadata.space}
123
+ )
124
+
125
+ spaces = SpaceApplyDict([dm.SpaceApply(space=space) for space in used_spaces])
125
126
  if self.instance_space and self.instance_space not in spaces:
126
127
  spaces[self.instance_space] = dm.SpaceApply(space=self.instance_space, name=self.instance_space)
127
128
  return spaces
@@ -1,3 +1,4 @@
1
+ import warnings
1
2
  from collections.abc import Hashable
2
3
  from typing import TYPE_CHECKING, Any, ClassVar, Literal
3
4
 
@@ -8,6 +9,7 @@ from pydantic_core.core_schema import SerializationInfo, ValidationInfo
8
9
 
9
10
  from cognite.neat._client.data_classes.schema import DMSSchema
10
11
  from cognite.neat._issues.errors import NeatValueError
12
+ from cognite.neat._issues.warnings._general import NeatValueWarning
11
13
  from cognite.neat._rules.models._base_rules import (
12
14
  BaseMetadata,
13
15
  BaseRules,
@@ -116,7 +118,7 @@ class DMSProperty(SheetRow):
116
118
  description="Used to indicate whether the property holds single or multiple values (list). "
117
119
  "Only applies to primitive types.",
118
120
  )
119
- default: str | int | dict | None = Field(
121
+ default: bool | str | int | float | dict | None = Field(
120
122
  None, alias="Default", description="Specifies default value for the property."
121
123
  )
122
124
  container: ContainerEntityType | None = Field(
@@ -168,6 +170,29 @@ class DMSProperty(SheetRow):
168
170
  raise ValueError(f"Reverse connection must have a value type that points to a view, got {value}")
169
171
  return value
170
172
 
173
+ @field_validator("default", mode="after")
174
+ def set_proper_type_on_default(cls, value: Any, info: ValidationInfo) -> Any:
175
+ if not value:
176
+ return value
177
+ value_type = info.data.get("value_type")
178
+ if not isinstance(value_type, DataType):
179
+ warnings.filterwarnings("default")
180
+ warnings.warn(
181
+ NeatValueWarning(f"Default value {value} set to connection {value_type} will be ignored"),
182
+ stacklevel=2,
183
+ )
184
+ return None
185
+ else:
186
+ try:
187
+ return value_type.convert_value(value)
188
+ except ValueError:
189
+ warnings.filterwarnings("default")
190
+ warnings.warn(
191
+ NeatValueWarning(f"Could not convert {value} to {value_type}"),
192
+ stacklevel=2,
193
+ )
194
+ return None
195
+
171
196
  @field_validator("container", "container_property", mode="after")
172
197
  def container_set_correctly(cls, value: Any, info: ValidationInfo) -> Any:
173
198
  if (connection := info.data.get("connection")) is None:
@@ -295,7 +320,9 @@ class DMSView(SheetRow):
295
320
  None, alias="Filter", description="Explicitly define the filter for the view."
296
321
  )
297
322
  in_model: bool = Field(
298
- True, alias="In Model", description="Indicates whether the view being defined is a part of the data model."
323
+ True,
324
+ alias="In Model",
325
+ description="Indicates whether the view being defined is a part of the data model.",
299
326
  )
300
327
  logical: URIRefType | None = Field(
301
328
  None,
@@ -21,6 +21,7 @@ from cognite.neat._rules.models.entities import (
21
21
  load_connection,
22
22
  load_dms_value_type,
23
23
  )
24
+ from cognite.neat._rules.models.entities._wrapped import DMSFilter
24
25
  from cognite.neat._utils.rdf_ import uri_display_name
25
26
 
26
27
  from ._rules import _DEFAULT_VERSION, DMSContainer, DMSEnum, DMSMetadata, DMSNode, DMSProperty, DMSRules, DMSView
@@ -69,7 +70,7 @@ class DMSInputMetadata(InputComponent[DMSMetadata]):
69
70
  def _get_description_and_creator(cls, description_raw: str | None) -> tuple[str | None, list[str]]:
70
71
  if description_raw and (description_match := re.search(r"Creator: (.+)", description_raw)):
71
72
  creator = description_match.group(1).split(", ")
72
- description = description_raw.replace(description_match.string, "").strip() or None
73
+ description = description_raw.replace(description_match[0], "").strip() or None
73
74
  elif description_raw:
74
75
  creator = ["MISSING"]
75
76
  description = description_raw
@@ -193,6 +194,10 @@ class DMSInputView(InputComponent[DMSView]):
193
194
  neatId: str | URIRef | None = None
194
195
  logical: str | URIRef | None = None
195
196
 
197
+ def __post_init__(self):
198
+ if self.in_model is None:
199
+ self.in_model = True
200
+
196
201
  @classmethod
197
202
  def _get_verified_cls(cls) -> type[DMSView]:
198
203
  return DMSView
@@ -207,6 +212,8 @@ class DMSInputView(InputComponent[DMSView]):
207
212
  return ViewEntity.load(self.view, strict=True, space=default_space, version=default_version)
208
213
 
209
214
  def _load_implements(self, default_space: str, default_version: str) -> list[ViewEntity] | None:
215
+ self.implements = self.implements.strip() if self.implements else None
216
+
210
217
  return (
211
218
  [
212
219
  ViewEntity.load(implement, strict=True, space=default_space, version=default_version)
@@ -230,6 +237,7 @@ class DMSInputView(InputComponent[DMSView]):
230
237
  implements=", ".join([str(ViewEntity.from_id(parent, _DEFAULT_VERSION)) for parent in view.implements])
231
238
  or None,
232
239
  in_model=in_model,
240
+ filter_=(str(DMSFilter.from_dms_filter(view.filter)) if view.filter else None),
233
241
  )
234
242
 
235
243