cognite-neat 0.86.0__py3-none-any.whl → 0.87.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (108) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/app/api/configuration.py +1 -10
  3. cognite/neat/app/api/routers/data_exploration.py +1 -1
  4. cognite/neat/config.py +84 -17
  5. cognite/neat/constants.py +11 -9
  6. cognite/neat/graph/extractors/_classic_cdf/_assets.py +1 -1
  7. cognite/neat/graph/extractors/_classic_cdf/_events.py +1 -1
  8. cognite/neat/graph/extractors/_classic_cdf/_files.py +1 -1
  9. cognite/neat/graph/extractors/_classic_cdf/_labels.py +1 -1
  10. cognite/neat/graph/extractors/_classic_cdf/_relationships.py +1 -1
  11. cognite/neat/graph/extractors/_classic_cdf/_sequences.py +1 -1
  12. cognite/neat/graph/extractors/_classic_cdf/_timeseries.py +1 -1
  13. cognite/neat/graph/extractors/_dexpi.py +1 -1
  14. cognite/neat/graph/extractors/_mock_graph_generator.py +8 -9
  15. cognite/neat/graph/loaders/__init__.py +5 -2
  16. cognite/neat/graph/loaders/_base.py +13 -5
  17. cognite/neat/graph/loaders/_rdf2asset.py +185 -55
  18. cognite/neat/graph/loaders/_rdf2dms.py +7 -7
  19. cognite/neat/graph/queries/_base.py +20 -11
  20. cognite/neat/graph/queries/_construct.py +5 -5
  21. cognite/neat/graph/queries/_shared.py +21 -7
  22. cognite/neat/graph/stores/_base.py +16 -4
  23. cognite/neat/graph/transformers/__init__.py +3 -0
  24. cognite/neat/graph/transformers/_rdfpath.py +42 -0
  25. cognite/neat/legacy/graph/extractors/_dexpi.py +0 -5
  26. cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +1 -1
  27. cognite/neat/legacy/graph/loaders/_asset_loader.py +2 -2
  28. cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +5 -2
  29. cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +4 -1
  30. cognite/neat/legacy/graph/loaders/rdf_to_dms.py +3 -1
  31. cognite/neat/legacy/graph/stores/_base.py +24 -8
  32. cognite/neat/legacy/graph/stores/_graphdb_store.py +3 -2
  33. cognite/neat/legacy/graph/stores/_memory_store.py +3 -3
  34. cognite/neat/legacy/graph/stores/_oxigraph_store.py +8 -4
  35. cognite/neat/legacy/graph/stores/_rdf_to_graph.py +5 -3
  36. cognite/neat/legacy/graph/transformations/query_generator/sparql.py +49 -16
  37. cognite/neat/legacy/graph/transformations/transformer.py +1 -1
  38. cognite/neat/legacy/rules/exporters/_rules2dms.py +8 -3
  39. cognite/neat/legacy/rules/exporters/_rules2graphql.py +1 -1
  40. cognite/neat/legacy/rules/exporters/_rules2ontology.py +2 -1
  41. cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +3 -4
  42. cognite/neat/legacy/rules/importers/_dms2rules.py +4 -1
  43. cognite/neat/legacy/rules/importers/_graph2rules.py +3 -3
  44. cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +1 -1
  45. cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +2 -1
  46. cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +1 -1
  47. cognite/neat/legacy/rules/models/raw_rules.py +19 -7
  48. cognite/neat/legacy/rules/models/rules.py +32 -12
  49. cognite/neat/rules/_shared.py +6 -1
  50. cognite/neat/rules/analysis/__init__.py +4 -4
  51. cognite/neat/rules/analysis/_asset.py +143 -0
  52. cognite/neat/rules/analysis/_base.py +385 -6
  53. cognite/neat/rules/analysis/_information.py +183 -0
  54. cognite/neat/rules/exporters/_rules2dms.py +1 -1
  55. cognite/neat/rules/exporters/_rules2ontology.py +6 -5
  56. cognite/neat/rules/importers/_dms2rules.py +3 -1
  57. cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +2 -8
  58. cognite/neat/rules/importers/_inference2rules.py +3 -7
  59. cognite/neat/rules/importers/_owl2rules/_owl2classes.py +1 -1
  60. cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +2 -1
  61. cognite/neat/rules/importers/_owl2rules/_owl2properties.py +1 -1
  62. cognite/neat/rules/issues/spreadsheet.py +35 -0
  63. cognite/neat/rules/models/_base.py +7 -7
  64. cognite/neat/rules/models/_rdfpath.py +17 -21
  65. cognite/neat/rules/models/asset/_rules.py +4 -5
  66. cognite/neat/rules/models/asset/_validation.py +38 -1
  67. cognite/neat/rules/models/dms/_converter.py +1 -2
  68. cognite/neat/rules/models/dms/_exporter.py +7 -3
  69. cognite/neat/rules/models/dms/_rules.py +3 -0
  70. cognite/neat/rules/models/dms/_schema.py +5 -4
  71. cognite/neat/rules/models/domain.py +5 -2
  72. cognite/neat/rules/models/entities.py +28 -17
  73. cognite/neat/rules/models/information/_rules.py +10 -8
  74. cognite/neat/rules/models/information/_rules_input.py +1 -2
  75. cognite/neat/rules/models/information/_validation.py +2 -2
  76. cognite/neat/utils/__init__.py +0 -3
  77. cognite/neat/utils/auth.py +47 -28
  78. cognite/neat/utils/auxiliary.py +141 -1
  79. cognite/neat/utils/cdf/__init__.py +0 -0
  80. cognite/neat/utils/{cdf_classes.py → cdf/data_classes.py} +122 -2
  81. cognite/neat/utils/{cdf_loaders → cdf/loaders}/_data_modeling.py +37 -0
  82. cognite/neat/utils/{cdf_loaders → cdf/loaders}/_ingestion.py +2 -1
  83. cognite/neat/utils/collection_.py +18 -0
  84. cognite/neat/utils/rdf_.py +165 -0
  85. cognite/neat/utils/text.py +4 -0
  86. cognite/neat/utils/time_.py +17 -0
  87. cognite/neat/utils/upload.py +13 -1
  88. cognite/neat/workflows/_exceptions.py +5 -5
  89. cognite/neat/workflows/base.py +1 -1
  90. cognite/neat/workflows/steps/lib/current/graph_store.py +28 -8
  91. cognite/neat/workflows/steps/lib/current/rules_validator.py +2 -2
  92. cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +130 -28
  93. cognite/neat/workflows/steps/lib/legacy/graph_loader.py +1 -1
  94. cognite/neat/workflows/steps/lib/legacy/graph_store.py +4 -4
  95. cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +1 -1
  96. cognite/neat/workflows/steps/lib/legacy/rules_importer.py +1 -1
  97. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/METADATA +2 -2
  98. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/RECORD +103 -102
  99. cognite/neat/rules/analysis/_information_rules.py +0 -476
  100. cognite/neat/utils/cdf.py +0 -59
  101. cognite/neat/utils/cdf_loaders/data_classes.py +0 -121
  102. cognite/neat/utils/exceptions.py +0 -41
  103. cognite/neat/utils/utils.py +0 -429
  104. /cognite/neat/utils/{cdf_loaders → cdf/loaders}/__init__.py +0 -0
  105. /cognite/neat/utils/{cdf_loaders → cdf/loaders}/_base.py +0 -0
  106. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/LICENSE +0 -0
  107. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/WHEEL +0 -0
  108. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,183 @@
1
+ import logging
2
+ import warnings
3
+ from typing import Any, cast
4
+
5
+ from pydantic import ValidationError
6
+ from rdflib import URIRef
7
+
8
+ from cognite.neat.rules.models import SchemaCompleteness
9
+ from cognite.neat.rules.models._rdfpath import (
10
+ Hop,
11
+ RDFPath,
12
+ SelfReferenceProperty,
13
+ SingleProperty,
14
+ )
15
+ from cognite.neat.rules.models.entities import ClassEntity, ReferenceEntity
16
+ from cognite.neat.rules.models.information import (
17
+ InformationClass,
18
+ InformationProperty,
19
+ InformationRules,
20
+ )
21
+ from cognite.neat.utils.rdf_ import get_inheritance_path
22
+
23
+ from ._base import BaseAnalysis
24
+
25
+
26
+ class InformationAnalysis(BaseAnalysis[InformationRules, InformationClass, InformationProperty, ClassEntity, str]):
27
+ """Assumes analysis over only the complete schema"""
28
+
29
+ def _get_object(self, property_: InformationProperty) -> ClassEntity | None:
30
+ return property_.value_type if isinstance(property_.value_type, ClassEntity) else None
31
+
32
+ def _get_max_occurrence(self, property_: InformationProperty) -> int | float | None:
33
+ return property_.max_count
34
+
35
+ def _get_reference(self, class_or_property: InformationClass | InformationProperty) -> ReferenceEntity | None:
36
+ return class_or_property.reference if isinstance(class_or_property.reference, ReferenceEntity) else None
37
+
38
+ def _get_cls_entity(self, class_: InformationClass | InformationProperty) -> ClassEntity:
39
+ return class_.class_
40
+
41
+ @classmethod
42
+ def _set_cls_entity(cls, property_: InformationProperty, class_: ClassEntity) -> None:
43
+ property_.class_ = class_
44
+
45
+ def _get_prop_entity(self, property_: InformationProperty) -> str:
46
+ return property_.property_
47
+
48
+ def _get_cls_parents(self, class_: InformationClass) -> list[ClassEntity] | None:
49
+ return list(class_.parent or []) or None
50
+
51
+ def _get_reference_rules(self) -> InformationRules | None:
52
+ return self.rules.reference
53
+
54
+ def _get_properties(self) -> list[InformationProperty]:
55
+ return list(self.rules.properties)
56
+
57
+ def _get_classes(self) -> list[InformationClass]:
58
+ return list(self.rules.classes)
59
+
60
+ def has_hop_transformations(self):
61
+ return any(
62
+ prop_.transformation and isinstance(prop_.transformation.traversal, Hop) for prop_ in self.rules.properties
63
+ )
64
+
65
+ def has_self_reference_property_transformations(self):
66
+ return any(
67
+ prop_.transformation and isinstance(prop_.transformation.traversal, SelfReferenceProperty)
68
+ for prop_ in self.rules.properties
69
+ )
70
+
71
+ def all_reference_transformations(self):
72
+ return [
73
+ prop_
74
+ for prop_ in self.rules.properties
75
+ if prop_.transformation and isinstance(prop_.transformation.traversal, SelfReferenceProperty)
76
+ ]
77
+
78
+ def define_property_renaming_config(self, class_: ClassEntity) -> dict[str | URIRef, str]:
79
+ property_renaming_configuration: dict[str | URIRef, str] = {}
80
+
81
+ if definitions := self.class_property_pairs(only_rdfpath=True, consider_inheritance=True).get(class_, None):
82
+ for property_id, definition in definitions.items():
83
+ transformation = cast(RDFPath, definition.transformation)
84
+
85
+ # use case we have a single property rdf path, and defined prefix
86
+ # in either metadata or prefixes of rules
87
+ if isinstance(
88
+ transformation.traversal,
89
+ SingleProperty,
90
+ ) and (
91
+ transformation.traversal.property.prefix in self.rules.prefixes
92
+ or transformation.traversal.property.prefix == self.rules.metadata.prefix
93
+ ):
94
+ namespace = (
95
+ self.rules.metadata.namespace
96
+ if transformation.traversal.property.prefix == self.rules.metadata.prefix
97
+ else self.rules.prefixes[transformation.traversal.property.prefix]
98
+ )
99
+
100
+ property_renaming_configuration[namespace[transformation.traversal.property.suffix]] = property_id
101
+
102
+ # otherwise we default to the property id
103
+ else:
104
+ property_renaming_configuration[property_id] = property_id
105
+
106
+ return property_renaming_configuration
107
+
108
+ def subset_rules(self, desired_classes: set[ClassEntity]) -> InformationRules:
109
+ """
110
+ Subset rules to only include desired classes and their properties.
111
+
112
+ Args:
113
+ desired_classes: Desired classes to include in the reduced data model
114
+
115
+ Returns:
116
+ Instance of InformationRules
117
+
118
+ !!! note "Inheritance"
119
+ If desired classes contain a class that is a subclass of another class(es), the parent class(es)
120
+ will be included in the reduced data model as well even though the parent class(es) are
121
+ not in the desired classes set. This is to ensure that the reduced data model is
122
+ consistent and complete.
123
+
124
+ !!! note "Partial Reduction"
125
+ This method does not perform checks if classes that are value types of desired classes
126
+ properties are part of desired classes. If a class is not part of desired classes, but it
127
+ is a value type of a property of a class that is part of desired classes, derived reduced
128
+ rules will be marked as partial.
129
+
130
+ !!! note "Validation"
131
+ This method will attempt to validate the reduced rules with custom validations.
132
+ If it fails, it will return a partial rules with a warning message, validated
133
+ only with base Pydantic validators.
134
+ """
135
+ if self.rules.metadata.schema_ is not SchemaCompleteness.complete:
136
+ raise ValueError("Rules are not complete cannot perform reduction!")
137
+ class_as_dict = self.as_class_dict()
138
+ class_parents_pairs = self.class_parent_pairs()
139
+ defined_classes = self.defined_classes(consider_inheritance=True)
140
+
141
+ possible_classes = defined_classes.intersection(desired_classes)
142
+ impossible_classes = desired_classes - possible_classes
143
+
144
+ # need to add all the parent classes of the desired classes to the possible classes
145
+ parents: set[ClassEntity] = set()
146
+ for class_ in possible_classes:
147
+ parents = parents.union({parent for parent in get_inheritance_path(class_, class_parents_pairs)})
148
+ possible_classes = possible_classes.union(parents)
149
+
150
+ if not possible_classes:
151
+ logging.error("None of the desired classes are defined in the data model!")
152
+ raise ValueError("None of the desired classes are defined in the data model!")
153
+
154
+ if impossible_classes:
155
+ logging.warning(f"Could not find the following classes defined in the data model: {impossible_classes}")
156
+ warnings.warn(
157
+ f"Could not find the following classes defined in the data model: {impossible_classes}",
158
+ stacklevel=2,
159
+ )
160
+
161
+ reduced_data_model: dict[str, Any] = {
162
+ "metadata": self.rules.metadata.model_copy(),
163
+ "prefixes": (self.rules.prefixes or {}).copy(),
164
+ "classes": [],
165
+ "properties": [],
166
+ }
167
+
168
+ logging.info(f"Reducing data model to only include the following classes: {possible_classes}")
169
+ for class_ in possible_classes:
170
+ reduced_data_model["classes"].append(class_as_dict[str(class_.suffix)])
171
+
172
+ class_property_pairs = self.classes_with_properties(consider_inheritance=False)
173
+
174
+ for class_, properties in class_property_pairs.items():
175
+ if class_ in possible_classes:
176
+ reduced_data_model["properties"].extend(properties)
177
+
178
+ try:
179
+ return type(self.rules)(**reduced_data_model)
180
+ except ValidationError as e:
181
+ warnings.warn(f"Reduced data model is not complete: {e}", stacklevel=2)
182
+ reduced_data_model["metadata"].schema_ = SchemaCompleteness.partial
183
+ return type(self.rules).model_construct(**reduced_data_model)
@@ -20,7 +20,7 @@ from cognite.neat.rules._shared import Rules
20
20
  from cognite.neat.rules.issues import IssueList
21
21
  from cognite.neat.rules.models import InformationRules
22
22
  from cognite.neat.rules.models.dms import DMSRules, DMSSchema, PipelineSchema
23
- from cognite.neat.utils.cdf_loaders import (
23
+ from cognite.neat.utils.cdf.loaders import (
24
24
  ContainerLoader,
25
25
  DataModelingLoader,
26
26
  DataModelLoader,
@@ -10,7 +10,7 @@ from rdflib.collection import Collection as GraphCollection
10
10
 
11
11
  from cognite.neat.constants import DEFAULT_NAMESPACE as NEAT_NAMESPACE
12
12
  from cognite.neat.rules import exceptions
13
- from cognite.neat.rules.analysis import InformationArchitectRulesAnalysis
13
+ from cognite.neat.rules.analysis import InformationAnalysis
14
14
  from cognite.neat.rules.models import DMSRules
15
15
  from cognite.neat.rules.models.data_types import DataType
16
16
  from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
@@ -20,7 +20,8 @@ from cognite.neat.rules.models.information import (
20
20
  InformationProperty,
21
21
  InformationRules,
22
22
  )
23
- from cognite.neat.utils.utils import generate_exception_report, remove_namespace_from_uri
23
+ from cognite.neat.utils.auxiliary import generate_exception_report
24
+ from cognite.neat.utils.rdf_ import remove_namespace_from_uri
24
25
 
25
26
  from ._base import BaseExporter
26
27
  from ._validation import are_properties_redefined
@@ -109,11 +110,11 @@ class Ontology(OntologyModel):
109
110
  if rules.metadata.namespace is None:
110
111
  raise exceptions.MissingDataModelPrefixOrNamespace()
111
112
 
112
- class_dict = InformationArchitectRulesAnalysis(rules).as_class_dict()
113
+ class_dict = InformationAnalysis(rules).as_class_dict()
113
114
  return cls(
114
115
  properties=[
115
116
  OWLProperty.from_list_of_properties(definition, rules.metadata.namespace)
116
- for definition in InformationArchitectRulesAnalysis(rules).as_property_dict().values()
117
+ for definition in InformationAnalysis(rules).as_property_dict().values()
117
118
  ],
118
119
  classes=[
119
120
  OWLClass.from_class(definition, rules.metadata.namespace, rules.prefixes)
@@ -125,7 +126,7 @@ class Ontology(OntologyModel):
125
126
  list(properties.values()),
126
127
  rules.metadata.namespace,
127
128
  )
128
- for class_, properties in InformationArchitectRulesAnalysis(rules).class_property_pairs().items()
129
+ for class_, properties in InformationAnalysis(rules).class_property_pairs().items()
129
130
  ]
130
131
  + [
131
132
  SHACLNodeShape.from_rules(
@@ -8,6 +8,7 @@ from cognite.client import CogniteClient
8
8
  from cognite.client import data_modeling as dm
9
9
  from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
10
10
  from cognite.client.data_classes.data_modeling.containers import BTreeIndex, InvertedIndex
11
+ from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
11
12
  from cognite.client.data_classes.data_modeling.views import (
12
13
  MultiEdgeConnectionApply,
13
14
  MultiReverseDirectRelationApply,
@@ -403,7 +404,8 @@ class DMSImporter(BaseImporter):
403
404
 
404
405
  def _get_is_list(self, prop: ViewPropertyApply) -> bool | None:
405
406
  if isinstance(prop, dm.MappedPropertyApply):
406
- return self._container_prop_unsafe(prop).type.is_list
407
+ prop_type = self._container_prop_unsafe(prop).type
408
+ return isinstance(prop_type, ListablePropertyType) and prop_type.is_list
407
409
  elif isinstance(prop, MultiEdgeConnectionApply | MultiReverseDirectRelationApply):
408
410
  return True
409
411
  elif isinstance(prop, SingleEdgeConnectionApply | SingleReverseDirectRelationApply):
@@ -1,6 +1,5 @@
1
1
  from collections import Counter
2
2
  from collections.abc import Callable, Sequence
3
- from typing import cast
4
3
 
5
4
  import cognite.neat.rules.issues.importing
6
5
  from cognite.neat.rules import issues
@@ -22,7 +21,7 @@ from cognite.neat.rules.importers._dtdl2rules.spec import (
22
21
  )
23
22
  from cognite.neat.rules.issues import IssueList, ValidationIssue
24
23
  from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_NAME, DataType, Json, String
25
- from cognite.neat.rules.models.entities import ClassEntity, ParentClassEntity
24
+ from cognite.neat.rules.models.entities import ClassEntity
26
25
  from cognite.neat.rules.models.information import InformationClass, InformationProperty
27
26
 
28
27
 
@@ -89,12 +88,7 @@ class _DTDLConverter:
89
88
  name=item.display_name,
90
89
  description=item.description,
91
90
  comment=item.comment,
92
- parent=[
93
- cast(ParentClassEntity, parent_entity)
94
- for parent in item.extends or []
95
- if isinstance(parent_entity := ParentClassEntity.load(parent.as_class_id()), ParentClassEntity)
96
- ]
97
- or None,
91
+ parent=[parent.as_class_id() for parent in item.extends or []] or None,
98
92
  )
99
93
  self.classes.append(class_)
100
94
  for sub_item_or_id in item.contents or []:
@@ -7,7 +7,7 @@ from rdflib import Graph, Namespace, URIRef
7
7
  from rdflib import Literal as RdfLiteral
8
8
 
9
9
  import cognite.neat.rules.issues as issues
10
- from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
10
+ from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
11
11
  from cognite.neat.graph.stores import NeatGraphStore
12
12
  from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
13
13
  from cognite.neat.rules.issues import IssueList
@@ -17,11 +17,7 @@ from cognite.neat.rules.models.information import (
17
17
  InformationMetadata,
18
18
  InformationRulesInput,
19
19
  )
20
- from cognite.neat.utils.utils import (
21
- get_namespace,
22
- remove_namespace_from_uri,
23
- uri_to_short_form,
24
- )
20
+ from cognite.neat.utils.rdf_ import get_namespace, remove_namespace_from_uri, uri_to_short_form
25
21
 
26
22
  ORDERED_CLASSES_QUERY = """SELECT ?class (count(?s) as ?instances )
27
23
  WHERE { ?s a ?class . }
@@ -204,7 +200,7 @@ class InferenceImporter(BaseImporter):
204
200
  """
205
201
  classes: dict[str, dict] = {}
206
202
  properties: dict[str, dict] = {}
207
- prefixes: dict[str, Namespace] = PREFIXES.copy()
203
+ prefixes: dict[str, Namespace] = get_default_prefixes()
208
204
 
209
205
  query = INSTANCE_PROPERTIES_JSON_DEFINITION if self.check_for_json_string else INSTANCE_PROPERTIES_DEFINITION
210
206
  # Adds default namespace to prefixes
@@ -5,7 +5,7 @@ import pandas as pd
5
5
  from rdflib import OWL, Graph
6
6
 
7
7
  from cognite.neat.rules.models._base import MatchType
8
- from cognite.neat.utils.utils import remove_namespace_from_uri
8
+ from cognite.neat.utils.rdf_ import remove_namespace_from_uri
9
9
 
10
10
 
11
11
  def parse_owl_classes(graph: Graph, language: str = "en") -> list[dict]:
@@ -9,7 +9,8 @@ from cognite.neat.rules.models._types._base import (
9
9
  PREFIX_COMPLIANCE_REGEX,
10
10
  VERSION_COMPLIANCE_REGEX,
11
11
  )
12
- from cognite.neat.utils.utils import convert_rdflib_content, remove_none_elements_from_set
12
+ from cognite.neat.utils.collection_ import remove_none_elements_from_set
13
+ from cognite.neat.utils.rdf_ import convert_rdflib_content
13
14
 
14
15
 
15
16
  def parse_owl_metadata(graph: Graph) -> dict:
@@ -5,7 +5,7 @@ import pandas as pd
5
5
  from rdflib import Graph
6
6
 
7
7
  from cognite.neat.rules.models._base import MatchType
8
- from cognite.neat.utils.utils import remove_namespace_from_uri
8
+ from cognite.neat.utils.rdf_ import remove_namespace_from_uri
9
9
 
10
10
  from ._owl2classes import _data_type_property_class, _object_property_class, _thing_class
11
11
 
@@ -295,6 +295,41 @@ class ClassNoPropertiesNoParentError(NeatValidationError):
295
295
  return f"Class {self.classes[0]} have no direct or inherited properties. This may be a mistake."
296
296
 
297
297
 
298
+ @dataclass(frozen=True)
299
+ class AssetRulesHaveCircularDependencyError(NeatValidationError):
300
+ description = "Asset rules have circular dependencies."
301
+ fix = "Linking between classes via property that maps to parent_external_id must yield hierarchy structure."
302
+
303
+ classes: list[str]
304
+
305
+ def dump(self) -> dict[str, list[tuple[str, str]]]:
306
+ output = super().dump()
307
+ output["classes"] = self.classes
308
+ return output
309
+
310
+ def message(self) -> str:
311
+ return f"Asset rules have circular dependencies between classes {', '.join(self.classes)}."
312
+
313
+
314
+ @dataclass(frozen=True)
315
+ class AssetParentPropertyPointsToDataValueTypeError(NeatValidationError):
316
+ description = "Parent property points to a data value type instead of a class."
317
+ fix = "Make sure that the parent property points to a class."
318
+
319
+ class_property_with_data_value_type: list[tuple[str, str]]
320
+
321
+ def dump(self) -> dict[str, list[tuple[str, str]]]:
322
+ output = super().dump()
323
+ output["class_property"] = self.class_property_with_data_value_type
324
+ return output
325
+
326
+ def message(self) -> str:
327
+ text = [
328
+ f"class {class_} property {property_}" for class_, property_ in self.class_property_with_data_value_type
329
+ ]
330
+ return f"Following {', and'.join(text)} point to data value type instead to classes. This is a mistake."
331
+
332
+
298
333
  @dataclass(frozen=True)
299
334
  class ParentClassesNotDefinedError(NeatValidationError):
300
335
  description = "Parent classes are not defined."
@@ -7,7 +7,7 @@ from __future__ import annotations
7
7
  import math
8
8
  import sys
9
9
  import types
10
- from abc import abstractmethod
10
+ from abc import ABC, abstractmethod
11
11
  from collections.abc import Callable, Iterator
12
12
  from functools import wraps
13
13
  from typing import Annotated, Any, ClassVar, Generic, Literal, TypeAlias, TypeVar
@@ -18,7 +18,6 @@ from pydantic import (
18
18
  BeforeValidator,
19
19
  ConfigDict,
20
20
  Field,
21
- HttpUrl,
22
21
  PlainSerializer,
23
22
  constr,
24
23
  field_validator,
@@ -219,10 +218,6 @@ class RuleModel(BaseModel):
219
218
  return headers_by_sheet
220
219
 
221
220
 
222
- class URL(BaseModel):
223
- url: HttpUrl
224
-
225
-
226
221
  class BaseMetadata(RuleModel):
227
222
  """
228
223
  Metadata model for data model
@@ -252,8 +247,13 @@ class BaseMetadata(RuleModel):
252
247
  """Returns a unique identifier for the metadata."""
253
248
  raise NotImplementedError()
254
249
 
250
+ @abstractmethod
251
+ def get_prefix(self) -> str:
252
+ """Returns the prefix for the metadata."""
253
+ raise NotImplementedError()
254
+
255
255
 
256
- class BaseRules(RuleModel):
256
+ class BaseRules(RuleModel, ABC):
257
257
  """
258
258
  Rules is a core concept in `neat`. This represents fusion of data model
259
259
  definitions and (optionally) the transformation rules used to transform the data/graph
@@ -232,21 +232,12 @@ class SingleProperty(Traversal):
232
232
  return f"{self.class_}({self.property})"
233
233
 
234
234
 
235
- class AllReferences(Traversal):
235
+ class SelfReferenceProperty(Traversal):
236
236
  @classmethod
237
237
  def from_string(cls, class_: str) -> Self:
238
238
  return cls(class_=Entity.from_string(class_))
239
239
 
240
240
 
241
- class AllProperties(Traversal):
242
- @classmethod
243
- def from_string(cls, class_: str) -> Self:
244
- return cls(class_=Entity.from_string(class_))
245
-
246
- def __str__(self) -> str:
247
- return f"{self.class_}(*)"
248
-
249
-
250
241
  class Origin(BaseModel):
251
242
  class_: Entity
252
243
 
@@ -290,7 +281,7 @@ class Query(BaseModel):
290
281
 
291
282
 
292
283
  class RDFPath(Rule):
293
- traversal: SingleProperty | AllProperties | AllReferences | Hop
284
+ traversal: SingleProperty | SelfReferenceProperty | Hop
294
285
 
295
286
  def __str__(self) -> str:
296
287
  return f"{self.traversal}"
@@ -311,14 +302,13 @@ class SPARQLQuery(RDFPath):
311
302
  traversal: Query
312
303
 
313
304
 
314
- def parse_traversal(raw: str) -> AllReferences | AllProperties | SingleProperty | Hop:
305
+ def parse_traversal(raw: str) -> SelfReferenceProperty | SingleProperty | Hop:
315
306
  if result := CLASS_ID_REGEX_COMPILED.match(raw):
316
- return AllReferences.from_string(class_=result.group(EntityTypes.class_))
317
- elif result := ALL_PROPERTIES_REGEX_COMPILED.match(raw):
318
- return AllProperties.from_string(class_=result.group(EntityTypes.class_))
307
+ return SelfReferenceProperty.from_string(class_=result.group(EntityTypes.class_))
319
308
  elif result := SINGLE_PROPERTY_REGEX_COMPILED.match(raw):
320
309
  return SingleProperty.from_string(
321
- class_=result.group(EntityTypes.class_), property_=result.group(EntityTypes.property_)
310
+ class_=result.group(EntityTypes.class_),
311
+ property_=result.group(EntityTypes.property_),
322
312
  )
323
313
  elif result := HOP_REGEX_COMPILED.match(raw):
324
314
  return Hop.from_string(class_=result.group("origin"), traversal=result.group(_traversal))
@@ -329,7 +319,9 @@ def parse_traversal(raw: str) -> AllReferences | AllProperties | SingleProperty
329
319
  def parse_table_lookup(raw: str) -> TableLookup:
330
320
  if result := TABLE_REGEX_COMPILED.match(raw):
331
321
  return TableLookup(
332
- name=result.group(Lookup.table), key=result.group(Lookup.key), value=result.group(Lookup.value)
322
+ name=result.group(Lookup.table),
323
+ key=result.group(Lookup.key),
324
+ value=result.group(Lookup.value),
333
325
  )
334
326
  raise exceptions.NotValidTableLookUp(raw).to_pydantic_custom_error()
335
327
 
@@ -344,7 +336,10 @@ def parse_rule(rule_raw: str, rule_type: TransformationRuleType | None) -> RDFPa
344
336
  if Counter(rule_raw).get("|") != 1:
345
337
  raise exceptions.NotValidRAWLookUp(rule_raw).to_pydantic_custom_error()
346
338
  traversal, table_lookup = rule_raw.split("|")
347
- return RawLookup(traversal=parse_traversal(traversal), table=parse_table_lookup(table_lookup))
339
+ return RawLookup(
340
+ traversal=parse_traversal(traversal),
341
+ table=parse_table_lookup(table_lookup),
342
+ )
348
343
  case TransformationRuleType.sparql:
349
344
  return SPARQLQuery(traversal=Query(query=rule_raw))
350
345
  case None:
@@ -352,9 +347,10 @@ def parse_rule(rule_raw: str, rule_type: TransformationRuleType | None) -> RDFPa
352
347
 
353
348
 
354
349
  def is_valid_rule(rule_type: TransformationRuleType, rule_raw: str) -> bool:
355
- is_valid_rule = {TransformationRuleType.rdfpath: is_rdfpath, TransformationRuleType.rawlookup: is_rawlookup}[
356
- rule_type
357
- ]
350
+ is_valid_rule = {
351
+ TransformationRuleType.rdfpath: is_rdfpath,
352
+ TransformationRuleType.rawlookup: is_rawlookup,
353
+ }[rule_type]
358
354
  return is_valid_rule(rule_raw)
359
355
 
360
356
 
@@ -5,7 +5,7 @@ from pydantic import Field, field_validator, model_validator
5
5
  from pydantic.main import IncEx
6
6
  from rdflib import Namespace
7
7
 
8
- from cognite.neat.constants import PREFIXES
8
+ from cognite.neat.constants import get_default_prefixes
9
9
  from cognite.neat.issues import MultiValueError
10
10
  from cognite.neat.rules import issues
11
11
  from cognite.neat.rules.models._base import BaseRules, RoleTypes, SheetList
@@ -14,7 +14,6 @@ from cognite.neat.rules.models.entities import (
14
14
  CdfResourceEntityList,
15
15
  ClassEntity,
16
16
  MultiValueTypeInfo,
17
- ParentClassEntity,
18
17
  Undefined,
19
18
  )
20
19
  from cognite.neat.rules.models.information import (
@@ -61,14 +60,14 @@ class AssetProperty(InformationProperty):
61
60
  implementation: Details on how given class-property is implemented in the classic CDF
62
61
  """
63
62
 
64
- implementation: CdfResourceEntityList | None = Field(alias="Implementation", default=None)
63
+ implementation: CdfResourceEntityList = Field(alias="Implementation")
65
64
 
66
65
 
67
66
  class AssetRules(BaseRules):
68
67
  metadata: AssetMetadata = Field(alias="Metadata")
69
68
  properties: SheetList[AssetProperty] = Field(alias="Properties")
70
69
  classes: SheetList[AssetClass] = Field(alias="Classes")
71
- prefixes: dict[str, Namespace] = Field(default_factory=lambda: PREFIXES.copy())
70
+ prefixes: dict[str, Namespace] = Field(default_factory=get_default_prefixes)
72
71
  last: "AssetRules | None" = Field(None, alias="Last")
73
72
  reference: "AssetRules | None" = Field(None, alias="Reference")
74
73
 
@@ -94,7 +93,7 @@ class AssetRules(BaseRules):
94
93
  # update parent classes
95
94
  for class_ in self.classes:
96
95
  if class_.parent:
97
- for parent in cast(list[ParentClassEntity], class_.parent):
96
+ for parent in class_.parent:
98
97
  if not isinstance(parent.prefix, str):
99
98
  parent.prefix = self.metadata.prefix
100
99
  if class_.class_.prefix is Undefined:
@@ -1,4 +1,41 @@
1
+ from graphlib import CycleError
2
+ from typing import cast
3
+
4
+ from cognite.neat.rules import issues
5
+ from cognite.neat.rules.issues.base import IssueList
6
+ from cognite.neat.rules.models._base import SheetList
7
+ from cognite.neat.rules.models.asset._rules import AssetProperty, AssetRules
8
+ from cognite.neat.rules.models.entities import AssetEntity, AssetFields, ClassEntity
1
9
  from cognite.neat.rules.models.information._validation import InformationPostValidation
2
10
 
3
11
 
4
- class AssetPostValidation(InformationPostValidation): ...
12
+ class AssetPostValidation(InformationPostValidation):
13
+ def validate(self) -> IssueList:
14
+ self.issue_list = super().validate()
15
+ self._parent_property_point_to_class()
16
+ self._circular_dependency()
17
+ return self.issue_list
18
+
19
+ def _parent_property_point_to_class(self) -> None:
20
+ class_property_with_data_value_type = []
21
+ for property_ in cast(SheetList[AssetProperty], self.properties):
22
+ for implementation in property_.implementation:
23
+ if (
24
+ isinstance(implementation, AssetEntity)
25
+ and implementation.property_ == AssetFields.parentExternalId
26
+ and not isinstance(property_.value_type, ClassEntity)
27
+ ):
28
+ class_property_with_data_value_type.append((property_.class_.suffix, property_.property_))
29
+
30
+ if class_property_with_data_value_type:
31
+ self.issue_list.append(
32
+ issues.spreadsheet.AssetParentPropertyPointsToDataValueTypeError(class_property_with_data_value_type)
33
+ )
34
+
35
+ def _circular_dependency(self) -> None:
36
+ from cognite.neat.rules.analysis import AssetAnalysis
37
+
38
+ try:
39
+ _ = AssetAnalysis(cast(AssetRules, self.rules)).class_topological_sort()
40
+ except CycleError as error:
41
+ self.issue_list.append(issues.spreadsheet.AssetRulesHaveCircularDependencyError(error.args[1]))
@@ -11,7 +11,6 @@ from cognite.neat.rules.models.entities import (
11
11
  ClassEntity,
12
12
  ContainerEntity,
13
13
  DMSUnknownEntity,
14
- ParentClassEntity,
15
14
  ReferenceEntity,
16
15
  UnknownEntity,
17
16
  ViewEntity,
@@ -52,7 +51,7 @@ class _DMSRulesConverter:
52
51
  description=view.description,
53
52
  parent=[
54
53
  # we do not want a version in class as we use URI for the class
55
- ParentClassEntity(prefix=implemented_view.prefix, suffix=implemented_view.suffix)
54
+ implemented_view.as_class(skip_version=True)
56
55
  # We only want parents in the same namespace, parent in a different namespace is a reference
57
56
  for implemented_view in view.implements or []
58
57
  if implemented_view.prefix == view.class_.prefix
@@ -5,6 +5,7 @@ from typing import Any, cast
5
5
 
6
6
  from cognite.client.data_classes import data_modeling as dm
7
7
  from cognite.client.data_classes.data_modeling.containers import BTreeIndex
8
+ from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
8
9
  from cognite.client.data_classes.data_modeling.views import (
9
10
  SingleEdgeConnectionApply,
10
11
  SingleReverseDirectRelationApply,
@@ -23,7 +24,7 @@ from cognite.neat.rules.models.entities import (
23
24
  ViewPropertyEntity,
24
25
  )
25
26
  from cognite.neat.rules.models.wrapped_entities import DMSFilter, HasDataFilter, NodeTypeFilter
26
- from cognite.neat.utils.cdf_classes import ContainerApplyDict, NodeApplyDict, SpaceApplyDict, ViewApplyDict
27
+ from cognite.neat.utils.cdf.data_classes import ContainerApplyDict, NodeApplyDict, SpaceApplyDict, ViewApplyDict
27
28
 
28
29
  from ._rules import DMSMetadata, DMSProperty, DMSRules, DMSView
29
30
  from ._schema import DMSSchema, PipelineSchema
@@ -282,8 +283,11 @@ class _DMSExporter:
282
283
  type_cls = prop.value_type.dms
283
284
  else:
284
285
  type_cls = dm.DirectRelation
285
-
286
- type_ = type_cls(is_list=prop.is_list or False)
286
+ type_: dm.PropertyType
287
+ if issubclass(type_cls, ListablePropertyType):
288
+ type_ = type_cls(is_list=prop.is_list or False)
289
+ else:
290
+ type_ = type_cls()
287
291
  container.properties[prop.container_property] = dm.ContainerProperty(
288
292
  type=type_,
289
293
  nullable=prop.nullable if prop.nullable is not None else True,
@@ -165,6 +165,9 @@ class DMSMetadata(BaseMetadata):
165
165
  updated=datetime.now(),
166
166
  )
167
167
 
168
+ def get_prefix(self) -> str:
169
+ return self.space
170
+
168
171
 
169
172
  class DMSProperty(SheetEntity):
170
173
  view: ViewEntity = Field(alias="View")