cognite-neat 0.88.0__py3-none-any.whl → 0.88.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (99) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/app/api/routers/configuration.py +1 -1
  3. cognite/neat/app/ui/neat-app/build/asset-manifest.json +7 -7
  4. cognite/neat/app/ui/neat-app/build/index.html +1 -1
  5. cognite/neat/app/ui/neat-app/build/static/css/{main.38a62222.css → main.72e3d92e.css} +2 -2
  6. cognite/neat/app/ui/neat-app/build/static/css/main.72e3d92e.css.map +1 -0
  7. cognite/neat/app/ui/neat-app/build/static/js/main.5a52cf09.js +3 -0
  8. cognite/neat/app/ui/neat-app/build/static/js/{main.ec7f72e2.js.LICENSE.txt → main.5a52cf09.js.LICENSE.txt} +0 -9
  9. cognite/neat/app/ui/neat-app/build/static/js/main.5a52cf09.js.map +1 -0
  10. cognite/neat/config.py +44 -27
  11. cognite/neat/exceptions.py +8 -2
  12. cognite/neat/graph/extractors/_classic_cdf/_assets.py +21 -73
  13. cognite/neat/graph/extractors/_classic_cdf/_base.py +102 -0
  14. cognite/neat/graph/extractors/_classic_cdf/_events.py +46 -42
  15. cognite/neat/graph/extractors/_classic_cdf/_files.py +41 -45
  16. cognite/neat/graph/extractors/_classic_cdf/_labels.py +75 -52
  17. cognite/neat/graph/extractors/_classic_cdf/_relationships.py +49 -27
  18. cognite/neat/graph/extractors/_classic_cdf/_sequences.py +47 -50
  19. cognite/neat/graph/extractors/_classic_cdf/_timeseries.py +47 -49
  20. cognite/neat/graph/loaders/_base.py +4 -4
  21. cognite/neat/graph/loaders/_rdf2asset.py +12 -14
  22. cognite/neat/graph/loaders/_rdf2dms.py +14 -10
  23. cognite/neat/graph/queries/_base.py +22 -29
  24. cognite/neat/graph/queries/_shared.py +1 -1
  25. cognite/neat/graph/stores/_base.py +19 -11
  26. cognite/neat/graph/transformers/_rdfpath.py +3 -2
  27. cognite/neat/issues/__init__.py +16 -0
  28. cognite/neat/{issues.py → issues/_base.py} +78 -2
  29. cognite/neat/issues/errors/external.py +21 -0
  30. cognite/neat/issues/errors/properties.py +75 -0
  31. cognite/neat/issues/errors/resources.py +123 -0
  32. cognite/neat/issues/errors/schema.py +0 -0
  33. cognite/neat/{rules/issues → issues}/formatters.py +9 -9
  34. cognite/neat/issues/neat_warnings/__init__.py +2 -0
  35. cognite/neat/issues/neat_warnings/identifier.py +27 -0
  36. cognite/neat/issues/neat_warnings/models.py +22 -0
  37. cognite/neat/issues/neat_warnings/properties.py +77 -0
  38. cognite/neat/issues/neat_warnings/resources.py +125 -0
  39. cognite/neat/rules/exporters/_rules2dms.py +3 -2
  40. cognite/neat/rules/exporters/_rules2ontology.py +28 -20
  41. cognite/neat/rules/exporters/_validation.py +15 -21
  42. cognite/neat/rules/importers/__init__.py +7 -3
  43. cognite/neat/rules/importers/_base.py +3 -3
  44. cognite/neat/rules/importers/_dms2rules.py +39 -18
  45. cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +44 -53
  46. cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +6 -5
  47. cognite/neat/rules/importers/_rdf/__init__.py +0 -0
  48. cognite/neat/rules/importers/_rdf/_imf2rules/__init__.py +3 -0
  49. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +82 -0
  50. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +34 -0
  51. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +123 -0
  52. cognite/neat/rules/importers/{_owl2rules/_owl2rules.py → _rdf/_imf2rules/_imf2rules.py} +15 -11
  53. cognite/neat/rules/importers/{_inference2rules.py → _rdf/_inference2rules.py} +1 -1
  54. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +57 -0
  55. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2metadata.py +68 -0
  56. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +59 -0
  57. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +76 -0
  58. cognite/neat/rules/importers/_rdf/_shared.py +586 -0
  59. cognite/neat/rules/importers/_spreadsheet2rules.py +31 -28
  60. cognite/neat/rules/importers/_yaml2rules.py +2 -1
  61. cognite/neat/rules/issues/__init__.py +1 -5
  62. cognite/neat/rules/issues/base.py +2 -21
  63. cognite/neat/rules/issues/dms.py +20 -134
  64. cognite/neat/rules/issues/ontology.py +298 -0
  65. cognite/neat/rules/issues/spreadsheet.py +51 -3
  66. cognite/neat/rules/issues/tables.py +72 -0
  67. cognite/neat/rules/models/_rdfpath.py +4 -4
  68. cognite/neat/rules/models/_types/_field.py +14 -21
  69. cognite/neat/rules/models/asset/_validation.py +1 -1
  70. cognite/neat/rules/models/dms/_schema.py +53 -30
  71. cognite/neat/rules/models/dms/_validation.py +2 -2
  72. cognite/neat/rules/models/entities.py +3 -0
  73. cognite/neat/rules/models/information/_rules.py +5 -4
  74. cognite/neat/rules/models/information/_validation.py +1 -1
  75. cognite/neat/utils/rdf_.py +17 -9
  76. cognite/neat/utils/regex_patterns.py +52 -0
  77. cognite/neat/workflows/steps/lib/current/rules_importer.py +73 -1
  78. cognite/neat/workflows/steps/lib/current/rules_validator.py +19 -7
  79. {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/METADATA +2 -6
  80. {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/RECORD +85 -72
  81. cognite/neat/app/ui/neat-app/build/static/css/main.38a62222.css.map +0 -1
  82. cognite/neat/app/ui/neat-app/build/static/js/main.ec7f72e2.js +0 -3
  83. cognite/neat/app/ui/neat-app/build/static/js/main.ec7f72e2.js.map +0 -1
  84. cognite/neat/graph/issues/loader.py +0 -104
  85. cognite/neat/graph/stores/_oxrdflib.py +0 -247
  86. cognite/neat/rules/exceptions.py +0 -2972
  87. cognite/neat/rules/importers/_owl2rules/_owl2classes.py +0 -215
  88. cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +0 -213
  89. cognite/neat/rules/importers/_owl2rules/_owl2properties.py +0 -203
  90. cognite/neat/rules/issues/importing.py +0 -408
  91. cognite/neat/rules/models/_types/_base.py +0 -16
  92. cognite/neat/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +0 -152
  93. cognite/neat/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +0 -139
  94. cognite/neat/workflows/examples/Ontology_to_Data_Model/workflow.yaml +0 -116
  95. /cognite/neat/{graph/issues → issues/errors}/__init__.py +0 -0
  96. /cognite/neat/rules/importers/{_owl2rules → _rdf/_owl2rules}/__init__.py +0 -0
  97. {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/LICENSE +0 -0
  98. {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/WHEEL +0 -0
  99. {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/entry_points.txt +0 -0
@@ -1,11 +1,11 @@
1
- import re
2
1
  import warnings
3
2
  from typing import Literal, overload
4
3
 
5
4
  from cognite.neat.exceptions import wrangle_warnings
6
- from cognite.neat.rules import exceptions
5
+ from cognite.neat.issues.neat_warnings.properties import PropertyRedefinedWarning
6
+ from cognite.neat.rules.issues.dms import EntityIDNotDMSCompliantWarning
7
7
  from cognite.neat.rules.models import InformationRules
8
- from cognite.neat.rules.models._types._base import DMS_PROPERTY_ID_COMPLIANCE_REGEX, VIEW_ID_COMPLIANCE_REGEX
8
+ from cognite.neat.utils.regex_patterns import DMS_PROPERTY_ID_COMPLIANCE_REGEX, PATTERNS, VIEW_ID_COMPLIANCE_REGEX
9
9
 
10
10
 
11
11
  @overload
@@ -26,35 +26,30 @@ def are_entity_names_dms_compliant(
26
26
  flag: bool = True
27
27
  with warnings.catch_warnings(record=True) as validation_warnings:
28
28
  for class_ in rules.classes:
29
- if not re.match(VIEW_ID_COMPLIANCE_REGEX, str(class_.class_.suffix)):
29
+ if not PATTERNS.view_id_compliance.match(class_.class_.suffix):
30
30
  warnings.warn(
31
- exceptions.EntityIDNotDMSCompliant(
32
- "Class", class_.class_.versioned_id, f"[Classes/Class/{class_.class_.versioned_id}]"
33
- ).message,
34
- category=exceptions.EntityIDNotDMSCompliant,
31
+ EntityIDNotDMSCompliantWarning(class_.class_.versioned_id, "Class", VIEW_ID_COMPLIANCE_REGEX),
35
32
  stacklevel=2,
36
33
  )
37
34
  flag = False
38
35
 
39
- for row, property_ in enumerate(rules.properties):
36
+ for _, property_ in enumerate(rules.properties):
40
37
  # check class id which would resolve as view/container id
41
- if not re.match(VIEW_ID_COMPLIANCE_REGEX, str(property_.class_.suffix)):
38
+ if not PATTERNS.view_id_compliance.match(property_.class_.suffix):
42
39
  warnings.warn(
43
- exceptions.EntityIDNotDMSCompliant(
44
- "Class", property_.class_.versioned_id, f"[Properties/Class/{row}]"
45
- ).message,
46
- category=exceptions.EntityIDNotDMSCompliant,
40
+ EntityIDNotDMSCompliantWarning(
41
+ property_.class_.versioned_id,
42
+ "Class",
43
+ VIEW_ID_COMPLIANCE_REGEX,
44
+ ),
47
45
  stacklevel=2,
48
46
  )
49
47
  flag = False
50
48
 
51
49
  # check property id which would resolve as view/container id
52
- if not re.match(DMS_PROPERTY_ID_COMPLIANCE_REGEX, property_.property_):
50
+ if not PATTERNS.dms_property_id_compliance.match(property_.property_):
53
51
  warnings.warn(
54
- exceptions.EntityIDNotDMSCompliant(
55
- "Property", property_.property_, f"[Properties/Property/{row}]"
56
- ).message,
57
- category=exceptions.EntityIDNotDMSCompliant,
52
+ EntityIDNotDMSCompliantWarning(property_.property_, "Property", DMS_PROPERTY_ID_COMPLIANCE_REGEX),
58
53
  stacklevel=2,
59
54
  )
60
55
  flag = False
@@ -83,8 +78,7 @@ def are_properties_redefined(rules: InformationRules, return_report: bool = Fals
83
78
  elif property_.class_ in analyzed_properties[property_.property_]:
84
79
  flag = True
85
80
  warnings.warn(
86
- exceptions.PropertyRedefined(property_.property_, property_.class_.versioned_id).message,
87
- category=exceptions.EntityIDNotDMSCompliant,
81
+ PropertyRedefinedWarning[str](property_.class_.versioned_id, "Class", property_.property_),
88
82
  stacklevel=2,
89
83
  )
90
84
 
@@ -1,14 +1,16 @@
1
1
  from ._base import BaseImporter
2
2
  from ._dms2rules import DMSImporter
3
3
  from ._dtdl2rules import DTDLImporter
4
- from ._inference2rules import InferenceImporter
5
- from ._owl2rules import OWLImporter
4
+ from ._rdf._imf2rules import IMFImporter
5
+ from ._rdf._inference2rules import InferenceImporter
6
+ from ._rdf._owl2rules import OWLImporter
6
7
  from ._spreadsheet2rules import ExcelImporter, GoogleSheetImporter
7
8
  from ._yaml2rules import YAMLImporter
8
9
 
9
10
  __all__ = [
10
11
  "BaseImporter",
11
12
  "OWLImporter",
13
+ "IMFImporter",
12
14
  "DMSImporter",
13
15
  "ExcelImporter",
14
16
  "GoogleSheetImporter",
@@ -25,7 +27,9 @@ def _repr_html_() -> str:
25
27
  [
26
28
  {
27
29
  "Importer": name,
28
- "Description": globals()[name].__doc__.strip().split("\n")[0] if globals()[name].__doc__ else "Missing",
30
+ "Description": (
31
+ globals()[name].__doc__.strip().split("\n")[0] if globals()[name].__doc__ else "Missing"
32
+ ),
29
33
  }
30
34
  for name in __all__
31
35
  if name != "BaseImporter"
@@ -9,9 +9,9 @@ from typing import Any, Literal, overload
9
9
  from pydantic import ValidationError
10
10
  from rdflib import Namespace
11
11
 
12
+ from cognite.neat.issues import IssueList, NeatError, NeatWarning
12
13
  from cognite.neat.rules._shared import Rules
13
14
  from cognite.neat.rules.issues.base import (
14
- IssueList,
15
15
  NeatValidationError,
16
16
  ValidationWarning,
17
17
  )
@@ -103,8 +103,8 @@ class _FutureResult:
103
103
  @contextmanager
104
104
  def _handle_issues(
105
105
  issues: IssueList,
106
- error_cls: type[NeatValidationError] = NeatValidationError,
107
- warning_cls: type[ValidationWarning] = ValidationWarning,
106
+ error_cls: type[NeatError] = NeatValidationError,
107
+ warning_cls: type[NeatWarning] = ValidationWarning,
108
108
  error_args: dict[str, Any] | None = None,
109
109
  ) -> Iterator[_FutureResult]:
110
110
  """This is an internal help function to handle issues and warnings.
@@ -18,9 +18,15 @@ from cognite.client.data_classes.data_modeling.views import (
18
18
  )
19
19
  from cognite.client.utils import ms_to_datetime
20
20
 
21
+ from cognite.neat.issues import IssueList, NeatIssue
22
+ from cognite.neat.issues.errors.resources import ResourceNotFoundError
23
+ from cognite.neat.issues.neat_warnings.properties import (
24
+ PropertyTypeNotSupportedWarning,
25
+ ReferredPropertyNotFoundWarning,
26
+ )
27
+ from cognite.neat.issues.neat_warnings.resources import ReferredResourceNotFoundWarning
21
28
  from cognite.neat.rules import issues
22
29
  from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
23
- from cognite.neat.rules.issues import IssueList, ValidationIssue
24
30
  from cognite.neat.rules.models import (
25
31
  DataModelType,
26
32
  DMSRules,
@@ -60,7 +66,7 @@ class DMSImporter(BaseImporter):
60
66
  def __init__(
61
67
  self,
62
68
  schema: DMSSchema,
63
- read_issues: Sequence[ValidationIssue] | None = None,
69
+ read_issues: Sequence[NeatIssue] | None = None,
64
70
  metadata: DMSMetadata | None = None,
65
71
  ref_metadata: DMSMetadata | None = None,
66
72
  ):
@@ -100,14 +106,28 @@ class DMSImporter(BaseImporter):
100
106
 
101
107
  user_models = cls._find_model_in_list(data_models, data_model_id)
102
108
  if len(user_models) == 0:
103
- return cls(DMSSchema(), [issues.importing.NoDataModelError(f"Data model {data_model_id} not found")])
109
+ return cls(
110
+ DMSSchema(),
111
+ [
112
+ ResourceNotFoundError[dm.DataModelId](
113
+ dm.DataModelId.load(reference_model_id), # type: ignore[arg-type]
114
+ "DataModel",
115
+ "Data Model is missing in CDF",
116
+ )
117
+ ],
118
+ )
104
119
  user_model = user_models.latest_version()
105
120
 
106
121
  if reference_model_id:
107
122
  ref_models = cls._find_model_in_list(data_models, reference_model_id)
108
123
  if len(ref_models) == 0:
109
124
  return cls(
110
- DMSSchema(), [issues.importing.NoDataModelError(f"Data model {reference_model_id} not found")]
125
+ DMSSchema(),
126
+ [
127
+ ResourceNotFoundError[dm.DataModelId](
128
+ dm.DataModelId.load(reference_model_id), "DataModel", "Data Model is missing in CDF"
129
+ )
130
+ ],
111
131
  )
112
132
  ref_model: dm.DataModel[dm.View] | None = ref_models.latest_version()
113
133
  else:
@@ -200,7 +220,7 @@ class DMSImporter(BaseImporter):
200
220
  return self._return_or_raise(self.issue_list, errors)
201
221
 
202
222
  if not self.root_schema.data_model:
203
- self.issue_list.append(issues.importing.NoDataModelError("No data model found."))
223
+ self.issue_list.append(ResourceNotFoundError[str]("Unknown", "DataModel", "Identifier is missing"))
204
224
  return self._return_or_raise(self.issue_list, errors)
205
225
  model = self.root_schema.data_model
206
226
  with _handle_issues(
@@ -301,10 +321,11 @@ class DMSImporter(BaseImporter):
301
321
  ) -> DMSProperty | None:
302
322
  if isinstance(prop, dm.MappedPropertyApply) and prop.container not in self._all_containers_by_id:
303
323
  self.issue_list.append(
304
- issues.importing.MissingContainerWarning(
305
- view_id=str(view_entity),
306
- property_=prop_id,
307
- container_id=str(ContainerEntity.from_id(prop.container)),
324
+ ReferredResourceNotFoundWarning[dm.ContainerId, dm.PropertyId](
325
+ dm.ContainerId.load(prop.container),
326
+ "Container",
327
+ view_entity.to_property_id(prop_id),
328
+ "View Property",
308
329
  )
309
330
  )
310
331
  return None
@@ -313,11 +334,9 @@ class DMSImporter(BaseImporter):
313
334
  and prop.container_property_identifier not in self._all_containers_by_id[prop.container].properties
314
335
  ):
315
336
  self.issue_list.append(
316
- issues.importing.MissingContainerPropertyWarning(
317
- view_id=str(view_entity),
318
- property_=prop_id,
319
- container_id=str(ContainerEntity.from_id(prop.container)),
320
- )
337
+ ReferredPropertyNotFoundWarning[dm.ContainerId, dm.ViewId](
338
+ prop.container, "Container", view_entity.as_id(), "View", prop_id
339
+ ),
321
340
  )
322
341
  return None
323
342
  if not isinstance(
@@ -329,7 +348,7 @@ class DMSImporter(BaseImporter):
329
348
  | MultiReverseDirectRelationApply,
330
349
  ):
331
350
  self.issue_list.append(
332
- issues.importing.UnknownPropertyTypeWarning(view_entity.versioned_id, prop_id, type(prop).__name__)
351
+ PropertyTypeNotSupportedWarning[dm.ViewId](view_entity.as_id(), "View", prop_id, type(prop).__name__)
333
352
  )
334
353
  return None
335
354
 
@@ -394,7 +413,9 @@ class DMSImporter(BaseImporter):
394
413
  else:
395
414
  return DataType.load(container_prop.type._type)
396
415
  else:
397
- self.issue_list.append(issues.importing.FailedToInferValueTypeWarning(str(view_entity), prop_id))
416
+ self.issue_list.append(
417
+ PropertyTypeNotSupportedWarning[dm.ViewId](view_entity.as_id(), "View", prop_id, type(prop).__name__)
418
+ )
398
419
  return None
399
420
 
400
421
  def _get_nullable(self, prop: ViewPropertyApply) -> bool | None:
@@ -453,8 +474,8 @@ class DMSImporter(BaseImporter):
453
474
  continue
454
475
  else:
455
476
  self.issue_list.append(
456
- issues.importing.UnknownContainerConstraintWarning(
457
- str(ContainerEntity.from_id(prop.container)), prop_id, type(constraint_obj).__name__
477
+ PropertyTypeNotSupportedWarning[dm.ContainerId](
478
+ prop.container, "Container", prop_id, type(constraint_obj).__name__
458
479
  )
459
480
  )
460
481
  return unique_constraints or None
@@ -1,14 +1,18 @@
1
1
  from collections import Counter
2
2
  from collections.abc import Callable, Sequence
3
3
 
4
- import cognite.neat.rules.issues.importing
5
- from cognite.neat.rules import issues
4
+ from cognite.neat.issues import IssueList, NeatIssue
5
+ from cognite.neat.issues.errors.properties import PropertyTypeNotSupportedError
6
+ from cognite.neat.issues.errors.resources import MissingIdentifierError, ResourceNotFoundError
7
+ from cognite.neat.issues.neat_warnings.properties import PropertyTypeNotSupportedWarning
8
+ from cognite.neat.issues.neat_warnings.resources import ResourceTypeNotSupportedWarning
6
9
  from cognite.neat.rules.importers._dtdl2rules.spec import (
7
10
  DTMI,
8
11
  Command,
9
12
  CommandV2,
10
13
  Component,
11
14
  DTDLBase,
15
+ DTDLBaseWithName,
12
16
  Enum,
13
17
  Interface,
14
18
  Object,
@@ -19,15 +23,14 @@ from cognite.neat.rules.importers._dtdl2rules.spec import (
19
23
  Telemetry,
20
24
  TelemetryV2,
21
25
  )
22
- from cognite.neat.rules.issues import IssueList, ValidationIssue
23
26
  from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_NAME, DataType, Json, String
24
27
  from cognite.neat.rules.models.entities import ClassEntity
25
28
  from cognite.neat.rules.models.information import InformationClass, InformationProperty
26
29
 
27
30
 
28
31
  class _DTDLConverter:
29
- def __init__(self, issues: list[ValidationIssue] | None = None) -> None:
30
- self.issues: IssueList = IssueList(issues or [])
32
+ def __init__(self, issues: list[NeatIssue] | None = None) -> None:
33
+ self.issues = IssueList(issues or [])
31
34
  self.properties: list[InformationProperty] = []
32
35
  self.classes: list[InformationClass] = []
33
36
  self._item_by_id: dict[DTMI, DTDLBase] = {}
@@ -75,11 +78,10 @@ class _DTDLConverter:
75
78
  convert_method(item, parent)
76
79
  else:
77
80
  self.issues.append(
78
- issues.importing.UnknownComponentWarning(
79
- component_type=item.type,
80
- instance_name=item.display_name,
81
- instance_id=item.id_.model_dump() if item.id_ else None,
82
- )
81
+ ResourceTypeNotSupportedWarning[str](
82
+ item.id_.model_dump() if item.id_ else item.display_name or "missing",
83
+ item.type,
84
+ ),
83
85
  )
84
86
 
85
87
  def convert_interface(self, item: Interface, _: str | None) -> None:
@@ -94,11 +96,11 @@ class _DTDLConverter:
94
96
  for sub_item_or_id in item.contents or []:
95
97
  if isinstance(sub_item_or_id, DTMI) and sub_item_or_id not in self._item_by_id:
96
98
  self.issues.append(
97
- issues.importing.UnknownPropertyWarning(
98
- component_type=item.type,
99
- property_name=sub_item_or_id.path[-1],
100
- instance_name=item.display_name,
101
- instance_id=item.id_.model_dump(),
99
+ PropertyTypeNotSupportedWarning(
100
+ item.id_.model_dump() or item.display_name or "missing",
101
+ item.type,
102
+ sub_item_or_id.path[-1],
103
+ ".".join(sub_item_or_id.path),
102
104
  )
103
105
  )
104
106
  elif isinstance(sub_item_or_id, DTMI):
@@ -130,12 +132,10 @@ class _DTDLConverter:
130
132
  )
131
133
  self.properties.append(prop)
132
134
 
133
- def _missing_parent_warning(self, item):
135
+ def _missing_parent_warning(self, item: DTDLBaseWithName):
134
136
  self.issues.append(
135
- cognite.neat.rules.issues.importing.MissingParentDefinitionError(
136
- component_type=item.type,
137
- instance_name=item.display_name,
138
- instance_id=item.id_.model_dump() if item.id_ else None,
137
+ ResourceNotFoundError[str](
138
+ (item.id_.model_dump() if item.id_ else item.display_name) or "missing", item.type, "parent missing"
139
139
  )
140
140
  )
141
141
 
@@ -151,22 +151,15 @@ class _DTDLConverter:
151
151
  return None
152
152
  if item.request is None:
153
153
  self.issues.append(
154
- issues.importing.UnknownSubComponentWarning(
155
- component_type=item.type,
156
- sub_component="request",
157
- instance_name=item.display_name,
158
- instance_id=item.id_.model_dump() if item.id_ else None,
159
- )
154
+ ResourceTypeNotSupportedWarning[str](
155
+ item.id_.model_dump() if item.id_ else item.display_name or "missing",
156
+ f"{item.type}.request",
157
+ ),
160
158
  )
161
159
  return None
162
160
  if item.response is not None:
163
161
  # Currently, we do not know how to handle response
164
- self.issues.append(
165
- issues.importing.IgnoredComponentWarning(
166
- identifier=f"{parent}.response",
167
- reason="Neat does not have a concept of response for commands. This will be ignored.",
168
- )
169
- )
162
+ self.issues.append(ResourceTypeNotSupportedWarning[str](f"{parent}.response", "Command.Response"))
170
163
  value_type = self.schema_to_value_type(item.request.schema_, item)
171
164
  if value_type is None:
172
165
  return
@@ -213,10 +206,9 @@ class _DTDLConverter:
213
206
  else:
214
207
  # Falling back to json
215
208
  self.issues.append(
216
- cognite.neat.rules.issues.importing.MissingIdentifierError(
217
- component_type="Unknown",
218
- instance_name=item.target.model_dump(),
219
- instance_id=item.target.model_dump(),
209
+ MissingIdentifierError(
210
+ "Unknown",
211
+ item.target.model_dump(),
220
212
  )
221
213
  )
222
214
  value_type = Json()
@@ -239,9 +231,9 @@ class _DTDLConverter:
239
231
  def convert_object(self, item: Object, _: str | None) -> None:
240
232
  if item.id_ is None:
241
233
  self.issues.append(
242
- cognite.neat.rules.issues.importing.MissingIdentifierError(
243
- component_type=item.type,
244
- instance_name=item.display_name,
234
+ MissingIdentifierError(
235
+ resource_type=item.type,
236
+ name=item.display_name,
245
237
  )
246
238
  )
247
239
  return None
@@ -280,21 +272,20 @@ class _DTDLConverter:
280
272
  return _DATA_TYPE_BY_NAME[input_type.casefold()]()
281
273
  elif isinstance(input_type, str):
282
274
  self.issues.append(
283
- cognite.neat.rules.issues.importing.UnsupportedPropertyTypeError(
284
- component_type=item.type,
285
- property_type=input_type,
286
- property_name="schema",
287
- instance_name=item.display_name,
288
- instance_id=item.id_.model_dump() if item.id_ else None,
275
+ PropertyTypeNotSupportedError[str](
276
+ (item.id_.model_dump() if item.id_ else item.display_name) or "missing",
277
+ item.type,
278
+ "schema",
279
+ input_type,
289
280
  )
290
281
  )
291
282
  return None
292
283
  elif isinstance(input_type, Object | Interface):
293
284
  if input_type.id_ is None:
294
285
  self.issues.append(
295
- cognite.neat.rules.issues.importing.MissingIdentifierError(
296
- component_type=input_type.type,
297
- instance_name=input_type.display_name,
286
+ MissingIdentifierError(
287
+ input_type.type,
288
+ input_type.display_name,
298
289
  )
299
290
  )
300
291
  return Json()
@@ -304,11 +295,11 @@ class _DTDLConverter:
304
295
  return ClassEntity.load(input_type.id_.as_class_id())
305
296
  else:
306
297
  self.issues.append(
307
- issues.importing.UnknownPropertyWarning(
308
- component_type=item.type,
309
- property_name="schema",
310
- instance_name=item.display_name,
311
- instance_id=item.id_.model_dump() if item.id_ else None,
298
+ PropertyTypeNotSupportedWarning(
299
+ item.id_.model_dump() if item.id_ else item.display_name or "missing",
300
+ item.type,
301
+ "schema",
302
+ input_type.type if input_type else "missing",
312
303
  )
313
304
  )
314
305
  return None
@@ -6,12 +6,13 @@ from typing import Literal, overload
6
6
 
7
7
  from pydantic import ValidationError
8
8
 
9
+ from cognite.neat.issues import IssueList, NeatIssue
9
10
  from cognite.neat.rules import issues
10
11
  from cognite.neat.rules._shared import Rules
11
12
  from cognite.neat.rules.importers._base import BaseImporter, _handle_issues
12
13
  from cognite.neat.rules.importers._dtdl2rules.dtdl_converter import _DTDLConverter
13
14
  from cognite.neat.rules.importers._dtdl2rules.spec import DTDL_CLS_BY_TYPE_BY_SPEC, DTDLBase, Interface
14
- from cognite.neat.rules.issues import IssueList, ValidationIssue
15
+ from cognite.neat.rules.issues import ValidationIssue
15
16
  from cognite.neat.rules.models import InformationRules, RoleTypes, SchemaCompleteness, SheetList
16
17
  from cognite.neat.rules.models.information import InformationClass, InformationProperty
17
18
  from cognite.neat.utils.text import to_pascal
@@ -36,7 +37,7 @@ class DTDLImporter(BaseImporter):
36
37
  self,
37
38
  items: Sequence[DTDLBase],
38
39
  title: str | None = None,
39
- read_issues: list[ValidationIssue] | None = None,
40
+ read_issues: list[NeatIssue] | None = None,
40
41
  schema: SchemaCompleteness = SchemaCompleteness.partial,
41
42
  ) -> None:
42
43
  self._items = items
@@ -94,10 +95,10 @@ class DTDLImporter(BaseImporter):
94
95
  @classmethod
95
96
  def from_directory(cls, directory: Path) -> "DTDLImporter":
96
97
  items: list[DTDLBase] = []
97
- issues: list[ValidationIssue] = []
98
+ issues: list[NeatIssue] = []
98
99
  for filepath in directory.glob("**/*.json"):
99
100
  for item in cls._from_file_content(filepath.read_text(), filepath):
100
- if isinstance(item, ValidationIssue):
101
+ if isinstance(item, NeatIssue):
101
102
  issues.append(item)
102
103
  else:
103
104
  items.append(item)
@@ -106,7 +107,7 @@ class DTDLImporter(BaseImporter):
106
107
  @classmethod
107
108
  def from_zip(cls, zip_file: Path) -> "DTDLImporter":
108
109
  items: list[DTDLBase] = []
109
- issues: list[ValidationIssue] = []
110
+ issues: list[NeatIssue] = []
110
111
  with zipfile.ZipFile(zip_file) as z:
111
112
  for filepath in z.namelist():
112
113
  if filepath.endswith(".json"):
File without changes
@@ -0,0 +1,3 @@
1
+ from ._imf2rules import IMFImporter
2
+
3
+ __all__ = ["IMFImporter"]
@@ -0,0 +1,82 @@
1
+ from typing import cast
2
+
3
+ from rdflib import Graph
4
+
5
+ from cognite.neat.rules.importers._rdf._shared import (
6
+ clean_up_classes,
7
+ make_classes_compliant,
8
+ parse_raw_classes_dataframe,
9
+ )
10
+
11
+
12
+ def parse_imf_to_classes(graph: Graph, language: str = "en") -> list[dict]:
13
+ """Parse IMF elements from RDF-graph and extract classes to pandas dataframe.
14
+
15
+ Args:
16
+ graph: Graph containing imf elements
17
+ language: Language to use for parsing, by default "en"
18
+
19
+ Returns:
20
+ Dataframe containing imf elements
21
+
22
+ !!! note "IMF Compliance"
23
+ The IMF elements are expressed in RDF, primarily using SHACL and OWL. To ensure
24
+ that the resulting classes are compliant with CDF, similar validation checks as
25
+ in the OWL ontology importer are applied.
26
+
27
+ For the IMF-types more of the compliance logic is placed directly in the SPARQL
28
+ query. Among these are the creation of class name not starting with a number,
29
+ and ensuring that all classes have a parent.
30
+
31
+ IMF-attributes are considered both classes and properties. This kind of punning
32
+ is necessary to capture additional information carried by attributes. They carry,
33
+ among other things, a set of relationsships to reference terms, units of measure,
34
+ and qualifiers that together make up the meaning of the attribute.
35
+ """
36
+
37
+ query = """
38
+ SELECT ?class ?name ?description ?parentClass ?reference ?match ?comment
39
+ WHERE {
40
+ #Finding IMF - elements
41
+ VALUES ?type { imf:BlockType imf:TerminalType imf:AttributeType }
42
+ ?imfClass a ?type .
43
+ OPTIONAL {?imfClass rdfs:subClassOf ?parent }.
44
+ OPTIONAL {?imfClass rdfs:label | skos:prefLabel ?name }.
45
+ OPTIONAL {?imfClass rdfs:comment | skos:description ?description} .
46
+
47
+ # Finding the last segment of the class IRI
48
+ BIND(STR(?imfClass) AS ?classString)
49
+ BIND(REPLACE(?classString, "^.*[/#]([^/#]*)$", "$1") AS ?classSegment)
50
+ BIND(IF(CONTAINS(?classString, "imf/"), CONCAT("IMF_", ?classSegment) , ?classSegment) AS ?class)
51
+
52
+ # Add imf:Attribute as parent class
53
+ BIND(IF(!bound(?parent) && ?type = imf:AttributeType, imf:Attribute, ?parent) AS ?parentClass)
54
+
55
+ # Rebind the IRI of the IMF-type to the ?reference variable to align with dataframe column headers
56
+ # This is solely for readability, the ?imfClass could have been returnered directly instead of ?reference
57
+ BIND(?imfClass AS ?reference)
58
+
59
+ FILTER (!isBlank(?class))
60
+ FILTER (!bound(?parentClass) || !isBlank(?parentClass))
61
+ FILTER (!bound(?name) || LANG(?name) = "" || LANGMATCHES(LANG(?name), "en"))
62
+ FILTER (!bound(?description) || LANG(?description) = "" || LANGMATCHES(LANG(?description), "en"))
63
+ }
64
+ """
65
+
66
+ # create raw dataframe
67
+ raw_df = parse_raw_classes_dataframe(cast(list[tuple], list(graph.query(query.replace("en", language)))))
68
+ if raw_df.empty:
69
+ return []
70
+
71
+ # group values and clean up
72
+ processed_df = clean_up_classes(raw_df)
73
+
74
+ # make compliant
75
+ processed_df = make_classes_compliant(processed_df, importer="IMF")
76
+
77
+ # Make Parent Class list elements into string joined with comma
78
+ processed_df["Parent Class"] = processed_df["Parent Class"].apply(
79
+ lambda x: ", ".join(x) if isinstance(x, list) and x else None
80
+ )
81
+
82
+ return processed_df.dropna(axis=0, how="all").replace(float("nan"), None).to_dict(orient="records")
@@ -0,0 +1,34 @@
1
+ from rdflib import Namespace
2
+
3
+ from cognite.neat.rules.importers._rdf._shared import make_metadata_compliant
4
+ from cognite.neat.rules.models import RoleTypes, SchemaCompleteness
5
+
6
+
7
+ def parse_imf_metadata() -> dict:
8
+ """Provide hardcoded IMF metadata to dict.
9
+
10
+ Returns:
11
+ Dictionary containing IMF metadata
12
+
13
+ !!! note "Compliant IMF metadata"
14
+ The current RDF provide IMF types as SHACL, but there are not any metadata describing
15
+ the actual content.
16
+
17
+ """
18
+
19
+ raw_metadata = {
20
+ "role": RoleTypes.information,
21
+ "schema": SchemaCompleteness.partial,
22
+ "prefix": "pca-imf",
23
+ "namespace": Namespace("http://posccaesar.org/imf/"),
24
+ "version": None,
25
+ "created": None,
26
+ "updated": None,
27
+ "title": "IMF - types",
28
+ "description": "IMF - types",
29
+ "creator": None,
30
+ "rights": None,
31
+ "license": None,
32
+ }
33
+
34
+ return make_metadata_compliant(raw_metadata)