cognite-neat 0.88.0__py3-none-any.whl → 0.88.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (99) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/app/api/routers/configuration.py +1 -1
  3. cognite/neat/app/ui/neat-app/build/asset-manifest.json +7 -7
  4. cognite/neat/app/ui/neat-app/build/index.html +1 -1
  5. cognite/neat/app/ui/neat-app/build/static/css/{main.38a62222.css → main.72e3d92e.css} +2 -2
  6. cognite/neat/app/ui/neat-app/build/static/css/main.72e3d92e.css.map +1 -0
  7. cognite/neat/app/ui/neat-app/build/static/js/main.5a52cf09.js +3 -0
  8. cognite/neat/app/ui/neat-app/build/static/js/{main.ec7f72e2.js.LICENSE.txt → main.5a52cf09.js.LICENSE.txt} +0 -9
  9. cognite/neat/app/ui/neat-app/build/static/js/main.5a52cf09.js.map +1 -0
  10. cognite/neat/config.py +44 -27
  11. cognite/neat/exceptions.py +8 -2
  12. cognite/neat/graph/extractors/_classic_cdf/_assets.py +21 -73
  13. cognite/neat/graph/extractors/_classic_cdf/_base.py +102 -0
  14. cognite/neat/graph/extractors/_classic_cdf/_events.py +46 -42
  15. cognite/neat/graph/extractors/_classic_cdf/_files.py +41 -45
  16. cognite/neat/graph/extractors/_classic_cdf/_labels.py +75 -52
  17. cognite/neat/graph/extractors/_classic_cdf/_relationships.py +49 -27
  18. cognite/neat/graph/extractors/_classic_cdf/_sequences.py +47 -50
  19. cognite/neat/graph/extractors/_classic_cdf/_timeseries.py +47 -49
  20. cognite/neat/graph/loaders/_base.py +4 -4
  21. cognite/neat/graph/loaders/_rdf2asset.py +12 -14
  22. cognite/neat/graph/loaders/_rdf2dms.py +14 -10
  23. cognite/neat/graph/queries/_base.py +22 -29
  24. cognite/neat/graph/queries/_shared.py +1 -1
  25. cognite/neat/graph/stores/_base.py +19 -11
  26. cognite/neat/graph/transformers/_rdfpath.py +3 -2
  27. cognite/neat/issues/__init__.py +16 -0
  28. cognite/neat/{issues.py → issues/_base.py} +78 -2
  29. cognite/neat/issues/errors/external.py +21 -0
  30. cognite/neat/issues/errors/properties.py +75 -0
  31. cognite/neat/issues/errors/resources.py +123 -0
  32. cognite/neat/issues/errors/schema.py +0 -0
  33. cognite/neat/{rules/issues → issues}/formatters.py +9 -9
  34. cognite/neat/issues/neat_warnings/__init__.py +2 -0
  35. cognite/neat/issues/neat_warnings/identifier.py +27 -0
  36. cognite/neat/issues/neat_warnings/models.py +22 -0
  37. cognite/neat/issues/neat_warnings/properties.py +77 -0
  38. cognite/neat/issues/neat_warnings/resources.py +125 -0
  39. cognite/neat/rules/exporters/_rules2dms.py +3 -2
  40. cognite/neat/rules/exporters/_rules2ontology.py +28 -20
  41. cognite/neat/rules/exporters/_validation.py +15 -21
  42. cognite/neat/rules/importers/__init__.py +7 -3
  43. cognite/neat/rules/importers/_base.py +3 -3
  44. cognite/neat/rules/importers/_dms2rules.py +39 -18
  45. cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +44 -53
  46. cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +6 -5
  47. cognite/neat/rules/importers/_rdf/__init__.py +0 -0
  48. cognite/neat/rules/importers/_rdf/_imf2rules/__init__.py +3 -0
  49. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +82 -0
  50. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +34 -0
  51. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +123 -0
  52. cognite/neat/rules/importers/{_owl2rules/_owl2rules.py → _rdf/_imf2rules/_imf2rules.py} +15 -11
  53. cognite/neat/rules/importers/{_inference2rules.py → _rdf/_inference2rules.py} +1 -1
  54. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +57 -0
  55. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2metadata.py +68 -0
  56. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +59 -0
  57. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +76 -0
  58. cognite/neat/rules/importers/_rdf/_shared.py +586 -0
  59. cognite/neat/rules/importers/_spreadsheet2rules.py +31 -28
  60. cognite/neat/rules/importers/_yaml2rules.py +2 -1
  61. cognite/neat/rules/issues/__init__.py +1 -5
  62. cognite/neat/rules/issues/base.py +2 -21
  63. cognite/neat/rules/issues/dms.py +20 -134
  64. cognite/neat/rules/issues/ontology.py +298 -0
  65. cognite/neat/rules/issues/spreadsheet.py +51 -3
  66. cognite/neat/rules/issues/tables.py +72 -0
  67. cognite/neat/rules/models/_rdfpath.py +4 -4
  68. cognite/neat/rules/models/_types/_field.py +14 -21
  69. cognite/neat/rules/models/asset/_validation.py +1 -1
  70. cognite/neat/rules/models/dms/_schema.py +53 -30
  71. cognite/neat/rules/models/dms/_validation.py +2 -2
  72. cognite/neat/rules/models/entities.py +3 -0
  73. cognite/neat/rules/models/information/_rules.py +5 -4
  74. cognite/neat/rules/models/information/_validation.py +1 -1
  75. cognite/neat/utils/rdf_.py +17 -9
  76. cognite/neat/utils/regex_patterns.py +52 -0
  77. cognite/neat/workflows/steps/lib/current/rules_importer.py +73 -1
  78. cognite/neat/workflows/steps/lib/current/rules_validator.py +19 -7
  79. {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/METADATA +2 -6
  80. {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/RECORD +85 -72
  81. cognite/neat/app/ui/neat-app/build/static/css/main.38a62222.css.map +0 -1
  82. cognite/neat/app/ui/neat-app/build/static/js/main.ec7f72e2.js +0 -3
  83. cognite/neat/app/ui/neat-app/build/static/js/main.ec7f72e2.js.map +0 -1
  84. cognite/neat/graph/issues/loader.py +0 -104
  85. cognite/neat/graph/stores/_oxrdflib.py +0 -247
  86. cognite/neat/rules/exceptions.py +0 -2972
  87. cognite/neat/rules/importers/_owl2rules/_owl2classes.py +0 -215
  88. cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +0 -213
  89. cognite/neat/rules/importers/_owl2rules/_owl2properties.py +0 -203
  90. cognite/neat/rules/issues/importing.py +0 -408
  91. cognite/neat/rules/models/_types/_base.py +0 -16
  92. cognite/neat/workflows/examples/Export_Rules_to_Ontology/workflow.yaml +0 -152
  93. cognite/neat/workflows/examples/Extract_DEXPI_Graph_and_Export_Rules/workflow.yaml +0 -139
  94. cognite/neat/workflows/examples/Ontology_to_Data_Model/workflow.yaml +0 -116
  95. /cognite/neat/{graph/issues → issues/errors}/__init__.py +0 -0
  96. /cognite/neat/rules/importers/{_owl2rules → _rdf/_owl2rules}/__init__.py +0 -0
  97. {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/LICENSE +0 -0
  98. {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/WHEEL +0 -0
  99. {cognite_neat-0.88.0.dist-info → cognite_neat-0.88.2.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,72 @@
1
+ from dataclasses import dataclass
2
+
3
+ from .base import NeatValidationError
4
+
5
+
6
+ @dataclass(frozen=True)
7
+ class NotValidRDFPathError(NeatValidationError):
8
+ """Provided `rdfpath` is not valid, i.e. it cannot be converted to SPARQL query.
9
+
10
+ Args:
11
+ rdf_path: `rdfpath` that raised exception
12
+
13
+ Notes:
14
+ Get familiar with `rdfpath` to avoid this exception.
15
+ """
16
+
17
+ description = "Provided `rdfpath` is not valid, i.e. it cannot be converted to SPARQL query"
18
+ fix = "Get familiar with `rdfpath` and check if provided path is valid!"
19
+ rdf_path: str
20
+
21
+ def message(self) -> str:
22
+ message = f"{self.rdf_path} is not a valid rdfpath!"
23
+
24
+ message += f"\nDescription: {self.description}"
25
+ message += f"\nFix: {self.fix}"
26
+ return message
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class NotValidTableLookUpError(NeatValidationError):
31
+ """Provided `table lookup` is not valid, i.e. it cannot be converted to CDF lookup.
32
+
33
+ Args:
34
+ table_look_up: `table_look_up`, a part of `rawlookup`, that raised exception
35
+
36
+ Notes:
37
+ Get familiar with `rawlookup` and `rdfpath` to avoid this exception.
38
+ """
39
+
40
+ description = "Provided table lookup is not valid, i.e. it cannot be converted to CDF lookup"
41
+ fix = "Get familiar with RAW look up and RDF paths and check if provided rawlookup is valid"
42
+ table_look_up: str
43
+
44
+ def message(self) -> str:
45
+ message = f"{self.table_look_up} is not a valid table lookup"
46
+
47
+ message += f"\nDescription: {self.description}"
48
+ message += f"\nFix: {self.fix}"
49
+ return message
50
+
51
+
52
+ @dataclass(frozen=True)
53
+ class NotValidRAWLookUpError(NeatValidationError):
54
+ """Provided `rawlookup` is not valid, i.e. it cannot be converted to SPARQL query and CDF lookup
55
+
56
+ Args:
57
+ raw_look_up: `rawlookup` rule that raised exception
58
+
59
+ Notes:
60
+ Get familiar with `rawlookup` and `rdfpath` to avoid this exception.
61
+ """
62
+
63
+ description = "Provided rawlookup is not valid, i.e. it cannot be converted to SPARQL query and CDF lookup"
64
+ fix = "Get familiar with `rawlookup` and `rdfpath` to avoid this exception"
65
+ raw_look_up: str
66
+
67
+ def message(self):
68
+ message = f"Invalid rawlookup expected traversal | table lookup, got {self.raw_look_up}"
69
+
70
+ message += f"\nDescription: {self.description}"
71
+ message += f"\nFix: {self.fix}"
72
+ return message
@@ -8,7 +8,7 @@ from typing import ClassVar, Literal
8
8
 
9
9
  from pydantic import BaseModel, field_validator, model_serializer
10
10
 
11
- from cognite.neat.rules import exceptions
11
+ from cognite.neat.rules.issues.tables import NotValidRAWLookUpError, NotValidRDFPathError, NotValidTableLookUpError
12
12
 
13
13
  if sys.version_info >= (3, 11):
14
14
  from enum import StrEnum
@@ -313,7 +313,7 @@ def parse_traversal(raw: str) -> SelfReferenceProperty | SingleProperty | Hop:
313
313
  elif result := HOP_REGEX_COMPILED.match(raw):
314
314
  return Hop.from_string(class_=result.group("origin"), traversal=result.group(_traversal))
315
315
  else:
316
- raise exceptions.NotValidRDFPath(raw).to_pydantic_custom_error()
316
+ raise NotValidRDFPathError(raw).as_pydantic_exception()
317
317
 
318
318
 
319
319
  def parse_table_lookup(raw: str) -> TableLookup:
@@ -323,7 +323,7 @@ def parse_table_lookup(raw: str) -> TableLookup:
323
323
  key=result.group(Lookup.key),
324
324
  value=result.group(Lookup.value),
325
325
  )
326
- raise exceptions.NotValidTableLookUp(raw).to_pydantic_custom_error()
326
+ raise NotValidTableLookUpError(raw).as_pydantic_exception()
327
327
 
328
328
 
329
329
  def parse_rule(rule_raw: str, rule_type: TransformationRuleType | None) -> RDFPath:
@@ -334,7 +334,7 @@ def parse_rule(rule_raw: str, rule_type: TransformationRuleType | None) -> RDFPa
334
334
  case TransformationRuleType.rawlookup:
335
335
  rule_raw = rule_raw.replace(" ", "")
336
336
  if Counter(rule_raw).get("|") != 1:
337
- raise exceptions.NotValidRAWLookUp(rule_raw).to_pydantic_custom_error()
337
+ raise NotValidRAWLookUpError(rule_raw).as_pydantic_exception()
338
338
  traversal, table_lookup = rule_raw.split("|")
339
339
  return RawLookup(
340
340
  traversal=parse_traversal(traversal),
@@ -1,7 +1,6 @@
1
- import re
2
1
  import warnings
3
2
  from collections.abc import Callable
4
- from typing import Annotated, Any, cast
3
+ from typing import Annotated, Any
5
4
 
6
5
  import rdflib
7
6
  from pydantic import (
@@ -17,11 +16,10 @@ from pydantic import (
17
16
  from pydantic.functional_serializers import PlainSerializer
18
17
  from pydantic_core import PydanticCustomError
19
18
 
20
- from cognite.neat.rules import exceptions
21
- from cognite.neat.rules.issues.importing import MoreThanOneNonAlphanumericCharacterWarning
22
-
23
- from ._base import (
24
- MORE_THAN_ONE_NONE_ALPHANUMERIC_REGEX,
19
+ from cognite.neat.issues.neat_warnings.identifier import RegexViolationWarning
20
+ from cognite.neat.rules.issues.spreadsheet import RegexViolationError
21
+ from cognite.neat.utils.regex_patterns import (
22
+ PATTERNS,
25
23
  PREFIX_COMPLIANCE_REGEX,
26
24
  PROPERTY_ID_COMPLIANCE_REGEX,
27
25
  VERSION_COMPLIANCE_REGEX,
@@ -74,11 +72,7 @@ NamespaceType = Annotated[
74
72
  PrefixType = Annotated[
75
73
  str,
76
74
  StringConstraints(pattern=PREFIX_COMPLIANCE_REGEX),
77
- _custom_error(
78
- lambda _, value: exceptions.PrefixesRegexViolation(
79
- cast(list[str], [value]), PREFIX_COMPLIANCE_REGEX
80
- ).to_pydantic_custom_error()
81
- ),
75
+ _custom_error(lambda _, value: RegexViolationError(value, PREFIX_COMPLIANCE_REGEX).as_pydantic_exception()),
82
76
  ]
83
77
 
84
78
  ExternalIdType = Annotated[
@@ -89,19 +83,18 @@ ExternalIdType = Annotated[
89
83
  VersionType = Annotated[
90
84
  str,
91
85
  StringConstraints(pattern=VERSION_COMPLIANCE_REGEX),
92
- _custom_error(
93
- lambda _, value: exceptions.VersionRegexViolation(
94
- version=cast(str, value), regex_expression=VERSION_COMPLIANCE_REGEX
95
- ).to_pydantic_custom_error()
96
- ),
86
+ _custom_error(lambda _, value: RegexViolationError(value, VERSION_COMPLIANCE_REGEX).as_pydantic_exception()),
97
87
  ]
98
88
 
99
89
 
100
90
  def _property_validation(value: str) -> str:
101
- if not re.match(PROPERTY_ID_COMPLIANCE_REGEX, value):
102
- _raise(exceptions.PropertyIDRegexViolation(value, PROPERTY_ID_COMPLIANCE_REGEX).to_pydantic_custom_error())
103
- if re.search(MORE_THAN_ONE_NONE_ALPHANUMERIC_REGEX, value):
104
- warnings.warn(MoreThanOneNonAlphanumericCharacterWarning("property", value), stacklevel=2)
91
+ if not PATTERNS.property_id_compliance.match(value):
92
+ _raise(RegexViolationError(value, PROPERTY_ID_COMPLIANCE_REGEX).as_pydantic_exception())
93
+ if PATTERNS.more_than_one_alphanumeric.search(value):
94
+ warnings.warn(
95
+ RegexViolationWarning(value, PROPERTY_ID_COMPLIANCE_REGEX, "property", "MoreThanOneNonAlphanumeric"),
96
+ stacklevel=2,
97
+ )
105
98
  return value
106
99
 
107
100
 
@@ -1,8 +1,8 @@
1
1
  from graphlib import CycleError
2
2
  from typing import cast
3
3
 
4
+ from cognite.neat.issues import IssueList
4
5
  from cognite.neat.rules import issues
5
- from cognite.neat.rules.issues.base import IssueList
6
6
  from cognite.neat.rules.models._base import SheetList
7
7
  from cognite.neat.rules.models.asset._rules import AssetProperty, AssetRules
8
8
  from cognite.neat.rules.models.entities import AssetEntity, AssetFields, ClassEntity
@@ -25,20 +25,16 @@ from cognite.client.data_classes.data_modeling.views import (
25
25
  )
26
26
  from cognite.client.data_classes.transformations.common import Edges, EdgeType, Nodes, ViewInfo
27
27
 
28
+ from cognite.neat.issues import NeatError
29
+ from cognite.neat.issues.errors.properties import ReferredPropertyNotFoundError
30
+ from cognite.neat.issues.errors.resources import ReferredResourceNotFoundError
31
+ from cognite.neat.issues.neat_warnings.resources import FailedLoadingResourcesWarning, MultipleResourcesWarning
28
32
  from cognite.neat.rules import issues
29
33
  from cognite.neat.rules.issues.dms import (
30
34
  ContainerPropertyUsedMultipleTimesError,
31
35
  DirectRelationMissingSourceWarning,
32
- DMSSchemaError,
33
36
  DuplicatedViewInDataModelError,
34
37
  IncompleteSchemaError,
35
- MissingContainerError,
36
- MissingContainerPropertyError,
37
- MissingEdgeViewError,
38
- MissingParentViewError,
39
- MissingSourceViewError,
40
- MissingSpaceError,
41
- MissingViewError,
42
38
  MissingViewInModelWarning,
43
39
  )
44
40
  from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE
@@ -168,14 +164,10 @@ class DMSSchema:
168
164
  connection_referenced_view_ids |= cls._connection_references(view)
169
165
  connection_referenced_view_ids = connection_referenced_view_ids - existing_view_ids
170
166
  if connection_referenced_view_ids:
171
- warnings.warn(
172
- MissingViewInModelWarning(data_model.as_id(), connection_referenced_view_ids), UserWarning, stacklevel=2
173
- )
167
+ warnings.warn(MissingViewInModelWarning(data_model.as_id(), connection_referenced_view_ids), stacklevel=2)
174
168
  connection_referenced_views = view_loader.retrieve(list(connection_referenced_view_ids))
175
169
  if failed := connection_referenced_view_ids - set(connection_referenced_views.as_ids()):
176
- warnings.warn(
177
- issues.importing.FailedImportWarning({repr(v) for v in failed}), UserWarning, stacklevel=2
178
- )
170
+ warnings.warn(FailedLoadingResourcesWarning[dm.ViewId](frozenset(failed), "View"), stacklevel=2)
179
171
  views.extend(connection_referenced_views)
180
172
 
181
173
  # We need to include parent views in the schema to make sure that the schema is valid.
@@ -432,8 +424,9 @@ class DMSSchema:
432
424
  if attr.name == "data_model":
433
425
  if isinstance(items, list) and len(items) > 1:
434
426
  warnings.warn(
435
- issues.importing.MultipleDataModelsWarning(
436
- [item.get("externalId", "Unknown") for item in items]
427
+ MultipleResourcesWarning[str](
428
+ frozenset([item.get("externalId", "Unknown") for item in items]),
429
+ "DataModel",
437
430
  ),
438
431
  stacklevel=2,
439
432
  )
@@ -522,8 +515,8 @@ class DMSSchema:
522
515
  else:
523
516
  raise ValueError(f"Cannot sort item of type {type(item)}")
524
517
 
525
- def validate(self) -> list[DMSSchemaError]:
526
- errors: set[DMSSchemaError] = set()
518
+ def validate(self) -> list[NeatError]:
519
+ errors: set[NeatError] = set()
527
520
  defined_spaces = self.spaces.copy()
528
521
  defined_containers = self.containers.copy()
529
522
  defined_views = self.views.copy()
@@ -535,28 +528,42 @@ class DMSSchema:
535
528
 
536
529
  for container in self.containers.values():
537
530
  if container.space not in defined_spaces:
538
- errors.add(MissingSpaceError(space=container.space, referred_by=container.as_id()))
531
+ errors.add(
532
+ ReferredResourceNotFoundError[str, dm.ContainerId](
533
+ container.space, "Space", container.as_id(), "Container"
534
+ )
535
+ )
539
536
 
540
537
  for view in self.views.values():
541
538
  view_id = view.as_id()
542
539
  if view.space not in defined_spaces:
543
- errors.add(MissingSpaceError(space=view.space, referred_by=view_id))
540
+ errors.add(ReferredResourceNotFoundError[str, dm.ViewId](view.space, "Space", view_id, "View"))
544
541
 
545
542
  for parent in view.implements or []:
546
543
  if parent not in defined_views:
547
- errors.add(MissingParentViewError(view=parent, referred_by=view_id))
544
+ errors.add(
545
+ ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
546
+ parent, "View", view_id, "View", property_name="implements"
547
+ )
548
+ )
548
549
 
549
550
  for prop_name, prop in (view.properties or {}).items():
550
551
  if isinstance(prop, dm.MappedPropertyApply):
551
552
  ref_container = defined_containers.get(prop.container)
552
553
  if ref_container is None:
553
- errors.add(MissingContainerError(container=prop.container, referred_by=view_id))
554
+ errors.add(
555
+ ReferredResourceNotFoundError[dm.ContainerId, dm.ViewId](
556
+ prop.container, "Container", view_id, "View"
557
+ )
558
+ )
554
559
  elif prop.container_property_identifier not in ref_container.properties:
555
560
  errors.add(
556
- MissingContainerPropertyError(
557
- container=prop.container,
558
- property=prop.container_property_identifier,
559
- referred_by=view_id,
561
+ ReferredPropertyNotFoundError[dm.ContainerId, dm.ViewId](
562
+ prop.container,
563
+ "Container",
564
+ view_id,
565
+ "View",
566
+ property_name=prop.container_property_identifier,
560
567
  )
561
568
  )
562
569
  else:
@@ -568,14 +575,22 @@ class DMSSchema:
568
575
  )
569
576
 
570
577
  if isinstance(prop, dm.EdgeConnectionApply) and prop.source not in defined_views:
571
- errors.add(MissingSourceViewError(view=prop.source, property=prop_name, referred_by=view_id))
578
+ errors.add(
579
+ ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
580
+ prop.source, "View", view_id, "View", property_name=prop_name
581
+ )
582
+ )
572
583
 
573
584
  if (
574
585
  isinstance(prop, dm.EdgeConnectionApply)
575
586
  and prop.edge_source is not None
576
587
  and prop.edge_source not in defined_views
577
588
  ):
578
- errors.add(MissingEdgeViewError(view=prop.edge_source, property=prop_name, referred_by=view_id))
589
+ errors.add(
590
+ ReferredPropertyNotFoundError[dm.ViewId, dm.ViewId](
591
+ prop.edge_source, "View", view_id, "View", property_name=prop_name
592
+ )
593
+ )
579
594
 
580
595
  # This allows for multiple view properties to be mapped to the same container property,
581
596
  # as long as they have different external_id, otherwise this will lead to raising
@@ -610,13 +625,21 @@ class DMSSchema:
610
625
  if self.data_model:
611
626
  model = self.data_model
612
627
  if model.space not in defined_spaces:
613
- errors.add(MissingSpaceError(space=model.space, referred_by=model.as_id()))
628
+ errors.add(
629
+ ReferredResourceNotFoundError[str, dm.DataModelId](
630
+ model.space, "Space", model.as_id(), "Data Model"
631
+ )
632
+ )
614
633
 
615
634
  view_counts: dict[dm.ViewId, int] = defaultdict(int)
616
635
  for view_id_or_class in model.views or []:
617
636
  view_id = view_id_or_class if isinstance(view_id_or_class, dm.ViewId) else view_id_or_class.as_id()
618
637
  if view_id not in defined_views:
619
- errors.add(MissingViewError(referred_by=model.as_id(), view=view_id))
638
+ errors.add(
639
+ ReferredResourceNotFoundError[dm.ViewId, dm.DataModelId](
640
+ view_id, "View", model.as_id(), "DataModel"
641
+ )
642
+ )
620
643
  view_counts[view_id] += 1
621
644
 
622
645
  for view_id, count in view_counts.items():
@@ -3,8 +3,8 @@ from typing import Any, ClassVar
3
3
 
4
4
  from cognite.client import data_modeling as dm
5
5
 
6
+ from cognite.neat.issues import IssueList, NeatIssueList
6
7
  from cognite.neat.rules import issues
7
- from cognite.neat.rules.issues import IssueList
8
8
  from cognite.neat.rules.models._base import DataModelType, ExtensionCategory, SchemaCompleteness
9
9
  from cognite.neat.rules.models._constants import DMS_CONTAINER_SIZE_LIMIT
10
10
  from cognite.neat.rules.models.data_types import DataType
@@ -31,7 +31,7 @@ class DMSPostValidation:
31
31
  self.views = rules.views
32
32
  self.issue_list = IssueList()
33
33
 
34
- def validate(self) -> IssueList:
34
+ def validate(self) -> NeatIssueList:
35
35
  self._validate_raw_filter()
36
36
  self._consistent_container_properties()
37
37
 
@@ -418,6 +418,9 @@ class ViewEntity(DMSVersionedEntity[ViewId]):
418
418
  ) -> ViewId:
419
419
  return ViewId(space=self.space, external_id=self.external_id, version=self.version)
420
420
 
421
+ def to_property_id(self, property_id: str) -> PropertyId:
422
+ return PropertyId(source=self.as_id(), property=property_id)
423
+
421
424
  @classmethod
422
425
  def from_id(cls, id: ViewId, default_version: str | None = None) -> "ViewEntity":
423
426
  if id.version is not None:
@@ -9,7 +9,8 @@ from rdflib import Namespace
9
9
 
10
10
  from cognite.neat.constants import get_default_prefixes
11
11
  from cognite.neat.issues import MultiValueError
12
- from cognite.neat.rules import exceptions, issues
12
+ from cognite.neat.rules import issues
13
+ from cognite.neat.rules.issues.spreadsheet import DefaultValueTypeNotProperError
13
14
  from cognite.neat.rules.models._base import (
14
15
  BaseMetadata,
15
16
  BaseRules,
@@ -227,11 +228,11 @@ class InformationProperty(SheetEntity):
227
228
  self.default = self.value_type.python(self.default)
228
229
 
229
230
  except Exception:
230
- exceptions.DefaultValueTypeNotProper(
231
+ raise DefaultValueTypeNotProperError(
231
232
  self.property_,
232
233
  type(self.default),
233
- self.value_type.python,
234
- )
234
+ str(self.value_type.python),
235
+ ).as_exception() from None
235
236
  return self
236
237
 
237
238
  @property
@@ -2,8 +2,8 @@ import itertools
2
2
  from collections import Counter
3
3
  from typing import cast
4
4
 
5
+ from cognite.neat.issues import IssueList
5
6
  from cognite.neat.rules import issues
6
- from cognite.neat.rules.issues import IssueList
7
7
  from cognite.neat.rules.models._base import DataModelType, SchemaCompleteness
8
8
  from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, UnknownEntity
9
9
  from cognite.neat.utils.rdf_ import get_inheritance_path
@@ -1,6 +1,8 @@
1
1
  import re
2
- from typing import Any, Literal, TypeAlias, cast, overload
2
+ from collections.abc import Iterable
3
+ from typing import Any, Literal, TypeAlias, overload
3
4
 
5
+ from cognite.client.utils.useful_types import SequenceNotStr
4
6
  from pydantic import HttpUrl, TypeAdapter, ValidationError
5
7
  from rdflib import Literal as RdfLiteral
6
8
  from rdflib import Namespace, URIRef
@@ -10,7 +12,8 @@ Triple: TypeAlias = tuple[URIRef, URIRef, RdfLiteral | URIRef]
10
12
 
11
13
  @overload
12
14
  def remove_namespace_from_uri(
13
- *URI: URIRef | str,
15
+ URI: URIRef | str,
16
+ *,
14
17
  special_separator: str = "#_",
15
18
  validation: Literal["full", "prefix"] = "prefix",
16
19
  ) -> str: ...
@@ -18,17 +21,19 @@ def remove_namespace_from_uri(
18
21
 
19
22
  @overload
20
23
  def remove_namespace_from_uri(
21
- *URI: tuple[URIRef | str, ...],
24
+ URI: SequenceNotStr[URIRef | str],
25
+ *,
22
26
  special_separator: str = "#_",
23
27
  validation: Literal["full", "prefix"] = "prefix",
24
- ) -> tuple[str, ...]: ...
28
+ ) -> list[str]: ...
25
29
 
26
30
 
27
31
  def remove_namespace_from_uri(
28
- *URI: URIRef | str | tuple[URIRef | str, ...],
32
+ URI: URIRef | str | SequenceNotStr[URIRef | str],
33
+ *,
29
34
  special_separator: str = "#_",
30
35
  validation: Literal["full", "prefix"] = "prefix",
31
- ) -> tuple[str, ...] | str:
36
+ ) -> str | list[str]:
32
37
  """Removes namespace from URI
33
38
 
34
39
  Args
@@ -51,11 +56,14 @@ def remove_namespace_from_uri(
51
56
  >>> remove_namespace_from_uri("http://www.example.org/index.html#section2", "http://www.example.org/index.html#section3")
52
57
  ('section2', 'section3')
53
58
  """
59
+ is_single = False
60
+ uris: Iterable[str | URIRef]
54
61
  if isinstance(URI, str | URIRef):
55
62
  uris = (URI,)
56
- elif isinstance(URI, tuple):
63
+ is_single = True
64
+ elif isinstance(URI, SequenceNotStr):
57
65
  # Assume that all elements in the tuple are of the same type following type hint
58
- uris = cast(tuple[URIRef | str, ...], URI)
66
+ uris = URI
59
67
  else:
60
68
  raise TypeError(f"URI must be of type URIRef or str, got {type(URI)}")
61
69
 
@@ -73,7 +81,7 @@ def remove_namespace_from_uri(
73
81
  else:
74
82
  output.append(str(u))
75
83
 
76
- return tuple(output) if len(output) > 1 else output[0]
84
+ return output[0] if is_single else output
77
85
 
78
86
 
79
87
  def get_namespace(URI: URIRef, special_separator: str = "#_") -> str:
@@ -0,0 +1,52 @@
1
+ import re
2
+ from functools import cached_property
3
+
4
+ MORE_THAN_ONE_NONE_ALPHANUMERIC_REGEX = r"([_-]{2,})"
5
+ PREFIX_COMPLIANCE_REGEX = r"^([a-zA-Z]+)([a-zA-Z0-9]*[_-]{0,1}[a-zA-Z0-9_-]*)([a-zA-Z0-9]*)$"
6
+
7
+ VIEW_ID_COMPLIANCE_REGEX = (
8
+ r"(?!^(Query|Mutation|Subscription|String|Int32|Int64|Int|Float32|Float64|Float|"
9
+ r"Timestamp|JSONObject|Date|Numeric|Boolean|PageInfo|File|Sequence|TimeSeries)$)"
10
+ r"(^[a-zA-Z][a-zA-Z0-9_]{0,253}[a-zA-Z0-9]?$)"
11
+ )
12
+ DMS_PROPERTY_ID_COMPLIANCE_REGEX = (
13
+ r"(?!^(space|externalId|createdTime|lastUpdatedTime|deletedTime|edge_id|"
14
+ r"node_id|project_id|property_group|seq|tg_table_name|extensions)$)"
15
+ r"(^[a-zA-Z][a-zA-Z0-9_]{0,253}[a-zA-Z0-9]?$)"
16
+ )
17
+ CLASS_ID_COMPLIANCE_REGEX = r"(?!^(Class|class)$)(^[a-zA-Z][a-zA-Z0-9._-]{0,253}[a-zA-Z0-9]?$)"
18
+ PROPERTY_ID_COMPLIANCE_REGEX = r"^(\*)|(?!^(Property|property)$)(^[a-zA-Z][a-zA-Z0-9._-]{0,253}[a-zA-Z0-9]?$)"
19
+ VERSION_COMPLIANCE_REGEX = r"^[a-zA-Z0-9]([.a-zA-Z0-9_-]{0,41}[a-zA-Z0-9])?$"
20
+
21
+
22
+ class _Patterns:
23
+ @cached_property
24
+ def more_than_one_alphanumeric(self) -> re.Pattern:
25
+ return re.compile(MORE_THAN_ONE_NONE_ALPHANUMERIC_REGEX)
26
+
27
+ @cached_property
28
+ def prefix_compliance(self) -> re.Pattern[str]:
29
+ return re.compile(PREFIX_COMPLIANCE_REGEX)
30
+
31
+ @cached_property
32
+ def view_id_compliance(self) -> re.Pattern[str]:
33
+ return re.compile(VIEW_ID_COMPLIANCE_REGEX)
34
+
35
+ @cached_property
36
+ def dms_property_id_compliance(self) -> re.Pattern[str]:
37
+ return re.compile(DMS_PROPERTY_ID_COMPLIANCE_REGEX)
38
+
39
+ @cached_property
40
+ def class_id_compliance(self) -> re.Pattern[str]:
41
+ return re.compile(CLASS_ID_COMPLIANCE_REGEX)
42
+
43
+ @cached_property
44
+ def property_id_compliance(self) -> re.Pattern[str]:
45
+ return re.compile(PROPERTY_ID_COMPLIANCE_REGEX)
46
+
47
+ @cached_property
48
+ def version_compliance(self) -> re.Pattern[str]:
49
+ return re.compile(VERSION_COMPLIANCE_REGEX)
50
+
51
+
52
+ PATTERNS = _Patterns()
@@ -5,8 +5,8 @@ from typing import ClassVar
5
5
  from cognite.client import CogniteClient
6
6
  from cognite.client.data_classes.data_modeling import DataModelId
7
7
 
8
+ from cognite.neat.issues.formatters import FORMATTER_BY_NAME
8
9
  from cognite.neat.rules import importers
9
- from cognite.neat.rules.issues.formatters import FORMATTER_BY_NAME
10
10
  from cognite.neat.rules.models import RoleTypes
11
11
  from cognite.neat.rules.models.entities import DataModelEntity, DMSUnknownEntity
12
12
  from cognite.neat.workflows._exceptions import StepNotInitialized
@@ -19,6 +19,7 @@ CATEGORY = __name__.split(".")[-1].replace("_", " ").title()
19
19
  __all__ = [
20
20
  "ExcelToRules",
21
21
  "OntologyToRules",
22
+ "IMFToRules",
22
23
  "DMSToRules",
23
24
  "RulesInferenceFromRdfFile",
24
25
  ]
@@ -162,6 +163,77 @@ class OntologyToRules(Step):
162
163
  return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
163
164
 
164
165
 
166
+ class IMFToRules(Step):
167
+ """This step import rules from the IMF-types and validates them."""
168
+
169
+ description = "This step imports rules from an RDF file "
170
+ version = "private-beta"
171
+ category = CATEGORY
172
+ configurables: ClassVar[list[Configurable]] = [
173
+ Configurable(
174
+ name="File name",
175
+ value="",
176
+ label="""Full file name of the RDF-file containing the IMF-types in the rules folder.
177
+ If not provided, step will attempt to get file name from payload
178
+ of 'File Uploader' step (if exist)""",
179
+ ),
180
+ Configurable(
181
+ name="Report formatter",
182
+ value=next(iter(FORMATTER_BY_NAME.keys())),
183
+ label="The format of the report for the validation of the rules",
184
+ options=list(FORMATTER_BY_NAME),
185
+ ),
186
+ Configurable(
187
+ name="Role",
188
+ value="infer",
189
+ label="For what role Rules are intended?",
190
+ options=["infer", *RoleTypes.__members__.keys()],
191
+ ),
192
+ ]
193
+
194
+ def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData): # type: ignore[syntax, override]
195
+ if self.configs is None or self.data_store_path is None:
196
+ raise StepNotInitialized(type(self).__name__)
197
+
198
+ file_name = self.configs.get("File name", None)
199
+
200
+ full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None
201
+
202
+ if file_name:
203
+ rules_file_path = self.config.rules_store_path / file_name
204
+
205
+ elif full_path:
206
+ rules_file_path = full_path
207
+ else:
208
+ error_text = "Expected either 'File name' in the step config or 'File uploader' step uploading Excel Rules."
209
+ return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
210
+
211
+ # if role is None, it will be inferred from the rules file
212
+ role = self.configs.get("Role")
213
+ role_enum = None
214
+ if role != "infer" and role is not None:
215
+ role_enum = RoleTypes[role]
216
+
217
+ ontology_importer = importers.IMFImporter(filepath=rules_file_path)
218
+ rules, issues = ontology_importer.to_rules(errors="continue", role=role_enum)
219
+
220
+ if rules is None:
221
+ output_dir = self.config.staging_path
222
+ report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
223
+ report_writer.write_to_file(issues, file_or_dir_path=output_dir)
224
+ report_file = report_writer.default_file_name
225
+ error_text = (
226
+ "<p></p>"
227
+ f'<a href="/data/staging/{report_file}?{time.time()}" '
228
+ f'target="_blank">Failed to validate rules, click here for report</a>'
229
+ )
230
+ return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
231
+
232
+ output_text = "Rules validation passed successfully!"
233
+
234
+ return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
235
+
236
+
165
237
  class DMSToRules(Step):
166
238
  """This step imports rules from CDF Data Model"""
167
239