cognite-neat 0.88.1__py3-none-any.whl → 0.88.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/graph/__init__.py +0 -3
  3. cognite/neat/graph/loaders/_base.py +6 -6
  4. cognite/neat/graph/loaders/_rdf2asset.py +28 -31
  5. cognite/neat/graph/loaders/_rdf2dms.py +24 -15
  6. cognite/neat/issues/__init__.py +14 -0
  7. cognite/neat/issues/_base.py +415 -0
  8. cognite/neat/issues/errors/__init__.py +72 -0
  9. cognite/neat/issues/errors/_external.py +67 -0
  10. cognite/neat/issues/errors/_general.py +28 -0
  11. cognite/neat/issues/errors/_properties.py +62 -0
  12. cognite/neat/issues/errors/_resources.py +111 -0
  13. cognite/neat/issues/errors/_workflow.py +36 -0
  14. cognite/neat/{rules/issues → issues}/formatters.py +10 -10
  15. cognite/neat/issues/warnings/__init__.py +66 -0
  16. cognite/neat/issues/warnings/_external.py +40 -0
  17. cognite/neat/issues/warnings/_general.py +29 -0
  18. cognite/neat/issues/warnings/_models.py +92 -0
  19. cognite/neat/issues/warnings/_properties.py +44 -0
  20. cognite/neat/issues/warnings/_resources.py +55 -0
  21. cognite/neat/issues/warnings/user_modeling.py +113 -0
  22. cognite/neat/rules/_shared.py +10 -2
  23. cognite/neat/rules/exporters/_base.py +6 -6
  24. cognite/neat/rules/exporters/_rules2dms.py +19 -11
  25. cognite/neat/rules/exporters/_rules2excel.py +4 -4
  26. cognite/neat/rules/exporters/_rules2ontology.py +74 -51
  27. cognite/neat/rules/exporters/_rules2yaml.py +3 -3
  28. cognite/neat/rules/exporters/_validation.py +11 -96
  29. cognite/neat/rules/importers/__init__.py +7 -3
  30. cognite/neat/rules/importers/_base.py +9 -13
  31. cognite/neat/rules/importers/_dms2rules.py +42 -24
  32. cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +49 -53
  33. cognite/neat/rules/importers/_dtdl2rules/dtdl_importer.py +31 -23
  34. cognite/neat/rules/importers/_dtdl2rules/spec.py +7 -0
  35. cognite/neat/rules/importers/_rdf/_imf2rules/__init__.py +3 -0
  36. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2classes.py +82 -0
  37. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2metadata.py +34 -0
  38. cognite/neat/rules/importers/_rdf/_imf2rules/_imf2properties.py +123 -0
  39. cognite/neat/rules/importers/{_owl2rules/_owl2rules.py → _rdf/_imf2rules/_imf2rules.py} +24 -18
  40. cognite/neat/rules/importers/{_inference2rules.py → _rdf/_inference2rules.py} +9 -9
  41. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2classes.py +58 -0
  42. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2metadata.py +68 -0
  43. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2properties.py +60 -0
  44. cognite/neat/rules/importers/_rdf/_owl2rules/_owl2rules.py +76 -0
  45. cognite/neat/rules/importers/_rdf/_shared.py +586 -0
  46. cognite/neat/rules/importers/_spreadsheet2rules.py +35 -22
  47. cognite/neat/rules/importers/_yaml2rules.py +23 -21
  48. cognite/neat/rules/models/_constants.py +2 -1
  49. cognite/neat/rules/models/_rdfpath.py +4 -4
  50. cognite/neat/rules/models/_types/_field.py +9 -11
  51. cognite/neat/rules/models/asset/_rules.py +1 -3
  52. cognite/neat/rules/models/asset/_validation.py +14 -10
  53. cognite/neat/rules/models/dms/_converter.py +2 -4
  54. cognite/neat/rules/models/dms/_exporter.py +30 -8
  55. cognite/neat/rules/models/dms/_rules.py +23 -7
  56. cognite/neat/rules/models/dms/_schema.py +94 -62
  57. cognite/neat/rules/models/dms/_validation.py +105 -66
  58. cognite/neat/rules/models/entities.py +3 -0
  59. cognite/neat/rules/models/information/_converter.py +2 -2
  60. cognite/neat/rules/models/information/_rules.py +7 -8
  61. cognite/neat/rules/models/information/_validation.py +48 -25
  62. cognite/neat/rules/transformers/__init__.py +0 -0
  63. cognite/neat/rules/transformers/_base.py +15 -0
  64. cognite/neat/utils/auxiliary.py +2 -35
  65. cognite/neat/utils/text.py +17 -0
  66. cognite/neat/workflows/base.py +4 -4
  67. cognite/neat/workflows/cdf_store.py +3 -3
  68. cognite/neat/workflows/steps/data_contracts.py +1 -1
  69. cognite/neat/workflows/steps/lib/current/graph_extractor.py +3 -3
  70. cognite/neat/workflows/steps/lib/current/graph_loader.py +2 -2
  71. cognite/neat/workflows/steps/lib/current/graph_store.py +1 -1
  72. cognite/neat/workflows/steps/lib/current/rules_exporter.py +10 -10
  73. cognite/neat/workflows/steps/lib/current/rules_importer.py +78 -6
  74. cognite/neat/workflows/steps/lib/current/rules_validator.py +20 -9
  75. cognite/neat/workflows/steps/lib/io/io_steps.py +5 -5
  76. cognite/neat/workflows/steps_registry.py +4 -5
  77. {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/METADATA +1 -1
  78. {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/RECORD +86 -77
  79. cognite/neat/exceptions.py +0 -145
  80. cognite/neat/graph/exceptions.py +0 -90
  81. cognite/neat/graph/issues/loader.py +0 -104
  82. cognite/neat/issues.py +0 -158
  83. cognite/neat/rules/importers/_owl2rules/_owl2classes.py +0 -215
  84. cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +0 -209
  85. cognite/neat/rules/importers/_owl2rules/_owl2properties.py +0 -203
  86. cognite/neat/rules/issues/__init__.py +0 -26
  87. cognite/neat/rules/issues/base.py +0 -82
  88. cognite/neat/rules/issues/dms.py +0 -683
  89. cognite/neat/rules/issues/fileread.py +0 -197
  90. cognite/neat/rules/issues/importing.py +0 -423
  91. cognite/neat/rules/issues/ontology.py +0 -298
  92. cognite/neat/rules/issues/spreadsheet.py +0 -563
  93. cognite/neat/rules/issues/spreadsheet_file.py +0 -151
  94. cognite/neat/rules/issues/tables.py +0 -72
  95. cognite/neat/workflows/_exceptions.py +0 -41
  96. /cognite/neat/{graph/issues → rules/importers/_rdf}/__init__.py +0 -0
  97. /cognite/neat/rules/importers/{_owl2rules → _rdf/_owl2rules}/__init__.py +0 -0
  98. /cognite/neat/{graph/stores → store}/__init__.py +0 -0
  99. /cognite/neat/{graph/stores → store}/_base.py +0 -0
  100. /cognite/neat/{graph/stores → store}/_provenance.py +0 -0
  101. {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/LICENSE +0 -0
  102. {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/WHEEL +0 -0
  103. {cognite_neat-0.88.1.dist-info → cognite_neat-0.88.3.dist-info}/entry_points.txt +0 -0
@@ -8,9 +8,7 @@ from pydantic.main import IncEx
8
8
  from rdflib import Namespace
9
9
 
10
10
  from cognite.neat.constants import get_default_prefixes
11
- from cognite.neat.issues import MultiValueError
12
- from cognite.neat.rules import issues
13
- from cognite.neat.rules.issues.spreadsheet import DefaultValueTypeNotProperError
11
+ from cognite.neat.issues.errors import PropertyDefinitionError
14
12
  from cognite.neat.rules.models._base import (
15
13
  BaseMetadata,
16
14
  BaseRules,
@@ -228,11 +226,12 @@ class InformationProperty(SheetEntity):
228
226
  self.default = self.value_type.python(self.default)
229
227
 
230
228
  except Exception:
231
- raise DefaultValueTypeNotProperError(
229
+ raise PropertyDefinitionError(
230
+ self.class_,
231
+ "Class",
232
232
  self.property_,
233
- type(self.default),
234
- str(self.value_type.python),
235
- ).as_exception() from None
233
+ f"Default value {self.default} is not of type {self.value_type.python}",
234
+ ) from None
236
235
  return self
237
236
 
238
237
  @property
@@ -306,7 +305,7 @@ class InformationRules(BaseRules):
306
305
  if issue_list.warnings:
307
306
  issue_list.trigger_warnings()
308
307
  if issue_list.has_errors:
309
- raise MultiValueError([error for error in issue_list if isinstance(error, issues.NeatValidationError)])
308
+ raise issue_list.as_exception()
310
309
  return self
311
310
 
312
311
  def dump(
@@ -2,8 +2,8 @@ import itertools
2
2
  from collections import Counter
3
3
  from typing import cast
4
4
 
5
- from cognite.neat.rules import issues
6
- from cognite.neat.rules.issues import IssueList
5
+ from cognite.neat.issues import IssueList
6
+ from cognite.neat.issues.errors import NeatValueError, ResourceNotDefinedError
7
7
  from cognite.neat.rules.models._base import DataModelType, SchemaCompleteness
8
8
  from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, UnknownEntity
9
9
  from cognite.neat.utils.rdf_ import get_inheritance_path
@@ -58,9 +58,9 @@ class InformationPostValidation:
58
58
  ):
59
59
  dangling_classes.add(class_)
60
60
 
61
- if dangling_classes:
61
+ for class_ in dangling_classes:
62
62
  self.issue_list.append(
63
- issues.spreadsheet.ClassNoPropertiesNoParentError([class_.versioned_id for class_ in dangling_classes])
63
+ NeatValueError(f"Class {class_} has no properties and is not a parent of any class with properties")
64
64
  )
65
65
 
66
66
  def _referenced_parent_classes_exist(self) -> None:
@@ -70,9 +70,15 @@ class InformationPostValidation:
70
70
  parents = set(itertools.chain.from_iterable(class_parent_pairs.values()))
71
71
 
72
72
  if undefined_parents := parents.difference(classes):
73
- self.issue_list.append(
74
- issues.spreadsheet.ParentClassesNotDefinedError([missing.versioned_id for missing in undefined_parents])
75
- )
73
+ for parent in undefined_parents:
74
+ # Todo: include row and column number
75
+ self.issue_list.append(
76
+ ResourceNotDefinedError[ClassEntity](
77
+ resource_type="class",
78
+ identifier=parent,
79
+ location="Classes sheet",
80
+ )
81
+ )
76
82
 
77
83
  def _referenced_classes_exist(self) -> None:
78
84
  # needs to be complete for this validation to pass
@@ -83,21 +89,27 @@ class InformationPostValidation:
83
89
  if self.metadata.schema_ == SchemaCompleteness.complete and (
84
90
  missing_classes := referred_classes.difference(defined_classes)
85
91
  ):
86
- self.issue_list.append(
87
- issues.spreadsheet.PropertiesDefinedForUndefinedClassesError(
88
- [missing.versioned_id for missing in missing_classes]
92
+ for class_ in missing_classes:
93
+ self.issue_list.append(
94
+ ResourceNotDefinedError[ClassEntity](
95
+ resource_type="class",
96
+ identifier=class_,
97
+ location="Classes sheet",
98
+ )
89
99
  )
90
- )
91
100
 
92
101
  # USE CASE: models are extended (user + last = complete)
93
102
  if self.metadata.schema_ == SchemaCompleteness.extended:
94
103
  defined_classes |= {class_.class_ for class_ in cast(InformationRules, self.rules.last).classes}
95
104
  if missing_classes := referred_classes.difference(defined_classes):
96
- self.issue_list.append(
97
- issues.spreadsheet.PropertiesDefinedForUndefinedClassesError(
98
- [missing.versioned_id for missing in missing_classes]
105
+ for class_ in missing_classes:
106
+ self.issue_list.append(
107
+ ResourceNotDefinedError[ClassEntity](
108
+ resource_type="class",
109
+ identifier=class_,
110
+ location="Classes sheet",
111
+ )
99
112
  )
100
- )
101
113
 
102
114
  def _referenced_value_types_exist(self) -> None:
103
115
  # adding UnknownEntity to the set of defined classes to handle the case where a property references an unknown
@@ -112,21 +124,29 @@ class InformationPostValidation:
112
124
  if self.metadata.schema_ == SchemaCompleteness.complete and (
113
125
  missing_value_types := referred_object_types.difference(defined_classes)
114
126
  ):
115
- self.issue_list.append(
116
- issues.spreadsheet.ValueTypeNotDefinedError(
117
- [cast(ClassEntity, missing).versioned_id for missing in missing_value_types]
127
+ # Todo: include row and column number
128
+ for missing in missing_value_types:
129
+ self.issue_list.append(
130
+ ResourceNotDefinedError[ClassEntity](
131
+ resource_type="class",
132
+ identifier=cast(ClassEntity, missing),
133
+ location="Classes sheet",
134
+ )
118
135
  )
119
- )
120
136
 
121
137
  # USE CASE: models are extended (user + last = complete)
122
138
  if self.metadata.schema_ == SchemaCompleteness.extended:
123
139
  defined_classes |= {class_.class_ for class_ in cast(InformationRules, self.rules.last).classes}
124
140
  if missing_value_types := referred_object_types.difference(defined_classes):
125
- self.issue_list.append(
126
- issues.spreadsheet.ValueTypeNotDefinedError(
127
- [cast(ClassEntity, missing).versioned_id for missing in missing_value_types]
141
+ # Todo: include row and column number
142
+ for missing in missing_value_types:
143
+ self.issue_list.append(
144
+ ResourceNotDefinedError(
145
+ resource_type="class",
146
+ identifier=cast(ClassEntity, missing),
147
+ location="Classes sheet",
148
+ )
128
149
  )
129
- )
130
150
 
131
151
  def _class_parent_pairs(self) -> dict[ClassEntity, list[ClassEntity]]:
132
152
  class_subclass_pairs: dict[ClassEntity, list[ClassEntity]] = {}
@@ -173,7 +193,10 @@ class InformationPostValidation:
173
193
  reused_namespaces = [value for value, count in Counter(prefixes.values()).items() if count > 1]
174
194
  impacted_prefixes = [key for key, value in prefixes.items() if value in reused_namespaces]
175
195
  self.issue_list.append(
176
- issues.spreadsheet.PrefixNamespaceCollisionError(
177
- prefixes=impacted_prefixes, namespaces=reused_namespaces
196
+ NeatValueError(
197
+ "Namespace collision detected. The following prefixes "
198
+ f"are assigned to the same namespace: {impacted_prefixes}"
199
+ f"\nImpacted namespaces: {reused_namespaces}"
200
+ "\nMake sure that each unique namespace is assigned to a unique prefix"
178
201
  )
179
202
  )
File without changes
@@ -0,0 +1,15 @@
1
+ from abc import ABC, abstractmethod
2
+ from typing import Generic, TypeVar
3
+
4
+ from cognite.neat.rules._shared import Rules
5
+
6
+ T_RulesIn = TypeVar("T_RulesIn", bound=Rules)
7
+ T_RulesOut = TypeVar("T_RulesOut", bound=Rules)
8
+
9
+
10
+ class RulesTransformer(ABC, Generic[T_RulesIn, T_RulesOut]):
11
+ """This is the base class for all rule transformers."""
12
+
13
+ @abstractmethod
14
+ def transform(self, rules: T_RulesIn) -> T_RulesOut:
15
+ raise NotImplementedError()
@@ -3,15 +3,14 @@ import importlib
3
3
  import inspect
4
4
  import logging
5
5
  import time
6
- from collections.abc import Callable, Iterable
6
+ from collections.abc import Callable
7
7
  from datetime import datetime
8
8
  from functools import wraps
9
9
  from types import ModuleType
10
10
 
11
11
  from cognite.client.exceptions import CogniteDuplicatedError, CogniteReadTimeout
12
- from pydantic_core import ErrorDetails
13
12
 
14
- from cognite.neat.exceptions import NeatImportError
13
+ from cognite.neat.issues.errors import NeatImportError
15
14
 
16
15
 
17
16
  def local_import(module: str, extra: str) -> ModuleType:
@@ -119,38 +118,6 @@ def create_sha256_hash(string: str) -> str:
119
118
  return hash_value
120
119
 
121
120
 
122
- # Will likely be removed with legacy code
123
- def generate_exception_report(exceptions: list[dict] | list[ErrorDetails] | None, category: str = "") -> str:
124
- exceptions_as_dict = _order_expectations_by_type(exceptions) if exceptions else {}
125
- report = ""
126
-
127
- for exception_type in exceptions_as_dict.keys():
128
- title = f"# {category}: {exception_type}" if category else ""
129
- warnings = "\n- " + "\n- ".join(exceptions_as_dict[exception_type])
130
- report += title + warnings + "\n\n"
131
-
132
- return report
133
-
134
-
135
- def _order_expectations_by_type(
136
- exceptions: list[dict] | list[ErrorDetails],
137
- ) -> dict[str, list[str]]:
138
- exception_dict: dict[str, list[str]] = {}
139
- for exception in exceptions:
140
- if not isinstance(exception["loc"], str) and isinstance(exception["loc"], Iterable):
141
- location = f"[{'/'.join(str(e) for e in exception['loc'])}]"
142
- else:
143
- location = ""
144
-
145
- issue = f"{exception['msg']} {location}"
146
-
147
- if exception_dict.get(exception["type"]) is None:
148
- exception_dict[exception["type"]] = [issue]
149
- else:
150
- exception_dict[exception["type"]].append(issue)
151
- return exception_dict
152
-
153
-
154
121
  def string_to_ideal_type(input_string: str) -> int | bool | float | datetime | str:
155
122
  try:
156
123
  # Try converting to int
@@ -1,4 +1,6 @@
1
1
  import re
2
+ from collections.abc import Collection
3
+ from typing import Any
2
4
 
3
5
 
4
6
  def to_camel(string: str) -> str:
@@ -106,3 +108,18 @@ def to_snake(string: str) -> str:
106
108
 
107
109
  def replace_non_alphanumeric_with_underscore(text: str) -> str:
108
110
  return re.sub(r"\W+", "_", text)
111
+
112
+
113
+ def humanize_collection(collection: Collection[Any], /, *, sort: bool = True) -> str:
114
+ if not collection:
115
+ return ""
116
+ elif len(collection) == 1:
117
+ return str(next(iter(collection)))
118
+
119
+ strings = (str(item) for item in collection)
120
+ if sort:
121
+ sequence = sorted(strings)
122
+ else:
123
+ sequence = list(strings)
124
+
125
+ return f"{', '.join(sequence[:-1])} and {sequence[-1]}"
@@ -13,9 +13,9 @@ from prometheus_client import Gauge
13
13
 
14
14
  from cognite.neat.app.monitoring.metrics import NeatMetricsCollector
15
15
  from cognite.neat.config import Config
16
+ from cognite.neat.issues.errors import WorkflowConfigurationNotSetError, WorkflowStepOutputError
16
17
  from cognite.neat.utils.auxiliary import retry_decorator
17
18
  from cognite.neat.workflows import cdf_store, utils
18
- from cognite.neat.workflows._exceptions import ConfigurationNotSet, InvalidStepOutputException
19
19
  from cognite.neat.workflows.cdf_store import CdfStore
20
20
  from cognite.neat.workflows.model import (
21
21
  FlowMessage,
@@ -316,7 +316,7 @@ class BaseWorkflow:
316
316
  elif isinstance(out_obj, DataContract):
317
317
  self.data[type(out_obj).__name__] = out_obj
318
318
  else:
319
- raise InvalidStepOutputException(step_type=type(out_obj).__name__)
319
+ raise WorkflowStepOutputError(step_type=type(out_obj).__name__)
320
320
 
321
321
  elif step.stype == StepType.START_WORKFLOW_TASK_STEP:
322
322
  if self.task_builder:
@@ -510,7 +510,7 @@ class BaseWorkflow:
510
510
  if storage_type == "transformation_rules":
511
511
  self.rules_storage_path = Path(storage_path)
512
512
  if self.default_dataset_id is None:
513
- raise ConfigurationNotSet("default_dataset_id")
513
+ raise WorkflowConfigurationNotSetError("default_dataset_id")
514
514
  self.cdf_store = cdf_store.CdfStore(
515
515
  self.cdf_client, data_set_id=self.default_dataset_id, rules_storage_path=self.rules_storage_path
516
516
  )
@@ -576,7 +576,7 @@ class BaseWorkflow:
576
576
 
577
577
  file_list: list[Path] = []
578
578
  if self.data_store_path is None:
579
- raise ConfigurationNotSet("data_store_path")
579
+ raise WorkflowConfigurationNotSetError("data_store_path")
580
580
  workflow_data_path = Path(self.data_store_path) / "workflows" / self.name
581
581
  try:
582
582
  for root, _dirs, files in os.walk(workflow_data_path):
@@ -11,7 +11,7 @@ from cognite.client.data_classes import Event, FileMetadataUpdate
11
11
  from fastapi.encoders import jsonable_encoder
12
12
  from pydantic import BaseModel
13
13
 
14
- from cognite.neat.workflows._exceptions import ConfigurationNotSet
14
+ from cognite.neat.issues.errors import WorkflowConfigurationNotSetError
15
15
  from cognite.neat.workflows.model import WorkflowFullStateReport, WorkflowState, WorkflowStepEvent
16
16
  from cognite.neat.workflows.utils import get_file_hash
17
17
 
@@ -53,7 +53,7 @@ class CdfStore:
53
53
  def package_workflow(self, workflow_name: str) -> str:
54
54
  """Creates a zip archive from a folder"""
55
55
  if self.workflows_storage_path is None:
56
- raise ConfigurationNotSet("workflows_storage_path")
56
+ raise WorkflowConfigurationNotSetError("workflows_storage_path")
57
57
  folder_path = self.workflows_storage_path / workflow_name
58
58
  archive_path = self.workflows_storage_path / f"{workflow_name}.zip"
59
59
  # Make sure the folder exists
@@ -77,7 +77,7 @@ class CdfStore:
77
77
  # Make sure the archive exists
78
78
  workflow_name = workflow_name.replace(".zip", "")
79
79
  if self.workflows_storage_path is None:
80
- raise ConfigurationNotSet("workflows_storage_path")
80
+ raise WorkflowConfigurationNotSetError("workflows_storage_path")
81
81
  package_full_path = Path(self.workflows_storage_path) / f"{workflow_name}.zip"
82
82
  output_folder = Path(self.workflows_storage_path) / workflow_name
83
83
  if not package_full_path.is_file():
@@ -9,13 +9,13 @@ from cognite.client.data_classes import (
9
9
  )
10
10
  from cognite.client.data_classes.data_modeling import EdgeApply, NodeApply
11
11
 
12
- from cognite.neat.graph.stores import NeatGraphStore
13
12
  from cognite.neat.rules.models import (
14
13
  AssetRules,
15
14
  DMSRules,
16
15
  DomainRules,
17
16
  InformationRules,
18
17
  )
18
+ from cognite.neat.store import NeatGraphStore
19
19
  from cognite.neat.workflows.steps.step_model import DataContract
20
20
 
21
21
 
@@ -8,8 +8,8 @@ from rdflib import URIRef
8
8
  from cognite.neat.constants import DEFAULT_NAMESPACE
9
9
  from cognite.neat.graph.extractors import RdfFileExtractor
10
10
  from cognite.neat.graph.extractors._mock_graph_generator import MockGraphGenerator
11
+ from cognite.neat.issues.errors import WorkflowStepNotInitializedError
11
12
  from cognite.neat.rules._shared import DMSRules, InformationRules
12
- from cognite.neat.workflows._exceptions import StepNotInitialized
13
13
  from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
14
14
  from cognite.neat.workflows.steps.data_contracts import MultiRuleData, NeatGraph
15
15
  from cognite.neat.workflows.steps.step_model import Configurable, Step
@@ -45,7 +45,7 @@ class GraphFromMockData(Step):
45
45
  self, rules: MultiRuleData, graph_store: NeatGraph
46
46
  ) -> FlowMessage:
47
47
  if self.configs is None:
48
- raise StepNotInitialized(type(self).__name__)
48
+ raise WorkflowStepNotInitializedError(type(self).__name__)
49
49
 
50
50
  if not rules.information and not rules.dms:
51
51
  return FlowMessage(
@@ -110,7 +110,7 @@ class GraphFromRdfFile(Step):
110
110
 
111
111
  def run(self, graph_store: NeatGraph) -> FlowMessage: # type: ignore[override, syntax]
112
112
  if self.configs is None or self.data_store_path is None:
113
- raise StepNotInitialized(type(self).__name__)
113
+ raise WorkflowStepNotInitializedError(type(self).__name__)
114
114
 
115
115
  if source_file := self.configs["File path"]:
116
116
  NeatGraph.graph.write(
@@ -2,7 +2,7 @@ import time
2
2
  from pathlib import Path
3
3
  from typing import ClassVar
4
4
 
5
- from cognite.neat.workflows._exceptions import StepNotInitialized
5
+ from cognite.neat.issues.errors import WorkflowStepNotInitializedError
6
6
  from cognite.neat.workflows.model import FlowMessage
7
7
  from cognite.neat.workflows.steps.data_contracts import NeatGraph
8
8
  from cognite.neat.workflows.steps.step_model import Configurable, Step
@@ -34,7 +34,7 @@ class GraphToRdfFile(Step):
34
34
  self, graph: NeatGraph
35
35
  ) -> FlowMessage: # type: ignore[syntax]
36
36
  if self.configs is None or self.data_store_path is None:
37
- raise StepNotInitialized(type(self).__name__)
37
+ raise WorkflowStepNotInitializedError(type(self).__name__)
38
38
 
39
39
  storage_path = self.data_store_path / Path(self.configs["File path"])
40
40
  relative_graph_file_path = str(storage_path).split("/data/")[1]
@@ -1,6 +1,6 @@
1
1
  from typing import ClassVar
2
2
 
3
- from cognite.neat.graph.stores import NeatGraphStore
3
+ from cognite.neat.store import NeatGraphStore
4
4
  from cognite.neat.workflows.model import FlowMessage
5
5
  from cognite.neat.workflows.steps.data_contracts import (
6
6
  MultiRuleData,
@@ -2,10 +2,10 @@ import time
2
2
  from pathlib import Path
3
3
  from typing import ClassVar, Literal, cast
4
4
 
5
+ from cognite.neat.issues.errors import WorkflowStepNotInitializedError
5
6
  from cognite.neat.rules import exporters
6
- from cognite.neat.rules._shared import DMSRules, InformationRules, Rules
7
+ from cognite.neat.rules._shared import DMSRules, InformationRules, VerifiedRules
7
8
  from cognite.neat.rules.models import RoleTypes
8
- from cognite.neat.workflows._exceptions import StepNotInitialized
9
9
  from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
10
10
  from cognite.neat.workflows.steps.data_contracts import CogniteClient, MultiRuleData
11
11
  from cognite.neat.workflows.steps.step_model import Configurable, Step
@@ -61,7 +61,7 @@ class DeleteDataModelFromCDF(Step):
61
61
 
62
62
  def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage: # type: ignore[override]
63
63
  if self.configs is None or self.data_store_path is None:
64
- raise StepNotInitialized(type(self).__name__)
64
+ raise WorkflowStepNotInitializedError(type(self).__name__)
65
65
  components_to_delete = {
66
66
  cast(Literal["all", "spaces", "data_models", "views", "containers"], key)
67
67
  for key, value in self.complex_configs["Components"].items()
@@ -168,7 +168,7 @@ class RulesToDMS(Step):
168
168
 
169
169
  def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage: # type: ignore[override]
170
170
  if self.configs is None or self.data_store_path is None:
171
- raise StepNotInitialized(type(self).__name__)
171
+ raise WorkflowStepNotInitializedError(type(self).__name__)
172
172
  existing_components_handling = cast(
173
173
  Literal["fail", "update", "skip", "force"], self.configs["Existing component handling"]
174
174
  )
@@ -291,7 +291,7 @@ class RulesToExcel(Step):
291
291
 
292
292
  def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
293
293
  if self.configs is None or self.data_store_path is None:
294
- raise StepNotInitialized(type(self).__name__)
294
+ raise WorkflowStepNotInitializedError(type(self).__name__)
295
295
 
296
296
  dump_format = self.configs.get("Dump Format", "user")
297
297
  styling = cast(exporters.ExcelExporter.Style, self.configs.get("Styling", "default"))
@@ -317,7 +317,7 @@ class RulesToExcel(Step):
317
317
  new_model_id=new_model_id,
318
318
  )
319
319
 
320
- rule_instance: Rules
320
+ rule_instance: VerifiedRules
321
321
  if rules.domain:
322
322
  rule_instance = rules.domain
323
323
  elif rules.information:
@@ -369,7 +369,7 @@ class RulesToOntology(Step):
369
369
 
370
370
  def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
371
371
  if self.configs is None or self.data_store_path is None:
372
- raise StepNotInitialized(type(self).__name__)
372
+ raise WorkflowStepNotInitializedError(type(self).__name__)
373
373
 
374
374
  if not rules.information and not rules.dms:
375
375
  return FlowMessage(
@@ -420,7 +420,7 @@ class RulesToSHACL(Step):
420
420
 
421
421
  def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
422
422
  if self.configs is None or self.data_store_path is None:
423
- raise StepNotInitialized(type(self).__name__)
423
+ raise WorkflowStepNotInitializedError(type(self).__name__)
424
424
 
425
425
  if not rules.information and not rules.dms:
426
426
  return FlowMessage(
@@ -471,7 +471,7 @@ class RulesToSemanticDataModel(Step):
471
471
 
472
472
  def run(self, rules: MultiRuleData) -> FlowMessage: # type: ignore[override, syntax]
473
473
  if self.configs is None or self.data_store_path is None:
474
- raise StepNotInitialized(type(self).__name__)
474
+ raise WorkflowStepNotInitializedError(type(self).__name__)
475
475
 
476
476
  if not rules.information and not rules.dms:
477
477
  return FlowMessage(
@@ -525,7 +525,7 @@ class RulesToCDFTransformations(Step):
525
525
 
526
526
  def run(self, rules: MultiRuleData, cdf_client: CogniteClient) -> FlowMessage: # type: ignore[override]
527
527
  if self.configs is None or self.data_store_path is None:
528
- raise StepNotInitialized(type(self).__name__)
528
+ raise WorkflowStepNotInitializedError(type(self).__name__)
529
529
 
530
530
  input_rules = rules.dms or rules.information
531
531
  if input_rules is None:
@@ -5,11 +5,11 @@ from typing import ClassVar
5
5
  from cognite.client import CogniteClient
6
6
  from cognite.client.data_classes.data_modeling import DataModelId
7
7
 
8
+ from cognite.neat.issues.errors import WorkflowStepNotInitializedError
9
+ from cognite.neat.issues.formatters import FORMATTER_BY_NAME
8
10
  from cognite.neat.rules import importers
9
- from cognite.neat.rules.issues.formatters import FORMATTER_BY_NAME
10
11
  from cognite.neat.rules.models import RoleTypes
11
12
  from cognite.neat.rules.models.entities import DataModelEntity, DMSUnknownEntity
12
- from cognite.neat.workflows._exceptions import StepNotInitialized
13
13
  from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
14
14
  from cognite.neat.workflows.steps.data_contracts import MultiRuleData
15
15
  from cognite.neat.workflows.steps.step_model import Configurable, Step
@@ -19,6 +19,7 @@ CATEGORY = __name__.split(".")[-1].replace("_", " ").title()
19
19
  __all__ = [
20
20
  "ExcelToRules",
21
21
  "OntologyToRules",
22
+ "IMFToRules",
22
23
  "DMSToRules",
23
24
  "RulesInferenceFromRdfFile",
24
25
  ]
@@ -54,7 +55,7 @@ class ExcelToRules(Step):
54
55
 
55
56
  def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData): # type: ignore[syntax, override]
56
57
  if self.configs is None or self.data_store_path is None:
57
- raise StepNotInitialized(type(self).__name__)
58
+ raise WorkflowStepNotInitializedError(type(self).__name__)
58
59
 
59
60
  file_name = self.configs.get("File name", None)
60
61
  full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None
@@ -123,7 +124,7 @@ class OntologyToRules(Step):
123
124
 
124
125
  def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData): # type: ignore[syntax, override]
125
126
  if self.configs is None or self.data_store_path is None:
126
- raise StepNotInitialized(type(self).__name__)
127
+ raise WorkflowStepNotInitializedError(type(self).__name__)
127
128
 
128
129
  file_name = self.configs.get("File name", None)
129
130
  full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None
@@ -162,6 +163,77 @@ class OntologyToRules(Step):
162
163
  return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
163
164
 
164
165
 
166
+ class IMFToRules(Step):
167
+ """This step import rules from the IMF-types and validates them."""
168
+
169
+ description = "This step imports rules from an RDF file "
170
+ version = "private-beta"
171
+ category = CATEGORY
172
+ configurables: ClassVar[list[Configurable]] = [
173
+ Configurable(
174
+ name="File name",
175
+ value="",
176
+ label="""Full file name of the RDF-file containing the IMF-types in the rules folder.
177
+ If not provided, step will attempt to get file name from payload
178
+ of 'File Uploader' step (if exist)""",
179
+ ),
180
+ Configurable(
181
+ name="Report formatter",
182
+ value=next(iter(FORMATTER_BY_NAME.keys())),
183
+ label="The format of the report for the validation of the rules",
184
+ options=list(FORMATTER_BY_NAME),
185
+ ),
186
+ Configurable(
187
+ name="Role",
188
+ value="infer",
189
+ label="For what role Rules are intended?",
190
+ options=["infer", *RoleTypes.__members__.keys()],
191
+ ),
192
+ ]
193
+
194
+ def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData): # type: ignore[syntax, override]
195
+ if self.configs is None or self.data_store_path is None:
196
+ raise WorkflowStepNotInitializedError(type(self).__name__)
197
+
198
+ file_name = self.configs.get("File name", None)
199
+
200
+ full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None
201
+
202
+ if file_name:
203
+ rules_file_path = self.config.rules_store_path / file_name
204
+
205
+ elif full_path:
206
+ rules_file_path = full_path
207
+ else:
208
+ error_text = "Expected either 'File name' in the step config or 'File uploader' step uploading Excel Rules."
209
+ return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
210
+
211
+ # if role is None, it will be inferred from the rules file
212
+ role = self.configs.get("Role")
213
+ role_enum = None
214
+ if role != "infer" and role is not None:
215
+ role_enum = RoleTypes[role]
216
+
217
+ ontology_importer = importers.IMFImporter(filepath=rules_file_path)
218
+ rules, issues = ontology_importer.to_rules(errors="continue", role=role_enum)
219
+
220
+ if rules is None:
221
+ output_dir = self.config.staging_path
222
+ report_writer = FORMATTER_BY_NAME[self.configs["Report formatter"]]()
223
+ report_writer.write_to_file(issues, file_or_dir_path=output_dir)
224
+ report_file = report_writer.default_file_name
225
+ error_text = (
226
+ "<p></p>"
227
+ f'<a href="/data/staging/{report_file}?{time.time()}" '
228
+ f'target="_blank">Failed to validate rules, click here for report</a>'
229
+ )
230
+ return FlowMessage(error_text=error_text, step_execution_status=StepExecutionStatus.ABORT_AND_FAIL)
231
+
232
+ output_text = "Rules validation passed successfully!"
233
+
234
+ return FlowMessage(output_text=output_text), MultiRuleData.from_rules(rules)
235
+
236
+
165
237
  class DMSToRules(Step):
166
238
  """This step imports rules from CDF Data Model"""
167
239
 
@@ -199,7 +271,7 @@ class DMSToRules(Step):
199
271
 
200
272
  def run(self, cdf_client: CogniteClient) -> (FlowMessage, MultiRuleData): # type: ignore[syntax, override]
201
273
  if self.configs is None or self.data_store_path is None:
202
- raise StepNotInitialized(type(self).__name__)
274
+ raise WorkflowStepNotInitializedError(type(self).__name__)
203
275
 
204
276
  datamodel_id_str = self.configs.get("Data model id")
205
277
  if datamodel_id_str is None:
@@ -288,7 +360,7 @@ class RulesInferenceFromRdfFile(Step):
288
360
 
289
361
  def run(self, flow_message: FlowMessage) -> (FlowMessage, MultiRuleData): # type: ignore[syntax, override]
290
362
  if self.configs is None or self.data_store_path is None:
291
- raise StepNotInitialized(type(self).__name__)
363
+ raise WorkflowStepNotInitializedError(type(self).__name__)
292
364
 
293
365
  file_path = self.configs.get("File path", None)
294
366
  full_path = flow_message.payload.get("full_path", None) if flow_message.payload else None