cognite-neat 0.121.1__py3-none-any.whl → 0.122.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (104) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/core/_client/_api/statistics.py +91 -0
  3. cognite/neat/core/_client/_api_client.py +2 -0
  4. cognite/neat/core/_client/data_classes/statistics.py +125 -0
  5. cognite/neat/core/_client/testing.py +4 -0
  6. cognite/neat/core/_constants.py +6 -7
  7. cognite/neat/core/_data_model/_constants.py +23 -16
  8. cognite/neat/core/_data_model/_shared.py +33 -17
  9. cognite/neat/core/_data_model/analysis/__init__.py +2 -2
  10. cognite/neat/core/_data_model/analysis/_base.py +186 -183
  11. cognite/neat/core/_data_model/catalog/__init__.py +2 -2
  12. cognite/neat/core/_data_model/exporters/__init__.py +6 -6
  13. cognite/neat/core/_data_model/exporters/_base.py +10 -8
  14. cognite/neat/core/_data_model/exporters/{_rules2dms.py → _data_model2dms.py} +22 -18
  15. cognite/neat/core/_data_model/exporters/{_rules2excel.py → _data_model2excel.py} +51 -51
  16. cognite/neat/core/_data_model/exporters/{_rules2instance_template.py → _data_model2instance_template.py} +14 -14
  17. cognite/neat/core/_data_model/exporters/{_rules2ontology.py → _data_model2ontology.py} +50 -50
  18. cognite/neat/core/_data_model/exporters/{_rules2yaml.py → _data_model2yaml.py} +21 -18
  19. cognite/neat/core/_data_model/importers/__init__.py +8 -8
  20. cognite/neat/core/_data_model/importers/_base.py +8 -6
  21. cognite/neat/core/_data_model/importers/_base_file_reader.py +56 -0
  22. cognite/neat/core/_data_model/importers/{_yaml2rules.py → _dict2data_model.py} +50 -25
  23. cognite/neat/core/_data_model/importers/{_dms2rules.py → _dms2data_model.py} +58 -49
  24. cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_converter.py +22 -22
  25. cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/dtdl_importer.py +7 -7
  26. cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/spec.py +3 -3
  27. cognite/neat/core/_data_model/importers/_rdf/__init__.py +3 -3
  28. cognite/neat/core/_data_model/importers/_rdf/_base.py +15 -15
  29. cognite/neat/core/_data_model/importers/_rdf/{_imf2rules.py → _imf2data_model.py} +17 -17
  30. cognite/neat/core/_data_model/importers/_rdf/{_inference2rules.py → _inference2rdata_model.py} +59 -59
  31. cognite/neat/core/_data_model/importers/_rdf/{_owl2rules.py → _owl2data_model.py} +17 -17
  32. cognite/neat/core/_data_model/importers/_rdf/_shared.py +25 -25
  33. cognite/neat/core/_data_model/importers/{_spreadsheet2rules.py → _spreadsheet2data_model.py} +76 -19
  34. cognite/neat/core/_data_model/models/__init__.py +11 -9
  35. cognite/neat/core/_data_model/models/_base_unverified.py +12 -12
  36. cognite/neat/core/_data_model/models/_base_verified.py +9 -14
  37. cognite/neat/core/_data_model/models/_types.py +6 -6
  38. cognite/neat/core/_data_model/models/conceptual/__init__.py +6 -6
  39. cognite/neat/core/_data_model/models/conceptual/_unverified.py +20 -20
  40. cognite/neat/core/_data_model/models/conceptual/_validation.py +88 -78
  41. cognite/neat/core/_data_model/models/conceptual/_verified.py +54 -52
  42. cognite/neat/core/_data_model/models/data_types.py +2 -2
  43. cognite/neat/core/_data_model/models/entities/__init__.py +8 -8
  44. cognite/neat/core/_data_model/models/entities/_loaders.py +11 -10
  45. cognite/neat/core/_data_model/models/entities/_multi_value.py +5 -5
  46. cognite/neat/core/_data_model/models/entities/_single_value.py +44 -38
  47. cognite/neat/core/_data_model/models/entities/_types.py +9 -3
  48. cognite/neat/core/_data_model/models/entities/_wrapped.py +3 -3
  49. cognite/neat/core/_data_model/models/mapping/_classic2core.py +12 -9
  50. cognite/neat/core/_data_model/models/physical/__init__.py +40 -0
  51. cognite/neat/core/_data_model/models/{dms → physical}/_exporter.py +75 -55
  52. cognite/neat/core/_data_model/models/{dms/_rules_input.py → physical/_unverified.py} +48 -39
  53. cognite/neat/core/_data_model/models/{dms → physical}/_validation.py +17 -15
  54. cognite/neat/core/_data_model/models/{dms/_rules.py → physical/_verified.py} +68 -60
  55. cognite/neat/core/_data_model/transformers/__init__.py +29 -25
  56. cognite/neat/core/_data_model/transformers/_base.py +27 -20
  57. cognite/neat/core/_data_model/transformers/_converters.py +707 -622
  58. cognite/neat/core/_data_model/transformers/_mapping.py +74 -55
  59. cognite/neat/core/_data_model/transformers/_verification.py +64 -55
  60. cognite/neat/core/_instances/extractors/_base.py +2 -2
  61. cognite/neat/core/_instances/extractors/_classic_cdf/_classic.py +9 -9
  62. cognite/neat/core/_instances/extractors/_dms_graph.py +42 -34
  63. cognite/neat/core/_instances/extractors/_mock_graph_generator.py +107 -103
  64. cognite/neat/core/_instances/loaders/_base.py +3 -3
  65. cognite/neat/core/_instances/loaders/_rdf2dms.py +22 -22
  66. cognite/neat/core/_instances/transformers/_base.py +7 -4
  67. cognite/neat/core/_instances/transformers/_rdfpath.py +1 -1
  68. cognite/neat/core/_instances/transformers/_value_type.py +2 -6
  69. cognite/neat/core/_issues/_base.py +4 -4
  70. cognite/neat/core/_issues/_factory.py +1 -1
  71. cognite/neat/core/_issues/errors/__init__.py +2 -2
  72. cognite/neat/core/_issues/errors/_resources.py +1 -1
  73. cognite/neat/core/_issues/errors/_wrapper.py +2 -2
  74. cognite/neat/core/_issues/warnings/_models.py +4 -4
  75. cognite/neat/core/_issues/warnings/_properties.py +1 -1
  76. cognite/neat/core/_store/__init__.py +3 -3
  77. cognite/neat/core/_store/{_rules_store.py → _data_model.py} +119 -112
  78. cognite/neat/core/_store/{_graph_store.py → _instance.py} +3 -4
  79. cognite/neat/core/_store/_provenance.py +2 -2
  80. cognite/neat/core/_store/exceptions.py +2 -2
  81. cognite/neat/core/_utils/rdf_.py +14 -0
  82. cognite/neat/core/_utils/text.py +1 -1
  83. cognite/neat/session/_base.py +42 -36
  84. cognite/neat/session/_drop.py +2 -2
  85. cognite/neat/session/_experimental.py +1 -1
  86. cognite/neat/session/_inspect.py +13 -13
  87. cognite/neat/session/_mapping.py +15 -9
  88. cognite/neat/session/_read.py +39 -37
  89. cognite/neat/session/_set.py +6 -6
  90. cognite/neat/session/_show.py +24 -21
  91. cognite/neat/session/_state/README.md +1 -1
  92. cognite/neat/session/_state.py +27 -27
  93. cognite/neat/session/_subset.py +14 -11
  94. cognite/neat/session/_template.py +23 -21
  95. cognite/neat/session/_to.py +42 -42
  96. {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/METADATA +14 -7
  97. {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/RECORD +102 -100
  98. cognite/neat/core/_data_model/exporters/_validation.py +0 -14
  99. cognite/neat/core/_data_model/models/dms/__init__.py +0 -32
  100. /cognite/neat/core/_data_model/catalog/{info-rules-imf.xlsx → conceptual-imf-data-model.xlsx} +0 -0
  101. /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/__init__.py +0 -0
  102. /cognite/neat/core/_data_model/importers/{_dtdl2rules → _dtdl2data_model}/_unit_lookup.py +0 -0
  103. {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/WHEEL +0 -0
  104. {cognite_neat-0.121.1.dist-info → cognite_neat-0.122.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,7 +1,7 @@
1
1
  # NeatState
2
2
 
3
3
  **IN DEVELOPMENT**
4
- The neat state controls the `NeatGraphStore` and `NeatRuleStore`. It is implementing a state machine pattern
4
+ The neat state controls the `NeatInstanceStore` and `NeatGraphStore`. It is implementing a state machine pattern
5
5
  to ensure valid state transitions. The diagram below shows the state machine:
6
6
 
7
7
  ```mermaid
@@ -5,13 +5,13 @@ from rdflib import URIRef
5
5
 
6
6
  from cognite.neat.core._client import NeatClient
7
7
  from cognite.neat.core._data_model.importers import BaseImporter, InferenceImporter
8
- from cognite.neat.core._data_model.models import ConceptualDataModel, DMSRules
8
+ from cognite.neat.core._data_model.models import ConceptualDataModel, PhysicalDataModel
9
9
  from cognite.neat.core._data_model.transformers import (
10
- VerifiedRulesTransformer,
10
+ VerifiedDataModelTransformer,
11
11
  )
12
12
  from cognite.neat.core._instances.extractors import KnowledgeGraphExtractor
13
13
  from cognite.neat.core._issues import IssueList
14
- from cognite.neat.core._store import NeatGraphStore, NeatRulesStore
14
+ from cognite.neat.core._store import NeatDataModelStore, NeatInstanceStore
15
15
  from cognite.neat.core._utils.upload import UploadResultList
16
16
 
17
17
  from .exceptions import NeatSessionError, _session_method_wrapper
@@ -25,31 +25,31 @@ class SessionState:
25
25
  client: NeatClient | None = None,
26
26
  ) -> None:
27
27
  self.instances = InstancesState(store_type, storage_path=storage_path)
28
- self.rule_store = NeatRulesStore()
29
- self.last_reference: DMSRules | ConceptualDataModel | None = None
28
+ self.data_model_store = NeatDataModelStore()
29
+ self.last_reference: PhysicalDataModel | ConceptualDataModel | None = None
30
30
  self.client = client
31
31
  self.quoted_source_identifiers = False
32
32
 
33
- def rule_transform(self, *transformer: VerifiedRulesTransformer) -> IssueList:
33
+ def rule_transform(self, *transformer: VerifiedDataModelTransformer) -> IssueList:
34
34
  if not transformer:
35
35
  raise NeatSessionError("No transformers provided.")
36
- start = self.rule_store.provenance[-1].target_entity.display_name
37
- issues = self.rule_store.transform(*transformer)
38
- last_entity = self.rule_store.provenance[-1].target_entity
36
+ start = self.data_model_store.provenance[-1].target_entity.display_name
37
+ issues = self.data_model_store.transform(*transformer)
38
+ last_entity = self.data_model_store.provenance[-1].target_entity
39
39
  issues.action = f"{start} → {last_entity.display_name}"
40
40
  issues.hint = "Use the .inspect.issues() for more details."
41
41
  return issues
42
42
 
43
43
  def rule_import(self, importer: BaseImporter, enable_manual_edit: bool = False) -> IssueList:
44
- issues = self.rule_store.import_rules(
44
+ issues = self.data_model_store.import_data_model(
45
45
  importer,
46
46
  client=self.client,
47
47
  enable_manual_edit=enable_manual_edit,
48
48
  )
49
- if self.rule_store.empty:
49
+ if self.data_model_store.empty:
50
50
  result = "failed"
51
51
  else:
52
- result = self.rule_store.provenance[-1].target_entity.display_name
52
+ result = self.data_model_store.provenance[-1].target_entity.display_name
53
53
  if isinstance(importer, InferenceImporter):
54
54
  issues.action = f"Inferred {result}"
55
55
  else:
@@ -60,19 +60,19 @@ class SessionState:
60
60
 
61
61
  def write_graph(self, extractor: KnowledgeGraphExtractor) -> IssueList:
62
62
  extract_issues = self.instances.store.write(extractor)
63
- issues = self.rule_store.import_graph(extractor)
63
+ issues = self.data_model_store.import_graph(extractor)
64
64
  issues.extend(extract_issues)
65
65
  return issues
66
66
 
67
67
  def _raise_exception_if_condition_not_met(
68
68
  self,
69
69
  activity: str,
70
- empty_rules_store_required: bool = False,
70
+ empty_data_model_store_required: bool = False,
71
71
  empty_instances_store_required: bool = False,
72
72
  instances_required: bool = False,
73
73
  client_required: bool = False,
74
- has_information_rules: bool | None = None,
75
- has_dms_rules: bool | None = None,
74
+ has_conceptual_data_model: bool | None = None,
75
+ has_physical_data_model: bool | None = None,
76
76
  ) -> None:
77
77
  """Set conditions for raising an error in the session that are used by various methods in the session."""
78
78
  condition = set()
@@ -81,14 +81,14 @@ class SessionState:
81
81
  if client_required and not self.client:
82
82
  condition.add(f"{activity} expects a client in NEAT session")
83
83
  suggestion.add("Please provide a client")
84
- if has_information_rules is True and self.rule_store.try_get_last_information_rules is None:
85
- condition.add(f"{activity} expects information rules in NEAT session")
86
- suggestion.add("Read in information rules to neat session")
87
- if has_dms_rules is False and self.rule_store.try_get_last_dms_rules is not None:
88
- condition.add(f"{activity} expects no DMS data model in NEAT session")
89
- suggestion.add("You already have a DMS data model in the session")
84
+ if has_conceptual_data_model is True and self.data_model_store.try_get_last_conceptual_data_model is None:
85
+ condition.add(f"{activity} expects conceptual data model in NEAT session")
86
+ suggestion.add("Read in conceptual data model to neat session")
87
+ if has_physical_data_model is False and self.data_model_store.try_get_last_physical_data_model is not None:
88
+ condition.add(f"{activity} expects no physical data model in NEAT session")
89
+ suggestion.add("You already have a physical data model in the session")
90
90
  try_again = False
91
- if empty_rules_store_required and not self.rule_store.empty:
91
+ if empty_data_model_store_required and not self.data_model_store.empty:
92
92
  condition.add(f"{activity} expects no data model in NEAT session")
93
93
  suggestion.add("Start new session")
94
94
  if empty_instances_store_required and not self.instances.empty:
@@ -121,18 +121,18 @@ class InstancesState:
121
121
  self.neat_prefix_by_type_uri: dict[URIRef, str] = {}
122
122
 
123
123
  # Ensure that error handling is done in the constructor
124
- self.store: NeatGraphStore = _session_method_wrapper(self._create_store, "NeatSession")()
124
+ self.store: NeatInstanceStore = _session_method_wrapper(self._create_store, "NeatSession")()
125
125
 
126
126
  if self.storage_path:
127
127
  print("Remember to close neat session .close() once you are done to avoid oxigraph lock.")
128
128
 
129
- def _create_store(self) -> NeatGraphStore:
129
+ def _create_store(self) -> NeatInstanceStore:
130
130
  if self.store_type == "oxigraph":
131
131
  if self.storage_path:
132
132
  self.storage_path.mkdir(parents=True, exist_ok=True)
133
- return NeatGraphStore.from_oxi_local_store(storage_dir=self.storage_path)
133
+ return NeatInstanceStore.from_oxi_local_store(storage_dir=self.storage_path)
134
134
  else:
135
- return NeatGraphStore.from_memory_store()
135
+ return NeatInstanceStore.from_memory_store()
136
136
 
137
137
  @property
138
138
  def empty(self) -> bool:
@@ -1,10 +1,13 @@
1
1
  import warnings
2
2
 
3
3
  from cognite.neat.core._data_model.models.entities._single_value import (
4
- ClassEntity,
4
+ ConceptEntity,
5
5
  ViewEntity,
6
6
  )
7
- from cognite.neat.core._data_model.transformers import SubsetDMSRules, SubsetInformationRules
7
+ from cognite.neat.core._data_model.transformers import (
8
+ SubsetConceptualDataModel,
9
+ SubsetPhysicalDataModel,
10
+ )
8
11
  from cognite.neat.core._issues._base import IssueList
9
12
  from cognite.neat.session._experimental import ExperimentalFlags
10
13
 
@@ -46,14 +49,14 @@ class SubsetAPI:
46
49
  neat.subset.data_model("CogniteAsset")
47
50
  ```
48
51
  """
49
- if self._state.rule_store.empty:
50
- raise NeatSessionError("No rules to set the data model ID.")
52
+ if self._state.data_model_store.empty:
53
+ raise NeatSessionError("No data model to set the data model ID.")
51
54
 
52
55
  warnings.filterwarnings("default")
53
56
  ExperimentalFlags.data_model_subsetting.warn()
54
57
 
55
- dms = self._state.rule_store.provenance[-1].target_entity.dms
56
- information = self._state.rule_store.provenance[-1].target_entity.information
58
+ dms = self._state.data_model_store.provenance[-1].target_entity.physical
59
+ information = self._state.data_model_store.provenance[-1].target_entity.conceptual
57
60
 
58
61
  if dms:
59
62
  views = {
@@ -65,16 +68,16 @@ class SubsetAPI:
65
68
  for concept in concepts
66
69
  }
67
70
 
68
- issues = self._state.rule_transform(SubsetDMSRules(views=views))
71
+ issues = self._state.rule_transform(SubsetPhysicalDataModel(views=views))
69
72
  if not issues:
70
- after = len(self._state.rule_store.last_verified_dms_rules.views)
73
+ after = len(self._state.data_model_store.last_verified_physical_data_model.views)
71
74
 
72
75
  elif information:
73
- classes = {ClassEntity(prefix=information.metadata.space, suffix=concept) for concept in concepts}
76
+ classes = {ConceptEntity(prefix=information.metadata.space, suffix=concept) for concept in concepts}
74
77
 
75
- issues = self._state.rule_transform(SubsetInformationRules(classes=classes))
78
+ issues = self._state.rule_transform(SubsetConceptualDataModel(concepts=classes))
76
79
  if not issues:
77
- after = len(self._state.rule_store.last_verified_information_rules.classes)
80
+ after = len(self._state.data_model_store.last_verified_conceptual_data_model.concepts)
78
81
 
79
82
  else:
80
83
  raise NeatSessionError("Something went terrible wrong. Please contact the neat team.")
@@ -4,18 +4,18 @@ from typing import Any, Literal
4
4
  from cognite.client.data_classes.data_modeling import DataModelIdentifier
5
5
 
6
6
  from cognite.neat.core._constants import BASE_MODEL
7
- from cognite.neat.core._data_model._shared import ReadRules
7
+ from cognite.neat.core._data_model._shared import ImportedDataModel
8
8
  from cognite.neat.core._data_model.exporters import ExcelExporter
9
9
  from cognite.neat.core._data_model.importers import ExcelImporter
10
10
  from cognite.neat.core._data_model.models import UnverifiedConceptualDataModel
11
11
  from cognite.neat.core._data_model.models._base_verified import RoleTypes
12
- from cognite.neat.core._data_model.models.dms import DMSValidation
12
+ from cognite.neat.core._data_model.models.physical import PhysicalValidation
13
13
  from cognite.neat.core._data_model.transformers import (
14
14
  AddCogniteProperties,
15
15
  IncludeReferenced,
16
16
  ToDataProductModel,
17
17
  ToEnterpriseModel,
18
- VerifiedRulesTransformer,
18
+ VerifiedDataModelTransformer,
19
19
  )
20
20
  from cognite.neat.core._issues import IssueList, catch_issues
21
21
  from cognite.neat.core._utils.reader import NeatReader, PathReader
@@ -69,7 +69,7 @@ class TemplateAPI:
69
69
  - Charts
70
70
 
71
71
  """
72
- last_rules = self._state.rule_store.last_verified_rules
72
+ last_dm = self._state.data_model_store.last_verified_data_model
73
73
  issues = self._state.rule_transform(
74
74
  ToEnterpriseModel(
75
75
  new_model_id=data_model_id,
@@ -78,8 +78,8 @@ class TemplateAPI:
78
78
  move_connections=True,
79
79
  )
80
80
  )
81
- if last_rules and not issues.has_errors:
82
- self._state.last_reference = last_rules
81
+ if last_dm and not issues.has_errors:
82
+ self._state.last_reference = last_dm
83
83
  return issues
84
84
 
85
85
  def data_product_model(
@@ -105,11 +105,11 @@ class TemplateAPI:
105
105
  If you set same-space, only the properties of the views in the same space as the data model
106
106
  will be included.
107
107
  """
108
- last_rules = self._state.rule_store.last_verified_rules
109
- view_ids, container_ids = DMSValidation(
110
- self._state.rule_store.last_verified_dms_rules
108
+ last_dm = self._state.data_model_store.last_verified_data_model
109
+ view_ids, container_ids = PhysicalValidation(
110
+ self._state.data_model_store.last_verified_physical_data_model
111
111
  ).imported_views_and_containers_ids()
112
- transformers: list[VerifiedRulesTransformer] = []
112
+ transformers: list[VerifiedDataModelTransformer] = []
113
113
  client = self._state.client
114
114
  if (view_ids or container_ids) and client is None:
115
115
  raise NeatSessionError(
@@ -123,8 +123,8 @@ class TemplateAPI:
123
123
  transformers.append(ToDataProductModel(new_model_id=data_model_id, include=include))
124
124
 
125
125
  issues = self._state.rule_transform(*transformers)
126
- if last_rules and not issues.has_errors:
127
- self._state.last_reference = last_rules
126
+ if last_dm and not issues.has_errors:
127
+ self._state.last_reference = last_dm
128
128
  return issues
129
129
 
130
130
  def conceptual_model(
@@ -185,22 +185,24 @@ class TemplateAPI:
185
185
  output_path = Path(output)
186
186
 
187
187
  with catch_issues() as issues:
188
- read: ReadRules[UnverifiedConceptualDataModel] = ExcelImporter(path).to_rules()
189
- if read.rules is not None:
190
- # If rules are None there will be issues that are already caught.
191
- if not isinstance(read.rules, UnverifiedConceptualDataModel):
192
- raise NeatSessionError(f"The input {reader.name} must contain an InformationInputRules object. ")
188
+ read: ImportedDataModel[UnverifiedConceptualDataModel] = ExcelImporter(path).to_data_model()
189
+ if read.unverified_data_model is not None:
190
+ # If data model arise None there will be issues that are already caught.
191
+ if not isinstance(read.unverified_data_model, UnverifiedConceptualDataModel):
192
+ raise NeatSessionError(
193
+ f"The input {reader.name} must contain an UnverifiedConceptualDataModel object. "
194
+ )
193
195
  if self._state.client is None:
194
196
  raise NeatSessionError("Client must be set in the session to run the extension.")
195
197
  modified = AddCogniteProperties(self._state.client, dummy_property).transform(read)
196
- if modified.rules is not None:
197
- # If rules are None there will be issues that are already caught.
198
- info = modified.rules.as_verified_rules()
198
+ if modified.unverified_data_model is not None:
199
+ # If data model is None there will be issues that are already caught.
200
+ info = modified.unverified_data_model.as_verified_data_model()
199
201
 
200
202
  ExcelExporter(styling="maximal").export_to_file(info, output_path)
201
203
  issues.action = "Created extension template"
202
204
 
203
205
  # Adding issues to the state in the rule store
204
206
  if issues:
205
- self._state.rule_store._last_issues = issues
207
+ self._state.data_model_store._last_issues = issues
206
208
  return issues
@@ -10,10 +10,10 @@ from cognite.neat.core._client._api_client import NeatClient
10
10
  from cognite.neat.core._constants import COGNITE_MODELS
11
11
  from cognite.neat.core._data_model import exporters
12
12
  from cognite.neat.core._data_model._constants import PATTERNS
13
- from cognite.neat.core._data_model._shared import VerifiedRules
13
+ from cognite.neat.core._data_model._shared import VerifiedDataModel
14
14
  from cognite.neat.core._data_model.importers import DMSImporter
15
- from cognite.neat.core._data_model.models import ConceptualDataModel, DMSRules
16
- from cognite.neat.core._data_model.models.dms import DMSMetadata
15
+ from cognite.neat.core._data_model.models import ConceptualDataModel, PhysicalDataModel
16
+ from cognite.neat.core._data_model.models.physical import PhysicalMetadata
17
17
  from cognite.neat.core._instances import loaders
18
18
  from cognite.neat.core._issues import IssueList, NeatIssue, catch_issues
19
19
  from cognite.neat.core._utils.upload import UploadResultList
@@ -25,8 +25,8 @@ from .exceptions import NeatSessionError, session_class_wrapper
25
25
 
26
26
  @session_class_wrapper
27
27
  class ToAPI:
28
- """API used to write the contents of a NeatSession to a specified destination. For instance writing information
29
- rules or DMS rules to a NEAT rules Excel spreadsheet, or writing a verified data model to CDF.
28
+ """API used to write the contents of a NeatSession to a specified destination. For instance writing
29
+ conceptual or physical data model to a NEAT formatted Excel spreadsheet, or writing a verified data model to CDF.
30
30
 
31
31
  """
32
32
 
@@ -58,7 +58,7 @@ class ToAPI:
58
58
  filepath = filepath.with_suffix(".ttl")
59
59
 
60
60
  exporter = exporters.OWLExporter()
61
- self._state.rule_store.export_to_file(exporter, Path(io))
61
+ self._state.data_model_store.export_to_file(exporter, Path(io))
62
62
  return None
63
63
 
64
64
  def excel(
@@ -81,10 +81,10 @@ class ToAPI:
81
81
  add_empty_rows: If True, empty rows will be added between each component. Defaults to False.
82
82
 
83
83
  Example:
84
- Export information model to excel rules sheet
84
+ Export conceptual data model to an Excel file
85
85
  ```python
86
- information_rules_file_name = "information_rules.xlsx"
87
- neat.to.excel(information_rules_file_name)
86
+ conceptual_dm_file_name = "conceptual_data_model.xlsx"
87
+ neat.to.excel(conceptual_dm_file_name)
88
88
  ```
89
89
 
90
90
  Example:
@@ -98,8 +98,8 @@ class ToAPI:
98
98
  data_model_id=("sp_doctrino_space", "ExtensionCore", "v1"),
99
99
  org_name="MyOrg",
100
100
  )
101
- dms_rules_file_name = "dms_rules.xlsx"
102
- neat.to.excel(dms_rules_file_name, include_reference=True)
101
+ physical_dm_file_name = "physical_dm.xlsx"
102
+ neat.to.excel(physical_dm_file_name, include_reference=True)
103
103
  ```
104
104
 
105
105
  Example:
@@ -110,36 +110,36 @@ class ToAPI:
110
110
  neat = NeatSession(client)
111
111
 
112
112
  neat.read.cdf(("my_space", "ISA95Model", "v5"))
113
- dms_rules_file_name = "dms_rules.xlsx"
114
- neat.to.excel(dms_rules_file_name, include_reference=("cdf_cdm", "CogniteCore", "v1"))
113
+ physical_dm_file_name = "physical_dm.xlsx"
114
+ neat.to.excel(physical_dm_file_name, include_reference=("cdf_cdm", "CogniteCore", "v1"))
115
115
  """
116
- reference_rules_with_prefix: tuple[VerifiedRules, str] | None = None
116
+ reference_data_model_with_prefix: tuple[VerifiedDataModel, str] | None = None
117
117
  include_properties = include_properties.strip().lower()
118
118
 
119
119
  if include_reference is not False:
120
120
  if include_reference is True and self._state.last_reference is not None:
121
- ref_rules: ConceptualDataModel | DMSRules | None = self._state.last_reference
121
+ ref_data_model: ConceptualDataModel | PhysicalDataModel | None = self._state.last_reference
122
122
  elif include_reference is True:
123
- ref_rules = None
123
+ ref_data_model = None
124
124
  else:
125
125
  if not self._state.client:
126
126
  raise NeatSessionError("No client provided!")
127
- ref_rules = None
127
+ ref_data_model = None
128
128
  with catch_issues() as issues:
129
- ref_read = DMSImporter.from_data_model_id(self._state.client, include_reference).to_rules()
130
- if ref_read.rules is not None:
131
- ref_rules = ref_read.rules.as_verified_rules()
132
- if ref_rules is None or issues.has_errors:
129
+ ref_read = DMSImporter.from_data_model_id(self._state.client, include_reference).to_data_model()
130
+ if ref_read.unverified_data_model is not None:
131
+ ref_data_model = ref_read.unverified_data_model.as_verified_data_model()
132
+ if ref_data_model is None or issues.has_errors:
133
133
  issues.action = f"Read {include_reference}"
134
134
  return issues
135
- if ref_rules is not None:
135
+ if ref_data_model is not None:
136
136
  prefix = "Ref"
137
137
  if (
138
- isinstance(ref_rules.metadata, DMSMetadata)
139
- and ref_rules.metadata.as_data_model_id() in COGNITE_MODELS
138
+ isinstance(ref_data_model.metadata, PhysicalMetadata)
139
+ and ref_data_model.metadata.as_data_model_id() in COGNITE_MODELS
140
140
  ):
141
141
  prefix = "CDM"
142
- reference_rules_with_prefix = ref_rules, prefix
142
+ reference_data_model_with_prefix = ref_data_model, prefix
143
143
 
144
144
  if include_properties == "same-space":
145
145
  warnings.filterwarnings("default")
@@ -147,11 +147,11 @@ class ToAPI:
147
147
 
148
148
  exporter = exporters.ExcelExporter(
149
149
  styling="maximal",
150
- reference_rules_with_prefix=reference_rules_with_prefix,
150
+ reference_data_model_with_prefix=reference_data_model_with_prefix,
151
151
  add_empty_rows=add_empty_rows,
152
152
  include_properties=include_properties, # type: ignore
153
153
  )
154
- self._state.rule_store.export_to_file(exporter, Path(io))
154
+ self._state.data_model_store.export_to_file(exporter, Path(io))
155
155
  return None
156
156
 
157
157
  def session(self, io: Any) -> None:
@@ -216,7 +216,7 @@ class ToAPI:
216
216
  Example:
217
217
  Export to yaml file in the case of "neat" format
218
218
  ```python
219
- your_yaml_file_name = "neat_rules.yaml"
219
+ your_yaml_file_name = "neat_dm.yaml"
220
220
  neat.to.yaml(your_yaml_file_name, format="neat")
221
221
  ```
222
222
 
@@ -238,9 +238,9 @@ class ToAPI:
238
238
  if format == "neat":
239
239
  exporter = exporters.YAMLExporter()
240
240
  if io is None:
241
- return self._state.rule_store.export(exporter)
241
+ return self._state.data_model_store.export(exporter)
242
242
 
243
- self._state.rule_store.export_to_file(exporter, Path(io))
243
+ self._state.data_model_store.export_to_file(exporter, Path(io))
244
244
  elif format == "toolkit":
245
245
  if io is None or not isinstance(io, str | Path):
246
246
  raise NeatSessionError(
@@ -250,7 +250,7 @@ class ToAPI:
250
250
  user_path = Path(io)
251
251
  if user_path.suffix == "" and not user_path.exists():
252
252
  user_path.mkdir(parents=True)
253
- self._state.rule_store.export_to_file(
253
+ self._state.data_model_store.export_to_file(
254
254
  exporters.DMSExporter(remove_cdf_spaces=skip_system_spaces), user_path
255
255
  )
256
256
  else:
@@ -310,10 +310,10 @@ class CDFToAPI:
310
310
  )
311
311
 
312
312
  client = cast(NeatClient, self._state.client)
313
- dms_rules = self._state.rule_store.last_verified_dms_rules
314
- instance_space = instance_space or f"{dms_rules.metadata.space}_instances"
313
+ physical_data_model = self._state.data_model_store.last_verified_physical_data_model
314
+ instance_space = instance_space or f"{physical_data_model.metadata.space}_instances"
315
315
 
316
- if instance_space and instance_space == dms_rules.metadata.space:
316
+ if instance_space and instance_space == physical_data_model.metadata.space:
317
317
  raise NeatSessionError("Space for instances must be different from the data model space.")
318
318
  elif not PATTERNS.space_compliance.match(str(instance_space)):
319
319
  raise NeatSessionError("Please provide a valid space name. {PATTERNS.space_compliance.pattern}")
@@ -322,8 +322,8 @@ class CDFToAPI:
322
322
  client.data_modeling.spaces.apply(dm.SpaceApply(space=instance_space))
323
323
 
324
324
  loader = loaders.DMSLoader(
325
- self._state.rule_store.last_verified_dms_rules,
326
- self._state.rule_store.last_verified_information_rules,
325
+ self._state.data_model_store.last_verified_physical_data_model,
326
+ self._state.data_model_store.last_verified_conceptual_data_model,
327
327
  self._state.instances.store,
328
328
  instance_space=instance_space,
329
329
  client=client,
@@ -375,7 +375,7 @@ class CDFToAPI:
375
375
 
376
376
  exporter = exporters.DMSExporter(existing=existing, drop_data=drop_data)
377
377
 
378
- result = self._state.rule_store.export_to_cdf(exporter, cast(NeatClient, self._state.client), dry_run)
378
+ result = self._state.data_model_store.export_to_cdf(exporter, cast(NeatClient, self._state.client), dry_run)
379
379
  print("You can inspect the details with the .inspect.outcome.data_model(...) method.")
380
380
  return result
381
381
 
@@ -418,17 +418,17 @@ class ToPythonAPI:
418
418
  instances = neat.to._python.instances(space_from_property="dataSetId")
419
419
  ```
420
420
  """
421
- dms_rules = self._state.rule_store.last_verified_dms_rules
422
- instance_space = instance_space or f"{dms_rules.metadata.space}_instances"
421
+ physical_data_model = self._state.data_model_store.last_verified_physical_data_model
422
+ instance_space = instance_space or f"{physical_data_model.metadata.space}_instances"
423
423
 
424
- if instance_space and instance_space == dms_rules.metadata.space:
424
+ if instance_space and instance_space == physical_data_model.metadata.space:
425
425
  raise NeatSessionError("Space for instances must be different from the data model space.")
426
426
  elif not PATTERNS.space_compliance.match(str(instance_space)):
427
427
  raise NeatSessionError(f"Please provide a valid space name. {PATTERNS.space_compliance.pattern}")
428
428
 
429
429
  loader = loaders.DMSLoader(
430
- self._state.rule_store.last_verified_dms_rules,
431
- self._state.rule_store.last_verified_information_rules,
430
+ self._state.data_model_store.last_verified_physical_data_model,
431
+ self._state.data_model_store.last_verified_conceptual_data_model,
432
432
  self._state.instances.store,
433
433
  instance_space=instance_space,
434
434
  space_property=space_from_property,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cognite-neat
3
- Version: 0.121.1
3
+ Version: 0.122.0
4
4
  Summary: Knowledge graph transformation
5
5
  Project-URL: Documentation, https://cognite-neat.readthedocs-hosted.com/
6
6
  Project-URL: Homepage, https://cognite-neat.readthedocs-hosted.com/
@@ -67,9 +67,9 @@ Description-Content-Type: text/markdown
67
67
 
68
68
  NEAT is a domain expert centric and developer friendly solution for rapid:
69
69
 
70
- - semantic data modeling
71
- - creation, transformation and enrichment of knowledge graphs
72
- - and ingestion of the models and graphs into [Cognite Data Fusion](https://www.cognite.com/en/product/cognite_data_fusion_industrial_dataops_platform)
70
+ - data modeling
71
+ - extraction, transformation and loading of instances
72
+ - and ingestion of the models and instances (i.e. knowledge graphs) into [Cognite Data Fusion](https://www.cognite.com/en/product/cognite_data_fusion_industrial_dataops_platform)
73
73
 
74
74
  NEAT is using open and globally recognized standards maintained by the [World Wide Web Consortium (W3C)](https://www.w3.org/RDF/).
75
75
  NEAT represents an essential tool for creation of standardized, machine-actionable, linked and semantic (meta)data.
@@ -85,13 +85,12 @@ By mid of 2022, sheet2rdf was used in several POCs in Cognite. As Cognite's Data
85
85
 
86
86
  Presented in various forums in 2022, this approach paved the way for NEAT’s formal development in November 2022 to enable cost-saving and empowerment of Cognite customers to self-sufficiently maintain and onboard knowledge graphs to Cognite Data Fusion.
87
87
 
88
-
89
88
  ## Authorship
90
89
 
91
90
  ### Authors
92
91
  The plot below shows the NEAT authorship from the start until present day.
93
92
 
94
- ![NEAT authorship](./docs/artifacts/figs/authorship.png)
93
+ ![NEAT authorship](./artifacts/figs/authorship.png)
95
94
 
96
95
  #### Current authors
97
96
  - [Nikola Vasiljević](www.linkedin.com/in/thisisnikola)
@@ -108,9 +107,17 @@ We are very grateful for the contributions made by:
108
107
 
109
108
  - [Marie Solvik Lepoutre](https://www.linkedin.com/in/mslepoutre/), who improved RDF triples projections to Cognite Data Fusion
110
109
  - [Bård Henning Tvedt](https://www.linkedin.com/in/bhtvedt/), who implemented IMF importer
110
+ - [Hassan Gomaa](https://www.linkedin.com/in/dr-hassan-gomaa-232638121/), who extended the DEXPI extractor
111
111
  - [Kristina Tomičić](https://www.linkedin.com/in/kristina-tomicic-6bb443108/), who implemented Data Model and Instances visualization
112
112
 
113
113
 
114
+ ## Sponsors
115
+ NEAT is developed and maintained by Cognite. We are grateful for the past support of our sponsors, who funded us to develop NEAT and to make it open source.
116
+
117
+ - [Statnett](https://www.statnett.no/) - the MIMIR team ([Ola Hagen Øyan](https://www.linkedin.com/in/ola-%C3%B8yan-b0205b19/), [Olav Westeng Alstad](https://www.linkedin.com/in/olav-w-alstad-52329191/),[Andreas Kimsås](https://www.linkedin.com/in/andreas-kims%C3%A5s-964a0b2/) and [Anders Willersrud](https://www.linkedin.com/in/anders-willersrud-13a20220/)) – who supported the development of NEAT from end of 2022 to mid of 2023 and its integration with Statnett's infrastructure, where NEAT was battle-tested as a tool for non-sematic experts to define data models and transform large knowledge graphs representing the entire Norwegian power grid system. Without Statnett's support NEAT would not exist in the first place, and would not be open-source.
118
+ - [Aker Solutions](https://www.akersolutions.com/) – the IMod Team (currently lead by [Maria Kenderkova](https://www.linkedin.com/in/maria-kenderkova/)), who funded development of NEAT from mid of 2023 till end of 2024 (multi-level and role-based data modeling, support for ontologies, IMF, DEXPI, AML,...) , as well who were early adopters and embraced NEAT and who were patient with us when things did not work so well. Aker Solutions was instrumental to elevating NEAT to a product level, and who selflessly advocate for NEAT globally.
119
+
120
+
114
121
  ## Installation
115
122
 
116
123
  ```bash
@@ -134,4 +141,4 @@ neat.read.cdf.data_model(("my_space", "MyDataModel", "v1"))
134
141
 
135
142
  ## Documentation
136
143
 
137
- For more information, see the [documentation](https://cognite-neat.readthedocs-hosted.com/en/latest/)
144
+ For more information, see the [documentation](https://cognite-neat.readthedocs-hosted.com/en/latest/)