cognite-neat 0.103.0__py3-none-any.whl → 0.104.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (64) hide show
  1. cognite/neat/_graph/extractors/_mock_graph_generator.py +1 -1
  2. cognite/neat/_graph/transformers/_base.py +109 -1
  3. cognite/neat/_graph/transformers/_classic_cdf.py +4 -0
  4. cognite/neat/_graph/transformers/_prune_graph.py +103 -47
  5. cognite/neat/_graph/transformers/_rdfpath.py +41 -17
  6. cognite/neat/_graph/transformers/_value_type.py +119 -154
  7. cognite/neat/_issues/_base.py +35 -8
  8. cognite/neat/_issues/warnings/_resources.py +1 -1
  9. cognite/neat/_rules/_shared.py +18 -34
  10. cognite/neat/_rules/exporters/_base.py +28 -2
  11. cognite/neat/_rules/exporters/_rules2dms.py +4 -0
  12. cognite/neat/_rules/exporters/_rules2excel.py +11 -0
  13. cognite/neat/_rules/exporters/_rules2instance_template.py +4 -0
  14. cognite/neat/_rules/exporters/_rules2ontology.py +13 -1
  15. cognite/neat/_rules/exporters/_rules2yaml.py +4 -0
  16. cognite/neat/_rules/importers/_base.py +9 -0
  17. cognite/neat/_rules/importers/_dms2rules.py +17 -5
  18. cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +5 -2
  19. cognite/neat/_rules/importers/_rdf/_base.py +10 -8
  20. cognite/neat/_rules/importers/_rdf/_imf2rules.py +4 -0
  21. cognite/neat/_rules/importers/_rdf/_inference2rules.py +7 -0
  22. cognite/neat/_rules/importers/_rdf/_owl2rules.py +4 -0
  23. cognite/neat/_rules/importers/_spreadsheet2rules.py +17 -8
  24. cognite/neat/_rules/importers/_yaml2rules.py +21 -7
  25. cognite/neat/_rules/models/_base_input.py +1 -1
  26. cognite/neat/_rules/models/_base_rules.py +5 -0
  27. cognite/neat/_rules/models/dms/_rules.py +4 -0
  28. cognite/neat/_rules/models/dms/_rules_input.py +9 -0
  29. cognite/neat/_rules/models/dms/_validation.py +2 -0
  30. cognite/neat/_rules/models/entities/_single_value.py +25 -5
  31. cognite/neat/_rules/models/information/_rules.py +4 -0
  32. cognite/neat/_rules/models/information/_rules_input.py +9 -0
  33. cognite/neat/_rules/models/mapping/_classic2core.py +2 -5
  34. cognite/neat/_rules/transformers/__init__.py +5 -4
  35. cognite/neat/_rules/transformers/_base.py +41 -65
  36. cognite/neat/_rules/transformers/_converters.py +149 -62
  37. cognite/neat/_rules/transformers/_mapping.py +17 -12
  38. cognite/neat/_rules/transformers/_verification.py +50 -37
  39. cognite/neat/_session/_base.py +32 -121
  40. cognite/neat/_session/_inspect.py +3 -3
  41. cognite/neat/_session/_mapping.py +17 -105
  42. cognite/neat/_session/_prepare.py +36 -254
  43. cognite/neat/_session/_read.py +11 -130
  44. cognite/neat/_session/_set.py +6 -30
  45. cognite/neat/_session/_show.py +49 -30
  46. cognite/neat/_session/_state.py +49 -107
  47. cognite/neat/_session/_to.py +42 -31
  48. cognite/neat/_shared.py +23 -2
  49. cognite/neat/_store/_provenance.py +3 -82
  50. cognite/neat/_store/_rules_store.py +372 -10
  51. cognite/neat/_store/exceptions.py +23 -0
  52. cognite/neat/_utils/graph_transformations_report.py +36 -0
  53. cognite/neat/_utils/io_.py +11 -0
  54. cognite/neat/_utils/rdf_.py +8 -0
  55. cognite/neat/_utils/spreadsheet.py +5 -4
  56. cognite/neat/_version.py +1 -1
  57. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +7 -7
  58. cognite/neat/_workflows/steps/lib/current/rules_importer.py +24 -99
  59. {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/METADATA +1 -1
  60. {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/RECORD +63 -61
  61. cognite/neat/_rules/transformers/_pipelines.py +0 -70
  62. {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/LICENSE +0 -0
  63. {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/WHEEL +0 -0
  64. {cognite_neat-0.103.0.dist-info → cognite_neat-0.104.0.dist-info}/entry_points.txt +0 -0
@@ -5,7 +5,6 @@ from typing import Any, cast
5
5
  import networkx as nx
6
6
  from IPython.display import HTML, display
7
7
  from pyvis.network import Network as PyVisNetwork # type: ignore
8
- from rdflib import URIRef
9
8
 
10
9
  from cognite.neat._constants import IN_NOTEBOOK, IN_PYODIDE
11
10
  from cognite.neat._rules._constants import EntityTypes
@@ -13,7 +12,8 @@ from cognite.neat._rules.models.dms._rules import DMSRules
13
12
  from cognite.neat._rules.models.entities._single_value import ClassEntity, ViewEntity
14
13
  from cognite.neat._rules.models.information._rules import InformationRules
15
14
  from cognite.neat._session.exceptions import NeatSessionError
16
- from cognite.neat._utils.rdf_ import remove_namespace_from_uri
15
+ from cognite.neat._utils.io_ import to_directory_compatible
16
+ from cognite.neat._utils.rdf_ import remove_namespace_from_uri, uri_display_name
17
17
 
18
18
  from ._state import SessionState
19
19
  from .exceptions import session_class_wrapper
@@ -98,24 +98,24 @@ class ShowDataModelAPI(ShowBaseAPI):
98
98
  self.implements = ShowDataModelImplementsAPI(self._state)
99
99
 
100
100
  def __call__(self) -> Any:
101
- if not self._state.data_model.has_verified_rules:
101
+ if not self._state.rule_store.has_verified_rules:
102
102
  raise NeatSessionError(
103
103
  "No verified data model available. Try using [bold].verify()[/bold] to verify data model."
104
104
  )
105
105
 
106
- rules = self._state.data_model.last_verified_rule[1]
106
+ rules = self._state.rule_store.last_verified_rule
107
107
 
108
108
  if isinstance(rules, DMSRules):
109
- di_graph = self._generate_dms_di_graph(self._state.data_model.last_verified_dms_rules[1])
110
- name = "dms_data_model.html"
109
+ di_graph = self._generate_dms_di_graph(rules)
111
110
  elif isinstance(rules, InformationRules):
112
- di_graph = self._generate_info_di_graph(self._state.data_model.last_verified_information_rules[1])
113
- name = "information_data_model.html"
111
+ di_graph = self._generate_info_di_graph(rules)
114
112
  else:
115
113
  # This should never happen, but we need to handle it to satisfy mypy
116
114
  raise NeatSessionError(
117
115
  f"Unsupported type {type(rules) }. Make sure you have either information or DMS rules."
118
116
  )
117
+ identifier = to_directory_compatible(str(rules.metadata.identifier))
118
+ name = f"{identifier}.html"
119
119
 
120
120
  return self._generate_visualization(di_graph, name)
121
121
 
@@ -187,25 +187,24 @@ class ShowDataModelImplementsAPI(ShowBaseAPI):
187
187
  self._state = state
188
188
 
189
189
  def __call__(self) -> Any:
190
- if not self._state.data_model.has_verified_rules:
190
+ if not self._state.rule_store.has_verified_rules:
191
191
  raise NeatSessionError(
192
192
  "No verified data model available. Try using [bold].verify()[/bold] to verify data model."
193
193
  )
194
194
 
195
- rules = self._state.data_model.last_verified_rule[1]
195
+ rules = self._state.rule_store.last_verified_rule
196
196
 
197
197
  if isinstance(rules, DMSRules):
198
- di_graph = self._generate_dms_di_graph(self._state.data_model.last_verified_dms_rules[1])
199
- name = "dms_data_model_implements.html"
198
+ di_graph = self._generate_dms_di_graph(rules)
200
199
  elif isinstance(rules, InformationRules):
201
- di_graph = self._generate_info_di_graph(self._state.data_model.last_verified_information_rules[1])
202
- name = "information_data_model_implements.html"
200
+ di_graph = self._generate_info_di_graph(rules)
203
201
  else:
204
202
  # This should never happen, but we need to handle it to satisfy mypy
205
203
  raise NeatSessionError(
206
204
  f"Unsupported type {type(rules) }. Make sure you have either information or DMS rules."
207
205
  )
208
-
206
+ identifier = to_directory_compatible(str(rules.metadata.identifier))
207
+ name = f"{identifier}_implements.html"
209
208
  return self._generate_visualization(di_graph, name)
210
209
 
211
210
  def _generate_dms_di_graph(self, rules: DMSRules) -> nx.DiGraph:
@@ -270,20 +269,21 @@ class ShowDataModelProvenanceAPI(ShowBaseAPI):
270
269
  self._state = state
271
270
 
272
271
  def __call__(self) -> Any:
273
- if not self._state.data_model.provenance:
272
+ if not self._state.rule_store.provenance:
274
273
  raise NeatSessionError("No data model available. Try using [bold].read[/bold] to load data model.")
275
274
 
276
275
  di_graph = self._generate_dm_provenance_di_graph_and_types()
277
- return self._generate_visualization(di_graph, name="data_model_provenance.html")
276
+ unique_hash = self._state.rule_store.calculate_provenance_hash(shorten=True)
277
+ return self._generate_visualization(di_graph, name=f"data_model_provenance_{unique_hash}.html")
278
278
 
279
279
  def _generate_dm_provenance_di_graph_and_types(self) -> nx.DiGraph:
280
280
  di_graph = nx.DiGraph()
281
- hex_colored_types = _generate_hex_color_per_type(["Agent", "Entity", "Activity"])
281
+ hex_colored_types = _generate_hex_color_per_type(["Agent", "Entity", "Activity", "Export", "Pruned"])
282
282
 
283
- for change in self._state.data_model.provenance:
284
- source = self._shorten_id(change.source_entity.id_)
285
- target = self._shorten_id(change.target_entity.id_)
286
- agent = self._shorten_id(change.agent.id_)
283
+ for change in self._state.rule_store.provenance:
284
+ source = uri_display_name(change.source_entity.id_)
285
+ target = uri_display_name(change.target_entity.id_)
286
+ agent = uri_display_name(change.agent.id_)
287
287
 
288
288
  di_graph.add_node(
289
289
  source,
@@ -312,15 +312,34 @@ class ShowDataModelProvenanceAPI(ShowBaseAPI):
312
312
  di_graph.add_edge(source, agent, label="used", color="grey")
313
313
  di_graph.add_edge(agent, target, label="generated", color="grey")
314
314
 
315
- return di_graph
315
+ for source_id, exports in self._state.rule_store.exports_by_source_entity_id.items():
316
+ source_shorten = uri_display_name(source_id)
317
+ for export in exports:
318
+ export_id = uri_display_name(export.target_entity.id_)
319
+ di_graph.add_node(
320
+ export_id,
321
+ label=export_id,
322
+ type="Export",
323
+ title="Export",
324
+ color=hex_colored_types["Export"],
325
+ )
326
+ di_graph.add_edge(source_shorten, export_id, label="exported", color="grey")
316
327
 
317
- @staticmethod
318
- def _shorten_id(thing: URIRef) -> str:
319
- if "https://cognitedata.com/dms/data-model/" in thing:
320
- return "DMS(" + ",".join(thing.replace("https://cognitedata.com/dms/data-model/", "").split("/")) + ")"
321
- elif "http://purl.org/cognite/neat/data-model/" in thing:
322
- return "NEAT(" + ",".join(thing.replace("http://purl.org/cognite/neat/data-model/", "").split("/")) + ")"
323
- return remove_namespace_from_uri(thing)
328
+ for pruned_lists in self._state.rule_store.pruned_by_source_entity_id.values():
329
+ for prune_path in pruned_lists:
330
+ for change in prune_path:
331
+ source = uri_display_name(change.source_entity.id_)
332
+ target = uri_display_name(change.target_entity.id_)
333
+ di_graph.add_node(
334
+ target,
335
+ label=target,
336
+ type="Pruned",
337
+ title="Pruned",
338
+ color=hex_colored_types["Pruned"],
339
+ )
340
+ di_graph.add_edge(source, target, label="pruned", color="grey")
341
+
342
+ return di_graph
324
343
 
325
344
 
326
345
  @session_class_wrapper
@@ -1,15 +1,14 @@
1
1
  from dataclasses import dataclass, field
2
2
  from typing import Literal, cast
3
3
 
4
- from rdflib import URIRef
5
-
6
4
  from cognite.neat._issues import IssueList
7
- from cognite.neat._rules._shared import JustRules, ReadRules, VerifiedRules
8
- from cognite.neat._rules.models.dms._rules import DMSRules
9
- from cognite.neat._rules.models.information._rules import InformationRules
10
- from cognite.neat._rules.models.information._rules_input import InformationInputRules
11
- from cognite.neat._store import NeatGraphStore
12
- from cognite.neat._store._provenance import Change, Provenance
5
+ from cognite.neat._rules.importers import BaseImporter, InferenceImporter
6
+ from cognite.neat._rules.models import DMSRules, InformationRules
7
+ from cognite.neat._rules.transformers import RulesTransformer, ToExtension
8
+ from cognite.neat._store import NeatGraphStore, NeatRulesStore
9
+ from cognite.neat._store._rules_store import ModelEntity
10
+ from cognite.neat._utils.rdf_ import uri_display_name
11
+ from cognite.neat._utils.text import humanize_collection
13
12
  from cognite.neat._utils.upload import UploadResultList
14
13
 
15
14
  from .exceptions import NeatSessionError
@@ -18,7 +17,48 @@ from .exceptions import NeatSessionError
18
17
  class SessionState:
19
18
  def __init__(self, store_type: Literal["memory", "oxigraph"]) -> None:
20
19
  self.instances = InstancesState(store_type)
21
- self.data_model = DataModelState()
20
+ self.rule_store = NeatRulesStore()
21
+ self.last_reference: DMSRules | InformationRules | None = None
22
+
23
+ def rule_transform(self, *transformer: RulesTransformer) -> IssueList:
24
+ if not transformer:
25
+ raise NeatSessionError("No transformers provided.")
26
+ first_transformer = transformer[0]
27
+ pruned = self.rule_store.prune_until_compatible(first_transformer)
28
+ if pruned:
29
+ type_hint = first_transformer.transform_type_hint()
30
+ action = uri_display_name(first_transformer.agent.id_)
31
+ location = cast(ModelEntity, self.rule_store.provenance[-1].target_entity).display_name
32
+ expected = humanize_collection([hint.display_type_name() for hint in type_hint]) # type: ignore[attr-defined]
33
+ step_str = "step" if len(pruned) == 1 else "steps"
34
+ print(
35
+ f"The {action} actions expects a {expected}. "
36
+ f"Moving back {len(pruned)} {step_str} to the last {location}."
37
+ )
38
+ if (
39
+ any(isinstance(t, ToExtension) for t in transformer)
40
+ and isinstance(self.rule_store.provenance[-1].target_entity, ModelEntity)
41
+ and isinstance(self.rule_store.provenance[-1].target_entity.result, DMSRules | InformationRules)
42
+ ):
43
+ self.last_reference = self.rule_store.provenance[-1].target_entity.result
44
+
45
+ start = cast(ModelEntity, self.rule_store.provenance[-1].target_entity).display_name
46
+ issues = self.rule_store.transform(*transformer)
47
+ end = cast(ModelEntity, self.rule_store.provenance[-1].target_entity).display_name
48
+ issues.action = f"{start} → {end}"
49
+ issues.hint = "Use the .inspect.issues() for more details."
50
+ return issues
51
+
52
+ def rule_import(self, importer: BaseImporter) -> IssueList:
53
+ issues = self.rule_store.import_(importer)
54
+ result = cast(ModelEntity, self.rule_store.provenance[-1].target_entity).display_name
55
+ if isinstance(importer, InferenceImporter):
56
+ issues.action = f"Inferred {result}"
57
+ else:
58
+ issues.action = f"Read {result}"
59
+ if issues:
60
+ issues.hint = "Use the .inspect.issues() for more details."
61
+ return issues
22
62
 
23
63
 
24
64
  @dataclass
@@ -48,101 +88,3 @@ class InstancesState:
48
88
  "No outcome available. Try using [bold].to.cdf.instances[/bold] to upload a data minstances."
49
89
  )
50
90
  return self.outcome[-1]
51
-
52
-
53
- @dataclass
54
- class DataModelState:
55
- _rules: dict[URIRef, ReadRules | JustRules | VerifiedRules] = field(init=False, default_factory=dict)
56
- issue_lists: list[IssueList] = field(default_factory=list)
57
- provenance: Provenance = field(default_factory=Provenance)
58
- outcome: list[UploadResultList] = field(default_factory=list)
59
-
60
- def write(self, rules: ReadRules | JustRules | VerifiedRules, change: Change) -> None:
61
- if change.target_entity.id_ in self._rules:
62
- raise NeatSessionError(f"Data model <{change.target_entity.id_}> already exists.")
63
-
64
- else:
65
- self._rules[change.target_entity.id_] = rules
66
- self.provenance.append(change)
67
-
68
- def read(self, id_: URIRef) -> ReadRules | JustRules | VerifiedRules:
69
- if id_ not in self._rules:
70
- raise NeatSessionError(f"Data model <{id_}> not found.")
71
- return self._rules[id_]
72
-
73
- @property
74
- def unverified_rules(self) -> dict[URIRef, ReadRules]:
75
- return {id_: rules for id_, rules in self._rules.items() if isinstance(rules, ReadRules)}
76
-
77
- @property
78
- def verified_rules(self) -> dict[URIRef, VerifiedRules]:
79
- return {id_: rules for id_, rules in self._rules.items() if isinstance(rules, VerifiedRules)}
80
-
81
- @property
82
- def last_unverified_rule(self) -> tuple[URIRef, ReadRules]:
83
- if not self.unverified_rules:
84
- raise NeatSessionError("No data model available. Try using [bold].read[/bold] to load a data model.")
85
- return next(reversed(self.unverified_rules.items()))
86
-
87
- @property
88
- def last_info_unverified_rule(self) -> tuple[URIRef, ReadRules]:
89
- if self.unverified_rules:
90
- for id_, rule in reversed(self.unverified_rules.items()):
91
- if isinstance(rule.rules, InformationInputRules):
92
- return id_, rule
93
-
94
- raise NeatSessionError("No data model available. Try using [bold].read[/bold] to load a data model.")
95
-
96
- @property
97
- def last_verified_rule(self) -> tuple[URIRef, VerifiedRules]:
98
- if not self.verified_rules:
99
- raise NeatSessionError(
100
- "No data model available to verify. Try using [bold].read[/bold] to load a data model."
101
- )
102
- return next(reversed(self.verified_rules.items()))
103
-
104
- @property
105
- def last_verified_information_rules(self) -> tuple[URIRef, InformationRules]:
106
- if self.verified_rules:
107
- for id_, rules in reversed(self.verified_rules.items()):
108
- if isinstance(rules, InformationRules):
109
- return id_, rules
110
-
111
- raise NeatSessionError(
112
- "No verified information data model. Try using [bold].verify()[/bold]"
113
- " to convert unverified information model to verified information model."
114
- )
115
-
116
- @property
117
- def last_verified_dms_rules(self) -> tuple[URIRef, DMSRules]:
118
- if self.verified_rules:
119
- for id_, rules in reversed(self.verified_rules.items()):
120
- if isinstance(rules, DMSRules):
121
- return id_, rules
122
-
123
- raise NeatSessionError(
124
- 'No verified DMS data model. Try using [bold].convert("DMS")[/bold]'
125
- " to convert verified information model to verified DMS model."
126
- )
127
-
128
- @property
129
- def has_unverified_rules(self) -> bool:
130
- return bool(self.unverified_rules)
131
-
132
- @property
133
- def has_verified_rules(self) -> bool:
134
- return bool(self.verified_rules)
135
-
136
- @property
137
- def last_issues(self) -> IssueList:
138
- if not self.issue_lists:
139
- raise NeatSessionError("No issues available. Try using [bold].verify()[/bold] to verify a data model.")
140
- return self.issue_lists[-1]
141
-
142
- @property
143
- def last_outcome(self) -> UploadResultList:
144
- if not self.outcome:
145
- raise NeatSessionError(
146
- "No outcome available. Try using [bold].to.cdf.data_model[/bold] to upload a data model."
147
- )
148
- return self.outcome[-1]
@@ -5,13 +5,14 @@ from typing import Any, Literal, overload
5
5
  from cognite.client.data_classes.data_modeling import SpaceApply
6
6
 
7
7
  from cognite.neat._client import NeatClient
8
+ from cognite.neat._constants import COGNITE_MODELS
8
9
  from cognite.neat._graph import loaders
9
- from cognite.neat._issues import IssueList, catch_warnings
10
10
  from cognite.neat._rules import exporters
11
11
  from cognite.neat._rules._constants import PATTERNS
12
12
  from cognite.neat._rules._shared import VerifiedRules
13
13
  from cognite.neat._rules.exporters._rules2dms import Component
14
- from cognite.neat._utils.upload import UploadResultCore, UploadResultList
14
+ from cognite.neat._rules.models.dms import DMSMetadata
15
+ from cognite.neat._utils.upload import UploadResultList
15
16
 
16
17
  from ._state import SessionState
17
18
  from .exceptions import NeatSessionError, session_class_wrapper
@@ -32,39 +33,53 @@ class ToAPI:
32
33
  def excel(
33
34
  self,
34
35
  io: Any,
35
- model: Literal["dms", "information", "logical", "physical"] | None,
36
+ include_reference: bool = True,
36
37
  ) -> None:
37
38
  """Export the verified data model to Excel.
38
39
 
39
40
  Args:
40
41
  io: The file path or file-like object to write the Excel file to.
41
- model: The format of the data model to export. Defaults to None.
42
+ include_reference: If True, the reference data model will be included. Defaults to True.
43
+ Note that this only applies if you have created the data model using the
44
+ .to_enterprise(), .to_solution(), or .to_data_product() methods.
42
45
 
43
46
  Example:
44
47
  Export information model to excel rules sheet
45
48
  ```python
46
49
  information_rules_file_name = "information_rules.xlsx"
47
- neat.to.excel(information_rules_file_name, model="information")
50
+ neat.to.excel(information_rules_file_name)
48
51
  ```
49
52
 
50
53
  Example:
51
- Export data model to excel rules sheet
54
+ Read CogniteCore model, convert it to an enterprise model, and export it to an excel file
52
55
  ```python
56
+ client = CogniteClient()
57
+ neat = NeatSession(client)
58
+
59
+ neat.read.cdf(("cdf_cdm", "CogniteCore", "v1"))
60
+ neat.verify()
61
+ neat.prepare.data_model.to_enterprise(
62
+ data_model_id=("sp_doctrino_space", "ExtensionCore", "v1"),
63
+ org_name="MyOrg",
64
+ move_connections=True
65
+ )
53
66
  dms_rules_file_name = "dms_rules.xlsx"
54
- neat.to.excel(information_rules_file_name, model="dms")
67
+ neat.to.excel(dms_rules_file_name, include_reference=True)
55
68
  ```
56
69
  """
57
- exporter = exporters.ExcelExporter(styling="maximal")
58
- rules: VerifiedRules
59
- if model == "information" or model == "logical":
60
- rules = self._state.data_model.last_verified_information_rules[1]
61
- elif model == "dms" or model == "physical":
62
- rules = self._state.data_model.last_verified_dms_rules[1]
63
- else:
64
- rules = self._state.data_model.last_verified_rule[1]
65
-
66
- exporter.export_to_file(rules, Path(io))
67
- return None
70
+ reference_rules_with_prefix: tuple[VerifiedRules, str] | None = None
71
+ if include_reference and self._state.last_reference:
72
+ if (
73
+ isinstance(self._state.last_reference.metadata, DMSMetadata)
74
+ and self._state.last_reference.metadata.as_data_model_id() in COGNITE_MODELS
75
+ ):
76
+ prefix = "CDM"
77
+ else:
78
+ prefix = "Ref"
79
+ reference_rules_with_prefix = self._state.last_reference, prefix
80
+
81
+ exporter = exporters.ExcelExporter(styling="maximal", reference_rules_with_prefix=reference_rules_with_prefix)
82
+ return self._state.rule_store.export_to_file(exporter, Path(io))
68
83
 
69
84
  @overload
70
85
  def yaml(self, io: None, format: Literal["neat"] = "neat") -> str: ...
@@ -112,22 +127,22 @@ class ToAPI:
112
127
  """
113
128
  if format == "neat":
114
129
  exporter = exporters.YAMLExporter()
115
- last_verified = self._state.data_model.last_verified_rule[1]
116
130
  if io is None:
117
- return exporter.export(last_verified)
131
+ return self._state.rule_store.export(exporter)
118
132
 
119
- exporter.export_to_file(last_verified, Path(io))
133
+ self._state.rule_store.export_to_file(exporter, Path(io))
120
134
  elif format == "toolkit":
121
135
  if io is None or not isinstance(io, str | Path):
122
136
  raise NeatSessionError(
123
137
  "Please provide a zip file or directory path to write the YAML files to."
124
138
  "This is required for the 'toolkit' format."
125
139
  )
126
- dms_rule = self._state.data_model.last_verified_dms_rules[1]
127
140
  user_path = Path(io)
128
141
  if user_path.suffix == "" and not user_path.exists():
129
142
  user_path.mkdir(parents=True)
130
- exporters.DMSExporter(remove_cdf_spaces=skip_system_spaces).export_to_file(dms_rule, user_path)
143
+ self._state.rule_store.export_to_file(
144
+ exporters.DMSExporter(remove_cdf_spaces=skip_system_spaces), user_path
145
+ )
131
146
  else:
132
147
  raise NeatSessionError("Please provide a valid format. 'neat' or 'toolkit'")
133
148
 
@@ -154,9 +169,9 @@ class CDFToAPI:
154
169
  if not self._client:
155
170
  raise NeatSessionError("No CDF client provided!")
156
171
 
157
- space = space or f"{self._state.data_model.last_verified_dms_rules[1].metadata.space}_instances"
172
+ space = space or f"{self._state.rule_store.last_verified_dms_rules.metadata.space}_instances"
158
173
 
159
- if space and space == self._state.data_model.last_verified_dms_rules[1].metadata.space:
174
+ if space and space == self._state.rule_store.last_verified_dms_rules.metadata.space:
160
175
  raise NeatSessionError("Space for instances must be different from the data model space.")
161
176
  elif not PATTERNS.space_compliance.match(str(space)):
162
177
  raise NeatSessionError("Please provide a valid space name. {PATTERNS.space_compliance.pattern}")
@@ -165,7 +180,7 @@ class CDFToAPI:
165
180
  self._client.data_modeling.spaces.apply(SpaceApply(space=space))
166
181
 
167
182
  loader = loaders.DMSLoader.from_rules(
168
- self._state.data_model.last_verified_dms_rules[1],
183
+ self._state.rule_store.last_verified_dms_rules,
169
184
  self._state.instances.store,
170
185
  instance_space=space,
171
186
  client=self._client,
@@ -207,10 +222,6 @@ class CDFToAPI:
207
222
  if not self._client:
208
223
  raise NeatSessionError("No client provided!")
209
224
 
210
- conversion_issues = IssueList(action="to.cdf.data_model")
211
- with catch_warnings(conversion_issues):
212
- result = exporter.export_to_cdf(self._state.data_model.last_verified_dms_rules[1], self._client, dry_run)
213
- result.insert(0, UploadResultCore(name="schema", issues=conversion_issues))
214
- self._state.data_model.outcome.append(result)
225
+ result = self._state.rule_store.export_to_cdf(exporter, self._client, dry_run)
215
226
  print("You can inspect the details with the .inspect.outcome.data_model(...) method.")
216
227
  return result
cognite/neat/_shared.py CHANGED
@@ -1,11 +1,17 @@
1
+ import sys
1
2
  from abc import abstractmethod
2
- from collections.abc import Hashable, Sequence
3
+ from collections.abc import Hashable, Iterator, Sequence
3
4
  from dataclasses import dataclass
4
- from typing import Any, TypeAlias, TypeVar
5
+ from typing import Any, SupportsIndex, TypeAlias, TypeVar, overload
5
6
 
6
7
  import pandas as pd
7
8
  from rdflib import Literal, URIRef
8
9
 
10
+ if sys.version_info <= (3, 11):
11
+ from typing_extensions import Self
12
+ else:
13
+ from typing import Self
14
+
9
15
  T_ID = TypeVar("T_ID", bound=Hashable)
10
16
 
11
17
 
@@ -51,6 +57,21 @@ class NeatList(list, Sequence[T_NeatObject]):
51
57
  def _repr_html_(self) -> str:
52
58
  return self.to_pandas()._repr_html_() # type: ignore[operator]
53
59
 
60
+ # Implemented to get correct type hints
61
+ def __iter__(self) -> Iterator[T_NeatObject]:
62
+ return super().__iter__()
63
+
64
+ @overload
65
+ def __getitem__(self, index: SupportsIndex) -> T_NeatObject: ...
66
+
67
+ @overload
68
+ def __getitem__(self, index: slice) -> Self: ...
69
+
70
+ def __getitem__(self, index: SupportsIndex | slice, /) -> T_NeatObject | Self:
71
+ if isinstance(index, slice):
72
+ return type(self)(super().__getitem__(index))
73
+ return super().__getitem__(index)
74
+
54
75
 
55
76
  Triple: TypeAlias = tuple[URIRef, URIRef, Literal | URIRef]
56
77
  InstanceType: TypeAlias = URIRef
@@ -20,14 +20,12 @@ import uuid
20
20
  from collections.abc import Iterable, Sequence
21
21
  from dataclasses import dataclass, field
22
22
  from datetime import datetime
23
- from typing import Generic, Optional, TypeVar
23
+ from typing import Optional
24
24
 
25
- from cognite.client.data_classes.data_modeling import DataModelId, DataModelIdentifier
26
25
  from rdflib import PROV, RDF, Literal, URIRef
27
26
 
28
27
  from cognite.neat._constants import CDF_NAMESPACE, DEFAULT_NAMESPACE
29
28
  from cognite.neat._issues import IssueList
30
- from cognite.neat._rules._shared import JustRules, ReadRules, VerifiedRules
31
29
  from cognite.neat._shared import FrozenNeatObject, NeatList, Triple
32
30
 
33
31
 
@@ -72,45 +70,6 @@ class Entity:
72
70
 
73
71
  return output
74
72
 
75
- @classmethod
76
- def from_data_model_id(cls, data_model_id: DataModelIdentifier) -> "Entity":
77
- data_model_id = DataModelId.load(data_model_id)
78
-
79
- return cls(
80
- was_attributed_to=CDF_AGENT,
81
- id_=CDF_NAMESPACE[
82
- f"dms/data-model/{data_model_id.space}/{data_model_id.external_id}/{data_model_id.version}"
83
- ],
84
- )
85
-
86
- @classmethod
87
- def from_rules(
88
- cls,
89
- rules: ReadRules | JustRules | VerifiedRules,
90
- agent: Agent | None = None,
91
- activity: "Activity | None" = None,
92
- ) -> "Entity":
93
- agent = agent or UNKNOWN_AGENT
94
- if isinstance(rules, VerifiedRules):
95
- return cls(
96
- was_attributed_to=agent,
97
- was_generated_by=activity,
98
- id_=rules.metadata.identifier,
99
- )
100
-
101
- elif isinstance(rules, ReadRules | JustRules) and rules.rules is not None:
102
- return cls(
103
- was_attributed_to=agent,
104
- was_generated_by=activity,
105
- id_=rules.rules.metadata.identifier,
106
- )
107
- else:
108
- return cls(
109
- was_attributed_to=agent,
110
- was_generated_by=activity,
111
- id_=DEFAULT_NAMESPACE["unknown-entity"],
112
- )
113
-
114
73
  @classmethod
115
74
  def new_unknown_entity(cls) -> "Entity":
116
75
  return cls(
@@ -119,15 +78,8 @@ class Entity:
119
78
  )
120
79
 
121
80
 
122
- T_Result = TypeVar("T_Result")
123
-
124
-
125
- @dataclass(frozen=True)
126
- class ModelEntity(Entity, Generic[T_Result]):
127
- result: T_Result | None = None
128
-
129
-
130
81
  INSTANCES_ENTITY = Entity(was_attributed_to=NEAT_AGENT, id_=CDF_NAMESPACE["instances"])
82
+ EMPTY_ENTITY = Entity(was_attributed_to=NEAT_AGENT, id_=DEFAULT_NAMESPACE["empty-entity"])
131
83
 
132
84
 
133
85
  @dataclass(frozen=True)
@@ -192,34 +144,6 @@ class Change(FrozenNeatObject):
192
144
  description=description,
193
145
  )
194
146
 
195
- @classmethod
196
- def from_rules_activity(
197
- cls,
198
- rules: ReadRules | JustRules | VerifiedRules,
199
- agent: Agent,
200
- start: datetime,
201
- end: datetime,
202
- description: str,
203
- source_entity: Entity | None = None,
204
- ) -> "Change":
205
- source_entity = source_entity or Entity.new_unknown_entity()
206
- activity = Activity(
207
- started_at_time=start,
208
- ended_at_time=end,
209
- was_associated_with=agent,
210
- used=source_entity,
211
- )
212
-
213
- target_entity = Entity.from_rules(rules, agent, activity)
214
-
215
- return cls(
216
- agent=agent,
217
- activity=activity,
218
- target_entity=target_entity,
219
- description=description,
220
- source_entity=source_entity,
221
- )
222
-
223
147
  def dump(self, aggregate: bool = True) -> dict[str, str]:
224
148
  return {
225
149
  "Source Entity": self.source_entity.id_,
@@ -230,11 +154,8 @@ class Change(FrozenNeatObject):
230
154
  }
231
155
 
232
156
 
233
- T_Change = TypeVar("T_Change", bound=Change)
234
-
235
-
236
157
  class Provenance(NeatList[Change]):
237
- def __init__(self, changes: Sequence[T_Change] | None = None):
158
+ def __init__(self, changes: Sequence[Change] | None = None):
238
159
  super().__init__(changes or [])
239
160
 
240
161
  def activity_took_place(self, activity: str) -> bool: