cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (109) hide show
  1. cognite/neat/_client/__init__.py +4 -0
  2. cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
  3. cognite/neat/_client/_api/schema.py +50 -0
  4. cognite/neat/_client/_api_client.py +17 -0
  5. cognite/neat/_client/data_classes/__init__.py +0 -0
  6. cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
  7. cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
  8. cognite/neat/_graph/_shared.py +14 -15
  9. cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
  10. cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
  11. cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
  12. cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
  13. cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
  14. cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
  15. cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
  16. cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
  17. cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
  18. cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
  19. cognite/neat/_graph/extractors/_rdf_file.py +6 -7
  20. cognite/neat/_graph/loaders/__init__.py +1 -2
  21. cognite/neat/_graph/queries/_base.py +17 -1
  22. cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
  23. cognite/neat/_graph/transformers/_prune_graph.py +1 -1
  24. cognite/neat/_graph/transformers/_rdfpath.py +1 -1
  25. cognite/neat/_issues/warnings/__init__.py +6 -0
  26. cognite/neat/_issues/warnings/_external.py +8 -0
  27. cognite/neat/_issues/warnings/_models.py +9 -0
  28. cognite/neat/_issues/warnings/_properties.py +16 -0
  29. cognite/neat/_rules/_constants.py +7 -6
  30. cognite/neat/_rules/_shared.py +3 -8
  31. cognite/neat/_rules/analysis/__init__.py +1 -2
  32. cognite/neat/_rules/analysis/_base.py +10 -27
  33. cognite/neat/_rules/analysis/_dms.py +4 -10
  34. cognite/neat/_rules/analysis/_information.py +2 -10
  35. cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
  36. cognite/neat/_rules/exporters/_base.py +3 -4
  37. cognite/neat/_rules/exporters/_rules2dms.py +29 -40
  38. cognite/neat/_rules/exporters/_rules2excel.py +15 -72
  39. cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
  40. cognite/neat/_rules/importers/_base.py +3 -4
  41. cognite/neat/_rules/importers/_dms2rules.py +21 -45
  42. cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
  43. cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
  44. cognite/neat/_rules/importers/_rdf/_base.py +17 -29
  45. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
  46. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
  47. cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
  48. cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
  49. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
  50. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
  51. cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
  52. cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
  53. cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
  54. cognite/neat/_rules/models/__init__.py +3 -17
  55. cognite/neat/_rules/models/_base_rules.py +118 -62
  56. cognite/neat/_rules/models/dms/__init__.py +2 -2
  57. cognite/neat/_rules/models/dms/_exporter.py +20 -178
  58. cognite/neat/_rules/models/dms/_rules.py +65 -128
  59. cognite/neat/_rules/models/dms/_rules_input.py +72 -56
  60. cognite/neat/_rules/models/dms/_validation.py +16 -109
  61. cognite/neat/_rules/models/entities/_single_value.py +32 -4
  62. cognite/neat/_rules/models/information/_rules.py +19 -122
  63. cognite/neat/_rules/models/information/_rules_input.py +32 -41
  64. cognite/neat/_rules/models/information/_validation.py +34 -102
  65. cognite/neat/_rules/models/mapping/__init__.py +2 -3
  66. cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
  67. cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
  68. cognite/neat/_rules/transformers/__init__.py +3 -6
  69. cognite/neat/_rules/transformers/_converters.py +128 -206
  70. cognite/neat/_rules/transformers/_mapping.py +105 -34
  71. cognite/neat/_rules/transformers/_verification.py +5 -16
  72. cognite/neat/_session/_base.py +83 -21
  73. cognite/neat/_session/_collector.py +126 -0
  74. cognite/neat/_session/_drop.py +35 -0
  75. cognite/neat/_session/_inspect.py +22 -10
  76. cognite/neat/_session/_mapping.py +39 -0
  77. cognite/neat/_session/_prepare.py +222 -27
  78. cognite/neat/_session/_read.py +109 -19
  79. cognite/neat/_session/_set.py +2 -2
  80. cognite/neat/_session/_show.py +11 -11
  81. cognite/neat/_session/_to.py +27 -14
  82. cognite/neat/_session/exceptions.py +20 -3
  83. cognite/neat/_store/_base.py +27 -24
  84. cognite/neat/_store/_provenance.py +2 -2
  85. cognite/neat/_utils/auxiliary.py +19 -0
  86. cognite/neat/_utils/rdf_.py +28 -1
  87. cognite/neat/_version.py +1 -1
  88. cognite/neat/_workflows/steps/data_contracts.py +2 -10
  89. cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
  90. cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
  91. cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
  92. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
  93. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
  94. cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
  95. cognite/neat/_rules/analysis/_asset.py +0 -173
  96. cognite/neat/_rules/models/asset/__init__.py +0 -13
  97. cognite/neat/_rules/models/asset/_rules.py +0 -109
  98. cognite/neat/_rules/models/asset/_rules_input.py +0 -101
  99. cognite/neat/_rules/models/asset/_validation.py +0 -45
  100. cognite/neat/_rules/models/domain.py +0 -136
  101. cognite/neat/_rules/models/mapping/_base.py +0 -131
  102. cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
  103. cognite/neat/_utils/cdf/loaders/_base.py +0 -54
  104. cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
  105. cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
  106. /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
  107. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
  108. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
  109. {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
@@ -1,13 +1,16 @@
1
+ import warnings
1
2
  from abc import ABC
2
3
  from collections import defaultdict
4
+ from functools import cached_property
5
+ from typing import Any, ClassVar, Literal
3
6
 
7
+ from cognite.neat._issues.errors import NeatValueError
8
+ from cognite.neat._issues.warnings import NeatValueWarning, PropertyOverwritingWarning
4
9
  from cognite.neat._rules._shared import JustRules, OutRules
5
- from cognite.neat._rules.models import DMSRules, InformationRules
6
- from cognite.neat._rules.models._base_rules import ClassRef
7
- from cognite.neat._rules.models.dms import DMSProperty
8
- from cognite.neat._rules.models.entities import ClassEntity, ReferenceEntity
9
- from cognite.neat._rules.models.information import InformationClass
10
- from cognite.neat._rules.models.mapping import RuleMapping
10
+ from cognite.neat._rules.models import DMSRules, SheetList
11
+ from cognite.neat._rules.models.data_types import Enum
12
+ from cognite.neat._rules.models.dms import DMSEnum, DMSProperty, DMSView
13
+ from cognite.neat._rules.models.entities import ViewEntity
11
14
 
12
15
  from ._base import RulesTransformer
13
16
 
@@ -51,9 +54,6 @@ class MapOneToOne(MapOntoTransformers):
51
54
 
52
55
  def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
53
56
  solution: DMSRules = self._to_rules(rules)
54
- if solution.reference is not None:
55
- raise ValueError("Reference already exists")
56
- solution.reference = self.reference
57
57
  view_by_external_id = {view.view.external_id: view for view in solution.views}
58
58
  ref_view_by_external_id = {view.view.external_id: view for view in self.reference.views}
59
59
 
@@ -95,12 +95,11 @@ class MapOneToOne(MapOntoTransformers):
95
95
  if ref_prop.container and ref_prop.container_property:
96
96
  prop.container = ref_prop.container
97
97
  prop.container_property = ref_prop.container_property
98
- prop.reference = ReferenceEntity.from_entity(ref_prop.view, ref_prop.view_property)
99
98
 
100
99
  return JustRules(solution)
101
100
 
102
101
 
103
- class RuleMapper(RulesTransformer[InformationRules, InformationRules]):
102
+ class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
104
103
  """Maps properties and classes using the given mapping.
105
104
 
106
105
  **Note**: This transformer mutates the input rules.
@@ -110,30 +109,102 @@ class RuleMapper(RulesTransformer[InformationRules, InformationRules]):
110
109
 
111
110
  """
112
111
 
113
- def __init__(self, mapping: RuleMapping) -> None:
112
+ _mapping_fields: ClassVar[frozenset[str]] = frozenset(
113
+ ["connection", "value_type", "nullable", "immutable", "is_list", "default", "index", "constraint"]
114
+ )
115
+
116
+ def __init__(self, mapping: DMSRules, data_type_conflict: Literal["overwrite"] = "overwrite") -> None:
114
117
  self.mapping = mapping
118
+ self.data_type_conflict = data_type_conflict
119
+
120
+ @cached_property
121
+ def _view_by_entity_id(self) -> dict[str, DMSView]:
122
+ return {view.view.external_id: view for view in self.mapping.views}
115
123
 
116
- def transform(self, rules: InformationRules | OutRules[InformationRules]) -> JustRules[InformationRules]:
124
+ @cached_property
125
+ def _property_by_view_property(self) -> dict[tuple[str, str], DMSProperty]:
126
+ return {(prop.view.external_id, prop.view_property): prop for prop in self.mapping.properties}
127
+
128
+ def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
129
+ if self.data_type_conflict != "overwrite":
130
+ raise NeatValueError(f"Invalid data_type_conflict: {self.data_type_conflict}")
117
131
  input_rules = self._to_rules(rules)
132
+ new_rules = input_rules.model_copy(deep=True)
133
+ new_rules.metadata.version += "_mapped"
118
134
 
119
- destination_by_source = self.mapping.properties.as_destination_by_source()
120
- destination_cls_by_source = self.mapping.classes.as_destination_by_source()
121
- used_destination_classes: set[ClassEntity] = set()
122
- for prop in input_rules.properties:
123
- if destination_prop := destination_by_source.get(prop.as_reference()):
124
- prop.class_ = destination_prop.class_
125
- prop.property_ = destination_prop.property_
126
- used_destination_classes.add(destination_prop.class_)
127
- elif destination_cls := destination_cls_by_source.get(ClassRef(Class=prop.class_)):
128
- # If the property is not in the mapping, but the class is,
129
- # then we should map the class to the destination
130
- prop.class_ = destination_cls.class_
131
-
132
- for cls_ in input_rules.classes:
133
- if destination_cls := destination_cls_by_source.get(cls_.as_reference()):
134
- cls_.class_ = destination_cls.class_
135
- existing_classes = {cls_.class_ for cls_ in input_rules.classes}
136
- for new_cls in used_destination_classes - existing_classes:
137
- input_rules.classes.append(InformationClass(class_=new_cls))
138
-
139
- return JustRules(input_rules)
135
+ for view in new_rules.views:
136
+ if mapping_view := self._view_by_entity_id.get(view.view.external_id):
137
+ view.implements = mapping_view.implements
138
+
139
+ for prop in new_rules.properties:
140
+ key = (prop.view.external_id, prop.view_property)
141
+ if key not in self._property_by_view_property:
142
+ continue
143
+ mapping_prop = self._property_by_view_property[key]
144
+ to_overwrite, conflicts = self._find_overwrites(prop, mapping_prop)
145
+ if conflicts and self.data_type_conflict == "overwrite":
146
+ warnings.warn(
147
+ PropertyOverwritingWarning(prop.view.as_id(), "view", prop.view_property, tuple(conflicts)),
148
+ stacklevel=2,
149
+ )
150
+ elif conflicts:
151
+ raise NeatValueError(f"Conflicting properties for {prop.view}.{prop.view_property}: {conflicts}")
152
+ for field_name, value in to_overwrite.items():
153
+ setattr(prop, field_name, value)
154
+ prop.container = mapping_prop.container
155
+ prop.container_property = mapping_prop.container_property
156
+
157
+ # Add missing views used as value types
158
+ existing_views = {view.view for view in new_rules.views}
159
+ new_value_types = {
160
+ prop.value_type
161
+ for prop in new_rules.properties
162
+ if isinstance(prop.value_type, ViewEntity) and prop.value_type not in existing_views
163
+ }
164
+ for new_value_type in new_value_types:
165
+ if mapping_view := self._view_by_entity_id.get(new_value_type.external_id):
166
+ new_rules.views.append(mapping_view)
167
+ else:
168
+ warnings.warn(NeatValueWarning(f"View {new_value_type} not found in mapping"), stacklevel=2)
169
+
170
+ # Add missing enums
171
+ existing_enum_collections = {item.collection for item in new_rules.enum or []}
172
+ new_enums = {
173
+ prop.value_type.collection
174
+ for prop in new_rules.properties
175
+ if isinstance(prop.value_type, Enum) and prop.value_type.collection not in existing_enum_collections
176
+ }
177
+ if new_enums:
178
+ new_rules.enum = new_rules.enum or SheetList[DMSEnum]([])
179
+ for item in self.mapping.enum or []:
180
+ if item.collection in new_enums:
181
+ new_rules.enum.append(item)
182
+
183
+ return JustRules(new_rules)
184
+
185
+ def _find_overwrites(self, prop: DMSProperty, mapping_prop: DMSProperty) -> tuple[dict[str, Any], list[str]]:
186
+ """Finds the properties that need to be overwritten and returns them.
187
+
188
+ In addition, conflicting properties are returned. Note that overwriting properties that are
189
+ originally None is not considered a conflict. Thus, you can have properties to overwrite but no
190
+ conflicts.
191
+
192
+ Args:
193
+ prop: The property to compare.
194
+ mapping_prop: The property to compare against.
195
+
196
+ Returns:
197
+ A tuple with the properties to overwrite and the conflicting properties.
198
+
199
+ """
200
+ to_overwrite: dict[str, Any] = {}
201
+ conflicts: list[str] = []
202
+ for field_name in self._mapping_fields:
203
+ mapping_value = getattr(mapping_prop, field_name)
204
+ source_value = getattr(prop, field_name)
205
+ if mapping_value != source_value:
206
+ to_overwrite[field_name] = mapping_value
207
+ if source_value is not None:
208
+ # These are used for warnings so we use the alias to make it more readable for the user
209
+ conflicts.append(mapping_prop.model_fields[field_name].alias or field_name)
210
+ return to_overwrite, conflicts
@@ -13,12 +13,8 @@ from cognite.neat._rules._shared import (
13
13
  VerifiedRules,
14
14
  )
15
15
  from cognite.neat._rules.models import (
16
- AssetInputRules,
17
- AssetRules,
18
16
  DMSInputRules,
19
17
  DMSRules,
20
- DomainInputRules,
21
- DomainRules,
22
18
  InformationInputRules,
23
19
  InformationRules,
24
20
  )
@@ -31,8 +27,9 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
31
27
 
32
28
  _rules_cls: type[T_VerifiedRules]
33
29
 
34
- def __init__(self, errors: Literal["raise", "continue"]) -> None:
30
+ def __init__(self, errors: Literal["raise", "continue"], post_validate: bool = True) -> None:
35
31
  self.errors = errors
32
+ self.post_validate = post_validate
36
33
 
37
34
  def transform(self, rules: T_InputRules | OutRules[T_InputRules]) -> MaybeRules[T_VerifiedRules]:
38
35
  issues = IssueList()
@@ -43,7 +40,9 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
43
40
  verified_rules: T_VerifiedRules | None = None
44
41
  with catch_issues(issues, NeatError, NeatWarning, error_args) as future:
45
42
  rules_cls = self._get_rules_cls(in_)
46
- verified_rules = rules_cls.model_validate(in_.dump()) # type: ignore[assignment]
43
+ dumped = in_.dump()
44
+ dumped["post_validate"] = self.post_validate
45
+ verified_rules = rules_cls.model_validate(dumped) # type: ignore[assignment]
47
46
 
48
47
  if (future.result == "failure" or issues.has_errors or verified_rules is None) and self.errors == "raise":
49
48
  raise issues.as_errors()
@@ -68,12 +67,6 @@ class VerifyInformationRules(VerificationTransformer[InformationInputRules, Info
68
67
  _rules_cls = InformationRules
69
68
 
70
69
 
71
- class VerifyAssetRules(VerificationTransformer[AssetInputRules, AssetRules]):
72
- """Class to verify Asset rules."""
73
-
74
- _rules_cls = AssetRules
75
-
76
-
77
70
  class VerifyAnyRules(VerificationTransformer[InputRules, VerifiedRules]):
78
71
  """Class to verify arbitrary rules"""
79
72
 
@@ -82,9 +75,5 @@ class VerifyAnyRules(VerificationTransformer[InputRules, VerifiedRules]):
82
75
  return InformationRules
83
76
  elif isinstance(in_, DMSInputRules):
84
77
  return DMSRules
85
- elif isinstance(in_, AssetInputRules):
86
- return AssetRules
87
- elif isinstance(in_, DomainInputRules):
88
- return DomainRules
89
78
  else:
90
79
  raise NeatTypeError(f"Unsupported rules type: {type(in_)}")
@@ -5,14 +5,13 @@ from cognite.client import CogniteClient
5
5
  from cognite.client import data_modeling as dm
6
6
 
7
7
  from cognite.neat import _version
8
+ from cognite.neat._client import NeatClient
8
9
  from cognite.neat._issues import IssueList, catch_issues
9
10
  from cognite.neat._issues.errors import RegexViolationError
10
11
  from cognite.neat._rules import importers
11
12
  from cognite.neat._rules._shared import ReadRules, VerifiedRules
12
- from cognite.neat._rules.importers._rdf._base import DEFAULT_NON_EXISTING_NODE_TYPE
13
- from cognite.neat._rules.models import DMSRules
14
- from cognite.neat._rules.models.data_types import AnyURI
15
- from cognite.neat._rules.models.entities._single_value import UnknownEntity
13
+ from cognite.neat._rules.importers import DMSImporter
14
+ from cognite.neat._rules.models import DMSInputRules, DMSRules, SheetList
16
15
  from cognite.neat._rules.models.information._rules import InformationRules
17
16
  from cognite.neat._rules.models.information._rules_input import InformationInputRules
18
17
  from cognite.neat._rules.transformers import ConvertToRules, VerifyAnyRules
@@ -21,9 +20,11 @@ from cognite.neat._store._provenance import (
21
20
  INSTANCES_ENTITY,
22
21
  Change,
23
22
  )
24
- from cognite.neat._utils.auth import _CLIENT_NAME
25
23
 
24
+ from ._collector import _COLLECTOR, Collector
25
+ from ._drop import DropAPI
26
26
  from ._inspect import InspectAPI
27
+ from ._mapping import MappingAPI
27
28
  from ._prepare import PrepareAPI
28
29
  from ._read import ReadAPI
29
30
  from ._set import SetAPI
@@ -31,10 +32,10 @@ from ._show import ShowAPI
31
32
  from ._state import SessionState
32
33
  from ._to import ToAPI
33
34
  from .engine import load_neat_engine
34
- from .exceptions import NeatSessionError, intercept_session_exceptions
35
+ from .exceptions import NeatSessionError, session_class_wrapper
35
36
 
36
37
 
37
- @intercept_session_exceptions
38
+ @session_class_wrapper
38
39
  class NeatSession:
39
40
  def __init__(
40
41
  self,
@@ -43,17 +44,19 @@ class NeatSession:
43
44
  verbose: bool = True,
44
45
  load_engine: Literal["newest", "cache", "skip"] = "cache",
45
46
  ) -> None:
46
- self._client = client
47
+ self._client = NeatClient(client) if client else None
47
48
  self._verbose = verbose
48
49
  self._state = SessionState(store_type=storage)
49
- self.read = ReadAPI(self._state, client, verbose)
50
- self.to = ToAPI(self._state, client, verbose)
51
- self.prepare = PrepareAPI(self._state, verbose)
50
+ self.read = ReadAPI(self._state, self._client, verbose)
51
+ self.to = ToAPI(self._state, self._client, verbose)
52
+ self.prepare = PrepareAPI(self._client, self._state, verbose)
52
53
  self.show = ShowAPI(self._state)
53
54
  self.set = SetAPI(self._state, verbose)
54
55
  self.inspect = InspectAPI(self._state)
55
- if self._client is not None and self._client._config is not None:
56
- self._client._config.client_name = _CLIENT_NAME
56
+ self.mapping = MappingAPI(self._state)
57
+ self.drop = DropAPI(self._state)
58
+ self.opt = OptAPI()
59
+ self.opt._display()
57
60
  if load_engine != "skip" and (engine_version := load_neat_engine(client, load_engine)):
58
61
  print(f"Neat Engine {engine_version} loaded.")
59
62
 
@@ -63,6 +66,28 @@ class NeatSession:
63
66
 
64
67
  def verify(self) -> IssueList:
65
68
  source_id, last_unverified_rule = self._state.data_model.last_unverified_rule
69
+
70
+ reference_rules: DMSInputRules | None = None
71
+ if isinstance(last_unverified_rule.rules, DMSInputRules):
72
+ dms_rules = last_unverified_rule.rules
73
+ views_ids, containers_ids = dms_rules.imported_views_and_containers_ids()
74
+ if views_ids or containers_ids:
75
+ if self._client is None:
76
+ raise NeatSessionError(
77
+ "No client provided. You are referencing unknown views and containers in your data model, "
78
+ "NEAT needs a client to lookup the definitions. "
79
+ "Please set the client in the session, NeatSession(client=client)."
80
+ )
81
+ schema = self._client.schema.retrieve(list(views_ids), list(containers_ids))
82
+
83
+ importer = DMSImporter(schema)
84
+ reference_rules = importer.to_rules().rules
85
+
86
+ if reference_rules is not None:
87
+ dms_rules.views.extend(reference_rules.views)
88
+ if dms_rules.containers:
89
+ dms_rules.containers.extend(reference_rules.containers or [])
90
+
66
91
  transformer = VerifyAnyRules("continue")
67
92
  start = datetime.now(timezone.utc)
68
93
  output = transformer.try_transform(last_unverified_rule)
@@ -74,10 +99,26 @@ class NeatSession:
74
99
  transformer.agent,
75
100
  start,
76
101
  end,
77
- f"Verified data model {source_id} as {output.rules.id_}",
102
+ f"Verified data model {source_id} as {output.rules.metadata.identifier}",
78
103
  self._state.data_model.provenance.source_entity(source_id)
79
104
  or self._state.data_model.provenance.target_entity(source_id),
80
105
  )
106
+ if reference_rules is not None and isinstance(output.rules, DMSRules):
107
+ # Remove the referenced views and containers from the rules
108
+ ref_view_ids = set(reference_rules.as_view_entities())
109
+ if ref_view_ids:
110
+ output.rules.views = SheetList(
111
+ [view for view in output.rules.views if view.view not in ref_view_ids]
112
+ )
113
+ ref_container_ids = reference_rules.as_container_entities()
114
+ if output.rules.containers and ref_container_ids:
115
+ output.rules.containers = SheetList(
116
+ [
117
+ container
118
+ for container in output.rules.containers
119
+ if container.container not in ref_container_ids
120
+ ]
121
+ )
81
122
 
82
123
  self._state.data_model.write(output.rules, change)
83
124
 
@@ -119,7 +160,7 @@ class NeatSession:
119
160
  converter.agent,
120
161
  start,
121
162
  end,
122
- f"Converted data model {source_id} to {converted_rules.id_}",
163
+ f"Converted data model {source_id} to {converted_rules.metadata.identifier}",
123
164
  self._state.data_model.provenance.source_entity(source_id)
124
165
  or self._state.data_model.provenance.target_entity(source_id),
125
166
  )
@@ -144,31 +185,28 @@ class NeatSession:
144
185
  "NeatInferredDataModel",
145
186
  "v1",
146
187
  ),
147
- non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
148
188
  max_number_of_instance: int = 100,
149
189
  ) -> IssueList:
150
190
  """Data model inference from instances.
151
191
 
152
192
  Args:
153
193
  model_id: The ID of the inferred data model.
154
- non_existing_node_type: The type of node to use when type of node is not possible to determine.
194
+ max_number_of_instance: The maximum number of instances to use for inference.
155
195
  """
156
-
157
196
  model_id = dm.DataModelId.load(model_id)
158
197
 
159
198
  start = datetime.now(timezone.utc)
160
199
  importer = importers.InferenceImporter.from_graph_store(
161
200
  store=self._state.instances.store,
162
- non_existing_node_type=non_existing_node_type,
163
201
  max_number_of_instance=max_number_of_instance,
164
202
  )
165
203
  inferred_rules: ReadRules = importer.to_rules()
166
204
  end = datetime.now(timezone.utc)
167
205
 
168
206
  if model_id.space:
169
- cast(InformationInputRules, inferred_rules.rules).metadata.prefix = model_id.space
207
+ cast(InformationInputRules, inferred_rules.rules).metadata.space = model_id.space
170
208
  if model_id.external_id:
171
- cast(InformationInputRules, inferred_rules.rules).metadata.name = model_id.external_id
209
+ cast(InformationInputRules, inferred_rules.rules).metadata.external_id = model_id.external_id
172
210
 
173
211
  if model_id.version:
174
212
  cast(InformationInputRules, inferred_rules.rules).metadata.version = model_id.version
@@ -208,3 +246,27 @@ class NeatSession:
208
246
  output.append(f"<H2>Instances</H2> {state.instances.store._repr_html_()}")
209
247
 
210
248
  return "<br />".join(output)
249
+
250
+
251
+ @session_class_wrapper
252
+ class OptAPI:
253
+ def __init__(self, collector: Collector | None = None) -> None:
254
+ self._collector = collector or _COLLECTOR
255
+
256
+ def _display(self) -> None:
257
+ if self._collector.opted_in or self._collector.opted_out:
258
+ return
259
+ print(
260
+ "For Neat to improve, we need to collect usage information. "
261
+ "You acknowledge and agree that neat may collect usage information."
262
+ "To remove this message run 'neat.opt.in_() "
263
+ "or to stop collecting usage information run 'neat.opt.out()'."
264
+ )
265
+
266
+ def in_(self) -> None:
267
+ self._collector.enable()
268
+ print("You have successfully opted in to data collection.")
269
+
270
+ def out(self) -> None:
271
+ self._collector.disable()
272
+ print("You have successfully opted out of data collection.")
@@ -0,0 +1,126 @@
1
+ import os
2
+ import platform
3
+ import tempfile
4
+ import threading
5
+ import uuid
6
+ from contextlib import suppress
7
+ from functools import cached_property
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from mixpanel import Consumer, Mixpanel # type: ignore[import-untyped]
12
+
13
+ from cognite.neat._constants import IN_NOTEBOOK, IN_PYODIDE
14
+ from cognite.neat._version import __version__
15
+
16
+ _NEAT_MIXPANEL_TOKEN: str = "bd630ad149d19999df3989c3a3750c4f"
17
+
18
+
19
+ class Collector:
20
+ def __init__(self, skip_tracking: bool = False) -> None:
21
+ self.mp = Mixpanel(_NEAT_MIXPANEL_TOKEN, consumer=Consumer(api_host="api-eu.mixpanel.com"))
22
+ tmp_dir = Path(tempfile.gettempdir()).resolve()
23
+ self._opt_status_file = tmp_dir / "neat-opt-status.bin"
24
+ self._distinct_id_file = tmp_dir / "neat-distinct-id.bin"
25
+ self.skip_tracking = self.opted_out or skip_tracking
26
+
27
+ @cached_property
28
+ def _opt_status(self) -> str:
29
+ if self._opt_status_file.exists():
30
+ return self._opt_status_file.read_text()
31
+ return ""
32
+
33
+ def _bust_opt_status(self) -> None:
34
+ self.__dict__.pop("_opt_status", None)
35
+
36
+ @property
37
+ def opted_out(self) -> bool:
38
+ return self._opt_status == "opted-out"
39
+
40
+ @property
41
+ def opted_in(self) -> bool:
42
+ return self._opt_status == "opted-in"
43
+
44
+ @staticmethod
45
+ def _get_environment() -> str:
46
+ if IN_PYODIDE:
47
+ return "pyodide"
48
+ if IN_NOTEBOOK:
49
+ return "notebook"
50
+ return "python"
51
+
52
+ def track_session_command(self, command: str, *args, **kwargs) -> None:
53
+ event_information = {
54
+ "neatVersion": __version__,
55
+ "$os": platform.system(),
56
+ "pythonVersion": platform.python_version(),
57
+ "environment": self._get_environment(),
58
+ }
59
+
60
+ if len(args) > 1:
61
+ # The first argument is self.
62
+ for i, arg in enumerate(args[1:]):
63
+ event_information[f"arg{i}"] = arg
64
+
65
+ if kwargs:
66
+ for key, value in kwargs.items():
67
+ event_information[key] = self._serialize_value(value)[:500]
68
+ self._track(command, event_information)
69
+
70
+ @staticmethod
71
+ def _serialize_value(value: Any) -> str:
72
+ if isinstance(value, (str | int | float | bool)):
73
+ return str(value)
74
+ if isinstance(value, list | tuple | dict):
75
+ return str(value)
76
+ return str(type(value))
77
+
78
+ def _track(self, event_name: str, event_information: dict[str, Any]) -> bool:
79
+ if self.skip_tracking or not self.opted_in or "PYTEST_CURRENT_TEST" in os.environ:
80
+ return False
81
+
82
+ distinct_id = self.get_distinct_id()
83
+
84
+ def track() -> None:
85
+ # If we are unable to connect to Mixpanel, we don't want to crash the program
86
+ with suppress(ConnectionError):
87
+ self.mp.track(
88
+ distinct_id,
89
+ event_name,
90
+ event_information,
91
+ )
92
+
93
+ thread = threading.Thread(
94
+ target=track,
95
+ daemon=False,
96
+ )
97
+ thread.start()
98
+ return True
99
+
100
+ def get_distinct_id(self) -> str:
101
+ if self._distinct_id_file.exists():
102
+ return self._distinct_id_file.read_text()
103
+
104
+ distinct_id = f"{platform.system()}-{platform.python_version()}-{uuid.uuid4()!s}"
105
+ self._distinct_id_file.write_text(distinct_id)
106
+ with suppress(ConnectionError):
107
+ self.mp.people_set(
108
+ distinct_id,
109
+ {
110
+ "$os": platform.system(),
111
+ "$python_version": platform.python_version(),
112
+ "$distinct_id": distinct_id,
113
+ },
114
+ )
115
+ return distinct_id
116
+
117
+ def enable(self) -> None:
118
+ self._opt_status_file.write_text("opted-in")
119
+ self._bust_opt_status()
120
+
121
+ def disable(self) -> None:
122
+ self._opt_status_file.write_text("opted-out")
123
+ self._bust_opt_status()
124
+
125
+
126
+ _COLLECTOR = Collector()
@@ -0,0 +1,35 @@
1
+ from rdflib import URIRef
2
+
3
+ from ._state import SessionState
4
+ from .exceptions import session_class_wrapper
5
+
6
+ try:
7
+ from rich import print
8
+ except ImportError:
9
+ ...
10
+
11
+
12
+ @session_class_wrapper
13
+ class DropAPI:
14
+ def __init__(self, state: SessionState):
15
+ self._state = state
16
+
17
+ def instances(self, type: str | list[str]) -> None:
18
+ """Drop instances from the session.
19
+
20
+ Args:
21
+ type: The type of instances to drop.
22
+ """
23
+ type_list = type if isinstance(type, list) else [type]
24
+ uri_type_type = dict((v, k) for k, v in self._state.instances.store.queries.types.items())
25
+ selected_uri_by_type: dict[URIRef, str] = {}
26
+ for type_item in type_list:
27
+ if type_item not in uri_type_type:
28
+ print(f"Type {type_item} not found.")
29
+ selected_uri_by_type[uri_type_type[type_item]] = type_item
30
+
31
+ result = self._state.instances.store.queries.drop_types(list(selected_uri_by_type.keys()))
32
+
33
+ for type_uri, count in result.items():
34
+ print(f"Dropped {count} instances of type {selected_uri_by_type[type_uri]}")
35
+ return None
@@ -9,10 +9,17 @@ from cognite.neat._issues import IssueList
9
9
  from cognite.neat._utils.upload import UploadResult, UploadResultCore, UploadResultList
10
10
 
11
11
  from ._state import SessionState
12
- from .exceptions import intercept_session_exceptions
12
+ from .exceptions import session_class_wrapper
13
13
 
14
+ try:
15
+ from rich.markdown import Markdown as RichMarkdown
14
16
 
15
- @intercept_session_exceptions
17
+ RICH_AVAILABLE = True
18
+ except ImportError:
19
+ RICH_AVAILABLE = False
20
+
21
+
22
+ @session_class_wrapper
16
23
  class InspectAPI:
17
24
  def __init__(self, state: SessionState) -> None:
18
25
  self._state = state
@@ -25,7 +32,7 @@ class InspectAPI:
25
32
  return self._state.data_model.last_verified_rule[1].properties.to_pandas()
26
33
 
27
34
 
28
- @intercept_session_exceptions
35
+ @session_class_wrapper
29
36
  class InspectIssues:
30
37
  """Inspect issues of the current data model."""
31
38
 
@@ -61,14 +68,19 @@ class InspectIssues:
61
68
  closest_match = set(difflib.get_close_matches(search, unique_types))
62
69
  issues = IssueList([issue for issue in issues if type(issue).__name__ in closest_match])
63
70
 
71
+ issue_str = "\n".join(
72
+ [f" * **{type(issue).__name__}**: {issue.as_message(include_type=False)}" for issue in issues]
73
+ )
74
+ markdown_str = f"### {len(issues)} issues found\n\n{issue_str}"
75
+
64
76
  if IN_NOTEBOOK:
65
77
  from IPython.display import Markdown, display
66
78
 
67
- issue_str = "\n".join(
68
- [f" * **{type(issue).__name__}**: {issue.as_message(include_type=False)}" for issue in issues]
69
- )
70
- message = f"### {len(issues)} issues found\n\n{issue_str}"
71
- display(Markdown(message))
79
+ display(Markdown(markdown_str))
80
+ elif RICH_AVAILABLE:
81
+ from rich import print
82
+
83
+ print(RichMarkdown(markdown_str))
72
84
 
73
85
  if return_dataframe:
74
86
  return issues.to_pandas()
@@ -92,14 +104,14 @@ class InspectIssues:
92
104
  )
93
105
 
94
106
 
95
- @intercept_session_exceptions
107
+ @session_class_wrapper
96
108
  class InspectOutcome:
97
109
  def __init__(self, state: SessionState) -> None:
98
110
  self.data_model = InspectUploadOutcome(lambda: state.data_model.last_outcome)
99
111
  self.instances = InspectUploadOutcome(lambda: state.instances.last_outcome)
100
112
 
101
113
 
102
- @intercept_session_exceptions
114
+ @session_class_wrapper
103
115
  class InspectUploadOutcome:
104
116
  def __init__(self, get_last_outcome: Callable[[], UploadResultList]) -> None:
105
117
  self._get_last_outcome = get_last_outcome