cognite-neat 0.97.3__py3-none-any.whl → 0.99.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_client/__init__.py +4 -0
- cognite/neat/_client/_api/data_modeling_loaders.py +512 -0
- cognite/neat/_client/_api/schema.py +50 -0
- cognite/neat/_client/_api_client.py +17 -0
- cognite/neat/_client/data_classes/__init__.py +0 -0
- cognite/neat/{_utils/cdf/data_classes.py → _client/data_classes/data_modeling.py} +8 -135
- cognite/neat/{_rules/models/dms/_schema.py → _client/data_classes/schema.py} +32 -281
- cognite/neat/_graph/_shared.py +14 -15
- cognite/neat/_graph/extractors/_classic_cdf/_assets.py +14 -154
- cognite/neat/_graph/extractors/_classic_cdf/_base.py +154 -7
- cognite/neat/_graph/extractors/_classic_cdf/_classic.py +23 -12
- cognite/neat/_graph/extractors/_classic_cdf/_data_sets.py +17 -92
- cognite/neat/_graph/extractors/_classic_cdf/_events.py +13 -162
- cognite/neat/_graph/extractors/_classic_cdf/_files.py +15 -179
- cognite/neat/_graph/extractors/_classic_cdf/_labels.py +32 -100
- cognite/neat/_graph/extractors/_classic_cdf/_relationships.py +27 -178
- cognite/neat/_graph/extractors/_classic_cdf/_sequences.py +14 -139
- cognite/neat/_graph/extractors/_classic_cdf/_timeseries.py +15 -173
- cognite/neat/_graph/extractors/_rdf_file.py +6 -7
- cognite/neat/_graph/loaders/__init__.py +1 -2
- cognite/neat/_graph/queries/_base.py +17 -1
- cognite/neat/_graph/transformers/_classic_cdf.py +50 -134
- cognite/neat/_graph/transformers/_prune_graph.py +1 -1
- cognite/neat/_graph/transformers/_rdfpath.py +1 -1
- cognite/neat/_issues/warnings/__init__.py +6 -0
- cognite/neat/_issues/warnings/_external.py +8 -0
- cognite/neat/_issues/warnings/_models.py +9 -0
- cognite/neat/_issues/warnings/_properties.py +16 -0
- cognite/neat/_rules/_constants.py +7 -6
- cognite/neat/_rules/_shared.py +3 -8
- cognite/neat/_rules/analysis/__init__.py +1 -2
- cognite/neat/_rules/analysis/_base.py +10 -27
- cognite/neat/_rules/analysis/_dms.py +4 -10
- cognite/neat/_rules/analysis/_information.py +2 -10
- cognite/neat/_rules/catalog/info-rules-imf.xlsx +0 -0
- cognite/neat/_rules/exporters/_base.py +3 -4
- cognite/neat/_rules/exporters/_rules2dms.py +29 -40
- cognite/neat/_rules/exporters/_rules2excel.py +15 -72
- cognite/neat/_rules/exporters/_rules2ontology.py +4 -4
- cognite/neat/_rules/importers/_base.py +3 -4
- cognite/neat/_rules/importers/_dms2rules.py +21 -45
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_converter.py +1 -7
- cognite/neat/_rules/importers/_dtdl2rules/dtdl_importer.py +7 -10
- cognite/neat/_rules/importers/_rdf/_base.py +17 -29
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2metadata.py +5 -10
- cognite/neat/_rules/importers/_rdf/_imf2rules/_imf2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_inference2rules.py +55 -51
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2classes.py +2 -2
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2metadata.py +5 -8
- cognite/neat/_rules/importers/_rdf/_owl2rules/_owl2properties.py +1 -2
- cognite/neat/_rules/importers/_rdf/_shared.py +25 -140
- cognite/neat/_rules/importers/_spreadsheet2rules.py +10 -41
- cognite/neat/_rules/models/__init__.py +3 -17
- cognite/neat/_rules/models/_base_rules.py +118 -62
- cognite/neat/_rules/models/dms/__init__.py +2 -2
- cognite/neat/_rules/models/dms/_exporter.py +20 -178
- cognite/neat/_rules/models/dms/_rules.py +65 -128
- cognite/neat/_rules/models/dms/_rules_input.py +72 -56
- cognite/neat/_rules/models/dms/_validation.py +16 -109
- cognite/neat/_rules/models/entities/_single_value.py +32 -4
- cognite/neat/_rules/models/information/_rules.py +19 -122
- cognite/neat/_rules/models/information/_rules_input.py +32 -41
- cognite/neat/_rules/models/information/_validation.py +34 -102
- cognite/neat/_rules/models/mapping/__init__.py +2 -3
- cognite/neat/_rules/models/mapping/_classic2core.py +36 -146
- cognite/neat/_rules/models/mapping/_classic2core.yaml +339 -0
- cognite/neat/_rules/transformers/__init__.py +3 -6
- cognite/neat/_rules/transformers/_converters.py +128 -206
- cognite/neat/_rules/transformers/_mapping.py +105 -34
- cognite/neat/_rules/transformers/_verification.py +5 -16
- cognite/neat/_session/_base.py +83 -21
- cognite/neat/_session/_collector.py +126 -0
- cognite/neat/_session/_drop.py +35 -0
- cognite/neat/_session/_inspect.py +22 -10
- cognite/neat/_session/_mapping.py +39 -0
- cognite/neat/_session/_prepare.py +222 -27
- cognite/neat/_session/_read.py +109 -19
- cognite/neat/_session/_set.py +2 -2
- cognite/neat/_session/_show.py +11 -11
- cognite/neat/_session/_to.py +27 -14
- cognite/neat/_session/exceptions.py +20 -3
- cognite/neat/_store/_base.py +27 -24
- cognite/neat/_store/_provenance.py +2 -2
- cognite/neat/_utils/auxiliary.py +19 -0
- cognite/neat/_utils/rdf_.py +28 -1
- cognite/neat/_version.py +1 -1
- cognite/neat/_workflows/steps/data_contracts.py +2 -10
- cognite/neat/_workflows/steps/lib/current/rules_exporter.py +14 -49
- cognite/neat/_workflows/steps/lib/current/rules_importer.py +4 -1
- cognite/neat/_workflows/steps/lib/current/rules_validator.py +5 -9
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/METADATA +4 -3
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/RECORD +97 -100
- cognite/neat/_graph/loaders/_rdf2asset.py +0 -416
- cognite/neat/_rules/analysis/_asset.py +0 -173
- cognite/neat/_rules/models/asset/__init__.py +0 -13
- cognite/neat/_rules/models/asset/_rules.py +0 -109
- cognite/neat/_rules/models/asset/_rules_input.py +0 -101
- cognite/neat/_rules/models/asset/_validation.py +0 -45
- cognite/neat/_rules/models/domain.py +0 -136
- cognite/neat/_rules/models/mapping/_base.py +0 -131
- cognite/neat/_utils/cdf/loaders/__init__.py +0 -25
- cognite/neat/_utils/cdf/loaders/_base.py +0 -54
- cognite/neat/_utils/cdf/loaders/_data_modeling.py +0 -339
- cognite/neat/_utils/cdf/loaders/_ingestion.py +0 -167
- /cognite/neat/{_utils/cdf → _client/_api}/__init__.py +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/LICENSE +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/WHEEL +0 -0
- {cognite_neat-0.97.3.dist-info → cognite_neat-0.99.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,13 +1,16 @@
|
|
|
1
|
+
import warnings
|
|
1
2
|
from abc import ABC
|
|
2
3
|
from collections import defaultdict
|
|
4
|
+
from functools import cached_property
|
|
5
|
+
from typing import Any, ClassVar, Literal
|
|
3
6
|
|
|
7
|
+
from cognite.neat._issues.errors import NeatValueError
|
|
8
|
+
from cognite.neat._issues.warnings import NeatValueWarning, PropertyOverwritingWarning
|
|
4
9
|
from cognite.neat._rules._shared import JustRules, OutRules
|
|
5
|
-
from cognite.neat._rules.models import DMSRules,
|
|
6
|
-
from cognite.neat._rules.models.
|
|
7
|
-
from cognite.neat._rules.models.dms import DMSProperty
|
|
8
|
-
from cognite.neat._rules.models.entities import
|
|
9
|
-
from cognite.neat._rules.models.information import InformationClass
|
|
10
|
-
from cognite.neat._rules.models.mapping import RuleMapping
|
|
10
|
+
from cognite.neat._rules.models import DMSRules, SheetList
|
|
11
|
+
from cognite.neat._rules.models.data_types import Enum
|
|
12
|
+
from cognite.neat._rules.models.dms import DMSEnum, DMSProperty, DMSView
|
|
13
|
+
from cognite.neat._rules.models.entities import ViewEntity
|
|
11
14
|
|
|
12
15
|
from ._base import RulesTransformer
|
|
13
16
|
|
|
@@ -51,9 +54,6 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
51
54
|
|
|
52
55
|
def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
|
|
53
56
|
solution: DMSRules = self._to_rules(rules)
|
|
54
|
-
if solution.reference is not None:
|
|
55
|
-
raise ValueError("Reference already exists")
|
|
56
|
-
solution.reference = self.reference
|
|
57
57
|
view_by_external_id = {view.view.external_id: view for view in solution.views}
|
|
58
58
|
ref_view_by_external_id = {view.view.external_id: view for view in self.reference.views}
|
|
59
59
|
|
|
@@ -95,12 +95,11 @@ class MapOneToOne(MapOntoTransformers):
|
|
|
95
95
|
if ref_prop.container and ref_prop.container_property:
|
|
96
96
|
prop.container = ref_prop.container
|
|
97
97
|
prop.container_property = ref_prop.container_property
|
|
98
|
-
prop.reference = ReferenceEntity.from_entity(ref_prop.view, ref_prop.view_property)
|
|
99
98
|
|
|
100
99
|
return JustRules(solution)
|
|
101
100
|
|
|
102
101
|
|
|
103
|
-
class RuleMapper(RulesTransformer[
|
|
102
|
+
class RuleMapper(RulesTransformer[DMSRules, DMSRules]):
|
|
104
103
|
"""Maps properties and classes using the given mapping.
|
|
105
104
|
|
|
106
105
|
**Note**: This transformer mutates the input rules.
|
|
@@ -110,30 +109,102 @@ class RuleMapper(RulesTransformer[InformationRules, InformationRules]):
|
|
|
110
109
|
|
|
111
110
|
"""
|
|
112
111
|
|
|
113
|
-
|
|
112
|
+
_mapping_fields: ClassVar[frozenset[str]] = frozenset(
|
|
113
|
+
["connection", "value_type", "nullable", "immutable", "is_list", "default", "index", "constraint"]
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
def __init__(self, mapping: DMSRules, data_type_conflict: Literal["overwrite"] = "overwrite") -> None:
|
|
114
117
|
self.mapping = mapping
|
|
118
|
+
self.data_type_conflict = data_type_conflict
|
|
119
|
+
|
|
120
|
+
@cached_property
|
|
121
|
+
def _view_by_entity_id(self) -> dict[str, DMSView]:
|
|
122
|
+
return {view.view.external_id: view for view in self.mapping.views}
|
|
115
123
|
|
|
116
|
-
|
|
124
|
+
@cached_property
|
|
125
|
+
def _property_by_view_property(self) -> dict[tuple[str, str], DMSProperty]:
|
|
126
|
+
return {(prop.view.external_id, prop.view_property): prop for prop in self.mapping.properties}
|
|
127
|
+
|
|
128
|
+
def transform(self, rules: DMSRules | OutRules[DMSRules]) -> JustRules[DMSRules]:
|
|
129
|
+
if self.data_type_conflict != "overwrite":
|
|
130
|
+
raise NeatValueError(f"Invalid data_type_conflict: {self.data_type_conflict}")
|
|
117
131
|
input_rules = self._to_rules(rules)
|
|
132
|
+
new_rules = input_rules.model_copy(deep=True)
|
|
133
|
+
new_rules.metadata.version += "_mapped"
|
|
118
134
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
135
|
+
for view in new_rules.views:
|
|
136
|
+
if mapping_view := self._view_by_entity_id.get(view.view.external_id):
|
|
137
|
+
view.implements = mapping_view.implements
|
|
138
|
+
|
|
139
|
+
for prop in new_rules.properties:
|
|
140
|
+
key = (prop.view.external_id, prop.view_property)
|
|
141
|
+
if key not in self._property_by_view_property:
|
|
142
|
+
continue
|
|
143
|
+
mapping_prop = self._property_by_view_property[key]
|
|
144
|
+
to_overwrite, conflicts = self._find_overwrites(prop, mapping_prop)
|
|
145
|
+
if conflicts and self.data_type_conflict == "overwrite":
|
|
146
|
+
warnings.warn(
|
|
147
|
+
PropertyOverwritingWarning(prop.view.as_id(), "view", prop.view_property, tuple(conflicts)),
|
|
148
|
+
stacklevel=2,
|
|
149
|
+
)
|
|
150
|
+
elif conflicts:
|
|
151
|
+
raise NeatValueError(f"Conflicting properties for {prop.view}.{prop.view_property}: {conflicts}")
|
|
152
|
+
for field_name, value in to_overwrite.items():
|
|
153
|
+
setattr(prop, field_name, value)
|
|
154
|
+
prop.container = mapping_prop.container
|
|
155
|
+
prop.container_property = mapping_prop.container_property
|
|
156
|
+
|
|
157
|
+
# Add missing views used as value types
|
|
158
|
+
existing_views = {view.view for view in new_rules.views}
|
|
159
|
+
new_value_types = {
|
|
160
|
+
prop.value_type
|
|
161
|
+
for prop in new_rules.properties
|
|
162
|
+
if isinstance(prop.value_type, ViewEntity) and prop.value_type not in existing_views
|
|
163
|
+
}
|
|
164
|
+
for new_value_type in new_value_types:
|
|
165
|
+
if mapping_view := self._view_by_entity_id.get(new_value_type.external_id):
|
|
166
|
+
new_rules.views.append(mapping_view)
|
|
167
|
+
else:
|
|
168
|
+
warnings.warn(NeatValueWarning(f"View {new_value_type} not found in mapping"), stacklevel=2)
|
|
169
|
+
|
|
170
|
+
# Add missing enums
|
|
171
|
+
existing_enum_collections = {item.collection for item in new_rules.enum or []}
|
|
172
|
+
new_enums = {
|
|
173
|
+
prop.value_type.collection
|
|
174
|
+
for prop in new_rules.properties
|
|
175
|
+
if isinstance(prop.value_type, Enum) and prop.value_type.collection not in existing_enum_collections
|
|
176
|
+
}
|
|
177
|
+
if new_enums:
|
|
178
|
+
new_rules.enum = new_rules.enum or SheetList[DMSEnum]([])
|
|
179
|
+
for item in self.mapping.enum or []:
|
|
180
|
+
if item.collection in new_enums:
|
|
181
|
+
new_rules.enum.append(item)
|
|
182
|
+
|
|
183
|
+
return JustRules(new_rules)
|
|
184
|
+
|
|
185
|
+
def _find_overwrites(self, prop: DMSProperty, mapping_prop: DMSProperty) -> tuple[dict[str, Any], list[str]]:
|
|
186
|
+
"""Finds the properties that need to be overwritten and returns them.
|
|
187
|
+
|
|
188
|
+
In addition, conflicting properties are returned. Note that overwriting properties that are
|
|
189
|
+
originally None is not considered a conflict. Thus, you can have properties to overwrite but no
|
|
190
|
+
conflicts.
|
|
191
|
+
|
|
192
|
+
Args:
|
|
193
|
+
prop: The property to compare.
|
|
194
|
+
mapping_prop: The property to compare against.
|
|
195
|
+
|
|
196
|
+
Returns:
|
|
197
|
+
A tuple with the properties to overwrite and the conflicting properties.
|
|
198
|
+
|
|
199
|
+
"""
|
|
200
|
+
to_overwrite: dict[str, Any] = {}
|
|
201
|
+
conflicts: list[str] = []
|
|
202
|
+
for field_name in self._mapping_fields:
|
|
203
|
+
mapping_value = getattr(mapping_prop, field_name)
|
|
204
|
+
source_value = getattr(prop, field_name)
|
|
205
|
+
if mapping_value != source_value:
|
|
206
|
+
to_overwrite[field_name] = mapping_value
|
|
207
|
+
if source_value is not None:
|
|
208
|
+
# These are used for warnings so we use the alias to make it more readable for the user
|
|
209
|
+
conflicts.append(mapping_prop.model_fields[field_name].alias or field_name)
|
|
210
|
+
return to_overwrite, conflicts
|
|
@@ -13,12 +13,8 @@ from cognite.neat._rules._shared import (
|
|
|
13
13
|
VerifiedRules,
|
|
14
14
|
)
|
|
15
15
|
from cognite.neat._rules.models import (
|
|
16
|
-
AssetInputRules,
|
|
17
|
-
AssetRules,
|
|
18
16
|
DMSInputRules,
|
|
19
17
|
DMSRules,
|
|
20
|
-
DomainInputRules,
|
|
21
|
-
DomainRules,
|
|
22
18
|
InformationInputRules,
|
|
23
19
|
InformationRules,
|
|
24
20
|
)
|
|
@@ -31,8 +27,9 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
|
|
|
31
27
|
|
|
32
28
|
_rules_cls: type[T_VerifiedRules]
|
|
33
29
|
|
|
34
|
-
def __init__(self, errors: Literal["raise", "continue"]) -> None:
|
|
30
|
+
def __init__(self, errors: Literal["raise", "continue"], post_validate: bool = True) -> None:
|
|
35
31
|
self.errors = errors
|
|
32
|
+
self.post_validate = post_validate
|
|
36
33
|
|
|
37
34
|
def transform(self, rules: T_InputRules | OutRules[T_InputRules]) -> MaybeRules[T_VerifiedRules]:
|
|
38
35
|
issues = IssueList()
|
|
@@ -43,7 +40,9 @@ class VerificationTransformer(RulesTransformer[T_InputRules, T_VerifiedRules], A
|
|
|
43
40
|
verified_rules: T_VerifiedRules | None = None
|
|
44
41
|
with catch_issues(issues, NeatError, NeatWarning, error_args) as future:
|
|
45
42
|
rules_cls = self._get_rules_cls(in_)
|
|
46
|
-
|
|
43
|
+
dumped = in_.dump()
|
|
44
|
+
dumped["post_validate"] = self.post_validate
|
|
45
|
+
verified_rules = rules_cls.model_validate(dumped) # type: ignore[assignment]
|
|
47
46
|
|
|
48
47
|
if (future.result == "failure" or issues.has_errors or verified_rules is None) and self.errors == "raise":
|
|
49
48
|
raise issues.as_errors()
|
|
@@ -68,12 +67,6 @@ class VerifyInformationRules(VerificationTransformer[InformationInputRules, Info
|
|
|
68
67
|
_rules_cls = InformationRules
|
|
69
68
|
|
|
70
69
|
|
|
71
|
-
class VerifyAssetRules(VerificationTransformer[AssetInputRules, AssetRules]):
|
|
72
|
-
"""Class to verify Asset rules."""
|
|
73
|
-
|
|
74
|
-
_rules_cls = AssetRules
|
|
75
|
-
|
|
76
|
-
|
|
77
70
|
class VerifyAnyRules(VerificationTransformer[InputRules, VerifiedRules]):
|
|
78
71
|
"""Class to verify arbitrary rules"""
|
|
79
72
|
|
|
@@ -82,9 +75,5 @@ class VerifyAnyRules(VerificationTransformer[InputRules, VerifiedRules]):
|
|
|
82
75
|
return InformationRules
|
|
83
76
|
elif isinstance(in_, DMSInputRules):
|
|
84
77
|
return DMSRules
|
|
85
|
-
elif isinstance(in_, AssetInputRules):
|
|
86
|
-
return AssetRules
|
|
87
|
-
elif isinstance(in_, DomainInputRules):
|
|
88
|
-
return DomainRules
|
|
89
78
|
else:
|
|
90
79
|
raise NeatTypeError(f"Unsupported rules type: {type(in_)}")
|
cognite/neat/_session/_base.py
CHANGED
|
@@ -5,14 +5,13 @@ from cognite.client import CogniteClient
|
|
|
5
5
|
from cognite.client import data_modeling as dm
|
|
6
6
|
|
|
7
7
|
from cognite.neat import _version
|
|
8
|
+
from cognite.neat._client import NeatClient
|
|
8
9
|
from cognite.neat._issues import IssueList, catch_issues
|
|
9
10
|
from cognite.neat._issues.errors import RegexViolationError
|
|
10
11
|
from cognite.neat._rules import importers
|
|
11
12
|
from cognite.neat._rules._shared import ReadRules, VerifiedRules
|
|
12
|
-
from cognite.neat._rules.importers
|
|
13
|
-
from cognite.neat._rules.models import DMSRules
|
|
14
|
-
from cognite.neat._rules.models.data_types import AnyURI
|
|
15
|
-
from cognite.neat._rules.models.entities._single_value import UnknownEntity
|
|
13
|
+
from cognite.neat._rules.importers import DMSImporter
|
|
14
|
+
from cognite.neat._rules.models import DMSInputRules, DMSRules, SheetList
|
|
16
15
|
from cognite.neat._rules.models.information._rules import InformationRules
|
|
17
16
|
from cognite.neat._rules.models.information._rules_input import InformationInputRules
|
|
18
17
|
from cognite.neat._rules.transformers import ConvertToRules, VerifyAnyRules
|
|
@@ -21,9 +20,11 @@ from cognite.neat._store._provenance import (
|
|
|
21
20
|
INSTANCES_ENTITY,
|
|
22
21
|
Change,
|
|
23
22
|
)
|
|
24
|
-
from cognite.neat._utils.auth import _CLIENT_NAME
|
|
25
23
|
|
|
24
|
+
from ._collector import _COLLECTOR, Collector
|
|
25
|
+
from ._drop import DropAPI
|
|
26
26
|
from ._inspect import InspectAPI
|
|
27
|
+
from ._mapping import MappingAPI
|
|
27
28
|
from ._prepare import PrepareAPI
|
|
28
29
|
from ._read import ReadAPI
|
|
29
30
|
from ._set import SetAPI
|
|
@@ -31,10 +32,10 @@ from ._show import ShowAPI
|
|
|
31
32
|
from ._state import SessionState
|
|
32
33
|
from ._to import ToAPI
|
|
33
34
|
from .engine import load_neat_engine
|
|
34
|
-
from .exceptions import NeatSessionError,
|
|
35
|
+
from .exceptions import NeatSessionError, session_class_wrapper
|
|
35
36
|
|
|
36
37
|
|
|
37
|
-
@
|
|
38
|
+
@session_class_wrapper
|
|
38
39
|
class NeatSession:
|
|
39
40
|
def __init__(
|
|
40
41
|
self,
|
|
@@ -43,17 +44,19 @@ class NeatSession:
|
|
|
43
44
|
verbose: bool = True,
|
|
44
45
|
load_engine: Literal["newest", "cache", "skip"] = "cache",
|
|
45
46
|
) -> None:
|
|
46
|
-
self._client = client
|
|
47
|
+
self._client = NeatClient(client) if client else None
|
|
47
48
|
self._verbose = verbose
|
|
48
49
|
self._state = SessionState(store_type=storage)
|
|
49
|
-
self.read = ReadAPI(self._state,
|
|
50
|
-
self.to = ToAPI(self._state,
|
|
51
|
-
self.prepare = PrepareAPI(self._state, verbose)
|
|
50
|
+
self.read = ReadAPI(self._state, self._client, verbose)
|
|
51
|
+
self.to = ToAPI(self._state, self._client, verbose)
|
|
52
|
+
self.prepare = PrepareAPI(self._client, self._state, verbose)
|
|
52
53
|
self.show = ShowAPI(self._state)
|
|
53
54
|
self.set = SetAPI(self._state, verbose)
|
|
54
55
|
self.inspect = InspectAPI(self._state)
|
|
55
|
-
|
|
56
|
-
|
|
56
|
+
self.mapping = MappingAPI(self._state)
|
|
57
|
+
self.drop = DropAPI(self._state)
|
|
58
|
+
self.opt = OptAPI()
|
|
59
|
+
self.opt._display()
|
|
57
60
|
if load_engine != "skip" and (engine_version := load_neat_engine(client, load_engine)):
|
|
58
61
|
print(f"Neat Engine {engine_version} loaded.")
|
|
59
62
|
|
|
@@ -63,6 +66,28 @@ class NeatSession:
|
|
|
63
66
|
|
|
64
67
|
def verify(self) -> IssueList:
|
|
65
68
|
source_id, last_unverified_rule = self._state.data_model.last_unverified_rule
|
|
69
|
+
|
|
70
|
+
reference_rules: DMSInputRules | None = None
|
|
71
|
+
if isinstance(last_unverified_rule.rules, DMSInputRules):
|
|
72
|
+
dms_rules = last_unverified_rule.rules
|
|
73
|
+
views_ids, containers_ids = dms_rules.imported_views_and_containers_ids()
|
|
74
|
+
if views_ids or containers_ids:
|
|
75
|
+
if self._client is None:
|
|
76
|
+
raise NeatSessionError(
|
|
77
|
+
"No client provided. You are referencing unknown views and containers in your data model, "
|
|
78
|
+
"NEAT needs a client to lookup the definitions. "
|
|
79
|
+
"Please set the client in the session, NeatSession(client=client)."
|
|
80
|
+
)
|
|
81
|
+
schema = self._client.schema.retrieve(list(views_ids), list(containers_ids))
|
|
82
|
+
|
|
83
|
+
importer = DMSImporter(schema)
|
|
84
|
+
reference_rules = importer.to_rules().rules
|
|
85
|
+
|
|
86
|
+
if reference_rules is not None:
|
|
87
|
+
dms_rules.views.extend(reference_rules.views)
|
|
88
|
+
if dms_rules.containers:
|
|
89
|
+
dms_rules.containers.extend(reference_rules.containers or [])
|
|
90
|
+
|
|
66
91
|
transformer = VerifyAnyRules("continue")
|
|
67
92
|
start = datetime.now(timezone.utc)
|
|
68
93
|
output = transformer.try_transform(last_unverified_rule)
|
|
@@ -74,10 +99,26 @@ class NeatSession:
|
|
|
74
99
|
transformer.agent,
|
|
75
100
|
start,
|
|
76
101
|
end,
|
|
77
|
-
f"Verified data model {source_id} as {output.rules.
|
|
102
|
+
f"Verified data model {source_id} as {output.rules.metadata.identifier}",
|
|
78
103
|
self._state.data_model.provenance.source_entity(source_id)
|
|
79
104
|
or self._state.data_model.provenance.target_entity(source_id),
|
|
80
105
|
)
|
|
106
|
+
if reference_rules is not None and isinstance(output.rules, DMSRules):
|
|
107
|
+
# Remove the referenced views and containers from the rules
|
|
108
|
+
ref_view_ids = set(reference_rules.as_view_entities())
|
|
109
|
+
if ref_view_ids:
|
|
110
|
+
output.rules.views = SheetList(
|
|
111
|
+
[view for view in output.rules.views if view.view not in ref_view_ids]
|
|
112
|
+
)
|
|
113
|
+
ref_container_ids = reference_rules.as_container_entities()
|
|
114
|
+
if output.rules.containers and ref_container_ids:
|
|
115
|
+
output.rules.containers = SheetList(
|
|
116
|
+
[
|
|
117
|
+
container
|
|
118
|
+
for container in output.rules.containers
|
|
119
|
+
if container.container not in ref_container_ids
|
|
120
|
+
]
|
|
121
|
+
)
|
|
81
122
|
|
|
82
123
|
self._state.data_model.write(output.rules, change)
|
|
83
124
|
|
|
@@ -119,7 +160,7 @@ class NeatSession:
|
|
|
119
160
|
converter.agent,
|
|
120
161
|
start,
|
|
121
162
|
end,
|
|
122
|
-
f"Converted data model {source_id} to {converted_rules.
|
|
163
|
+
f"Converted data model {source_id} to {converted_rules.metadata.identifier}",
|
|
123
164
|
self._state.data_model.provenance.source_entity(source_id)
|
|
124
165
|
or self._state.data_model.provenance.target_entity(source_id),
|
|
125
166
|
)
|
|
@@ -144,31 +185,28 @@ class NeatSession:
|
|
|
144
185
|
"NeatInferredDataModel",
|
|
145
186
|
"v1",
|
|
146
187
|
),
|
|
147
|
-
non_existing_node_type: UnknownEntity | AnyURI = DEFAULT_NON_EXISTING_NODE_TYPE,
|
|
148
188
|
max_number_of_instance: int = 100,
|
|
149
189
|
) -> IssueList:
|
|
150
190
|
"""Data model inference from instances.
|
|
151
191
|
|
|
152
192
|
Args:
|
|
153
193
|
model_id: The ID of the inferred data model.
|
|
154
|
-
|
|
194
|
+
max_number_of_instance: The maximum number of instances to use for inference.
|
|
155
195
|
"""
|
|
156
|
-
|
|
157
196
|
model_id = dm.DataModelId.load(model_id)
|
|
158
197
|
|
|
159
198
|
start = datetime.now(timezone.utc)
|
|
160
199
|
importer = importers.InferenceImporter.from_graph_store(
|
|
161
200
|
store=self._state.instances.store,
|
|
162
|
-
non_existing_node_type=non_existing_node_type,
|
|
163
201
|
max_number_of_instance=max_number_of_instance,
|
|
164
202
|
)
|
|
165
203
|
inferred_rules: ReadRules = importer.to_rules()
|
|
166
204
|
end = datetime.now(timezone.utc)
|
|
167
205
|
|
|
168
206
|
if model_id.space:
|
|
169
|
-
cast(InformationInputRules, inferred_rules.rules).metadata.
|
|
207
|
+
cast(InformationInputRules, inferred_rules.rules).metadata.space = model_id.space
|
|
170
208
|
if model_id.external_id:
|
|
171
|
-
cast(InformationInputRules, inferred_rules.rules).metadata.
|
|
209
|
+
cast(InformationInputRules, inferred_rules.rules).metadata.external_id = model_id.external_id
|
|
172
210
|
|
|
173
211
|
if model_id.version:
|
|
174
212
|
cast(InformationInputRules, inferred_rules.rules).metadata.version = model_id.version
|
|
@@ -208,3 +246,27 @@ class NeatSession:
|
|
|
208
246
|
output.append(f"<H2>Instances</H2> {state.instances.store._repr_html_()}")
|
|
209
247
|
|
|
210
248
|
return "<br />".join(output)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
@session_class_wrapper
|
|
252
|
+
class OptAPI:
|
|
253
|
+
def __init__(self, collector: Collector | None = None) -> None:
|
|
254
|
+
self._collector = collector or _COLLECTOR
|
|
255
|
+
|
|
256
|
+
def _display(self) -> None:
|
|
257
|
+
if self._collector.opted_in or self._collector.opted_out:
|
|
258
|
+
return
|
|
259
|
+
print(
|
|
260
|
+
"For Neat to improve, we need to collect usage information. "
|
|
261
|
+
"You acknowledge and agree that neat may collect usage information."
|
|
262
|
+
"To remove this message run 'neat.opt.in_() "
|
|
263
|
+
"or to stop collecting usage information run 'neat.opt.out()'."
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
def in_(self) -> None:
|
|
267
|
+
self._collector.enable()
|
|
268
|
+
print("You have successfully opted in to data collection.")
|
|
269
|
+
|
|
270
|
+
def out(self) -> None:
|
|
271
|
+
self._collector.disable()
|
|
272
|
+
print("You have successfully opted out of data collection.")
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import platform
|
|
3
|
+
import tempfile
|
|
4
|
+
import threading
|
|
5
|
+
import uuid
|
|
6
|
+
from contextlib import suppress
|
|
7
|
+
from functools import cached_property
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from mixpanel import Consumer, Mixpanel # type: ignore[import-untyped]
|
|
12
|
+
|
|
13
|
+
from cognite.neat._constants import IN_NOTEBOOK, IN_PYODIDE
|
|
14
|
+
from cognite.neat._version import __version__
|
|
15
|
+
|
|
16
|
+
_NEAT_MIXPANEL_TOKEN: str = "bd630ad149d19999df3989c3a3750c4f"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Collector:
|
|
20
|
+
def __init__(self, skip_tracking: bool = False) -> None:
|
|
21
|
+
self.mp = Mixpanel(_NEAT_MIXPANEL_TOKEN, consumer=Consumer(api_host="api-eu.mixpanel.com"))
|
|
22
|
+
tmp_dir = Path(tempfile.gettempdir()).resolve()
|
|
23
|
+
self._opt_status_file = tmp_dir / "neat-opt-status.bin"
|
|
24
|
+
self._distinct_id_file = tmp_dir / "neat-distinct-id.bin"
|
|
25
|
+
self.skip_tracking = self.opted_out or skip_tracking
|
|
26
|
+
|
|
27
|
+
@cached_property
|
|
28
|
+
def _opt_status(self) -> str:
|
|
29
|
+
if self._opt_status_file.exists():
|
|
30
|
+
return self._opt_status_file.read_text()
|
|
31
|
+
return ""
|
|
32
|
+
|
|
33
|
+
def _bust_opt_status(self) -> None:
|
|
34
|
+
self.__dict__.pop("_opt_status", None)
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def opted_out(self) -> bool:
|
|
38
|
+
return self._opt_status == "opted-out"
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def opted_in(self) -> bool:
|
|
42
|
+
return self._opt_status == "opted-in"
|
|
43
|
+
|
|
44
|
+
@staticmethod
|
|
45
|
+
def _get_environment() -> str:
|
|
46
|
+
if IN_PYODIDE:
|
|
47
|
+
return "pyodide"
|
|
48
|
+
if IN_NOTEBOOK:
|
|
49
|
+
return "notebook"
|
|
50
|
+
return "python"
|
|
51
|
+
|
|
52
|
+
def track_session_command(self, command: str, *args, **kwargs) -> None:
|
|
53
|
+
event_information = {
|
|
54
|
+
"neatVersion": __version__,
|
|
55
|
+
"$os": platform.system(),
|
|
56
|
+
"pythonVersion": platform.python_version(),
|
|
57
|
+
"environment": self._get_environment(),
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if len(args) > 1:
|
|
61
|
+
# The first argument is self.
|
|
62
|
+
for i, arg in enumerate(args[1:]):
|
|
63
|
+
event_information[f"arg{i}"] = arg
|
|
64
|
+
|
|
65
|
+
if kwargs:
|
|
66
|
+
for key, value in kwargs.items():
|
|
67
|
+
event_information[key] = self._serialize_value(value)[:500]
|
|
68
|
+
self._track(command, event_information)
|
|
69
|
+
|
|
70
|
+
@staticmethod
|
|
71
|
+
def _serialize_value(value: Any) -> str:
|
|
72
|
+
if isinstance(value, (str | int | float | bool)):
|
|
73
|
+
return str(value)
|
|
74
|
+
if isinstance(value, list | tuple | dict):
|
|
75
|
+
return str(value)
|
|
76
|
+
return str(type(value))
|
|
77
|
+
|
|
78
|
+
def _track(self, event_name: str, event_information: dict[str, Any]) -> bool:
|
|
79
|
+
if self.skip_tracking or not self.opted_in or "PYTEST_CURRENT_TEST" in os.environ:
|
|
80
|
+
return False
|
|
81
|
+
|
|
82
|
+
distinct_id = self.get_distinct_id()
|
|
83
|
+
|
|
84
|
+
def track() -> None:
|
|
85
|
+
# If we are unable to connect to Mixpanel, we don't want to crash the program
|
|
86
|
+
with suppress(ConnectionError):
|
|
87
|
+
self.mp.track(
|
|
88
|
+
distinct_id,
|
|
89
|
+
event_name,
|
|
90
|
+
event_information,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
thread = threading.Thread(
|
|
94
|
+
target=track,
|
|
95
|
+
daemon=False,
|
|
96
|
+
)
|
|
97
|
+
thread.start()
|
|
98
|
+
return True
|
|
99
|
+
|
|
100
|
+
def get_distinct_id(self) -> str:
|
|
101
|
+
if self._distinct_id_file.exists():
|
|
102
|
+
return self._distinct_id_file.read_text()
|
|
103
|
+
|
|
104
|
+
distinct_id = f"{platform.system()}-{platform.python_version()}-{uuid.uuid4()!s}"
|
|
105
|
+
self._distinct_id_file.write_text(distinct_id)
|
|
106
|
+
with suppress(ConnectionError):
|
|
107
|
+
self.mp.people_set(
|
|
108
|
+
distinct_id,
|
|
109
|
+
{
|
|
110
|
+
"$os": platform.system(),
|
|
111
|
+
"$python_version": platform.python_version(),
|
|
112
|
+
"$distinct_id": distinct_id,
|
|
113
|
+
},
|
|
114
|
+
)
|
|
115
|
+
return distinct_id
|
|
116
|
+
|
|
117
|
+
def enable(self) -> None:
|
|
118
|
+
self._opt_status_file.write_text("opted-in")
|
|
119
|
+
self._bust_opt_status()
|
|
120
|
+
|
|
121
|
+
def disable(self) -> None:
|
|
122
|
+
self._opt_status_file.write_text("opted-out")
|
|
123
|
+
self._bust_opt_status()
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
_COLLECTOR = Collector()
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from rdflib import URIRef
|
|
2
|
+
|
|
3
|
+
from ._state import SessionState
|
|
4
|
+
from .exceptions import session_class_wrapper
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
from rich import print
|
|
8
|
+
except ImportError:
|
|
9
|
+
...
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@session_class_wrapper
|
|
13
|
+
class DropAPI:
|
|
14
|
+
def __init__(self, state: SessionState):
|
|
15
|
+
self._state = state
|
|
16
|
+
|
|
17
|
+
def instances(self, type: str | list[str]) -> None:
|
|
18
|
+
"""Drop instances from the session.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
type: The type of instances to drop.
|
|
22
|
+
"""
|
|
23
|
+
type_list = type if isinstance(type, list) else [type]
|
|
24
|
+
uri_type_type = dict((v, k) for k, v in self._state.instances.store.queries.types.items())
|
|
25
|
+
selected_uri_by_type: dict[URIRef, str] = {}
|
|
26
|
+
for type_item in type_list:
|
|
27
|
+
if type_item not in uri_type_type:
|
|
28
|
+
print(f"Type {type_item} not found.")
|
|
29
|
+
selected_uri_by_type[uri_type_type[type_item]] = type_item
|
|
30
|
+
|
|
31
|
+
result = self._state.instances.store.queries.drop_types(list(selected_uri_by_type.keys()))
|
|
32
|
+
|
|
33
|
+
for type_uri, count in result.items():
|
|
34
|
+
print(f"Dropped {count} instances of type {selected_uri_by_type[type_uri]}")
|
|
35
|
+
return None
|
|
@@ -9,10 +9,17 @@ from cognite.neat._issues import IssueList
|
|
|
9
9
|
from cognite.neat._utils.upload import UploadResult, UploadResultCore, UploadResultList
|
|
10
10
|
|
|
11
11
|
from ._state import SessionState
|
|
12
|
-
from .exceptions import
|
|
12
|
+
from .exceptions import session_class_wrapper
|
|
13
13
|
|
|
14
|
+
try:
|
|
15
|
+
from rich.markdown import Markdown as RichMarkdown
|
|
14
16
|
|
|
15
|
-
|
|
17
|
+
RICH_AVAILABLE = True
|
|
18
|
+
except ImportError:
|
|
19
|
+
RICH_AVAILABLE = False
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@session_class_wrapper
|
|
16
23
|
class InspectAPI:
|
|
17
24
|
def __init__(self, state: SessionState) -> None:
|
|
18
25
|
self._state = state
|
|
@@ -25,7 +32,7 @@ class InspectAPI:
|
|
|
25
32
|
return self._state.data_model.last_verified_rule[1].properties.to_pandas()
|
|
26
33
|
|
|
27
34
|
|
|
28
|
-
@
|
|
35
|
+
@session_class_wrapper
|
|
29
36
|
class InspectIssues:
|
|
30
37
|
"""Inspect issues of the current data model."""
|
|
31
38
|
|
|
@@ -61,14 +68,19 @@ class InspectIssues:
|
|
|
61
68
|
closest_match = set(difflib.get_close_matches(search, unique_types))
|
|
62
69
|
issues = IssueList([issue for issue in issues if type(issue).__name__ in closest_match])
|
|
63
70
|
|
|
71
|
+
issue_str = "\n".join(
|
|
72
|
+
[f" * **{type(issue).__name__}**: {issue.as_message(include_type=False)}" for issue in issues]
|
|
73
|
+
)
|
|
74
|
+
markdown_str = f"### {len(issues)} issues found\n\n{issue_str}"
|
|
75
|
+
|
|
64
76
|
if IN_NOTEBOOK:
|
|
65
77
|
from IPython.display import Markdown, display
|
|
66
78
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
79
|
+
display(Markdown(markdown_str))
|
|
80
|
+
elif RICH_AVAILABLE:
|
|
81
|
+
from rich import print
|
|
82
|
+
|
|
83
|
+
print(RichMarkdown(markdown_str))
|
|
72
84
|
|
|
73
85
|
if return_dataframe:
|
|
74
86
|
return issues.to_pandas()
|
|
@@ -92,14 +104,14 @@ class InspectIssues:
|
|
|
92
104
|
)
|
|
93
105
|
|
|
94
106
|
|
|
95
|
-
@
|
|
107
|
+
@session_class_wrapper
|
|
96
108
|
class InspectOutcome:
|
|
97
109
|
def __init__(self, state: SessionState) -> None:
|
|
98
110
|
self.data_model = InspectUploadOutcome(lambda: state.data_model.last_outcome)
|
|
99
111
|
self.instances = InspectUploadOutcome(lambda: state.instances.last_outcome)
|
|
100
112
|
|
|
101
113
|
|
|
102
|
-
@
|
|
114
|
+
@session_class_wrapper
|
|
103
115
|
class InspectUploadOutcome:
|
|
104
116
|
def __init__(self, get_last_outcome: Callable[[], UploadResultList]) -> None:
|
|
105
117
|
self._get_last_outcome = get_last_outcome
|