cognite-neat 0.86.0__py3-none-any.whl → 0.87.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cognite-neat might be problematic. Click here for more details.
- cognite/neat/_version.py +1 -1
- cognite/neat/app/api/configuration.py +1 -10
- cognite/neat/app/api/routers/data_exploration.py +1 -1
- cognite/neat/config.py +84 -17
- cognite/neat/constants.py +11 -9
- cognite/neat/graph/extractors/_classic_cdf/_assets.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_events.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_files.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_labels.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_relationships.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_sequences.py +1 -1
- cognite/neat/graph/extractors/_classic_cdf/_timeseries.py +1 -1
- cognite/neat/graph/extractors/_dexpi.py +1 -1
- cognite/neat/graph/extractors/_mock_graph_generator.py +8 -9
- cognite/neat/graph/loaders/__init__.py +5 -2
- cognite/neat/graph/loaders/_base.py +13 -5
- cognite/neat/graph/loaders/_rdf2asset.py +185 -55
- cognite/neat/graph/loaders/_rdf2dms.py +7 -7
- cognite/neat/graph/queries/_base.py +20 -11
- cognite/neat/graph/queries/_construct.py +5 -5
- cognite/neat/graph/queries/_shared.py +21 -7
- cognite/neat/graph/stores/_base.py +16 -4
- cognite/neat/graph/transformers/__init__.py +3 -0
- cognite/neat/graph/transformers/_rdfpath.py +42 -0
- cognite/neat/legacy/graph/extractors/_dexpi.py +0 -5
- cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +1 -1
- cognite/neat/legacy/graph/loaders/_asset_loader.py +2 -2
- cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +5 -2
- cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +4 -1
- cognite/neat/legacy/graph/loaders/rdf_to_dms.py +3 -1
- cognite/neat/legacy/graph/stores/_base.py +24 -8
- cognite/neat/legacy/graph/stores/_graphdb_store.py +3 -2
- cognite/neat/legacy/graph/stores/_memory_store.py +3 -3
- cognite/neat/legacy/graph/stores/_oxigraph_store.py +8 -4
- cognite/neat/legacy/graph/stores/_rdf_to_graph.py +5 -3
- cognite/neat/legacy/graph/transformations/query_generator/sparql.py +49 -16
- cognite/neat/legacy/graph/transformations/transformer.py +1 -1
- cognite/neat/legacy/rules/exporters/_rules2dms.py +8 -3
- cognite/neat/legacy/rules/exporters/_rules2graphql.py +1 -1
- cognite/neat/legacy/rules/exporters/_rules2ontology.py +2 -1
- cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +3 -4
- cognite/neat/legacy/rules/importers/_dms2rules.py +4 -1
- cognite/neat/legacy/rules/importers/_graph2rules.py +3 -3
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +1 -1
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +2 -1
- cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +1 -1
- cognite/neat/legacy/rules/models/raw_rules.py +19 -7
- cognite/neat/legacy/rules/models/rules.py +32 -12
- cognite/neat/rules/_shared.py +6 -1
- cognite/neat/rules/analysis/__init__.py +4 -4
- cognite/neat/rules/analysis/_asset.py +143 -0
- cognite/neat/rules/analysis/_base.py +385 -6
- cognite/neat/rules/analysis/_information.py +183 -0
- cognite/neat/rules/exporters/_rules2dms.py +1 -1
- cognite/neat/rules/exporters/_rules2ontology.py +6 -5
- cognite/neat/rules/importers/_dms2rules.py +3 -1
- cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +2 -8
- cognite/neat/rules/importers/_inference2rules.py +3 -7
- cognite/neat/rules/importers/_owl2rules/_owl2classes.py +1 -1
- cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +2 -1
- cognite/neat/rules/importers/_owl2rules/_owl2properties.py +1 -1
- cognite/neat/rules/issues/spreadsheet.py +35 -0
- cognite/neat/rules/models/_base.py +7 -7
- cognite/neat/rules/models/_rdfpath.py +17 -21
- cognite/neat/rules/models/asset/_rules.py +4 -5
- cognite/neat/rules/models/asset/_validation.py +38 -1
- cognite/neat/rules/models/dms/_converter.py +1 -2
- cognite/neat/rules/models/dms/_exporter.py +7 -3
- cognite/neat/rules/models/dms/_rules.py +3 -0
- cognite/neat/rules/models/dms/_schema.py +5 -4
- cognite/neat/rules/models/domain.py +5 -2
- cognite/neat/rules/models/entities.py +28 -17
- cognite/neat/rules/models/information/_rules.py +10 -8
- cognite/neat/rules/models/information/_rules_input.py +1 -2
- cognite/neat/rules/models/information/_validation.py +2 -2
- cognite/neat/utils/__init__.py +0 -3
- cognite/neat/utils/auth.py +47 -28
- cognite/neat/utils/auxiliary.py +141 -1
- cognite/neat/utils/cdf/__init__.py +0 -0
- cognite/neat/utils/{cdf_classes.py → cdf/data_classes.py} +122 -2
- cognite/neat/utils/{cdf_loaders → cdf/loaders}/_data_modeling.py +37 -0
- cognite/neat/utils/{cdf_loaders → cdf/loaders}/_ingestion.py +2 -1
- cognite/neat/utils/collection_.py +18 -0
- cognite/neat/utils/rdf_.py +165 -0
- cognite/neat/utils/text.py +4 -0
- cognite/neat/utils/time_.py +17 -0
- cognite/neat/utils/upload.py +13 -1
- cognite/neat/workflows/_exceptions.py +5 -5
- cognite/neat/workflows/base.py +1 -1
- cognite/neat/workflows/steps/lib/current/graph_store.py +28 -8
- cognite/neat/workflows/steps/lib/current/rules_validator.py +2 -2
- cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +130 -28
- cognite/neat/workflows/steps/lib/legacy/graph_loader.py +1 -1
- cognite/neat/workflows/steps/lib/legacy/graph_store.py +4 -4
- cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +1 -1
- cognite/neat/workflows/steps/lib/legacy/rules_importer.py +1 -1
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/METADATA +2 -2
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/RECORD +103 -102
- cognite/neat/rules/analysis/_information_rules.py +0 -476
- cognite/neat/utils/cdf.py +0 -59
- cognite/neat/utils/cdf_loaders/data_classes.py +0 -121
- cognite/neat/utils/exceptions.py +0 -41
- cognite/neat/utils/utils.py +0 -429
- /cognite/neat/utils/{cdf_loaders → cdf/loaders}/__init__.py +0 -0
- /cognite/neat/utils/{cdf_loaders → cdf/loaders}/_base.py +0 -0
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/LICENSE +0 -0
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/WHEEL +0 -0
- {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/entry_points.txt +0 -0
|
@@ -42,17 +42,18 @@ from cognite.neat.rules.issues.dms import (
|
|
|
42
42
|
MissingViewInModelWarning,
|
|
43
43
|
)
|
|
44
44
|
from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE
|
|
45
|
-
from cognite.neat.utils.
|
|
45
|
+
from cognite.neat.utils.cdf.data_classes import (
|
|
46
46
|
CogniteResourceDict,
|
|
47
47
|
ContainerApplyDict,
|
|
48
48
|
NodeApplyDict,
|
|
49
|
+
RawTableWrite,
|
|
50
|
+
RawTableWriteList,
|
|
49
51
|
SpaceApplyDict,
|
|
50
52
|
ViewApplyDict,
|
|
51
53
|
)
|
|
52
|
-
from cognite.neat.utils.
|
|
53
|
-
from cognite.neat.utils.
|
|
54
|
+
from cognite.neat.utils.cdf.loaders import ViewLoader
|
|
55
|
+
from cognite.neat.utils.rdf_ import get_inheritance_path
|
|
54
56
|
from cognite.neat.utils.text import to_camel
|
|
55
|
-
from cognite.neat.utils.utils import get_inheritance_path
|
|
56
57
|
|
|
57
58
|
if sys.version_info >= (3, 11):
|
|
58
59
|
from typing import Self
|
|
@@ -5,7 +5,7 @@ from pydantic import Field, field_serializer, field_validator, model_serializer
|
|
|
5
5
|
from pydantic_core.core_schema import SerializationInfo
|
|
6
6
|
|
|
7
7
|
from cognite.neat.rules.models.data_types import DataType
|
|
8
|
-
from cognite.neat.rules.models.entities import ClassEntity,
|
|
8
|
+
from cognite.neat.rules.models.entities import ClassEntity, ClassEntityList
|
|
9
9
|
|
|
10
10
|
from ._base import (
|
|
11
11
|
BaseMetadata,
|
|
@@ -24,6 +24,9 @@ class DomainMetadata(BaseMetadata):
|
|
|
24
24
|
def as_identifier(self) -> str:
|
|
25
25
|
return "DomainRules"
|
|
26
26
|
|
|
27
|
+
def get_prefix(self) -> str:
|
|
28
|
+
return "domain"
|
|
29
|
+
|
|
27
30
|
|
|
28
31
|
class DomainProperty(SheetEntity):
|
|
29
32
|
class_: ClassEntity = Field(alias="Class")
|
|
@@ -51,7 +54,7 @@ class DomainClass(SheetEntity):
|
|
|
51
54
|
class_: ClassEntity = Field(alias="Class")
|
|
52
55
|
name: str | None = Field(alias="Name", default=None)
|
|
53
56
|
description: str | None = Field(None, alias="Description")
|
|
54
|
-
parent:
|
|
57
|
+
parent: ClassEntityList | None = Field(alias="Parent Class")
|
|
55
58
|
|
|
56
59
|
|
|
57
60
|
class DomainRules(BaseRules):
|
|
@@ -4,7 +4,13 @@ from abc import ABC, abstractmethod
|
|
|
4
4
|
from functools import total_ordering
|
|
5
5
|
from typing import Annotated, Any, ClassVar, Generic, TypeVar, cast
|
|
6
6
|
|
|
7
|
-
from cognite.client.data_classes.data_modeling.ids import
|
|
7
|
+
from cognite.client.data_classes.data_modeling.ids import (
|
|
8
|
+
ContainerId,
|
|
9
|
+
DataModelId,
|
|
10
|
+
NodeId,
|
|
11
|
+
PropertyId,
|
|
12
|
+
ViewId,
|
|
13
|
+
)
|
|
8
14
|
from pydantic import (
|
|
9
15
|
AnyHttpUrl,
|
|
10
16
|
BaseModel,
|
|
@@ -16,7 +22,7 @@ from pydantic import (
|
|
|
16
22
|
)
|
|
17
23
|
|
|
18
24
|
from cognite.neat.rules.models.data_types import DataType
|
|
19
|
-
from cognite.neat.utils.
|
|
25
|
+
from cognite.neat.utils.text import replace_non_alphanumeric_with_underscore
|
|
20
26
|
|
|
21
27
|
if sys.version_info >= (3, 11):
|
|
22
28
|
from enum import StrEnum
|
|
@@ -252,13 +258,6 @@ class ClassEntity(Entity):
|
|
|
252
258
|
return ContainerEntity(space=space, externalId=str(self.suffix))
|
|
253
259
|
|
|
254
260
|
|
|
255
|
-
class ParentClassEntity(ClassEntity):
|
|
256
|
-
type_: ClassVar[EntityTypes] = EntityTypes.parent_class
|
|
257
|
-
|
|
258
|
-
def as_class_entity(self) -> ClassEntity:
|
|
259
|
-
return ClassEntity(prefix=self.prefix, suffix=self.suffix, version=self.version)
|
|
260
|
-
|
|
261
|
-
|
|
262
261
|
class UnknownEntity(ClassEntity):
|
|
263
262
|
type_: ClassVar[EntityTypes] = EntityTypes.undefined
|
|
264
263
|
prefix: _UndefinedType = Undefined
|
|
@@ -270,9 +269,9 @@ class UnknownEntity(ClassEntity):
|
|
|
270
269
|
|
|
271
270
|
|
|
272
271
|
class AssetFields(StrEnum):
|
|
273
|
-
|
|
272
|
+
externalId = "externalId"
|
|
274
273
|
name = "name"
|
|
275
|
-
|
|
274
|
+
parentExternalId = "parentExternalId"
|
|
276
275
|
description = "description"
|
|
277
276
|
metadata = "metadata"
|
|
278
277
|
|
|
@@ -338,7 +337,8 @@ class MultiValueTypeInfo(BaseModel):
|
|
|
338
337
|
else:
|
|
339
338
|
return {
|
|
340
339
|
"types": [
|
|
341
|
-
DataType.load(type_) if DataType.is_data_type(type_) else ClassEntity.load(type_)
|
|
340
|
+
(DataType.load(type_) if DataType.is_data_type(type_) else ClassEntity.load(type_))
|
|
341
|
+
for type_ in types
|
|
342
342
|
]
|
|
343
343
|
}
|
|
344
344
|
|
|
@@ -454,7 +454,10 @@ class ViewPropertyEntity(DMSVersionedEntity[PropertyId]):
|
|
|
454
454
|
property_: str = Field(alias="property")
|
|
455
455
|
|
|
456
456
|
def as_id(self) -> PropertyId:
|
|
457
|
-
return PropertyId(
|
|
457
|
+
return PropertyId(
|
|
458
|
+
source=ViewId(self.space, self.external_id, self.version),
|
|
459
|
+
property=self.property_,
|
|
460
|
+
)
|
|
458
461
|
|
|
459
462
|
def as_view_id(self) -> ViewId:
|
|
460
463
|
return ViewId(space=self.space, external_id=self.external_id, version=self.version)
|
|
@@ -466,7 +469,10 @@ class ViewPropertyEntity(DMSVersionedEntity[PropertyId]):
|
|
|
466
469
|
if id.source.version is None:
|
|
467
470
|
raise ValueError("Version must be specified")
|
|
468
471
|
return cls(
|
|
469
|
-
space=id.source.space,
|
|
472
|
+
space=id.source.space,
|
|
473
|
+
externalId=id.source.external_id,
|
|
474
|
+
version=id.source.version,
|
|
475
|
+
property=id.property,
|
|
470
476
|
)
|
|
471
477
|
|
|
472
478
|
|
|
@@ -502,7 +508,12 @@ class ReferenceEntity(ClassEntity):
|
|
|
502
508
|
@classmethod
|
|
503
509
|
def from_entity(cls, entity: Entity, property_: str) -> "ReferenceEntity":
|
|
504
510
|
if isinstance(entity, ClassEntity):
|
|
505
|
-
return cls(
|
|
511
|
+
return cls(
|
|
512
|
+
prefix=str(entity.prefix),
|
|
513
|
+
suffix=entity.suffix,
|
|
514
|
+
version=entity.version,
|
|
515
|
+
property=property_,
|
|
516
|
+
)
|
|
506
517
|
else:
|
|
507
518
|
return cls(prefix=str(entity.prefix), suffix=entity.suffix, property=property_)
|
|
508
519
|
|
|
@@ -555,8 +566,8 @@ def _generate_cdf_resource_list(v: Any) -> list[AssetEntity | RelationshipEntity
|
|
|
555
566
|
return results # type: ignore
|
|
556
567
|
|
|
557
568
|
|
|
558
|
-
|
|
559
|
-
list[
|
|
569
|
+
ClassEntityList = Annotated[
|
|
570
|
+
list[ClassEntity],
|
|
560
571
|
BeforeValidator(_split_str),
|
|
561
572
|
PlainSerializer(
|
|
562
573
|
_join_str,
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import math
|
|
2
2
|
import sys
|
|
3
3
|
from datetime import datetime
|
|
4
|
-
from typing import TYPE_CHECKING, Any, ClassVar, Literal
|
|
4
|
+
from typing import TYPE_CHECKING, Any, ClassVar, Literal
|
|
5
5
|
|
|
6
6
|
from pydantic import Field, field_serializer, field_validator, model_validator
|
|
7
7
|
from pydantic.main import IncEx
|
|
8
8
|
from rdflib import Namespace
|
|
9
9
|
|
|
10
|
-
from cognite.neat.constants import
|
|
10
|
+
from cognite.neat.constants import get_default_prefixes
|
|
11
11
|
from cognite.neat.issues import MultiValueError
|
|
12
12
|
from cognite.neat.rules import exceptions, issues
|
|
13
13
|
from cognite.neat.rules.models._base import (
|
|
@@ -38,10 +38,9 @@ from cognite.neat.rules.models.data_types import DataType
|
|
|
38
38
|
from cognite.neat.rules.models.domain import DomainRules
|
|
39
39
|
from cognite.neat.rules.models.entities import (
|
|
40
40
|
ClassEntity,
|
|
41
|
+
ClassEntityList,
|
|
41
42
|
EntityTypes,
|
|
42
43
|
MultiValueTypeInfo,
|
|
43
|
-
ParentClassEntity,
|
|
44
|
-
ParentEntityList,
|
|
45
44
|
ReferenceEntity,
|
|
46
45
|
Undefined,
|
|
47
46
|
UnknownEntity,
|
|
@@ -114,6 +113,9 @@ class InformationMetadata(BaseMetadata):
|
|
|
114
113
|
def as_identifier(self) -> str:
|
|
115
114
|
return f"{self.prefix}:{self.name}"
|
|
116
115
|
|
|
116
|
+
def get_prefix(self) -> str:
|
|
117
|
+
return self.prefix
|
|
118
|
+
|
|
117
119
|
|
|
118
120
|
class InformationClass(SheetEntity):
|
|
119
121
|
"""
|
|
@@ -130,7 +132,7 @@ class InformationClass(SheetEntity):
|
|
|
130
132
|
class_: ClassEntity = Field(alias="Class")
|
|
131
133
|
name: str | None = Field(alias="Name", default=None)
|
|
132
134
|
description: str | None = Field(alias="Description", default=None)
|
|
133
|
-
parent:
|
|
135
|
+
parent: ClassEntityList | None = Field(alias="Parent Class", default=None)
|
|
134
136
|
reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
|
|
135
137
|
match_type: MatchType | None = Field(alias="Match Type", default=None)
|
|
136
138
|
comment: str | None = Field(alias="Comment", default=None)
|
|
@@ -259,7 +261,7 @@ class InformationRules(BaseRules):
|
|
|
259
261
|
metadata: InformationMetadata = Field(alias="Metadata")
|
|
260
262
|
properties: SheetList[InformationProperty] = Field(alias="Properties")
|
|
261
263
|
classes: SheetList[InformationClass] = Field(alias="Classes")
|
|
262
|
-
prefixes: dict[str, Namespace] = Field(default_factory=
|
|
264
|
+
prefixes: dict[str, Namespace] = Field(default_factory=get_default_prefixes, alias="Prefixes")
|
|
263
265
|
last: "InformationRules | None" = Field(None, alias="Last")
|
|
264
266
|
reference: "InformationRules | None" = Field(None, alias="Reference")
|
|
265
267
|
|
|
@@ -268,7 +270,7 @@ class InformationRules(BaseRules):
|
|
|
268
270
|
if isinstance(values, dict):
|
|
269
271
|
return {key: Namespace(value) if isinstance(value, str) else value for key, value in values.items()}
|
|
270
272
|
elif values is None:
|
|
271
|
-
values =
|
|
273
|
+
values = get_default_prefixes()
|
|
272
274
|
return values
|
|
273
275
|
|
|
274
276
|
@model_validator(mode="after")
|
|
@@ -287,7 +289,7 @@ class InformationRules(BaseRules):
|
|
|
287
289
|
# update parent classes
|
|
288
290
|
for class_ in self.classes:
|
|
289
291
|
if class_.parent:
|
|
290
|
-
for parent in
|
|
292
|
+
for parent in class_.parent:
|
|
291
293
|
if not isinstance(parent.prefix, str):
|
|
292
294
|
parent.prefix = self.metadata.prefix
|
|
293
295
|
if class_.class_.prefix is Undefined:
|
|
@@ -15,7 +15,6 @@ from cognite.neat.rules.models.data_types import DataType
|
|
|
15
15
|
from cognite.neat.rules.models.entities import (
|
|
16
16
|
ClassEntity,
|
|
17
17
|
MultiValueTypeInfo,
|
|
18
|
-
ParentClassEntity,
|
|
19
18
|
Unknown,
|
|
20
19
|
UnknownEntity,
|
|
21
20
|
)
|
|
@@ -227,7 +226,7 @@ class InformationClassInput:
|
|
|
227
226
|
"Reference": self.reference,
|
|
228
227
|
"Match Type": self.match_type,
|
|
229
228
|
"Parent Class": (
|
|
230
|
-
[
|
|
229
|
+
[ClassEntity.load(parent, prefix=default_prefix) for parent in self.parent.split(",")]
|
|
231
230
|
if self.parent
|
|
232
231
|
else None
|
|
233
232
|
),
|
|
@@ -6,7 +6,7 @@ from cognite.neat.rules import issues
|
|
|
6
6
|
from cognite.neat.rules.issues import IssueList
|
|
7
7
|
from cognite.neat.rules.models._base import DataModelType, SchemaCompleteness
|
|
8
8
|
from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, UnknownEntity
|
|
9
|
-
from cognite.neat.utils.
|
|
9
|
+
from cognite.neat.utils.rdf_ import get_inheritance_path
|
|
10
10
|
|
|
11
11
|
from ._rules import InformationRules
|
|
12
12
|
|
|
@@ -161,7 +161,7 @@ class InformationPostValidation:
|
|
|
161
161
|
class_subclass_pairs[class_.class_] = []
|
|
162
162
|
if class_.parent is None:
|
|
163
163
|
continue
|
|
164
|
-
class_subclass_pairs[class_.class_].extend(
|
|
164
|
+
class_subclass_pairs[class_.class_].extend(class_.parent)
|
|
165
165
|
|
|
166
166
|
return class_subclass_pairs
|
|
167
167
|
|
cognite/neat/utils/__init__.py
CHANGED
cognite/neat/utils/auth.py
CHANGED
|
@@ -5,13 +5,13 @@ from dataclasses import dataclass, fields
|
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
from typing import Literal, TypeAlias, get_args
|
|
7
7
|
|
|
8
|
-
from cognite.client import CogniteClient
|
|
8
|
+
from cognite.client import ClientConfig, CogniteClient
|
|
9
9
|
from cognite.client.credentials import CredentialProvider, OAuthClientCredentials, OAuthInteractive, Token
|
|
10
10
|
|
|
11
11
|
from cognite.neat import _version
|
|
12
12
|
from cognite.neat.utils.auxiliary import local_import
|
|
13
13
|
|
|
14
|
-
__all__ = ["get_cognite_client"]
|
|
14
|
+
__all__ = ["get_cognite_client", "EnvironmentVariables"]
|
|
15
15
|
|
|
16
16
|
_LOGIN_FLOW: TypeAlias = Literal["infer", "client_credentials", "interactive", "token"]
|
|
17
17
|
_VALID_LOGIN_FLOWS = get_args(_LOGIN_FLOW)
|
|
@@ -22,7 +22,7 @@ def get_cognite_client(env_file_name: str = ".env") -> CogniteClient:
|
|
|
22
22
|
if not env_file_name.endswith(".env"):
|
|
23
23
|
raise ValueError("env_file_name must end with '.env'")
|
|
24
24
|
with suppress(KeyError):
|
|
25
|
-
variables =
|
|
25
|
+
variables = EnvironmentVariables.create_from_environ()
|
|
26
26
|
return variables.get_client()
|
|
27
27
|
|
|
28
28
|
repo_root = _repo_root()
|
|
@@ -51,7 +51,7 @@ def get_cognite_client(env_file_name: str = ".env") -> CogniteClient:
|
|
|
51
51
|
|
|
52
52
|
|
|
53
53
|
@dataclass
|
|
54
|
-
class
|
|
54
|
+
class EnvironmentVariables:
|
|
55
55
|
CDF_CLUSTER: str
|
|
56
56
|
CDF_PROJECT: str
|
|
57
57
|
LOGIN_FLOW: _LOGIN_FLOW = "infer"
|
|
@@ -66,10 +66,17 @@ class _EnvironmentVariables:
|
|
|
66
66
|
IDP_AUDIENCE: str | None = None
|
|
67
67
|
IDP_SCOPES: str | None = None
|
|
68
68
|
IDP_AUTHORITY_URL: str | None = None
|
|
69
|
+
CDF_MAX_WORKERS: int | None = None
|
|
70
|
+
CDF_TIMEOUT: int | None = None
|
|
71
|
+
CDF_REDIRECT_PORT: int = 53_000
|
|
69
72
|
|
|
70
73
|
def __post_init__(self):
|
|
71
74
|
if self.LOGIN_FLOW.lower() not in _VALID_LOGIN_FLOWS:
|
|
72
75
|
raise ValueError(f"LOGIN_FLOW must be one of {_VALID_LOGIN_FLOWS}")
|
|
76
|
+
if self.IDP_TOKEN_URL and not self.IDP_TENANT_ID:
|
|
77
|
+
prefix, suffix = "https://login.microsoftonline.com/", "/oauth2/v2.0/token"
|
|
78
|
+
if self.IDP_TOKEN_URL.startswith(prefix) and self.IDP_TOKEN_URL.endswith(suffix):
|
|
79
|
+
self.IDP_TENANT_ID = self.IDP_TOKEN_URL.removeprefix(prefix).removesuffix(suffix)
|
|
73
80
|
|
|
74
81
|
@property
|
|
75
82
|
def cdf_url(self) -> str:
|
|
@@ -102,7 +109,7 @@ class _EnvironmentVariables:
|
|
|
102
109
|
return f"https://login.microsoftonline.com/{self.IDP_TENANT_ID}"
|
|
103
110
|
|
|
104
111
|
@classmethod
|
|
105
|
-
def create_from_environ(cls) -> "
|
|
112
|
+
def create_from_environ(cls) -> "EnvironmentVariables":
|
|
106
113
|
if "CDF_CLUSTER" not in os.environ or "CDF_PROJECT" not in os.environ:
|
|
107
114
|
raise KeyError("CDF_CLUSTER and CDF_PROJECT must be set in the environment.", "CDF_CLUSTER", "CDF_PROJECT")
|
|
108
115
|
|
|
@@ -119,6 +126,25 @@ class _EnvironmentVariables:
|
|
|
119
126
|
IDP_AUDIENCE=os.environ.get("IDP_AUDIENCE"),
|
|
120
127
|
IDP_SCOPES=os.environ.get("IDP_SCOPES"),
|
|
121
128
|
IDP_AUTHORITY_URL=os.environ.get("IDP_AUTHORITY_URL"),
|
|
129
|
+
CDF_MAX_WORKERS=int(os.environ["CDF_MAX_WORKERS"]) if "CDF_MAX_WORKERS" in os.environ else None,
|
|
130
|
+
CDF_TIMEOUT=int(os.environ["CDF_TIMEOUT"]) if "CDF_TIMEOUT" in os.environ else None,
|
|
131
|
+
CDF_REDIRECT_PORT=int(os.environ.get("CDF_REDIRECT_PORT", 53_000)),
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
@classmethod
|
|
135
|
+
def default(cls) -> "EnvironmentVariables":
|
|
136
|
+
# This method is for backwards compatibility with the old config
|
|
137
|
+
# It is not recommended to use this method.
|
|
138
|
+
return cls(
|
|
139
|
+
LOGIN_FLOW="client_credentials",
|
|
140
|
+
CDF_CLUSTER="api",
|
|
141
|
+
CDF_PROJECT="dev",
|
|
142
|
+
IDP_TENANT_ID="common",
|
|
143
|
+
IDP_CLIENT_ID="neat",
|
|
144
|
+
IDP_CLIENT_SECRET="secret",
|
|
145
|
+
IDP_SCOPES="project:read,project:write",
|
|
146
|
+
CDF_TIMEOUT=60,
|
|
147
|
+
CDF_MAX_WORKERS=3,
|
|
122
148
|
)
|
|
123
149
|
|
|
124
150
|
def get_credentials(self) -> CredentialProvider:
|
|
@@ -161,7 +187,7 @@ class _EnvironmentVariables:
|
|
|
161
187
|
return OAuthInteractive(
|
|
162
188
|
client_id=self.IDP_CLIENT_ID,
|
|
163
189
|
authority_url=self.idp_authority_url,
|
|
164
|
-
redirect_port=
|
|
190
|
+
redirect_port=self.CDF_REDIRECT_PORT,
|
|
165
191
|
scopes=self.idp_scopes,
|
|
166
192
|
)
|
|
167
193
|
|
|
@@ -171,9 +197,15 @@ class _EnvironmentVariables:
|
|
|
171
197
|
return Token(self.TOKEN)
|
|
172
198
|
|
|
173
199
|
def get_client(self) -> CogniteClient:
|
|
174
|
-
|
|
175
|
-
|
|
200
|
+
config = ClientConfig(
|
|
201
|
+
client_name=_CLIENT_NAME,
|
|
202
|
+
project=self.CDF_PROJECT,
|
|
203
|
+
credentials=self.get_credentials(),
|
|
204
|
+
base_url=self.cdf_url,
|
|
205
|
+
max_workers=self.CDF_MAX_WORKERS,
|
|
206
|
+
timeout=self.CDF_TIMEOUT,
|
|
176
207
|
)
|
|
208
|
+
return CogniteClient(config)
|
|
177
209
|
|
|
178
210
|
def create_env_file(self) -> str:
|
|
179
211
|
lines: list[str] = []
|
|
@@ -194,11 +226,11 @@ class _EnvironmentVariables:
|
|
|
194
226
|
return "\n".join(lines)
|
|
195
227
|
|
|
196
228
|
|
|
197
|
-
def _from_dotenv(evn_file: Path) ->
|
|
229
|
+
def _from_dotenv(evn_file: Path) -> EnvironmentVariables:
|
|
198
230
|
if not evn_file.exists():
|
|
199
231
|
raise FileNotFoundError(f"{evn_file} does not exist.")
|
|
200
232
|
content = evn_file.read_text()
|
|
201
|
-
valid_variables = {f.name for f in fields(
|
|
233
|
+
valid_variables = {f.name for f in fields(EnvironmentVariables)}
|
|
202
234
|
variables: dict[str, str] = {}
|
|
203
235
|
for line in content.splitlines():
|
|
204
236
|
if line.startswith("#") or "=" not in line:
|
|
@@ -206,15 +238,15 @@ def _from_dotenv(evn_file: Path) -> _EnvironmentVariables:
|
|
|
206
238
|
key, value = line.split("=", 1)
|
|
207
239
|
if key in valid_variables:
|
|
208
240
|
variables[key] = value
|
|
209
|
-
return
|
|
241
|
+
return EnvironmentVariables(**variables) # type: ignore[arg-type]
|
|
210
242
|
|
|
211
243
|
|
|
212
|
-
def _prompt_user() ->
|
|
244
|
+
def _prompt_user() -> EnvironmentVariables:
|
|
213
245
|
local_import("rich", "jupyter")
|
|
214
246
|
from rich.prompt import Prompt
|
|
215
247
|
|
|
216
248
|
try:
|
|
217
|
-
variables =
|
|
249
|
+
variables = EnvironmentVariables.create_from_environ()
|
|
218
250
|
continue_ = Prompt.ask(
|
|
219
251
|
f"Use environment variables for CDF Cluster '{variables.CDF_CLUSTER}' "
|
|
220
252
|
f"and Project '{variables.CDF_PROJECT}'? [y/n]",
|
|
@@ -263,25 +295,12 @@ def _prompt_user() -> _EnvironmentVariables:
|
|
|
263
295
|
return variables
|
|
264
296
|
|
|
265
297
|
|
|
266
|
-
def _prompt_cluster_and_project() ->
|
|
298
|
+
def _prompt_cluster_and_project() -> EnvironmentVariables:
|
|
267
299
|
from rich.prompt import Prompt
|
|
268
300
|
|
|
269
301
|
cluster = Prompt.ask("Enter CDF Cluster (example 'greenfield', 'bluefield', 'westeurope-1)")
|
|
270
302
|
project = Prompt.ask("Enter CDF Project")
|
|
271
|
-
return
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
def _is_notebook() -> bool:
|
|
275
|
-
try:
|
|
276
|
-
shell = get_ipython().__class__.__name__ # type: ignore[name-defined]
|
|
277
|
-
if shell == "ZMQInteractiveShell":
|
|
278
|
-
return True # Jupyter notebook or qtconsole
|
|
279
|
-
elif shell == "TerminalInteractiveShell":
|
|
280
|
-
return False # Terminal running IPython
|
|
281
|
-
else:
|
|
282
|
-
return False # Other type (?)
|
|
283
|
-
except NameError:
|
|
284
|
-
return False # Probably standard Python interpreter
|
|
303
|
+
return EnvironmentVariables(cluster, project)
|
|
285
304
|
|
|
286
305
|
|
|
287
306
|
def _repo_root() -> Path | None:
|
cognite/neat/utils/auxiliary.py
CHANGED
|
@@ -1,8 +1,16 @@
|
|
|
1
|
+
import hashlib
|
|
1
2
|
import importlib
|
|
2
3
|
import inspect
|
|
3
|
-
|
|
4
|
+
import logging
|
|
5
|
+
import time
|
|
6
|
+
from collections.abc import Callable, Iterable
|
|
7
|
+
from datetime import datetime
|
|
8
|
+
from functools import wraps
|
|
4
9
|
from types import ModuleType
|
|
5
10
|
|
|
11
|
+
from cognite.client.exceptions import CogniteDuplicatedError, CogniteReadTimeout
|
|
12
|
+
from pydantic_core import ErrorDetails
|
|
13
|
+
|
|
6
14
|
from cognite.neat.exceptions import NeatImportError
|
|
7
15
|
|
|
8
16
|
|
|
@@ -33,3 +41,135 @@ def class_html_doc(cls: type, include_factory_methods: bool = True) -> str:
|
|
|
33
41
|
f'<ul style="list-style-type:circle;">{factory_methods_str}</ul>'
|
|
34
42
|
)
|
|
35
43
|
return f"<h3>{cls.__name__}</h3><p>{docstring}</p>"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def retry_decorator(max_retries=2, retry_delay=3, component_name=""):
|
|
47
|
+
def decorator(func):
|
|
48
|
+
@wraps(func)
|
|
49
|
+
def wrapper(*args, **kwargs):
|
|
50
|
+
previous_exception = None
|
|
51
|
+
for attempt in range(max_retries + 1):
|
|
52
|
+
try:
|
|
53
|
+
logging.debug(f"Attempt {attempt + 1} of {max_retries + 1} for {component_name}")
|
|
54
|
+
return func(*args, **kwargs)
|
|
55
|
+
except CogniteReadTimeout as e:
|
|
56
|
+
previous_exception = e
|
|
57
|
+
if attempt < max_retries:
|
|
58
|
+
logging.error(
|
|
59
|
+
f"""CogniteReadTimeout retry attempt {attempt + 1} failed for {component_name} .
|
|
60
|
+
Retrying in {retry_delay} second(s). Error:"""
|
|
61
|
+
)
|
|
62
|
+
logging.error(e)
|
|
63
|
+
time.sleep(retry_delay)
|
|
64
|
+
else:
|
|
65
|
+
raise e
|
|
66
|
+
except CogniteDuplicatedError as e:
|
|
67
|
+
if isinstance(previous_exception, CogniteReadTimeout):
|
|
68
|
+
# if previous exception was CogniteReadTimeout,
|
|
69
|
+
# we can't be sure if the items were created or not
|
|
70
|
+
if len(e.successful) == 0 and len(e.failed) == 0 and len(e.duplicated) >= 0:
|
|
71
|
+
logging.warning(
|
|
72
|
+
f"Duplicate error for {component_name} . All items already exist in CDF. "
|
|
73
|
+
"Suppressing error."
|
|
74
|
+
)
|
|
75
|
+
return
|
|
76
|
+
else:
|
|
77
|
+
# can happend because of eventual consistency. Retry with delay to allow for CDF to catch up
|
|
78
|
+
if attempt < max_retries:
|
|
79
|
+
logging.error(
|
|
80
|
+
f"""CogniteDuplicatedError retry attempt {attempt + 1} failed for {component_name} .
|
|
81
|
+
Retrying in {retry_delay} second(s). Error:"""
|
|
82
|
+
)
|
|
83
|
+
logging.error(e)
|
|
84
|
+
# incerasing delay to allow for CDF to catch up
|
|
85
|
+
time.sleep(retry_delay)
|
|
86
|
+
else:
|
|
87
|
+
raise e
|
|
88
|
+
else:
|
|
89
|
+
# no point in retrying duplicate error if previous exception was not a timeout
|
|
90
|
+
raise e
|
|
91
|
+
|
|
92
|
+
except Exception as e:
|
|
93
|
+
previous_exception = e
|
|
94
|
+
if attempt < max_retries:
|
|
95
|
+
logging.error(
|
|
96
|
+
f"Retry attempt {attempt + 1} failed for {component_name}. "
|
|
97
|
+
f"Retrying in {retry_delay} second(s)."
|
|
98
|
+
)
|
|
99
|
+
logging.error(e)
|
|
100
|
+
time.sleep(retry_delay)
|
|
101
|
+
else:
|
|
102
|
+
raise e
|
|
103
|
+
|
|
104
|
+
return wrapper
|
|
105
|
+
|
|
106
|
+
return decorator
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def create_sha256_hash(string: str) -> str:
|
|
110
|
+
# Create a SHA-256 hash object
|
|
111
|
+
sha256_hash = hashlib.sha256()
|
|
112
|
+
|
|
113
|
+
# Convert the string to bytes and update the hash object
|
|
114
|
+
sha256_hash.update(string.encode("utf-8"))
|
|
115
|
+
|
|
116
|
+
# Get the hexadecimal representation of the hash
|
|
117
|
+
hash_value = sha256_hash.hexdigest()
|
|
118
|
+
|
|
119
|
+
return hash_value
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# Will likely be removed with legacy code
|
|
123
|
+
def generate_exception_report(exceptions: list[dict] | list[ErrorDetails] | None, category: str = "") -> str:
|
|
124
|
+
exceptions_as_dict = _order_expectations_by_type(exceptions) if exceptions else {}
|
|
125
|
+
report = ""
|
|
126
|
+
|
|
127
|
+
for exception_type in exceptions_as_dict.keys():
|
|
128
|
+
title = f"# {category}: {exception_type}" if category else ""
|
|
129
|
+
warnings = "\n- " + "\n- ".join(exceptions_as_dict[exception_type])
|
|
130
|
+
report += title + warnings + "\n\n"
|
|
131
|
+
|
|
132
|
+
return report
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _order_expectations_by_type(
|
|
136
|
+
exceptions: list[dict] | list[ErrorDetails],
|
|
137
|
+
) -> dict[str, list[str]]:
|
|
138
|
+
exception_dict: dict[str, list[str]] = {}
|
|
139
|
+
for exception in exceptions:
|
|
140
|
+
if not isinstance(exception["loc"], str) and isinstance(exception["loc"], Iterable):
|
|
141
|
+
location = f"[{'/'.join(str(e) for e in exception['loc'])}]"
|
|
142
|
+
else:
|
|
143
|
+
location = ""
|
|
144
|
+
|
|
145
|
+
issue = f"{exception['msg']} {location}"
|
|
146
|
+
|
|
147
|
+
if exception_dict.get(exception["type"]) is None:
|
|
148
|
+
exception_dict[exception["type"]] = [issue]
|
|
149
|
+
else:
|
|
150
|
+
exception_dict[exception["type"]].append(issue)
|
|
151
|
+
return exception_dict
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def string_to_ideal_type(input_string: str) -> int | bool | float | datetime | str:
|
|
155
|
+
try:
|
|
156
|
+
# Try converting to int
|
|
157
|
+
return int(input_string)
|
|
158
|
+
except ValueError:
|
|
159
|
+
try:
|
|
160
|
+
# Try converting to float
|
|
161
|
+
return float(input_string) # type: ignore
|
|
162
|
+
except ValueError:
|
|
163
|
+
if input_string.lower() == "true":
|
|
164
|
+
# Return True if input is 'true'
|
|
165
|
+
return True
|
|
166
|
+
elif input_string.lower() == "false":
|
|
167
|
+
# Return False if input is 'false'
|
|
168
|
+
return False
|
|
169
|
+
else:
|
|
170
|
+
try:
|
|
171
|
+
# Try converting to datetime
|
|
172
|
+
return datetime.fromisoformat(input_string) # type: ignore
|
|
173
|
+
except ValueError:
|
|
174
|
+
# Return the input string if no conversion is possible
|
|
175
|
+
return input_string
|
|
File without changes
|