cognite-neat 0.86.0__py3-none-any.whl → 0.87.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (108) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/app/api/configuration.py +1 -10
  3. cognite/neat/app/api/routers/data_exploration.py +1 -1
  4. cognite/neat/config.py +84 -17
  5. cognite/neat/constants.py +11 -9
  6. cognite/neat/graph/extractors/_classic_cdf/_assets.py +1 -1
  7. cognite/neat/graph/extractors/_classic_cdf/_events.py +1 -1
  8. cognite/neat/graph/extractors/_classic_cdf/_files.py +1 -1
  9. cognite/neat/graph/extractors/_classic_cdf/_labels.py +1 -1
  10. cognite/neat/graph/extractors/_classic_cdf/_relationships.py +1 -1
  11. cognite/neat/graph/extractors/_classic_cdf/_sequences.py +1 -1
  12. cognite/neat/graph/extractors/_classic_cdf/_timeseries.py +1 -1
  13. cognite/neat/graph/extractors/_dexpi.py +1 -1
  14. cognite/neat/graph/extractors/_mock_graph_generator.py +8 -9
  15. cognite/neat/graph/loaders/__init__.py +5 -2
  16. cognite/neat/graph/loaders/_base.py +13 -5
  17. cognite/neat/graph/loaders/_rdf2asset.py +185 -55
  18. cognite/neat/graph/loaders/_rdf2dms.py +7 -7
  19. cognite/neat/graph/queries/_base.py +20 -11
  20. cognite/neat/graph/queries/_construct.py +5 -5
  21. cognite/neat/graph/queries/_shared.py +21 -7
  22. cognite/neat/graph/stores/_base.py +16 -4
  23. cognite/neat/graph/transformers/__init__.py +3 -0
  24. cognite/neat/graph/transformers/_rdfpath.py +42 -0
  25. cognite/neat/legacy/graph/extractors/_dexpi.py +0 -5
  26. cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +1 -1
  27. cognite/neat/legacy/graph/loaders/_asset_loader.py +2 -2
  28. cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +5 -2
  29. cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +4 -1
  30. cognite/neat/legacy/graph/loaders/rdf_to_dms.py +3 -1
  31. cognite/neat/legacy/graph/stores/_base.py +24 -8
  32. cognite/neat/legacy/graph/stores/_graphdb_store.py +3 -2
  33. cognite/neat/legacy/graph/stores/_memory_store.py +3 -3
  34. cognite/neat/legacy/graph/stores/_oxigraph_store.py +8 -4
  35. cognite/neat/legacy/graph/stores/_rdf_to_graph.py +5 -3
  36. cognite/neat/legacy/graph/transformations/query_generator/sparql.py +49 -16
  37. cognite/neat/legacy/graph/transformations/transformer.py +1 -1
  38. cognite/neat/legacy/rules/exporters/_rules2dms.py +8 -3
  39. cognite/neat/legacy/rules/exporters/_rules2graphql.py +1 -1
  40. cognite/neat/legacy/rules/exporters/_rules2ontology.py +2 -1
  41. cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +3 -4
  42. cognite/neat/legacy/rules/importers/_dms2rules.py +4 -1
  43. cognite/neat/legacy/rules/importers/_graph2rules.py +3 -3
  44. cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +1 -1
  45. cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +2 -1
  46. cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +1 -1
  47. cognite/neat/legacy/rules/models/raw_rules.py +19 -7
  48. cognite/neat/legacy/rules/models/rules.py +32 -12
  49. cognite/neat/rules/_shared.py +6 -1
  50. cognite/neat/rules/analysis/__init__.py +4 -4
  51. cognite/neat/rules/analysis/_asset.py +143 -0
  52. cognite/neat/rules/analysis/_base.py +385 -6
  53. cognite/neat/rules/analysis/_information.py +183 -0
  54. cognite/neat/rules/exporters/_rules2dms.py +1 -1
  55. cognite/neat/rules/exporters/_rules2ontology.py +6 -5
  56. cognite/neat/rules/importers/_dms2rules.py +3 -1
  57. cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +2 -8
  58. cognite/neat/rules/importers/_inference2rules.py +3 -7
  59. cognite/neat/rules/importers/_owl2rules/_owl2classes.py +1 -1
  60. cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +2 -1
  61. cognite/neat/rules/importers/_owl2rules/_owl2properties.py +1 -1
  62. cognite/neat/rules/issues/spreadsheet.py +35 -0
  63. cognite/neat/rules/models/_base.py +7 -7
  64. cognite/neat/rules/models/_rdfpath.py +17 -21
  65. cognite/neat/rules/models/asset/_rules.py +4 -5
  66. cognite/neat/rules/models/asset/_validation.py +38 -1
  67. cognite/neat/rules/models/dms/_converter.py +1 -2
  68. cognite/neat/rules/models/dms/_exporter.py +7 -3
  69. cognite/neat/rules/models/dms/_rules.py +3 -0
  70. cognite/neat/rules/models/dms/_schema.py +5 -4
  71. cognite/neat/rules/models/domain.py +5 -2
  72. cognite/neat/rules/models/entities.py +28 -17
  73. cognite/neat/rules/models/information/_rules.py +10 -8
  74. cognite/neat/rules/models/information/_rules_input.py +1 -2
  75. cognite/neat/rules/models/information/_validation.py +2 -2
  76. cognite/neat/utils/__init__.py +0 -3
  77. cognite/neat/utils/auth.py +47 -28
  78. cognite/neat/utils/auxiliary.py +141 -1
  79. cognite/neat/utils/cdf/__init__.py +0 -0
  80. cognite/neat/utils/{cdf_classes.py → cdf/data_classes.py} +122 -2
  81. cognite/neat/utils/{cdf_loaders → cdf/loaders}/_data_modeling.py +37 -0
  82. cognite/neat/utils/{cdf_loaders → cdf/loaders}/_ingestion.py +2 -1
  83. cognite/neat/utils/collection_.py +18 -0
  84. cognite/neat/utils/rdf_.py +165 -0
  85. cognite/neat/utils/text.py +4 -0
  86. cognite/neat/utils/time_.py +17 -0
  87. cognite/neat/utils/upload.py +13 -1
  88. cognite/neat/workflows/_exceptions.py +5 -5
  89. cognite/neat/workflows/base.py +1 -1
  90. cognite/neat/workflows/steps/lib/current/graph_store.py +28 -8
  91. cognite/neat/workflows/steps/lib/current/rules_validator.py +2 -2
  92. cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +130 -28
  93. cognite/neat/workflows/steps/lib/legacy/graph_loader.py +1 -1
  94. cognite/neat/workflows/steps/lib/legacy/graph_store.py +4 -4
  95. cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +1 -1
  96. cognite/neat/workflows/steps/lib/legacy/rules_importer.py +1 -1
  97. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/METADATA +2 -2
  98. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/RECORD +103 -102
  99. cognite/neat/rules/analysis/_information_rules.py +0 -476
  100. cognite/neat/utils/cdf.py +0 -59
  101. cognite/neat/utils/cdf_loaders/data_classes.py +0 -121
  102. cognite/neat/utils/exceptions.py +0 -41
  103. cognite/neat/utils/utils.py +0 -429
  104. /cognite/neat/utils/{cdf_loaders → cdf/loaders}/__init__.py +0 -0
  105. /cognite/neat/utils/{cdf_loaders → cdf/loaders}/_base.py +0 -0
  106. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/LICENSE +0 -0
  107. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/WHEEL +0 -0
  108. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/entry_points.txt +0 -0
@@ -42,17 +42,18 @@ from cognite.neat.rules.issues.dms import (
42
42
  MissingViewInModelWarning,
43
43
  )
44
44
  from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE
45
- from cognite.neat.utils.cdf_classes import (
45
+ from cognite.neat.utils.cdf.data_classes import (
46
46
  CogniteResourceDict,
47
47
  ContainerApplyDict,
48
48
  NodeApplyDict,
49
+ RawTableWrite,
50
+ RawTableWriteList,
49
51
  SpaceApplyDict,
50
52
  ViewApplyDict,
51
53
  )
52
- from cognite.neat.utils.cdf_loaders import ViewLoader
53
- from cognite.neat.utils.cdf_loaders.data_classes import RawTableWrite, RawTableWriteList
54
+ from cognite.neat.utils.cdf.loaders import ViewLoader
55
+ from cognite.neat.utils.rdf_ import get_inheritance_path
54
56
  from cognite.neat.utils.text import to_camel
55
- from cognite.neat.utils.utils import get_inheritance_path
56
57
 
57
58
  if sys.version_info >= (3, 11):
58
59
  from typing import Self
@@ -5,7 +5,7 @@ from pydantic import Field, field_serializer, field_validator, model_serializer
5
5
  from pydantic_core.core_schema import SerializationInfo
6
6
 
7
7
  from cognite.neat.rules.models.data_types import DataType
8
- from cognite.neat.rules.models.entities import ClassEntity, ParentEntityList
8
+ from cognite.neat.rules.models.entities import ClassEntity, ClassEntityList
9
9
 
10
10
  from ._base import (
11
11
  BaseMetadata,
@@ -24,6 +24,9 @@ class DomainMetadata(BaseMetadata):
24
24
  def as_identifier(self) -> str:
25
25
  return "DomainRules"
26
26
 
27
+ def get_prefix(self) -> str:
28
+ return "domain"
29
+
27
30
 
28
31
  class DomainProperty(SheetEntity):
29
32
  class_: ClassEntity = Field(alias="Class")
@@ -51,7 +54,7 @@ class DomainClass(SheetEntity):
51
54
  class_: ClassEntity = Field(alias="Class")
52
55
  name: str | None = Field(alias="Name", default=None)
53
56
  description: str | None = Field(None, alias="Description")
54
- parent: ParentEntityList | None = Field(alias="Parent Class")
57
+ parent: ClassEntityList | None = Field(alias="Parent Class")
55
58
 
56
59
 
57
60
  class DomainRules(BaseRules):
@@ -4,7 +4,13 @@ from abc import ABC, abstractmethod
4
4
  from functools import total_ordering
5
5
  from typing import Annotated, Any, ClassVar, Generic, TypeVar, cast
6
6
 
7
- from cognite.client.data_classes.data_modeling.ids import ContainerId, DataModelId, NodeId, PropertyId, ViewId
7
+ from cognite.client.data_classes.data_modeling.ids import (
8
+ ContainerId,
9
+ DataModelId,
10
+ NodeId,
11
+ PropertyId,
12
+ ViewId,
13
+ )
8
14
  from pydantic import (
9
15
  AnyHttpUrl,
10
16
  BaseModel,
@@ -16,7 +22,7 @@ from pydantic import (
16
22
  )
17
23
 
18
24
  from cognite.neat.rules.models.data_types import DataType
19
- from cognite.neat.utils.utils import replace_non_alphanumeric_with_underscore
25
+ from cognite.neat.utils.text import replace_non_alphanumeric_with_underscore
20
26
 
21
27
  if sys.version_info >= (3, 11):
22
28
  from enum import StrEnum
@@ -252,13 +258,6 @@ class ClassEntity(Entity):
252
258
  return ContainerEntity(space=space, externalId=str(self.suffix))
253
259
 
254
260
 
255
- class ParentClassEntity(ClassEntity):
256
- type_: ClassVar[EntityTypes] = EntityTypes.parent_class
257
-
258
- def as_class_entity(self) -> ClassEntity:
259
- return ClassEntity(prefix=self.prefix, suffix=self.suffix, version=self.version)
260
-
261
-
262
261
  class UnknownEntity(ClassEntity):
263
262
  type_: ClassVar[EntityTypes] = EntityTypes.undefined
264
263
  prefix: _UndefinedType = Undefined
@@ -270,9 +269,9 @@ class UnknownEntity(ClassEntity):
270
269
 
271
270
 
272
271
  class AssetFields(StrEnum):
273
- external_id = "external_id"
272
+ externalId = "externalId"
274
273
  name = "name"
275
- parent_external_id = "parent_external_id"
274
+ parentExternalId = "parentExternalId"
276
275
  description = "description"
277
276
  metadata = "metadata"
278
277
 
@@ -338,7 +337,8 @@ class MultiValueTypeInfo(BaseModel):
338
337
  else:
339
338
  return {
340
339
  "types": [
341
- DataType.load(type_) if DataType.is_data_type(type_) else ClassEntity.load(type_) for type_ in types
340
+ (DataType.load(type_) if DataType.is_data_type(type_) else ClassEntity.load(type_))
341
+ for type_ in types
342
342
  ]
343
343
  }
344
344
 
@@ -454,7 +454,10 @@ class ViewPropertyEntity(DMSVersionedEntity[PropertyId]):
454
454
  property_: str = Field(alias="property")
455
455
 
456
456
  def as_id(self) -> PropertyId:
457
- return PropertyId(source=ViewId(self.space, self.external_id, self.version), property=self.property_)
457
+ return PropertyId(
458
+ source=ViewId(self.space, self.external_id, self.version),
459
+ property=self.property_,
460
+ )
458
461
 
459
462
  def as_view_id(self) -> ViewId:
460
463
  return ViewId(space=self.space, external_id=self.external_id, version=self.version)
@@ -466,7 +469,10 @@ class ViewPropertyEntity(DMSVersionedEntity[PropertyId]):
466
469
  if id.source.version is None:
467
470
  raise ValueError("Version must be specified")
468
471
  return cls(
469
- space=id.source.space, externalId=id.source.external_id, version=id.source.version, property=id.property
472
+ space=id.source.space,
473
+ externalId=id.source.external_id,
474
+ version=id.source.version,
475
+ property=id.property,
470
476
  )
471
477
 
472
478
 
@@ -502,7 +508,12 @@ class ReferenceEntity(ClassEntity):
502
508
  @classmethod
503
509
  def from_entity(cls, entity: Entity, property_: str) -> "ReferenceEntity":
504
510
  if isinstance(entity, ClassEntity):
505
- return cls(prefix=str(entity.prefix), suffix=entity.suffix, version=entity.version, property=property_)
511
+ return cls(
512
+ prefix=str(entity.prefix),
513
+ suffix=entity.suffix,
514
+ version=entity.version,
515
+ property=property_,
516
+ )
506
517
  else:
507
518
  return cls(prefix=str(entity.prefix), suffix=entity.suffix, property=property_)
508
519
 
@@ -555,8 +566,8 @@ def _generate_cdf_resource_list(v: Any) -> list[AssetEntity | RelationshipEntity
555
566
  return results # type: ignore
556
567
 
557
568
 
558
- ParentEntityList = Annotated[
559
- list[ParentClassEntity],
569
+ ClassEntityList = Annotated[
570
+ list[ClassEntity],
560
571
  BeforeValidator(_split_str),
561
572
  PlainSerializer(
562
573
  _join_str,
@@ -1,13 +1,13 @@
1
1
  import math
2
2
  import sys
3
3
  from datetime import datetime
4
- from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
4
+ from typing import TYPE_CHECKING, Any, ClassVar, Literal
5
5
 
6
6
  from pydantic import Field, field_serializer, field_validator, model_validator
7
7
  from pydantic.main import IncEx
8
8
  from rdflib import Namespace
9
9
 
10
- from cognite.neat.constants import PREFIXES
10
+ from cognite.neat.constants import get_default_prefixes
11
11
  from cognite.neat.issues import MultiValueError
12
12
  from cognite.neat.rules import exceptions, issues
13
13
  from cognite.neat.rules.models._base import (
@@ -38,10 +38,9 @@ from cognite.neat.rules.models.data_types import DataType
38
38
  from cognite.neat.rules.models.domain import DomainRules
39
39
  from cognite.neat.rules.models.entities import (
40
40
  ClassEntity,
41
+ ClassEntityList,
41
42
  EntityTypes,
42
43
  MultiValueTypeInfo,
43
- ParentClassEntity,
44
- ParentEntityList,
45
44
  ReferenceEntity,
46
45
  Undefined,
47
46
  UnknownEntity,
@@ -114,6 +113,9 @@ class InformationMetadata(BaseMetadata):
114
113
  def as_identifier(self) -> str:
115
114
  return f"{self.prefix}:{self.name}"
116
115
 
116
+ def get_prefix(self) -> str:
117
+ return self.prefix
118
+
117
119
 
118
120
  class InformationClass(SheetEntity):
119
121
  """
@@ -130,7 +132,7 @@ class InformationClass(SheetEntity):
130
132
  class_: ClassEntity = Field(alias="Class")
131
133
  name: str | None = Field(alias="Name", default=None)
132
134
  description: str | None = Field(alias="Description", default=None)
133
- parent: ParentEntityList | None = Field(alias="Parent Class", default=None)
135
+ parent: ClassEntityList | None = Field(alias="Parent Class", default=None)
134
136
  reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
135
137
  match_type: MatchType | None = Field(alias="Match Type", default=None)
136
138
  comment: str | None = Field(alias="Comment", default=None)
@@ -259,7 +261,7 @@ class InformationRules(BaseRules):
259
261
  metadata: InformationMetadata = Field(alias="Metadata")
260
262
  properties: SheetList[InformationProperty] = Field(alias="Properties")
261
263
  classes: SheetList[InformationClass] = Field(alias="Classes")
262
- prefixes: dict[str, Namespace] = Field(default_factory=lambda: PREFIXES.copy(), alias="Prefixes")
264
+ prefixes: dict[str, Namespace] = Field(default_factory=get_default_prefixes, alias="Prefixes")
263
265
  last: "InformationRules | None" = Field(None, alias="Last")
264
266
  reference: "InformationRules | None" = Field(None, alias="Reference")
265
267
 
@@ -268,7 +270,7 @@ class InformationRules(BaseRules):
268
270
  if isinstance(values, dict):
269
271
  return {key: Namespace(value) if isinstance(value, str) else value for key, value in values.items()}
270
272
  elif values is None:
271
- values = PREFIXES.copy()
273
+ values = get_default_prefixes()
272
274
  return values
273
275
 
274
276
  @model_validator(mode="after")
@@ -287,7 +289,7 @@ class InformationRules(BaseRules):
287
289
  # update parent classes
288
290
  for class_ in self.classes:
289
291
  if class_.parent:
290
- for parent in cast(list[ParentClassEntity], class_.parent):
292
+ for parent in class_.parent:
291
293
  if not isinstance(parent.prefix, str):
292
294
  parent.prefix = self.metadata.prefix
293
295
  if class_.class_.prefix is Undefined:
@@ -15,7 +15,6 @@ from cognite.neat.rules.models.data_types import DataType
15
15
  from cognite.neat.rules.models.entities import (
16
16
  ClassEntity,
17
17
  MultiValueTypeInfo,
18
- ParentClassEntity,
19
18
  Unknown,
20
19
  UnknownEntity,
21
20
  )
@@ -227,7 +226,7 @@ class InformationClassInput:
227
226
  "Reference": self.reference,
228
227
  "Match Type": self.match_type,
229
228
  "Parent Class": (
230
- [ParentClassEntity.load(parent, prefix=default_prefix) for parent in self.parent.split(",")]
229
+ [ClassEntity.load(parent, prefix=default_prefix) for parent in self.parent.split(",")]
231
230
  if self.parent
232
231
  else None
233
232
  ),
@@ -6,7 +6,7 @@ from cognite.neat.rules import issues
6
6
  from cognite.neat.rules.issues import IssueList
7
7
  from cognite.neat.rules.models._base import DataModelType, SchemaCompleteness
8
8
  from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, UnknownEntity
9
- from cognite.neat.utils.utils import get_inheritance_path
9
+ from cognite.neat.utils.rdf_ import get_inheritance_path
10
10
 
11
11
  from ._rules import InformationRules
12
12
 
@@ -161,7 +161,7 @@ class InformationPostValidation:
161
161
  class_subclass_pairs[class_.class_] = []
162
162
  if class_.parent is None:
163
163
  continue
164
- class_subclass_pairs[class_.class_].extend([parent.as_class_entity() for parent in class_.parent])
164
+ class_subclass_pairs[class_.class_].extend(class_.parent)
165
165
 
166
166
  return class_subclass_pairs
167
167
 
@@ -1,3 +0,0 @@
1
- from .utils import remove_namespace_from_uri
2
-
3
- __all__ = ["remove_namespace_from_uri"]
@@ -5,13 +5,13 @@ from dataclasses import dataclass, fields
5
5
  from pathlib import Path
6
6
  from typing import Literal, TypeAlias, get_args
7
7
 
8
- from cognite.client import CogniteClient
8
+ from cognite.client import ClientConfig, CogniteClient
9
9
  from cognite.client.credentials import CredentialProvider, OAuthClientCredentials, OAuthInteractive, Token
10
10
 
11
11
  from cognite.neat import _version
12
12
  from cognite.neat.utils.auxiliary import local_import
13
13
 
14
- __all__ = ["get_cognite_client"]
14
+ __all__ = ["get_cognite_client", "EnvironmentVariables"]
15
15
 
16
16
  _LOGIN_FLOW: TypeAlias = Literal["infer", "client_credentials", "interactive", "token"]
17
17
  _VALID_LOGIN_FLOWS = get_args(_LOGIN_FLOW)
@@ -22,7 +22,7 @@ def get_cognite_client(env_file_name: str = ".env") -> CogniteClient:
22
22
  if not env_file_name.endswith(".env"):
23
23
  raise ValueError("env_file_name must end with '.env'")
24
24
  with suppress(KeyError):
25
- variables = _EnvironmentVariables.create_from_environ()
25
+ variables = EnvironmentVariables.create_from_environ()
26
26
  return variables.get_client()
27
27
 
28
28
  repo_root = _repo_root()
@@ -51,7 +51,7 @@ def get_cognite_client(env_file_name: str = ".env") -> CogniteClient:
51
51
 
52
52
 
53
53
  @dataclass
54
- class _EnvironmentVariables:
54
+ class EnvironmentVariables:
55
55
  CDF_CLUSTER: str
56
56
  CDF_PROJECT: str
57
57
  LOGIN_FLOW: _LOGIN_FLOW = "infer"
@@ -66,10 +66,17 @@ class _EnvironmentVariables:
66
66
  IDP_AUDIENCE: str | None = None
67
67
  IDP_SCOPES: str | None = None
68
68
  IDP_AUTHORITY_URL: str | None = None
69
+ CDF_MAX_WORKERS: int | None = None
70
+ CDF_TIMEOUT: int | None = None
71
+ CDF_REDIRECT_PORT: int = 53_000
69
72
 
70
73
  def __post_init__(self):
71
74
  if self.LOGIN_FLOW.lower() not in _VALID_LOGIN_FLOWS:
72
75
  raise ValueError(f"LOGIN_FLOW must be one of {_VALID_LOGIN_FLOWS}")
76
+ if self.IDP_TOKEN_URL and not self.IDP_TENANT_ID:
77
+ prefix, suffix = "https://login.microsoftonline.com/", "/oauth2/v2.0/token"
78
+ if self.IDP_TOKEN_URL.startswith(prefix) and self.IDP_TOKEN_URL.endswith(suffix):
79
+ self.IDP_TENANT_ID = self.IDP_TOKEN_URL.removeprefix(prefix).removesuffix(suffix)
73
80
 
74
81
  @property
75
82
  def cdf_url(self) -> str:
@@ -102,7 +109,7 @@ class _EnvironmentVariables:
102
109
  return f"https://login.microsoftonline.com/{self.IDP_TENANT_ID}"
103
110
 
104
111
  @classmethod
105
- def create_from_environ(cls) -> "_EnvironmentVariables":
112
+ def create_from_environ(cls) -> "EnvironmentVariables":
106
113
  if "CDF_CLUSTER" not in os.environ or "CDF_PROJECT" not in os.environ:
107
114
  raise KeyError("CDF_CLUSTER and CDF_PROJECT must be set in the environment.", "CDF_CLUSTER", "CDF_PROJECT")
108
115
 
@@ -119,6 +126,25 @@ class _EnvironmentVariables:
119
126
  IDP_AUDIENCE=os.environ.get("IDP_AUDIENCE"),
120
127
  IDP_SCOPES=os.environ.get("IDP_SCOPES"),
121
128
  IDP_AUTHORITY_URL=os.environ.get("IDP_AUTHORITY_URL"),
129
+ CDF_MAX_WORKERS=int(os.environ["CDF_MAX_WORKERS"]) if "CDF_MAX_WORKERS" in os.environ else None,
130
+ CDF_TIMEOUT=int(os.environ["CDF_TIMEOUT"]) if "CDF_TIMEOUT" in os.environ else None,
131
+ CDF_REDIRECT_PORT=int(os.environ.get("CDF_REDIRECT_PORT", 53_000)),
132
+ )
133
+
134
+ @classmethod
135
+ def default(cls) -> "EnvironmentVariables":
136
+ # This method is for backwards compatibility with the old config
137
+ # It is not recommended to use this method.
138
+ return cls(
139
+ LOGIN_FLOW="client_credentials",
140
+ CDF_CLUSTER="api",
141
+ CDF_PROJECT="dev",
142
+ IDP_TENANT_ID="common",
143
+ IDP_CLIENT_ID="neat",
144
+ IDP_CLIENT_SECRET="secret",
145
+ IDP_SCOPES="project:read,project:write",
146
+ CDF_TIMEOUT=60,
147
+ CDF_MAX_WORKERS=3,
122
148
  )
123
149
 
124
150
  def get_credentials(self) -> CredentialProvider:
@@ -161,7 +187,7 @@ class _EnvironmentVariables:
161
187
  return OAuthInteractive(
162
188
  client_id=self.IDP_CLIENT_ID,
163
189
  authority_url=self.idp_authority_url,
164
- redirect_port=53_000,
190
+ redirect_port=self.CDF_REDIRECT_PORT,
165
191
  scopes=self.idp_scopes,
166
192
  )
167
193
 
@@ -171,9 +197,15 @@ class _EnvironmentVariables:
171
197
  return Token(self.TOKEN)
172
198
 
173
199
  def get_client(self) -> CogniteClient:
174
- return CogniteClient.default(
175
- self.CDF_PROJECT, self.CDF_CLUSTER, credentials=self.get_credentials(), client_name=_CLIENT_NAME
200
+ config = ClientConfig(
201
+ client_name=_CLIENT_NAME,
202
+ project=self.CDF_PROJECT,
203
+ credentials=self.get_credentials(),
204
+ base_url=self.cdf_url,
205
+ max_workers=self.CDF_MAX_WORKERS,
206
+ timeout=self.CDF_TIMEOUT,
176
207
  )
208
+ return CogniteClient(config)
177
209
 
178
210
  def create_env_file(self) -> str:
179
211
  lines: list[str] = []
@@ -194,11 +226,11 @@ class _EnvironmentVariables:
194
226
  return "\n".join(lines)
195
227
 
196
228
 
197
- def _from_dotenv(evn_file: Path) -> _EnvironmentVariables:
229
+ def _from_dotenv(evn_file: Path) -> EnvironmentVariables:
198
230
  if not evn_file.exists():
199
231
  raise FileNotFoundError(f"{evn_file} does not exist.")
200
232
  content = evn_file.read_text()
201
- valid_variables = {f.name for f in fields(_EnvironmentVariables)}
233
+ valid_variables = {f.name for f in fields(EnvironmentVariables)}
202
234
  variables: dict[str, str] = {}
203
235
  for line in content.splitlines():
204
236
  if line.startswith("#") or "=" not in line:
@@ -206,15 +238,15 @@ def _from_dotenv(evn_file: Path) -> _EnvironmentVariables:
206
238
  key, value = line.split("=", 1)
207
239
  if key in valid_variables:
208
240
  variables[key] = value
209
- return _EnvironmentVariables(**variables) # type: ignore[arg-type]
241
+ return EnvironmentVariables(**variables) # type: ignore[arg-type]
210
242
 
211
243
 
212
- def _prompt_user() -> _EnvironmentVariables:
244
+ def _prompt_user() -> EnvironmentVariables:
213
245
  local_import("rich", "jupyter")
214
246
  from rich.prompt import Prompt
215
247
 
216
248
  try:
217
- variables = _EnvironmentVariables.create_from_environ()
249
+ variables = EnvironmentVariables.create_from_environ()
218
250
  continue_ = Prompt.ask(
219
251
  f"Use environment variables for CDF Cluster '{variables.CDF_CLUSTER}' "
220
252
  f"and Project '{variables.CDF_PROJECT}'? [y/n]",
@@ -263,25 +295,12 @@ def _prompt_user() -> _EnvironmentVariables:
263
295
  return variables
264
296
 
265
297
 
266
- def _prompt_cluster_and_project() -> _EnvironmentVariables:
298
+ def _prompt_cluster_and_project() -> EnvironmentVariables:
267
299
  from rich.prompt import Prompt
268
300
 
269
301
  cluster = Prompt.ask("Enter CDF Cluster (example 'greenfield', 'bluefield', 'westeurope-1)")
270
302
  project = Prompt.ask("Enter CDF Project")
271
- return _EnvironmentVariables(cluster, project)
272
-
273
-
274
- def _is_notebook() -> bool:
275
- try:
276
- shell = get_ipython().__class__.__name__ # type: ignore[name-defined]
277
- if shell == "ZMQInteractiveShell":
278
- return True # Jupyter notebook or qtconsole
279
- elif shell == "TerminalInteractiveShell":
280
- return False # Terminal running IPython
281
- else:
282
- return False # Other type (?)
283
- except NameError:
284
- return False # Probably standard Python interpreter
303
+ return EnvironmentVariables(cluster, project)
285
304
 
286
305
 
287
306
  def _repo_root() -> Path | None:
@@ -1,8 +1,16 @@
1
+ import hashlib
1
2
  import importlib
2
3
  import inspect
3
- from collections.abc import Callable
4
+ import logging
5
+ import time
6
+ from collections.abc import Callable, Iterable
7
+ from datetime import datetime
8
+ from functools import wraps
4
9
  from types import ModuleType
5
10
 
11
+ from cognite.client.exceptions import CogniteDuplicatedError, CogniteReadTimeout
12
+ from pydantic_core import ErrorDetails
13
+
6
14
  from cognite.neat.exceptions import NeatImportError
7
15
 
8
16
 
@@ -33,3 +41,135 @@ def class_html_doc(cls: type, include_factory_methods: bool = True) -> str:
33
41
  f'<ul style="list-style-type:circle;">{factory_methods_str}</ul>'
34
42
  )
35
43
  return f"<h3>{cls.__name__}</h3><p>{docstring}</p>"
44
+
45
+
46
+ def retry_decorator(max_retries=2, retry_delay=3, component_name=""):
47
+ def decorator(func):
48
+ @wraps(func)
49
+ def wrapper(*args, **kwargs):
50
+ previous_exception = None
51
+ for attempt in range(max_retries + 1):
52
+ try:
53
+ logging.debug(f"Attempt {attempt + 1} of {max_retries + 1} for {component_name}")
54
+ return func(*args, **kwargs)
55
+ except CogniteReadTimeout as e:
56
+ previous_exception = e
57
+ if attempt < max_retries:
58
+ logging.error(
59
+ f"""CogniteReadTimeout retry attempt {attempt + 1} failed for {component_name} .
60
+ Retrying in {retry_delay} second(s). Error:"""
61
+ )
62
+ logging.error(e)
63
+ time.sleep(retry_delay)
64
+ else:
65
+ raise e
66
+ except CogniteDuplicatedError as e:
67
+ if isinstance(previous_exception, CogniteReadTimeout):
68
+ # if previous exception was CogniteReadTimeout,
69
+ # we can't be sure if the items were created or not
70
+ if len(e.successful) == 0 and len(e.failed) == 0 and len(e.duplicated) >= 0:
71
+ logging.warning(
72
+ f"Duplicate error for {component_name} . All items already exist in CDF. "
73
+ "Suppressing error."
74
+ )
75
+ return
76
+ else:
77
+ # can happend because of eventual consistency. Retry with delay to allow for CDF to catch up
78
+ if attempt < max_retries:
79
+ logging.error(
80
+ f"""CogniteDuplicatedError retry attempt {attempt + 1} failed for {component_name} .
81
+ Retrying in {retry_delay} second(s). Error:"""
82
+ )
83
+ logging.error(e)
84
+ # incerasing delay to allow for CDF to catch up
85
+ time.sleep(retry_delay)
86
+ else:
87
+ raise e
88
+ else:
89
+ # no point in retrying duplicate error if previous exception was not a timeout
90
+ raise e
91
+
92
+ except Exception as e:
93
+ previous_exception = e
94
+ if attempt < max_retries:
95
+ logging.error(
96
+ f"Retry attempt {attempt + 1} failed for {component_name}. "
97
+ f"Retrying in {retry_delay} second(s)."
98
+ )
99
+ logging.error(e)
100
+ time.sleep(retry_delay)
101
+ else:
102
+ raise e
103
+
104
+ return wrapper
105
+
106
+ return decorator
107
+
108
+
109
+ def create_sha256_hash(string: str) -> str:
110
+ # Create a SHA-256 hash object
111
+ sha256_hash = hashlib.sha256()
112
+
113
+ # Convert the string to bytes and update the hash object
114
+ sha256_hash.update(string.encode("utf-8"))
115
+
116
+ # Get the hexadecimal representation of the hash
117
+ hash_value = sha256_hash.hexdigest()
118
+
119
+ return hash_value
120
+
121
+
122
+ # Will likely be removed with legacy code
123
+ def generate_exception_report(exceptions: list[dict] | list[ErrorDetails] | None, category: str = "") -> str:
124
+ exceptions_as_dict = _order_expectations_by_type(exceptions) if exceptions else {}
125
+ report = ""
126
+
127
+ for exception_type in exceptions_as_dict.keys():
128
+ title = f"# {category}: {exception_type}" if category else ""
129
+ warnings = "\n- " + "\n- ".join(exceptions_as_dict[exception_type])
130
+ report += title + warnings + "\n\n"
131
+
132
+ return report
133
+
134
+
135
+ def _order_expectations_by_type(
136
+ exceptions: list[dict] | list[ErrorDetails],
137
+ ) -> dict[str, list[str]]:
138
+ exception_dict: dict[str, list[str]] = {}
139
+ for exception in exceptions:
140
+ if not isinstance(exception["loc"], str) and isinstance(exception["loc"], Iterable):
141
+ location = f"[{'/'.join(str(e) for e in exception['loc'])}]"
142
+ else:
143
+ location = ""
144
+
145
+ issue = f"{exception['msg']} {location}"
146
+
147
+ if exception_dict.get(exception["type"]) is None:
148
+ exception_dict[exception["type"]] = [issue]
149
+ else:
150
+ exception_dict[exception["type"]].append(issue)
151
+ return exception_dict
152
+
153
+
154
+ def string_to_ideal_type(input_string: str) -> int | bool | float | datetime | str:
155
+ try:
156
+ # Try converting to int
157
+ return int(input_string)
158
+ except ValueError:
159
+ try:
160
+ # Try converting to float
161
+ return float(input_string) # type: ignore
162
+ except ValueError:
163
+ if input_string.lower() == "true":
164
+ # Return True if input is 'true'
165
+ return True
166
+ elif input_string.lower() == "false":
167
+ # Return False if input is 'false'
168
+ return False
169
+ else:
170
+ try:
171
+ # Try converting to datetime
172
+ return datetime.fromisoformat(input_string) # type: ignore
173
+ except ValueError:
174
+ # Return the input string if no conversion is possible
175
+ return input_string
File without changes