cognite-neat 0.86.0__py3-none-any.whl → 0.87.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cognite-neat might be problematic. Click here for more details.

Files changed (108) hide show
  1. cognite/neat/_version.py +1 -1
  2. cognite/neat/app/api/configuration.py +1 -10
  3. cognite/neat/app/api/routers/data_exploration.py +1 -1
  4. cognite/neat/config.py +84 -17
  5. cognite/neat/constants.py +11 -9
  6. cognite/neat/graph/extractors/_classic_cdf/_assets.py +1 -1
  7. cognite/neat/graph/extractors/_classic_cdf/_events.py +1 -1
  8. cognite/neat/graph/extractors/_classic_cdf/_files.py +1 -1
  9. cognite/neat/graph/extractors/_classic_cdf/_labels.py +1 -1
  10. cognite/neat/graph/extractors/_classic_cdf/_relationships.py +1 -1
  11. cognite/neat/graph/extractors/_classic_cdf/_sequences.py +1 -1
  12. cognite/neat/graph/extractors/_classic_cdf/_timeseries.py +1 -1
  13. cognite/neat/graph/extractors/_dexpi.py +1 -1
  14. cognite/neat/graph/extractors/_mock_graph_generator.py +8 -9
  15. cognite/neat/graph/loaders/__init__.py +5 -2
  16. cognite/neat/graph/loaders/_base.py +13 -5
  17. cognite/neat/graph/loaders/_rdf2asset.py +185 -55
  18. cognite/neat/graph/loaders/_rdf2dms.py +7 -7
  19. cognite/neat/graph/queries/_base.py +20 -11
  20. cognite/neat/graph/queries/_construct.py +5 -5
  21. cognite/neat/graph/queries/_shared.py +21 -7
  22. cognite/neat/graph/stores/_base.py +16 -4
  23. cognite/neat/graph/transformers/__init__.py +3 -0
  24. cognite/neat/graph/transformers/_rdfpath.py +42 -0
  25. cognite/neat/legacy/graph/extractors/_dexpi.py +0 -5
  26. cognite/neat/legacy/graph/extractors/_mock_graph_generator.py +1 -1
  27. cognite/neat/legacy/graph/loaders/_asset_loader.py +2 -2
  28. cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py +5 -2
  29. cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py +4 -1
  30. cognite/neat/legacy/graph/loaders/rdf_to_dms.py +3 -1
  31. cognite/neat/legacy/graph/stores/_base.py +24 -8
  32. cognite/neat/legacy/graph/stores/_graphdb_store.py +3 -2
  33. cognite/neat/legacy/graph/stores/_memory_store.py +3 -3
  34. cognite/neat/legacy/graph/stores/_oxigraph_store.py +8 -4
  35. cognite/neat/legacy/graph/stores/_rdf_to_graph.py +5 -3
  36. cognite/neat/legacy/graph/transformations/query_generator/sparql.py +49 -16
  37. cognite/neat/legacy/graph/transformations/transformer.py +1 -1
  38. cognite/neat/legacy/rules/exporters/_rules2dms.py +8 -3
  39. cognite/neat/legacy/rules/exporters/_rules2graphql.py +1 -1
  40. cognite/neat/legacy/rules/exporters/_rules2ontology.py +2 -1
  41. cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py +3 -4
  42. cognite/neat/legacy/rules/importers/_dms2rules.py +4 -1
  43. cognite/neat/legacy/rules/importers/_graph2rules.py +3 -3
  44. cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py +1 -1
  45. cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py +2 -1
  46. cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py +1 -1
  47. cognite/neat/legacy/rules/models/raw_rules.py +19 -7
  48. cognite/neat/legacy/rules/models/rules.py +32 -12
  49. cognite/neat/rules/_shared.py +6 -1
  50. cognite/neat/rules/analysis/__init__.py +4 -4
  51. cognite/neat/rules/analysis/_asset.py +143 -0
  52. cognite/neat/rules/analysis/_base.py +385 -6
  53. cognite/neat/rules/analysis/_information.py +183 -0
  54. cognite/neat/rules/exporters/_rules2dms.py +1 -1
  55. cognite/neat/rules/exporters/_rules2ontology.py +6 -5
  56. cognite/neat/rules/importers/_dms2rules.py +3 -1
  57. cognite/neat/rules/importers/_dtdl2rules/dtdl_converter.py +2 -8
  58. cognite/neat/rules/importers/_inference2rules.py +3 -7
  59. cognite/neat/rules/importers/_owl2rules/_owl2classes.py +1 -1
  60. cognite/neat/rules/importers/_owl2rules/_owl2metadata.py +2 -1
  61. cognite/neat/rules/importers/_owl2rules/_owl2properties.py +1 -1
  62. cognite/neat/rules/issues/spreadsheet.py +35 -0
  63. cognite/neat/rules/models/_base.py +7 -7
  64. cognite/neat/rules/models/_rdfpath.py +17 -21
  65. cognite/neat/rules/models/asset/_rules.py +4 -5
  66. cognite/neat/rules/models/asset/_validation.py +38 -1
  67. cognite/neat/rules/models/dms/_converter.py +1 -2
  68. cognite/neat/rules/models/dms/_exporter.py +7 -3
  69. cognite/neat/rules/models/dms/_rules.py +3 -0
  70. cognite/neat/rules/models/dms/_schema.py +5 -4
  71. cognite/neat/rules/models/domain.py +5 -2
  72. cognite/neat/rules/models/entities.py +28 -17
  73. cognite/neat/rules/models/information/_rules.py +10 -8
  74. cognite/neat/rules/models/information/_rules_input.py +1 -2
  75. cognite/neat/rules/models/information/_validation.py +2 -2
  76. cognite/neat/utils/__init__.py +0 -3
  77. cognite/neat/utils/auth.py +47 -28
  78. cognite/neat/utils/auxiliary.py +141 -1
  79. cognite/neat/utils/cdf/__init__.py +0 -0
  80. cognite/neat/utils/{cdf_classes.py → cdf/data_classes.py} +122 -2
  81. cognite/neat/utils/{cdf_loaders → cdf/loaders}/_data_modeling.py +37 -0
  82. cognite/neat/utils/{cdf_loaders → cdf/loaders}/_ingestion.py +2 -1
  83. cognite/neat/utils/collection_.py +18 -0
  84. cognite/neat/utils/rdf_.py +165 -0
  85. cognite/neat/utils/text.py +4 -0
  86. cognite/neat/utils/time_.py +17 -0
  87. cognite/neat/utils/upload.py +13 -1
  88. cognite/neat/workflows/_exceptions.py +5 -5
  89. cognite/neat/workflows/base.py +1 -1
  90. cognite/neat/workflows/steps/lib/current/graph_store.py +28 -8
  91. cognite/neat/workflows/steps/lib/current/rules_validator.py +2 -2
  92. cognite/neat/workflows/steps/lib/legacy/graph_extractor.py +130 -28
  93. cognite/neat/workflows/steps/lib/legacy/graph_loader.py +1 -1
  94. cognite/neat/workflows/steps/lib/legacy/graph_store.py +4 -4
  95. cognite/neat/workflows/steps/lib/legacy/rules_exporter.py +1 -1
  96. cognite/neat/workflows/steps/lib/legacy/rules_importer.py +1 -1
  97. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/METADATA +2 -2
  98. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/RECORD +103 -102
  99. cognite/neat/rules/analysis/_information_rules.py +0 -476
  100. cognite/neat/utils/cdf.py +0 -59
  101. cognite/neat/utils/cdf_loaders/data_classes.py +0 -121
  102. cognite/neat/utils/exceptions.py +0 -41
  103. cognite/neat/utils/utils.py +0 -429
  104. /cognite/neat/utils/{cdf_loaders → cdf/loaders}/__init__.py +0 -0
  105. /cognite/neat/utils/{cdf_loaders → cdf/loaders}/_base.py +0 -0
  106. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/LICENSE +0 -0
  107. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/WHEEL +0 -0
  108. {cognite_neat-0.86.0.dist-info → cognite_neat-0.87.3.dist-info}/entry_points.txt +0 -0
@@ -18,7 +18,10 @@ from rdflib.term import URIRef
18
18
  from cognite.neat.legacy.graph.loaders.core.models import AssetTemplate
19
19
  from cognite.neat.legacy.graph.stores import NeatGraphStoreBase
20
20
  from cognite.neat.legacy.rules.models.rules import Property, Rules
21
- from cognite.neat.utils.utils import chunker, datetime_utc_now, remove_namespace_from_uri, retry_decorator
21
+ from cognite.neat.utils.auxiliary import retry_decorator
22
+ from cognite.neat.utils.collection_ import chunker
23
+ from cognite.neat.utils.rdf_ import remove_namespace_from_uri
24
+ from cognite.neat.utils.time_ import datetime_utc_now
22
25
 
23
26
  if sys.version_info >= (3, 11):
24
27
  from datetime import UTC
@@ -899,7 +902,7 @@ def _micro_batch_push(
899
902
  except CogniteDuplicatedError:
900
903
  # this is handling of very rare case when some assets might be lost . Normally this should not happen.
901
904
  # Last attempt to recover
902
- client.assets.create_hierarchy(batch, upsert=True)
905
+ client.assets.create_hierarchy(batch, upsert=True) # type: ignore[arg-type]
903
906
 
904
907
  delta_time = (datetime_utc_now() - start_time).seconds
905
908
 
@@ -14,7 +14,10 @@ from cognite.neat.legacy.graph.loaders.core.models import RelationshipDefinition
14
14
  from cognite.neat.legacy.graph.loaders.core.rdf_to_assets import _categorize_cdf_assets
15
15
  from cognite.neat.legacy.graph.stores import NeatGraphStoreBase
16
16
  from cognite.neat.legacy.rules.models.rules import Rules
17
- from cognite.neat.utils.utils import chunker, datetime_utc_now, epoch_now_ms, remove_namespace_from_uri, retry_decorator
17
+ from cognite.neat.utils.auxiliary import retry_decorator
18
+ from cognite.neat.utils.collection_ import chunker
19
+ from cognite.neat.utils.rdf_ import remove_namespace_from_uri
20
+ from cognite.neat.utils.time_ import datetime_utc_now, epoch_now_ms
18
21
 
19
22
 
20
23
  def define_relationships(rules: Rules, data_set_id: int, stop_on_exception: bool = False) -> RelationshipDefinitions:
@@ -13,7 +13,9 @@ from cognite.neat.legacy.graph.transformations.query_generator.sparql import tri
13
13
  from cognite.neat.legacy.rules.exporters._rules2dms import DMSSchemaComponents
14
14
  from cognite.neat.legacy.rules.exporters._rules2pydantic_models import add_class_prefix_to_xid, rules_to_pydantic_models
15
15
  from cognite.neat.legacy.rules.models.rules import Rules
16
- from cognite.neat.utils.utils import chunker, datetime_utc_now, retry_decorator
16
+ from cognite.neat.utils.auxiliary import retry_decorator
17
+ from cognite.neat.utils.collection_ import chunker
18
+ from cognite.neat.utils.time_ import datetime_utc_now
17
19
 
18
20
  from ._base import CogniteLoader
19
21
 
@@ -11,7 +11,7 @@ from prometheus_client import Gauge, Summary
11
11
  from rdflib import Graph, Namespace, URIRef
12
12
  from rdflib.query import Result, ResultRow
13
13
 
14
- from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
14
+ from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
15
15
  from cognite.neat.legacy.graph.models import Triple
16
16
  from cognite.neat.legacy.graph.stores._rdf_to_graph import rdf_file_to_graph
17
17
  from cognite.neat.legacy.rules.models.rules import Rules
@@ -25,7 +25,11 @@ prom_qsm = Summary("store_query_time_summary_legacy", "Time spent processing que
25
25
  prom_sq = Gauge("store_single_query_time_legacy", "Time spent processing a single query", ["query"])
26
26
 
27
27
  MIMETypes: TypeAlias = Literal[
28
- "application/rdf+xml", "text/turtle", "application/n-triple", "application/n-quads", "application/trig"
28
+ "application/rdf+xml",
29
+ "text/turtle",
30
+ "application/n-triple",
31
+ "application/n-quads",
32
+ "application/trig",
29
33
  ]
30
34
 
31
35
 
@@ -35,7 +39,8 @@ class NeatGraphStoreBase(ABC):
35
39
 
36
40
  Args:
37
41
  graph : Instance of rdflib.Graph class for graph storage
38
- base_prefix : Used as a base prefix for graph namespace, allowing querying graph data using a shortform of a URI
42
+ base_prefix : Used as a base prefix for graph namespace,
43
+ allowing querying graph data using a short form of a URI
39
44
  namespace : Namespace (aka URI) used to resolve any relative URI in the graph
40
45
  prefixes : Dictionary of additional prefixes used and bounded to the graph
41
46
  """
@@ -47,12 +52,12 @@ class NeatGraphStoreBase(ABC):
47
52
  graph: Graph | None = None,
48
53
  base_prefix: str = "", # usually empty
49
54
  namespace: Namespace = DEFAULT_NAMESPACE,
50
- prefixes: dict = PREFIXES,
55
+ prefixes: dict[str, Namespace] | None = None,
51
56
  ):
52
57
  self.graph = graph or Graph()
53
58
  self.base_prefix: str = base_prefix
54
59
  self.namespace: Namespace = namespace
55
- self.prefixes: dict[str, Namespace] = prefixes
60
+ self.prefixes: dict[str, Namespace] = prefixes or get_default_prefixes()
56
61
 
57
62
  self.rdf_store_query_url: str | None = None
58
63
  self.rdf_store_update_url: str | None = None
@@ -164,7 +169,10 @@ class NeatGraphStoreBase(ABC):
164
169
  return None
165
170
 
166
171
  def import_from_file(
167
- self, graph_file: Path, mime_type: MIMETypes = "application/rdf+xml", add_base_iri: bool = True
172
+ self,
173
+ graph_file: Path,
174
+ mime_type: MIMETypes = "application/rdf+xml",
175
+ add_base_iri: bool = True,
168
176
  ) -> None:
169
177
  """Imports graph data from file.
170
178
 
@@ -175,7 +183,10 @@ class NeatGraphStoreBase(ABC):
175
183
  """
176
184
  if add_base_iri:
177
185
  self.graph = rdf_file_to_graph(
178
- self.graph, graph_file, base_namespace=self.namespace, prefixes=self.prefixes
186
+ self.graph,
187
+ graph_file,
188
+ base_namespace=self.namespace,
189
+ prefixes=self.prefixes,
179
190
  )
180
191
  else:
181
192
  self.graph = rdf_file_to_graph(self.graph, graph_file, prefixes=self.prefixes)
@@ -290,7 +301,12 @@ class NeatGraphStoreBase(ABC):
290
301
  "rdf_store_update_url": self.rdf_store_update_url,
291
302
  }
292
303
 
293
- def add_triples(self, triples: list[Triple] | set[Triple], batch_size: int = 10_000, verbose: bool = False):
304
+ def add_triples(
305
+ self,
306
+ triples: list[Triple] | set[Triple],
307
+ batch_size: int = 10_000,
308
+ verbose: bool = False,
309
+ ):
294
310
  """Adds triples to the graph store in batches.
295
311
 
296
312
  Args:
@@ -4,7 +4,7 @@ import requests
4
4
  from rdflib import Graph, Namespace
5
5
  from rdflib.plugins.stores.sparqlstore import SPARQLUpdateStore
6
6
 
7
- from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
7
+ from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
8
8
 
9
9
  from ._base import NeatGraphStoreBase
10
10
 
@@ -28,8 +28,9 @@ class GraphDBStore(NeatGraphStoreBase):
28
28
  graph: Graph | None = None,
29
29
  base_prefix: str = "", # usually empty
30
30
  namespace: Namespace = DEFAULT_NAMESPACE,
31
- prefixes: dict = PREFIXES,
31
+ prefixes: dict[str, Namespace] | None = None,
32
32
  ):
33
+ prefixes = prefixes if prefixes else get_default_prefixes()
33
34
  super().__init__(graph, base_prefix, namespace, prefixes)
34
35
  self.graph_db_rest_url: str = "http://localhost:7200"
35
36
 
@@ -2,7 +2,7 @@ import logging
2
2
 
3
3
  from rdflib import Graph, Namespace
4
4
 
5
- from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
5
+ from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
6
6
 
7
7
  from ._base import NeatGraphStoreBase
8
8
 
@@ -26,9 +26,9 @@ class MemoryStore(NeatGraphStoreBase):
26
26
  graph: Graph | None = None,
27
27
  base_prefix: str = "", # usually empty
28
28
  namespace: Namespace = DEFAULT_NAMESPACE,
29
- prefixes: dict = PREFIXES,
29
+ prefixes: dict[str, Namespace] | None = None,
30
30
  ):
31
- # Init repeated to get nice docstring
31
+ prefixes = prefixes if prefixes else get_default_prefixes() # Init repeated to get nice docstring
32
32
  super().__init__(graph, base_prefix, namespace, prefixes)
33
33
 
34
34
  def _set_graph(self):
@@ -5,7 +5,7 @@ from pathlib import Path
5
5
 
6
6
  from rdflib import Graph, Namespace
7
7
 
8
- from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
8
+ from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
9
9
  from cognite.neat.utils.auxiliary import local_import
10
10
 
11
11
  from ._base import MIMETypes, NeatGraphStoreBase
@@ -29,8 +29,9 @@ class OxiGraphStore(NeatGraphStoreBase):
29
29
  graph: Graph | None = None,
30
30
  base_prefix: str = "", # usually empty
31
31
  namespace: Namespace = DEFAULT_NAMESPACE,
32
- prefixes: dict = PREFIXES,
32
+ prefixes: dict[str, Namespace] | None = None,
33
33
  ):
34
+ prefixes = prefixes if prefixes else get_default_prefixes()
34
35
  super().__init__(graph, base_prefix, namespace, prefixes)
35
36
 
36
37
  def _set_graph(self) -> None:
@@ -44,7 +45,7 @@ class OxiGraphStore(NeatGraphStoreBase):
44
45
  for i in range(4):
45
46
  try:
46
47
  oxstore = pyoxigraph.Store(
47
- path=str(self.internal_storage_dir) if self.internal_storage_dir else None
48
+ path=(str(self.internal_storage_dir) if self.internal_storage_dir else None)
48
49
  ) # Store (Rust object) accepts only str as path and not Path.
49
50
  break
50
51
  except OSError as e:
@@ -76,7 +77,10 @@ class OxiGraphStore(NeatGraphStoreBase):
76
77
  logging.info("GraphStore restarted")
77
78
 
78
79
  def import_from_file(
79
- self, graph_file: Path, mime_type: MIMETypes = "application/rdf+xml", add_base_iri: bool = True
80
+ self,
81
+ graph_file: Path,
82
+ mime_type: MIMETypes = "application/rdf+xml",
83
+ add_base_iri: bool = True,
80
84
  ) -> None:
81
85
  """Imports graph data from file.
82
86
 
@@ -2,7 +2,7 @@ from pathlib import Path
2
2
 
3
3
  from rdflib import Graph, Namespace
4
4
 
5
- from cognite.neat.constants import PREFIXES
5
+ from cognite.neat.constants import get_default_prefixes
6
6
 
7
7
 
8
8
  def rdf_file_to_graph(
@@ -10,7 +10,7 @@ def rdf_file_to_graph(
10
10
  filepath: Path,
11
11
  base_prefix: str | None = None,
12
12
  base_namespace: Namespace | None = None,
13
- prefixes: dict[str, Namespace] = PREFIXES,
13
+ prefixes: dict[str, Namespace] | None = None,
14
14
  ) -> Graph:
15
15
  """Created rdflib Graph instance loaded with RDF triples from file
16
16
 
@@ -18,13 +18,15 @@ def rdf_file_to_graph(
18
18
  filepath: Path to the RDF file
19
19
  base_prefix: base prefix for URIs. Defaults to None.
20
20
  base_namespace: base namespace for URIs . Defaults to None.
21
- prefixes: Dictionary of prefixes to bind to graph. Defaults to PREFIXES.
21
+ prefixes: Dictionary of prefixes to bind to graph. Defaults to internal set of prefixes.
22
22
  graph: Graph instance to load RDF triples into. Defaults to None.
23
23
 
24
24
  Returns:
25
25
  Graph instance loaded with RDF triples from file
26
26
  """
27
27
 
28
+ prefixes = prefixes if prefixes else get_default_prefixes()
29
+
28
30
  if filepath.is_file():
29
31
  graph.parse(filepath, publicID=base_namespace)
30
32
  else:
@@ -6,7 +6,7 @@ from typing import cast
6
6
  from rdflib import Graph, Namespace
7
7
  from rdflib.term import URIRef
8
8
 
9
- from cognite.neat.constants import PREFIXES
9
+ from cognite.neat.constants import get_default_prefixes
10
10
  from cognite.neat.legacy.rules.analysis import get_classes_with_properties
11
11
  from cognite.neat.legacy.rules.models._base import Triple
12
12
  from cognite.neat.legacy.rules.models.rdfpath import (
@@ -21,10 +21,10 @@ from cognite.neat.legacy.rules.models.rdfpath import (
21
21
  parse_traversal,
22
22
  )
23
23
  from cognite.neat.legacy.rules.models.rules import Rules
24
- from cognite.neat.utils.utils import remove_namespace_from_uri
24
+ from cognite.neat.utils.rdf_ import remove_namespace_from_uri
25
25
 
26
26
 
27
- def _generate_prefix_header(prefixes: dict[str, Namespace] = PREFIXES) -> str:
27
+ def _generate_prefix_header(prefixes: dict[str, Namespace] | None = None) -> str:
28
28
  """Generate prefix header which is added to SPARQL query and allows for shorten query statements
29
29
 
30
30
  Parameters
@@ -37,11 +37,15 @@ def _generate_prefix_header(prefixes: dict[str, Namespace] = PREFIXES) -> str:
37
37
  str
38
38
  Prefix header
39
39
  """
40
+ prefixes = prefixes if prefixes else get_default_prefixes()
40
41
  return "".join(f"PREFIX {key}:<{value}>\n" for key, value in prefixes.items())
41
42
 
42
43
 
43
44
  def _get_predicate_id(
44
- graph: Graph, subject_type_id: str, object_type_id: str, prefixes: dict[str, Namespace] = PREFIXES
45
+ graph: Graph,
46
+ subject_type_id: str,
47
+ object_type_id: str,
48
+ prefixes: dict[str, Namespace] | None = None,
45
49
  ) -> URIRef:
46
50
  """Returns predicate (aka property) URI (i.e., ID) that connects subject and object
47
51
 
@@ -61,6 +65,7 @@ def _get_predicate_id(
61
65
  URIRef
62
66
  ID of predicate (aka property) connecting subject and object
63
67
  """
68
+ prefixes = prefixes if prefixes else get_default_prefixes()
64
69
  query = """
65
70
 
66
71
  SELECT ?predicateTypeID
@@ -122,11 +127,21 @@ def _get_hop_triples(graph, path: Hop, prefixes) -> list[Triple]:
122
127
  if previous_step.property:
123
128
  triples.extend(
124
129
  [
125
- Triple(subject=f"?{previous_step.class_.suffix}ID", predicate="a", object=previous_step.class_.id),
126
130
  Triple(
127
- subject=f"?{previous_step.class_.suffix}ID", predicate=previous_step.property.id, object="?object"
131
+ subject=f"?{previous_step.class_.suffix}ID",
132
+ predicate="a",
133
+ object=previous_step.class_.id,
134
+ ),
135
+ Triple(
136
+ subject=f"?{previous_step.class_.suffix}ID",
137
+ predicate=previous_step.property.id,
138
+ object="?object",
139
+ ),
140
+ Triple(
141
+ subject="?predicate",
142
+ predicate="a",
143
+ object=previous_step.property.id,
128
144
  ),
129
- Triple(subject="?predicate", predicate="a", object=previous_step.property.id),
130
145
  ]
131
146
  )
132
147
  else:
@@ -139,7 +154,7 @@ def _get_hop_triples(graph, path: Hop, prefixes) -> list[Triple]:
139
154
  return triples
140
155
 
141
156
 
142
- def _get_path_triples(graph: Graph, traversal: Traversal, prefixes: dict[str, Namespace] = PREFIXES) -> list[Triple]:
157
+ def _get_path_triples(graph: Graph, traversal: Traversal, prefixes: dict[str, Namespace] | None = None) -> list[Triple]:
143
158
  """Creates triples subject-predicate-object from declarative graph traversal path
144
159
 
145
160
  Parameters
@@ -227,7 +242,9 @@ def _generate_all_references_query_statement(
227
242
 
228
243
 
229
244
  def _generate_single_property_query_statement(
230
- subject: str, predicate: str, query_template: str = SINGLE_PROPERTY_SPARQL_QUERY_TEMPLATE
245
+ subject: str,
246
+ predicate: str,
247
+ query_template: str = SINGLE_PROPERTY_SPARQL_QUERY_TEMPLATE,
231
248
  ) -> str:
232
249
  query_insertions = "\n".join([f"\t\t?subject a {subject} .", f"\t\t?subject {predicate} ?object ."])
233
250
 
@@ -257,7 +274,7 @@ def _generate_hop_query_statement(triples: list[Triple], query_template: str = B
257
274
  def build_sparql_query(
258
275
  graph: Graph,
259
276
  traversal_path: str | Traversal,
260
- prefixes: dict[str, Namespace] = PREFIXES,
277
+ prefixes: dict[str, Namespace] | None = None,
261
278
  insert_prefixes: bool = False,
262
279
  ) -> str:
263
280
  """Builds SPARQL query based on declarative traversal path
@@ -276,7 +293,7 @@ def build_sparql_query(
276
293
  str
277
294
  SPARQL query
278
295
  """
279
-
296
+ prefixes = prefixes if prefixes else get_default_prefixes()
280
297
  traversal = parse_traversal(traversal_path) if isinstance(traversal_path, str) else traversal_path
281
298
  triples = _get_path_triples(graph, traversal, prefixes)
282
299
 
@@ -297,7 +314,10 @@ def build_sparql_query(
297
314
  for prefix, URI in prefixes.items():
298
315
  query = query.replace(URI, f"{prefix}:")
299
316
 
300
- return query.replace("insertPrefixes\n\n", _generate_prefix_header(prefixes) if insert_prefixes else "")
317
+ return query.replace(
318
+ "insertPrefixes\n\n",
319
+ _generate_prefix_header(prefixes) if insert_prefixes else "",
320
+ )
301
321
 
302
322
 
303
323
  def compress_uri(uri: URIRef, prefixes: dict) -> str:
@@ -420,7 +440,8 @@ def _add_filter(class_instances, query_template):
420
440
  if class_instances:
421
441
  class_instances_formatted = [f"<{instance}>" for instance in class_instances]
422
442
  query_template = query_template.replace(
423
- "insert_filter", f"\n\nFILTER (?subject IN ({', '.join(class_instances_formatted)}))"
443
+ "insert_filter",
444
+ f"\n\nFILTER (?subject IN ({', '.join(class_instances_formatted)}))",
424
445
  )
425
446
  else:
426
447
  query_template = query_template.replace("insert_filter", "")
@@ -439,7 +460,10 @@ def _triples2sparql_statement(triples: list[Triple]):
439
460
 
440
461
 
441
462
  def _to_construct_triples(
442
- graph: Graph, class_: str, transformation_rules: Rules, properties_optional: bool = True
463
+ graph: Graph,
464
+ class_: str,
465
+ transformation_rules: Rules,
466
+ properties_optional: bool = True,
443
467
  ) -> tuple[list[Triple], list[Triple]]:
444
468
  """Converts class definition to CONSTRUCT triples which are used to generate CONSTRUCT query
445
469
 
@@ -485,7 +509,10 @@ def _to_construct_triples(
485
509
  # by binding them to certain property
486
510
  if isinstance(traversal, AllReferences):
487
511
  graph_pattern_triple = Triple(
488
- subject="BIND(?subject", predicate="AS", object=f"{graph_template_triple.object})", optional=False
512
+ subject="BIND(?subject",
513
+ predicate="AS",
514
+ object=f"{graph_template_triple.object})",
515
+ optional=False,
489
516
  )
490
517
 
491
518
  elif isinstance(traversal, SingleProperty):
@@ -510,7 +537,13 @@ def _to_construct_triples(
510
537
 
511
538
  # add first triple for graph pattern stating type of object
512
539
  patterns.insert(
513
- 0, Triple(subject="?subject", predicate="a", object=_most_occurring_element(class_ids), optional=False)
540
+ 0,
541
+ Triple(
542
+ subject="?subject",
543
+ predicate="a",
544
+ object=_most_occurring_element(class_ids),
545
+ optional=False,
546
+ ),
514
547
  )
515
548
 
516
549
  return templates, patterns
@@ -24,7 +24,7 @@ from cognite.neat.legacy.rules.models.rdfpath import (
24
24
  parse_rule,
25
25
  )
26
26
  from cognite.neat.legacy.rules.models.rules import Rules
27
- from cognite.neat.utils.utils import remove_namespace_from_uri
27
+ from cognite.neat.utils.rdf_ import remove_namespace_from_uri
28
28
 
29
29
  prom_total_proc_rules_g = Gauge("neat_total_processed_rules", "Number of processed rules", ["state"])
30
30
  rules_processing_timing_metric = Gauge(
@@ -8,6 +8,7 @@ from pathlib import Path
8
8
  from typing import ClassVar, Literal, cast, no_type_check
9
9
 
10
10
  import yaml
11
+ from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
11
12
 
12
13
  from cognite.neat.legacy.rules.models.value_types import ValueTypeMapping
13
14
 
@@ -45,7 +46,7 @@ from cognite.neat.legacy.rules.exporters._base import BaseExporter
45
46
  from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant
46
47
  from cognite.neat.legacy.rules.models._base import ContainerEntity, EntityTypes, ParentClass
47
48
  from cognite.neat.legacy.rules.models.rules import Class, Property, Rules
48
- from cognite.neat.utils.utils import generate_exception_report
49
+ from cognite.neat.utils.auxiliary import generate_exception_report
49
50
 
50
51
  if sys.version_info < (3, 11):
51
52
  from exceptiongroup import ExceptionGroup
@@ -278,13 +279,17 @@ class DMSSchemaComponents(BaseModel):
278
279
  existing_property, default_value=None
279
280
  )
280
281
 
281
- # scenario: property hold multiple values -> set is_list to True
282
+ # scenario: property holds multiple values -> set is_list to True
282
283
  if (
283
284
  not isinstance(existing_property.type, DirectRelation)
284
285
  and not isinstance(api_container_property.type, DirectRelation)
286
+ and isinstance(existing_property.type, ListablePropertyType)
287
+ and isinstance(api_container_property.type, ListablePropertyType)
285
288
  and existing_property.type.is_list != api_container_property.type.is_list
286
289
  ):
287
- containers[container_id].properties[container_property_id].type.is_list = True
290
+ type_ = containers[container_id].properties[container_property_id].type
291
+ if isinstance(type_, ListablePropertyType):
292
+ type_.is_list = True
288
293
 
289
294
  if errors:
290
295
  raise ExceptionGroup("Properties value types have been redefined! This is prohibited! Aborting!", errors)
@@ -14,7 +14,7 @@ from cognite.neat.legacy.rules.analysis import to_class_property_pairs
14
14
  from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant, are_properties_redefined
15
15
  from cognite.neat.legacy.rules.models.rules import Rules
16
16
  from cognite.neat.legacy.rules.models.value_types import XSD_VALUE_TYPE_MAPPINGS
17
- from cognite.neat.utils.utils import generate_exception_report
17
+ from cognite.neat.utils.auxiliary import generate_exception_report
18
18
 
19
19
  from ._base import BaseExporter
20
20
 
@@ -14,7 +14,8 @@ from cognite.neat.legacy.rules.exporters._base import BaseExporter
14
14
  from cognite.neat.legacy.rules.exporters._validation import are_properties_redefined
15
15
  from cognite.neat.legacy.rules.models.rules import Class, Metadata, Property, Rules
16
16
  from cognite.neat.legacy.rules.models.value_types import XSD_VALUE_TYPE_MAPPINGS
17
- from cognite.neat.utils.utils import generate_exception_report, remove_namespace_from_uri
17
+ from cognite.neat.utils.auxiliary import generate_exception_report
18
+ from cognite.neat.utils.rdf_ import remove_namespace_from_uri
18
19
 
19
20
  if sys.version_info >= (3, 11):
20
21
  from typing import Self
@@ -11,7 +11,6 @@ from cognite.client.data_classes.data_modeling.views import SingleHopConnectionD
11
11
  from pydantic import BaseModel, ConfigDict, Field, create_model
12
12
  from pydantic._internal._model_construction import ModelMetaclass
13
13
  from rdflib import Graph, URIRef
14
- from typing_extensions import TypeAliasType
15
14
 
16
15
  from cognite.neat.legacy.graph.loaders.core.rdf_to_assets import NeatMetadataKeys
17
16
  from cognite.neat.legacy.graph.transformations.query_generator.sparql import build_construct_query, triples2dictionary
@@ -21,7 +20,7 @@ from cognite.neat.legacy.rules.exporters._rules2dms import DMSSchemaComponents
21
20
  from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant
22
21
  from cognite.neat.legacy.rules.models.rules import Property, Rules
23
22
  from cognite.neat.legacy.rules.models.value_types import ValueTypeMapping
24
- from cognite.neat.utils.utils import create_sha256_hash, generate_exception_report
23
+ from cognite.neat.utils.auxiliary import create_sha256_hash, generate_exception_report
25
24
 
26
25
  if sys.version_info >= (3, 11):
27
26
  from datetime import UTC
@@ -30,8 +29,8 @@ else:
30
29
 
31
30
  UTC = timezone.utc
32
31
 
33
- EdgeOneToOne: TypeAlias = TypeAliasType("EdgeOneToOne", str) # type: ignore[valid-type]
34
- EdgeOneToMany: TypeAlias = TypeAliasType("EdgeOneToMany", list[str]) # type: ignore[valid-type]
32
+ EdgeOneToOne: TypeAlias = str # type: ignore[valid-type]
33
+ EdgeOneToMany: TypeAlias = list[str] # type: ignore[valid-type]
35
34
 
36
35
 
37
36
  def default_model_configuration(
@@ -13,6 +13,7 @@ from cognite.client.data_classes.data_modeling import (
13
13
  SingleHopConnectionDefinition,
14
14
  View,
15
15
  )
16
+ from cognite.client.data_classes.data_modeling.data_types import ListablePropertyType
16
17
  from cognite.client.data_classes.data_modeling.ids import DataModelIdentifier, ViewId
17
18
 
18
19
  from cognite.neat.legacy.rules.models.tables import Tables
@@ -129,7 +130,9 @@ class DMSImporter(BaseImporter):
129
130
 
130
131
  max_count: str | float = "1"
131
132
  if isinstance(prop, SingleHopConnectionDefinition) or (
132
- isinstance(prop, MappedProperty) and prop.type.is_list
133
+ isinstance(prop, MappedProperty)
134
+ and isinstance(prop.type, ListablePropertyType)
135
+ and prop.type.is_list
133
136
  ):
134
137
  max_count = float("nan")
135
138
 
@@ -10,11 +10,11 @@ from typing import cast
10
10
  import pandas as pd
11
11
  from rdflib import Graph, Literal, Namespace, URIRef
12
12
 
13
- from cognite.neat.constants import PREFIXES
13
+ from cognite.neat.constants import get_default_prefixes
14
14
  from cognite.neat.legacy.rules import exceptions
15
15
  from cognite.neat.legacy.rules.exporters._rules2rules import to_dms_name
16
16
  from cognite.neat.legacy.rules.models.tables import Tables
17
- from cognite.neat.utils.utils import get_namespace, remove_namespace_from_uri, uri_to_short_form
17
+ from cognite.neat.utils.rdf_ import get_namespace, remove_namespace_from_uri, uri_to_short_form
18
18
 
19
19
  from ._base import BaseImporter
20
20
 
@@ -169,7 +169,7 @@ def _graph_to_data_model_dict(graph: Graph, max_number_of_instance: int = -1) ->
169
169
  """
170
170
  data_model: dict[str, dict] = {}
171
171
 
172
- prefixes: dict[str, Namespace] = PREFIXES
172
+ prefixes: dict[str, Namespace] = get_default_prefixes()
173
173
 
174
174
  for class_ in _get_class_ids(graph):
175
175
  _add_uri_namespace_to_prefixes(class_, prefixes)
@@ -4,7 +4,7 @@ import numpy as np
4
4
  import pandas as pd
5
5
  from rdflib import OWL, Graph
6
6
 
7
- from cognite.neat.utils.utils import remove_namespace_from_uri
7
+ from cognite.neat.utils.rdf_ import remove_namespace_from_uri
8
8
 
9
9
 
10
10
  def parse_owl_classes(graph: Graph, make_compliant: bool = False, language: str = "en") -> pd.DataFrame:
@@ -10,7 +10,8 @@ from cognite.neat.legacy.rules.models.rules import (
10
10
  prefix_compliance_regex,
11
11
  version_compliance_regex,
12
12
  )
13
- from cognite.neat.utils.utils import convert_rdflib_content, remove_none_elements_from_set
13
+ from cognite.neat.utils.collection_ import remove_none_elements_from_set
14
+ from cognite.neat.utils.rdf_ import convert_rdflib_content
14
15
 
15
16
 
16
17
  def parse_owl_metadata(graph: Graph, make_compliant: bool = False) -> pd.DataFrame:
@@ -4,7 +4,7 @@ import numpy as np
4
4
  import pandas as pd
5
5
  from rdflib import Graph
6
6
 
7
- from cognite.neat.utils.utils import remove_namespace_from_uri
7
+ from cognite.neat.utils.rdf_ import remove_namespace_from_uri
8
8
 
9
9
  from ._owl2classes import _data_type_property_class, _object_property_class, _thing_class
10
10
 
@@ -9,16 +9,22 @@ from pydantic import field_validator
9
9
  from pydantic_core import ErrorDetails, ValidationError
10
10
  from rdflib import Namespace
11
11
 
12
- from cognite.neat.constants import PREFIXES
12
+ from cognite.neat.constants import get_default_prefixes
13
13
  from cognite.neat.exceptions import wrangle_warnings
14
14
 
15
15
  # rules model and model components:
16
- from cognite.neat.legacy.rules.models.rules import Class, Metadata, Property, RuleModel, Rules
16
+ from cognite.neat.legacy.rules.models.rules import (
17
+ Class,
18
+ Metadata,
19
+ Property,
20
+ RuleModel,
21
+ Rules,
22
+ )
17
23
  from cognite.neat.legacy.rules.models.tables import Tables
18
24
 
19
25
  # importers:
20
26
  from cognite.neat.rules import exceptions
21
- from cognite.neat.utils.utils import generate_exception_report
27
+ from cognite.neat.utils.auxiliary import generate_exception_report
22
28
 
23
29
  __all__ = ["RawRules"]
24
30
 
@@ -36,7 +42,7 @@ class RawRules(RuleModel):
36
42
  classes: Classes defined in the data model
37
43
  properties: Class properties defined in the data model with accompanying transformation rules
38
44
  to transform data from source to target representation
39
- prefixes: Prefixes used in the data model. Defaults to PREFIXES
45
+ prefixes: Prefixes used in the data model. Defaults to internal set of prefixes
40
46
  instances: Instances defined in the data model. Defaults to None
41
47
  """
42
48
 
@@ -242,7 +248,9 @@ def _raw_tables_to_rules_dict(raw_tables: RawRules, validators_to_skip: set | No
242
248
  "metadata": _metadata_table2dict(raw_tables.Metadata),
243
249
  "classes": _classes_table2dict(raw_tables.Classes),
244
250
  "properties": _properties_table2dict(raw_tables.Properties),
245
- "prefixes": PREFIXES if raw_tables.Prefixes.empty else _prefixes_table2dict(raw_tables.Prefixes),
251
+ "prefixes": (
252
+ get_default_prefixes() if raw_tables.Prefixes.empty else _prefixes_table2dict(raw_tables.Prefixes)
253
+ ),
246
254
  }
247
255
 
248
256
  rules_dict["instances"] = (
@@ -272,11 +280,15 @@ def _metadata_table2dict(meta_df: pd.DataFrame) -> dict[str, Any]:
272
280
  return metadata_dict
273
281
 
274
282
 
275
- def _classes_table2dict(classes_df: pd.DataFrame) -> dict[Any | None, dict[Hashable, Any]]:
283
+ def _classes_table2dict(
284
+ classes_df: pd.DataFrame,
285
+ ) -> dict[Any | None, dict[Hashable, Any]]:
276
286
  return {class_.get("Class"): class_ for class_ in classes_df.to_dict(orient="records")}
277
287
 
278
288
 
279
- def _properties_table2dict(properties_df: pd.DataFrame) -> dict[str, dict[Hashable, Any]]:
289
+ def _properties_table2dict(
290
+ properties_df: pd.DataFrame,
291
+ ) -> dict[str, dict[Hashable, Any]]:
280
292
  return {f"row {i+3}": property_ for i, property_ in enumerate(properties_df.to_dict(orient="records"))}
281
293
 
282
294