cognite-neat 0.87.6__py3-none-any.whl → 0.88.1__py3-none-any.whl

cognite/neat/workflows/steps/data_contracts.py

@@ -1,27 +1,24 @@
 from pathlib import Path
 
 from cognite.client import CogniteClient
-from cognite.client.data_classes import Asset, AssetUpdate, Relationship, RelationshipUpdate
+from cognite.client.data_classes import (
+    Asset,
+    AssetUpdate,
+    Relationship,
+    RelationshipUpdate,
+)
 from cognite.client.data_classes.data_modeling import EdgeApply, NodeApply
 
-from cognite.neat.legacy.graph.stores import NeatGraphStoreBase
-from cognite.neat.legacy.rules.exporters._rules2dms import DMSSchemaComponents
-from cognite.neat.legacy.rules.models.rules import Rules
-from cognite.neat.rules.models import AssetRules, DMSRules, DomainRules, InformationRules
+from cognite.neat.graph.stores import NeatGraphStore
+from cognite.neat.rules.models import (
+    AssetRules,
+    DMSRules,
+    DomainRules,
+    InformationRules,
+)
 from cognite.neat.workflows.steps.step_model import DataContract
 
 
-class RulesData(DataContract):
-    """
-    This represents the TransformationRules object.
-
-    Args:
-        rules: The TransformationRules object.
-    """
-
-    rules: Rules
-
-
 class MultiRuleData(DataContract):
     domain: DomainRules | None = None
     information: InformationRules | None = None
@@ -53,27 +50,15 @@ class PathData(DataContract):
     excel_file_path: Path
 
 
-class SourceGraph(DataContract):
-    """
-    This represents the source graph.
-
-    Args:
-        graph: The source graph.
-    """
-
-    graph: NeatGraphStoreBase
-
-
-class SolutionGraph(DataContract):
+class NeatGraph(DataContract):
     """
-    This represents the solution graph.
+    This represents the neat graph.
 
     Args:
-        graph (NeatGraphStoreBase): The solution graph.
-
+        graph: The neat graph store.
     """
 
-    graph: NeatGraphStoreBase
+    graph: NeatGraphStore
 
 
 class ClientData(DataContract):
@@ -130,14 +115,3 @@ class Edges(DataContract):
     """
 
     edges: list[EdgeApply]
-
-
-class DMSSchemaComponentsData(DataContract):
-    """
-    This represents DMS Schema Model.
-
-    Args:
-        components: DMS Schema Components model.
-    """
-
-    components: DMSSchemaComponents

cognite/neat/workflows/steps/lib/current/graph_extractor.py

@@ -3,11 +3,15 @@ import logging
 from pathlib import Path
 from typing import ClassVar, cast
 
+from rdflib import URIRef
+
+from cognite.neat.constants import DEFAULT_NAMESPACE
+from cognite.neat.graph.extractors import RdfFileExtractor
 from cognite.neat.graph.extractors._mock_graph_generator import MockGraphGenerator
 from cognite.neat.rules._shared import DMSRules, InformationRules
 from cognite.neat.workflows._exceptions import StepNotInitialized
 from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
-from cognite.neat.workflows.steps.data_contracts import MultiRuleData, SolutionGraph, SourceGraph
+from cognite.neat.workflows.steps.data_contracts import MultiRuleData, NeatGraph
 from cognite.neat.workflows.steps.step_model import Configurable, Step
 
 __all__ = ["GraphFromRdfFile", "GraphFromMockData"]
@@ -29,11 +33,16 @@ class GraphFromMockData(Step):
             value="",
             label="Target number of instances for each class",
         ),
-        Configurable(name="Graph", value="solution", label="The name of target graph.", options=["source", "solution"]),
+        Configurable(
+            name="Graph",
+            value="solution",
+            label="The name of target graph.",
+            options=["source", "solution"],
+        ),
     ]
 
     def run(  # type: ignore[override, syntax]
-        self, rules: MultiRuleData, graph_store: SolutionGraph | SourceGraph
+        self, rules: MultiRuleData, graph_store: NeatGraph
     ) -> FlowMessage:
         if self.configs is None:
             raise StepNotInitialized(type(self).__name__)
@@ -55,17 +64,12 @@ class GraphFromMockData(Step):
                 step_execution_status=StepExecutionStatus.ABORT_AND_FAIL,
             )
 
-        if self.configs["Graph"] == "solution":
-            # Todo Anders: Why is the graph fetched from context when it is passed as an argument?
-            graph_store = cast(SourceGraph | SolutionGraph, self.flow_context["SolutionGraph"])
-        else:
-            graph_store = cast(SourceGraph | SolutionGraph, self.flow_context["SourceGraph"])
-
-        logging.info("Initiated generation of mock triples")
+        extractor = MockGraphGenerator(
+            cast(InformationRules | DMSRules, rules.information or rules.dms),
+            class_count,
+        )
 
-        extractor = MockGraphGenerator(cast(InformationRules | DMSRules, rules.information or rules.dms), class_count)
-
-        graph_store.graph.add_triples(extractor.extract())
+        NeatGraph.graph.write(extractor)
 
         return FlowMessage(output_text=f"Instances loaded to the {graph_store.__class__.__name__}")
 
@@ -104,20 +108,20 @@ class GraphFromRdfFile(Step):
         ),
     ]
 
-    def run(self, source_graph: SourceGraph) -> FlowMessage:  # type: ignore[override, syntax]
+    def run(self, graph_store: NeatGraph) -> FlowMessage:  # type: ignore[override, syntax]
         if self.configs is None or self.data_store_path is None:
             raise StepNotInitialized(type(self).__name__)
-        if source_graph.graph.rdf_store_type.lower() in ("memory", "oxigraph"):
-            if source_file := self.configs["File path"]:
-                source_graph.graph.import_from_file(
-                    self.data_store_path / Path(source_file),
+
+        if source_file := self.configs["File path"]:
+            NeatGraph.graph.write(
+                RdfFileExtractor(  # type: ignore[abstract]
+                    filepath=self.data_store_path / Path(source_file),
                     mime_type=self.configs["MIME type"],  # type: ignore[arg-type]
-                    add_base_iri=self.configs["Add base URI"] == "True",
+                    base_uri=(URIRef(DEFAULT_NAMESPACE) if self.configs["Add base URI"] == "True" else None),
                 )
-                logging.info(f"Loaded {source_file} into source graph.")
-            else:
-                raise ValueError("You need a source_rdf_store.file specified for source_rdf_store.type=memory")
+            )
+
         else:
-            raise NotImplementedError(f"Graph type {source_graph.graph.rdf_store_type} is not supported.")
+            raise ValueError("You need a valid file path to be specified")
 
-        return FlowMessage(output_text="Instances loaded to source graph")
+        return FlowMessage(output_text="Instances loaded to NeatGraph!")

cognite/neat/workflows/steps/lib/current/graph_loader.py

@@ -1,13 +1,10 @@
 import time
 from pathlib import Path
-from typing import ClassVar, cast
+from typing import ClassVar
 
 from cognite.neat.workflows._exceptions import StepNotInitialized
 from cognite.neat.workflows.model import FlowMessage
-from cognite.neat.workflows.steps.data_contracts import (
-    SolutionGraph,
-    SourceGraph,
-)
+from cognite.neat.workflows.steps.data_contracts import NeatGraph
 from cognite.neat.workflows.steps.step_model import Configurable, Step
 
 __all__ = [
@@ -26,12 +23,6 @@ class GraphToRdfFile(Step):
     category = CATEGORY
     version = "private-beta"
     configurables: ClassVar[list[Configurable]] = [
-        Configurable(
-            name="Graph",
-            value="source",
-            options=["source", "solution"],
-            label=("The graph to be used for loading RDF File." " Supported options : source, solution"),
-        ),
         Configurable(
             name="File path",
             value="staging/graph_export.ttl",
@@ -40,7 +31,7 @@ class GraphToRdfFile(Step):
     ]
 
     def run(  # type: ignore[override, syntax]
-        self, graph: SourceGraph | SolutionGraph
+        self, graph: NeatGraph
     ) -> FlowMessage:  # type: ignore[syntax]
         if self.configs is None or self.data_store_path is None:
             raise StepNotInitialized(type(self).__name__)
@@ -48,15 +39,7 @@ class GraphToRdfFile(Step):
         storage_path = self.data_store_path / Path(self.configs["File path"])
         relative_graph_file_path = str(storage_path).split("/data/")[1]
 
-        graph_name = self.configs["Graph"] or "source"
-
-        if graph_name == "solution":
-            # Todo Anders: Why is the graph fetched from context when it is passed as an argument?
-            graph = cast(SourceGraph | SolutionGraph, self.flow_context["SolutionGraph"])
-        else:
-            graph = cast(SourceGraph | SolutionGraph, self.flow_context["SourceGraph"])
-
-        graph.graph.serialize(str(storage_path), format="turtle")
+        graph.graph.graph.serialize(str(storage_path), format="turtle")
 
         output_text = (
             "<p></p>"

cognite/neat/workflows/steps/lib/current/graph_store.py

@@ -1,19 +1,14 @@
-import logging
-from pathlib import Path
-from typing import ClassVar, cast
+from typing import ClassVar
 
-from cognite.neat.constants import DEFAULT_NAMESPACE, get_default_prefixes
-from cognite.neat.legacy.graph import stores
-from cognite.neat.workflows._exceptions import StepNotInitialized
+from cognite.neat.graph.stores import NeatGraphStore
 from cognite.neat.workflows.model import FlowMessage
 from cognite.neat.workflows.steps.data_contracts import (
-    RulesData,
-    SolutionGraph,
-    SourceGraph,
+    MultiRuleData,
+    NeatGraph,
 )
 from cognite.neat.workflows.steps.step_model import Configurable, Step
 
-__all__ = ["GraphStoreConfiguration", "GraphStoreReset"]
+__all__ = ["GraphStoreConfiguration"]
 
 CATEGORY = __name__.split(".")[-1].replace("_", " ").title()
 
@@ -27,138 +22,27 @@ class GraphStoreConfiguration(Step):
     version = "private-beta"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
-        Configurable(
-            name="Graph",
-            value="source",
-            label="Graph categorization, supported: source, solution ",
-            options=["source", "solution"],
-        ),
         Configurable(
             name="Graph store type",
-            value=stores.OxiGraphStore.rdf_store_type,
-            label="Graph store type, supported: oxigraph, memory,file, graphdb, sparql. ",
-            options=["oxigraph", "memory", "file", "graphdb", "sparql"],
-        ),
-        Configurable(
-            name="Disk storage directory",
-            value="source-graph-store",
-            label="Local directory that is used as local graph store.Only for oxigraph, file store types",
-        ),
-        Configurable(
-            name="Query URL",
-            value="",
-            label="Query URL for SPARQL endpoint. Only for SPARQL store type",
-        ),
-        Configurable(
-            name="Update URL",
-            value="",
-            label="Update URL for SPARQL endpoint. Only for SPARQL store type",
-        ),
-        Configurable(
-            name="GraphDB API root URL",
-            value="",
-            label="Root url for GraphDB. Only for graphdb",
-        ),
-        Configurable(
-            name="Init procedure",
-            value="reset",
-            label="Operations to be performed on the graph store as part of init and configuration process. \
-              Supported options : reset, clear, none",
-            options=["reset", "none"],
+            value="oxigraph",
+            label="Graph store type, supported: oxigraph, rdflib",
+            options=["oxigraph", "rdflib"],
         ),
     ]
 
     def run(  # type: ignore[override]
-        self, rules_data: RulesData | None = None
-    ) -> (FlowMessage, SourceGraph | SolutionGraph):  # type: ignore[syntax]
-        if self.configs is None or self.data_store_path is None:
-            raise StepNotInitialized(type(self).__name__)
-        logging.info("Initializing graph")
-        store_dir = self.data_store_path / Path(value) if (value := self.configs["Disk storage directory"]) else None
-        store_type = self.configs["Graph store type"]
-        graph_name_mapping = {"source": "SourceGraph", "solution": "SolutionGraph"}
+        self, rules_data: MultiRuleData | None = None
+    ) -> (FlowMessage, NeatGraph):  # type: ignore[syntax]
+        store_type = self.configs.get("Graph store type", "oxigraph")
 
-        graph_name = graph_name_mapping[self.configs["Graph"]]
-        graph_store = cast(SourceGraph | SolutionGraph | None, self.flow_context.get(graph_name, None))
-        if self.configs["Init procedure"] == "reset":
-            logging.info("Resetting graph")
-            reset_store(store_dir, graph_store.graph if graph_store else None)
-            if graph_name in self.flow_context:
-                del self.flow_context[graph_name]
-            graph_store = None
-            logging.info("Graph reset complete")
-
-        prefixes = rules_data.rules.prefixes if rules_data else get_default_prefixes()
-
-        if store_type == stores.OxiGraphStore.rdf_store_type and graph_store is not None:
-            # OXIGRAPH doesn't like to be initialized twice without a good reason
-            graph_store.graph.upsert_prefixes(prefixes)
-            return FlowMessage(output_text="Stores already configured")
-        try:
-            store_cls = stores.STORE_BY_TYPE[store_type]
-        except KeyError:
-            return FlowMessage(output_text="Invalid store type")
-
-        new_graph_store = store_cls(prefixes=prefixes, base_prefix="neat", namespace=DEFAULT_NAMESPACE)
-        new_graph_store.init_graph(
-            self.configs["Query URL"],
-            self.configs["Update URL"],
-            "neat-tnt",
-            internal_storage_dir=store_dir,
-        )
+        if store_type == "oxigraph":
+            store = NeatGraph(graph=NeatGraphStore.from_oxi_store(rules=rules_data.information if rules_data else None))
+        else:
+            store = NeatGraph(
+                graph=NeatGraphStore.from_memory_store(rules=rules_data.information if rules_data else None)
+            )
 
         return (
             FlowMessage(output_text="Graph store configured successfully"),
-            (
-                SourceGraph(graph=new_graph_store)
-                if graph_name == "SourceGraph"
-                else SolutionGraph(graph=new_graph_store)
-            ),
-        )
-
-
-def reset_store(data_store_dir: Path | None, graph_store: stores.NeatGraphStoreBase | None = None):
-    if isinstance(graph_store, stores.OxiGraphStore):
-        if graph_store:
-            graph_store.close()
-            graph_store.drop_graph_store_storage(data_store_dir)
-        elif data_store_dir:
-            graph_store.drop_graph_store_storage(data_store_dir)
-    elif isinstance(graph_store, stores.GraphDBStore):
-        if graph_store:
-            graph_store.drop()
-            graph_store.reinitialize_graph()
-    return None
-
-
-class GraphStoreReset(Step):
-    """
-    This step resets graph stores to their initial state (clears all data)
-    """
-
-    description = "This step resets graph stores to their initial state (clears all data)."
-    category = CATEGORY
-    version = "private-alpha"
-    configurables: ClassVar[list[Configurable]] = [
-        Configurable(
-            name="Graph",
-            value="source",
-            label="Graph store to be reset. Supported: solution, source ",
-            options=["source", "solution"],
+            store,
         )
-    ]
-
-    def run(self) -> FlowMessage:  # type: ignore[override, syntax]
-        if self.configs is None or self.data_store_path is None:
-            raise StepNotInitialized(type(self).__name__)
-        graph_name_mapping = {"source": "SourceGraph", "solution": "SolutionGraph"}
-
-        graph_name = graph_name_mapping[self.configs["Graph"]]
-        graph_store = cast(SourceGraph | SolutionGraph | None, self.flow_context.get(graph_name, None))
-        if graph_store is not None:
-            reset_store(graph_store.graph.internal_storage_dir, graph_store.graph)
-            if graph_name in self.flow_context:
-                del self.flow_context[graph_name]
-            return FlowMessage(output_text="Reset operation completed")
-        else:
-            return FlowMessage(output_text="Stores already reset")

cognite/neat/workflows/steps_registry.py

@@ -12,7 +12,6 @@ from pydantic import BaseModel
 # steps
 import cognite.neat.workflows.steps.lib.current
 import cognite.neat.workflows.steps.lib.io
-import cognite.neat.workflows.steps.lib.legacy
 from cognite.neat.app.monitoring.metrics import NeatMetricsCollector
 from cognite.neat.config import Config
 from cognite.neat.exceptions import InvalidWorkFlowError
@@ -42,7 +41,11 @@ class StepsRegistry:
         self._step_classes: list[type[Step]] = []
         self.user_steps_path: Path = config.data_store_path / "steps"
         self.data_store_path: str = str(config.data_store_path)
-        self.categorized_steps: dict[str, set] = {"legacy": set(), "current": set(), "io": set()}
+        self.categorized_steps: dict[str, set] = {
+            "legacy": set(),
+            "current": set(),
+            "io": set(),
+        }
 
     def load_step_classes(self):
         if self._step_classes:
@@ -58,11 +61,6 @@ class StepsRegistry:
                 logging.info(f"Loading NEAT step {name}")
                 self._step_classes.append(step_cls)
                 self.categorized_steps["io"].add(name)
-        for name, step_cls in inspect.getmembers(cognite.neat.workflows.steps.lib.legacy):
-            if inspect.isclass(step_cls):
-                logging.info(f"Loading NEAT step {name}")
-                self._step_classes.append(step_cls)
-                self.categorized_steps["legacy"].add(name)
         sys.path.append(str(Path(self.data_store_path) / "workflows"))
         try:
             if self.user_steps_path:

{cognite_neat-0.87.6.dist-info → cognite_neat-0.88.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.87.6
+Version: 0.88.1
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0
@@ -146,9 +146,6 @@ Open API docs :
 
 `http://localhost:8000/docs`
 
-Prom metrics :
-
-`http://localhost:8000/metrics`
 
 ## Workflows and configurations
 
@@ -194,8 +191,7 @@ cdf_client:
   client_id: "623c2450-cfc2-43d6-9036-10e14dad8ccf"
   client_secret: "my-super-secret"
   base_url: https://az-power-no-northeurope.cognitedata.com
-  scopes:
-    - https://az-power-no-northeurope.cognitedata.com/.default
+  scopes: [https://az-power-no-northeurope.cognitedata.com/.default]
   token_url: https://login.microsoftonline.com/e55e1701-82f8-4c52-af10-28e4d942c589/oauth2/v2.0/token
 
 cdf_default_dataset_id: 2626756768281823