cognite-neat 0.75.7__py3-none-any.whl → 0.75.9__py3-none-any.whl

cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_extractor.py

@@ -29,7 +29,7 @@ __all__ = [
     "ExtractGraphFromDexpiFile",
 ]
 
-CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [VERSION 1]"
+CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
 
 
 class ExtractGraphFromRdfFile(Step):
@@ -38,7 +38,7 @@ class ExtractGraphFromRdfFile(Step):
     """
 
     description = "This step extract instances from a file into the source graph. The file must be in RDF format."
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -91,7 +91,7 @@ class ExtractGraphFromDexpiFile(Step):
     """
 
     description = "This step converts DEXPI P&ID (XML) into Knowledge Graph"
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -130,7 +130,7 @@ class ExtractGraphFromGraphCapturingSheet(Step):
     """
 
     description = "This step extracts nodes and edges from graph capturing spreadsheet and load them into graph"
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -187,7 +187,7 @@ class ExtractGraphFromMockGraph(Step):
     """
 
     description = "This step generate mock graph based on the defined classes and target number of instances"
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -239,7 +239,7 @@ class ExtractGraphFromRulesInstanceSheet(Step):
 
     description = "This step extracts instances from Rules object and loads them into the graph."
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
 
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -276,7 +276,7 @@ class ExtractGraphFromRulesDataModel(Step):
 
     description = "This step extracts data model from rules file and loads it into source graph."
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
 
     def run(  # type: ignore[override, syntax]
         self, transformation_rules: RulesData, source_graph: SourceGraph
@@ -321,7 +321,7 @@ class ExtractGraphFromJsonFile(Step):
 
     description = "This step extracts instances from json file and loads them into a graph store"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="file_name", value="data_dump.json", label="Full path to the file containing data dump in JSON format"
@@ -504,7 +504,7 @@ class ExtractGraphFromAvevaPiAssetFramework(Step):
 
     description = "This step extracts instances from Aveva PI AF and loads them into a graph store"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="file_name",

cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_loader.py

@@ -52,7 +52,7 @@ __all__ = [
     "LoadGraphToRdfFile",
 ]
 
-CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [VERSION 1]"
+CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
 
 
 class LoadLabelsToCDF(Step):
@@ -62,7 +62,7 @@ class LoadLabelsToCDF(Step):
 
     description = "This step creates default NEAT labels in CDF"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(name="data_set_id", value="", label=("CDF dataset id to which the labels will be added."))
     ]
@@ -176,7 +176,7 @@ class LoadGraphToRdfFile(Step):
 
     description = "The step generates nodes and edges from the graph"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="graph_name",
@@ -227,7 +227,7 @@ class LoadNodesToCDF(Step):
 
     description = "This step uploads nodes to CDF"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
 
     def run(self, cdf_client: CogniteClient, nodes: Nodes) -> FlowMessage:  # type: ignore[override, syntax]
         if nodes.nodes:
@@ -244,7 +244,7 @@ class LoadEdgesToCDF(Step):
 
     description = "This step uploads edges to CDF"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
 
     def run(self, cdf_client: CogniteClient, edges: Edges) -> FlowMessage:  # type: ignore[override, syntax]
         if edges.edges:
@@ -263,7 +263,7 @@ class GenerateAssetsFromGraph(Step):
         "The step generates assets from the graph ,categorizes them and stores them in CategorizedAssets object"
     )
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(name="data_set_id", value="", label=("CDF dataset id to which the labels will be added.")),
         Configurable(
@@ -469,7 +469,7 @@ class LoadAssetsToCDF(Step):
 
     description = "This step uploads categorized assets to CDF"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
 
     def run(  # type: ignore[override]
         self, cdf_client: CogniteClient, categorized_assets: CategorizedAssets, flow_msg: FlowMessage
@@ -528,7 +528,7 @@ class GenerateRelationshipsFromGraph(Step):
 
     description = "This step generates relationships from the graph and saves them to CategorizedRelationships object"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(name="data_set_id", value="", label=("CDF dataset id to which the labels will be added.")),
         Configurable(
@@ -597,7 +597,7 @@ class LoadRelationshipsToCDF(Step):
 
     description = "This step uploads relationships to CDF"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
 
     def run(  # type: ignore[override, syntax]
         self, client: CogniteClient, categorized_relationships: CategorizedRelationships

cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_store.py

@@ -11,7 +11,7 @@ from cognite.neat.workflows.steps.step_model import Configurable, Step
 
 __all__ = ["ResetGraphStores", "ConfigureGraphStore"]
 
-CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [VERSION 1]"
+CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
 
 
 class ConfigureDefaultGraphStores(Step):
@@ -21,7 +21,7 @@ class ConfigureDefaultGraphStores(Step):
 
     description = "This step initializes the source and solution graph stores."
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="source_rdf_store.type",
@@ -154,7 +154,7 @@ class ResetGraphStores(Step):
 
     description = "This step resets graph stores to their initial state (clears all data)."
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="graph_name",
@@ -186,7 +186,7 @@ class ConfigureGraphStore(Step):
     """
 
     description = "This step initializes the source and solution graph stores."
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(

cognite/neat/workflows/steps/lib/{v1 → legacy}/graph_transformer.py

@@ -10,7 +10,7 @@ from cognite.neat.workflows.steps.step_model import Configurable, Step
 
 __all__ = ["TransformSourceToSolutionGraph"]
 
-CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [VERSION 1]"
+CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
 
 
 class TransformSourceToSolutionGraph(Step):
@@ -20,7 +20,7 @@ class TransformSourceToSolutionGraph(Step):
 
     description = "The step transforms source graph to solution graph"
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="cdf_lookup_database",

cognite/neat/workflows/steps/lib/{v1 → legacy}/rules_exporter.py

@@ -30,7 +30,7 @@ __all__ = [
     "DeleteDMSSchemaComponents",
 ]
 
-CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [VERSION 1]"
+CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
 
 
 class GenerateDMSSchemaComponentsFromRules(Step):
@@ -39,7 +39,7 @@ class GenerateDMSSchemaComponentsFromRules(Step):
     """
 
     description = "This step generates DMS Schema components, such as data model, views, containers, etc. from Rules."
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
 
     def run(self, rules: RulesData) -> (FlowMessage, DMSSchemaComponentsData):  # type: ignore[override, syntax]
@@ -66,10 +66,9 @@ class ExportDMSSchemaComponentsToYAML(Step):
     """
 
     description = "This step exports DMS schema components as YAML files"
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
-        Configurable(name="storage_dir", value="staging", label="Directory to store DMS schema files"),
         Configurable(
             name="format",
             value="yaml-dump",
@@ -82,10 +81,9 @@ class ExportDMSSchemaComponentsToYAML(Step):
         if self.configs is None or self.data_store_path is None:
             raise StepNotInitialized(type(self).__name__)
 
-        staging_dir_str = self.configs["storage_dir"]
         format_ = self.configs["format"]
 
-        staging_dir = self.data_store_path / Path(staging_dir_str)
+        staging_dir = self.config.staging_path
         staging_dir.mkdir(parents=True, exist_ok=True)
 
         if format_ in ["yaml-dump", "all"]:
@@ -119,10 +117,10 @@ class ExportDMSSchemaComponentsToYAML(Step):
                 "<p></p>"
                 "DMS Schema exported and can be downloaded here : "
                 "<p></p>"
-                f'- <a href="/data/{staging_dir_str}/{_data_model_file_name}?{time.time()}" '
+                f'- <a href="/data/{self.config.staging_path.name}/{_data_model_file_name}?{time.time()}" '
                 f'target="_blank">{_data_model_file_name}</a>'
                 "<p></p>"
-                f'- <a href="/data/{staging_dir_str}/{_container_file_name}?{time.time()}" '
+                f'- <a href="/data/{self.config.staging_path.name}/{_container_file_name}?{time.time()}" '
                 f'target="_blank">{_container_file_name}</a>'
             )
 
@@ -140,7 +138,7 @@ class ExportDMSSchemaComponentsToCDF(Step):
     """
 
     description = "This step exports generated DMS Schema components to CDF."
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
 
     configurables: ClassVar[list[Configurable]] = [
@@ -205,7 +203,7 @@ class ExportDMSSchemaComponentsToCDF(Step):
 
         # report
         report_file = "dms_component_creation_report.txt"
-        report_dir = self.data_store_path / Path("staging")
+        report_dir = self.config.staging_path
         report_dir.mkdir(parents=True, exist_ok=True)
         report_full_path = report_dir / report_file
         report_full_path.write_text(report)
@@ -229,7 +227,7 @@ class DeleteDMSSchemaComponents(Step):
     """
 
     description = "This step deletes DMS Data model and all underlying containers and views."
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
 
     configurables: ClassVar[list[Configurable]] = [
@@ -280,7 +278,7 @@ class DeleteDMSSchemaComponents(Step):
 
         # report
         report_file = "dms_component_removal_report.txt"
-        report_dir = self.data_store_path / Path("staging")
+        report_dir = self.config.staging_path
         report_dir.mkdir(parents=True, exist_ok=True)
         report_full_path = report_dir / report_file
         report_full_path.write_text(report)
@@ -304,7 +302,7 @@ class ExportRulesToGraphQLSchema(Step):
     """
 
     description = "This step generates GraphQL schema from data model defined in transformation rules."
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -354,7 +352,7 @@ class ExportRulesToOntology(Step):
     """
 
     description = "This step exports Rules to OWL ontology"
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -410,7 +408,7 @@ class ExportRulesToSHACL(Step):
     """
 
     description = "This step exports Rules to SHACL"
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -463,7 +461,7 @@ class ExportRulesToGraphCapturingSheet(Step):
     """
 
     description = "This step generates graph capturing sheet"
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(name="file_name", value="graph_capture_sheet.xlsx", label="File name of the data capture sheet"),
@@ -499,7 +497,7 @@ class ExportRulesToGraphCapturingSheet(Step):
 
 class ExportRulesToExcel(Step):
     description = "This step export Rules to Excel representation"
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(

cognite/neat/workflows/steps/lib/{v1 → legacy}/rules_importer.py

@@ -21,7 +21,7 @@ from cognite.neat.workflows.model import FlowMessage, StepExecutionStatus
 from cognite.neat.workflows.steps.data_contracts import RulesData, SolutionGraph, SourceGraph
 from cognite.neat.workflows.steps.step_model import Configurable, Step
 
-CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [VERSION 1]"
+CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
 
 __all__ = [
     "ImportExcelToRules",
@@ -37,7 +37,7 @@ class ImportExcelToRules(Step):
     """
 
     description = "This step import rules from the Excel file"
-    version = "private-alpha"
+    version = "legacy"
     category = CATEGORY
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
@@ -68,7 +68,7 @@ class ImportExcelToRules(Step):
             raise ValueError(f"Step {type(self).__name__} has not been configured.")
         rules_file = Path(self.configs["file_name"])
         if str(rules_file.parent) == ".":
-            rules_file_path = Path(self.data_store_path) / "rules" / rules_file
+            rules_file_path = self.config.rules_store_path / rules_file
         else:
             rules_file_path = Path(self.data_store_path) / rules_file
 
@@ -134,7 +134,7 @@ class ImportOntologyToRules(Step):
     description = "The step extracts NEAT rules object from OWL Ontology and \
         exports them as an Excel rules files for further editing."
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="ontology_file_path", value="staging/ontology.ttl", label="Relative path to the OWL ontology file."
@@ -191,7 +191,7 @@ class ImportOpenApiToRules(Step):
     description = "The step extracts schema from OpenAPI specification and generates NEAT transformation rules object. \
     The rules object can be serialized to excel file or used directly in other steps."
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="openapi_spec_file_path",
@@ -377,7 +377,7 @@ class ImportArbitraryJsonYamlToRules(Step):
 
     description = "The step extracts schema from arbitrary json file and generates NEAT transformation rules object."
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="file_path",
@@ -564,7 +564,7 @@ class ImportGraphToRules(Step):
     description = "The step extracts data model from RDF graph and generates NEAT transformation rules object. \
     The rules object can be serialized to excel file or used directly in other steps."
     category = CATEGORY
-    version = "private-alpha"
+    version = "legacy"
     configurables: ClassVar[list[Configurable]] = [
         Configurable(
             name="excel_file_path", value="staging/rules.xlsx", label="Relative path for the Excel rules storage."

cognite/neat/workflows/steps/step_model.py

@@ -1,18 +1,14 @@
 import typing
 from abc import ABC, abstractmethod
-from pathlib import Path
 from typing import ClassVar, TypeVar
 
 from pydantic import BaseModel, ConfigDict
 
 from cognite.neat.app.monitoring.metrics import NeatMetricsCollector
+from cognite.neat.config import Config
 from cognite.neat.workflows.model import FlowMessage, WorkflowConfigs
 
 
-class Config(BaseModel):
-    ...
-
-
 class Configurable(BaseModel):
     name: str
     value: str | None = None
@@ -45,7 +41,7 @@ class Step(ABC):
     )
     docs_url: str = "https://cognite-neat.readthedocs-hosted.com/en/latest/"  # url to the documentation of the step
 
-    def __init__(self, data_store_path: Path | None = None):
+    def __init__(self, config: Config):
         self.log: bool = False
         self.configs: dict[str, str] = {}
         self.complex_configs: dict[
@@ -53,7 +49,8 @@ class Step(ABC):
         ] = {}  # complex configs are meant for more complex configurations. Value can be any type.
         self.workflow_id: str = ""
         self.workflow_run_id: str = ""
-        self.data_store_path = Path(data_store_path) if data_store_path is not None else Path.cwd()
+        self.config = config
+        self.data_store_path = config.data_store_path
 
     @property
     def _not_configured_message(self) -> str:
@@ -79,5 +76,4 @@ class Step(ABC):
         self.flow_context = context
 
     @abstractmethod
-    def run(self, *input_data: DataContract) -> DataContract | tuple[FlowMessage, DataContract] | FlowMessage:
-        ...
+    def run(self, *input_data: DataContract) -> DataContract | tuple[FlowMessage, DataContract] | FlowMessage: ...

cognite/neat/workflows/steps_registry.py

@@ -9,9 +9,12 @@ from typing import Any
 
 from pydantic import BaseModel
 
-import cognite.neat.workflows.steps.lib
-import cognite.neat.workflows.steps.lib.v1
+# steps
+import cognite.neat.workflows.steps.lib.current
+import cognite.neat.workflows.steps.lib.io
+import cognite.neat.workflows.steps.lib.legacy
 from cognite.neat.app.monitoring.metrics import NeatMetricsCollector
+from cognite.neat.config import Config
 from cognite.neat.exceptions import InvalidWorkFlowError
 from cognite.neat.workflows._exceptions import ConfigurationNotSet
 from cognite.neat.workflows.model import FlowMessage, WorkflowConfigs
@@ -34,26 +37,32 @@ class StepMetadata(BaseModel):
 
 
 class StepsRegistry:
-    def __init__(self, data_store_path: Path | None = None):
+    def __init__(self, config: Config):
+        self.config = config
         self._step_classes: list[type[Step]] = []
-        if data_store_path:
-            self.user_steps_path: Path | None = Path(data_store_path) / "steps"
-        else:
-            self.user_steps_path = None
-        self.data_store_path: str = str(data_store_path)
+        self.user_steps_path: Path = config.data_store_path / "steps"
+        self.data_store_path: str = str(config.data_store_path)
+        self.categorized_steps: dict[str, set] = {"legacy": set(), "current": set(), "io": set()}
 
     def load_step_classes(self):
         if self._step_classes:
             # classes already loaded - no need to reload
             return
-        for name, step_cls in inspect.getmembers(cognite.neat.workflows.steps.lib):
+        for name, step_cls in inspect.getmembers(cognite.neat.workflows.steps.lib.current):
             if inspect.isclass(step_cls):
                 logging.info(f"Loading NEAT step {name}")
                 self._step_classes.append(step_cls)
-        for name, step_cls in inspect.getmembers(cognite.neat.workflows.steps.lib.v1):
+                self.categorized_steps["current"].add(name)
+        for name, step_cls in inspect.getmembers(cognite.neat.workflows.steps.lib.io):
             if inspect.isclass(step_cls):
                 logging.info(f"Loading NEAT step {name}")
                 self._step_classes.append(step_cls)
+                self.categorized_steps["io"].add(name)
+        for name, step_cls in inspect.getmembers(cognite.neat.workflows.steps.lib.legacy):
+            if inspect.isclass(step_cls):
+                logging.info(f"Loading NEAT step {name}")
+                self._step_classes.append(step_cls)
+                self.categorized_steps["legacy"].add(name)
         sys.path.append(str(Path(self.data_store_path) / "workflows"))
         try:
             if self.user_steps_path:
@@ -124,7 +133,7 @@ class StepsRegistry:
             step_complex_configs = {}
         for step_cls in self._step_classes:
             if step_cls.__name__ == step_name:
-                step_obj: Step = step_cls(Path(self.data_store_path))
+                step_obj: Step = step_cls(config=self.config)
                 step_obj.configure(step_configs, step_complex_configs)
                 step_obj.set_flow_context(flow_context)
                 step_obj.set_metrics(metrics)

{cognite_neat-0.75.7.dist-info → cognite_neat-0.75.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cognite-neat
-Version: 0.75.7
+Version: 0.75.9
 Summary: Knowledge graph transformation
 Home-page: https://cognite-neat.readthedocs-hosted.com/
 License: Apache-2.0