cognite-neat 0.75.8__py3-none-any.whl → 0.76.0__py3-none-any.whl

cognite/neat/legacy/workflows/examples/Ontology_to_Data_Model/workflow.yaml

@@ -0,0 +1,116 @@
+configs: []
+description: null
+implementation_module: null
+name: Ontology to Data Model
+steps:
+  - complex_configs: {}
+    configs: {}
+    description: null
+    enabled: true
+    id: step_trigger
+    label: Trigger
+    max_retries: 0
+    method: null
+    params: {}
+    retry_delay: 3
+    stype: http_trigger
+    system_component_id: null
+    transition_to:
+      - step_128344
+      - dowload_ontology
+    trigger: true
+    ui_config:
+      pos_x: 524
+      pos_y: 259
+  - complex_configs: {}
+    configs:
+      excel_file_path: rules/LIS-14-rules.xlsx
+      make_compliant: "True"
+      ontology_file_path: staging/LIS-14.ttl
+    description: null
+    enabled: true
+    id: onotology2rules
+    label: Import Ontology to Rules
+    max_retries: 0
+    method: ImportOntologyToRules
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to:
+      - load_generated_rules
+      - import_generated_rules
+    trigger: false
+    ui_config:
+      pos_x: 524
+      pos_y: 403
+  - complex_configs: {}
+    configs:
+      file_name: LIS-14-rules.xlsx
+      validation_report_file: rules_validation_report.txt
+      validation_report_storage_dir: rules_validation_report
+      version: ""
+    description: null
+    enabled: true
+    id: import_generated_rules
+    label: Import Generated Rules
+    max_retries: 0
+    method: ImportExcelToRules
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to:
+      - step_59017
+      - step_829062
+      - export_rules_to_graphql_schema
+    trigger: false
+    ui_config:
+      pos_x: 524
+      pos_y: 490
+  - complex_configs: {}
+    configs:
+      api_url: https://rds.posccaesar.org/ontology/lis14/ont/core/2.0/LIS-14.ttl
+      auth_mode: none
+      http_method: GET
+      output_file_path: staging/LIS-14.ttl
+      password: ""
+      response_destination: file
+      token: ""
+      username: ""
+    description: null
+    enabled: true
+    id: dowload_ontology
+    label: Download Ontology
+    max_retries: 0
+    method: DownloadDataFromRestApiToFile
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to:
+      - onotology2rules
+    trigger: false
+    ui_config:
+      pos_x: 524
+      pos_y: 333
+  - complex_configs: {}
+    configs:
+      file_name: ""
+      storage_dir: staging
+    description: null
+    enabled: true
+    id: export_rules_to_graphql_schema
+    label: Export Rules to GraphQL Schema
+    max_retries: 0
+    method: ExportRulesToGraphQLSchema
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to: []
+    trigger: false
+    ui_config:
+      pos_x: 525
+      pos_y: 577
+system_components: []

cognite/neat/legacy/workflows/examples/Validate_Rules/workflow.yaml

@@ -0,0 +1,67 @@
+configs: []
+description: null
+implementation_module: null
+name: Validate Rules
+steps:
+-   complex_configs: {}
+    configs:
+        File name: ''
+        Report formatter: BasicHTML
+        Role: infer
+    description: null
+    enabled: true
+    id: step_validate_rules
+    label: Validate Rules
+    max_retries: 0
+    method: ExcelToRules
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to:
+    - step_942973
+    - step_715590
+    trigger: false
+    ui_config:
+        pos_x: 558
+        pos_y: 97
+-   complex_configs: {}
+    configs: {}
+    description: null
+    enabled: true
+    id: step_upload_rules
+    label: Upload Rules Spreadsheets
+    max_retries: 0
+    method: null
+    params:
+        file_type: rules
+    retry_delay: 3
+    stype: file_uploader
+    system_component_id: null
+    transition_to:
+    - step_31642
+    - step_validate_rules
+    trigger: true
+    ui_config:
+        pos_x: 558
+        pos_y: -14
+-   complex_configs: {}
+    configs:
+        Output role format: input
+        Styling: default
+    description: null
+    enabled: true
+    id: step_715590
+    label: Convert Rules
+    max_retries: 0
+    method: RulesToExcel
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to: []
+    trigger: false
+    ui_config:
+        pos_x: 558
+        pos_y: 217
+system_components: []

cognite/neat/legacy/workflows/examples/Validate_Solution_Model/workflow.yaml

@@ -0,0 +1,64 @@
+configs: []
+description: null
+implementation_module: null
+name: Validate Solution Model
+steps:
+-   complex_configs: {}
+    configs: {}
+    description: null
+    enabled: true
+    id: step_769298
+    label: Upload File
+    max_retries: 0
+    method: null
+    params:
+        file_type: rules
+    retry_delay: 3
+    stype: file_uploader
+    system_component_id: null
+    transition_to:
+    - step_399494
+    trigger: true
+    ui_config:
+        pos_x: 627
+        pos_y: -19
+-   complex_configs: {}
+    configs:
+        File name: ''
+        Report formatter: BasicHTML
+        Role: infer
+    description: null
+    enabled: true
+    id: step_399494
+    label: Import Excel
+    max_retries: 0
+    method: ExcelToRules
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to:
+    - step_273233
+    trigger: false
+    ui_config:
+        pos_x: 627
+        pos_y: 66
+-   complex_configs: {}
+    configs:
+        Report Formatter: BasicHTML
+    description: null
+    enabled: true
+    id: step_273233
+    label: Validate Against CDF
+    max_retries: 0
+    method: ValidateRulesAgainstCDF
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to: []
+    trigger: false
+    ui_config:
+        pos_x: 627
+        pos_y: 154
+system_components: []

cognite/neat/legacy/workflows/examples/Visualize_Data_Model_Using_Mock_Graph/workflow.yaml

@@ -0,0 +1,95 @@
+configs: []
+description: null
+implementation_module: null
+name: Visualize Data Model Using Mock Graph
+steps:
+  - complex_configs: {}
+    configs:
+      File name: information-architect-david.xlsx
+      Report formatter: BasicHTML
+      Role: infer
+    description: null
+    enabled: true
+    id: step_verify_rules
+    label: Verify Rules
+    max_retries: 0
+    method: ExcelToRules
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to:
+      - step_configure_graph_store
+    trigger: false
+    ui_config:
+      pos_x: 507
+      pos_y: 250
+  - complex_configs: {}
+    configs:
+      Disk storage directory: mock-graph-store
+      Graph: source
+      Graph store type: oxigraph
+      GraphDB API root URL: ""
+      Init procedure: reset
+      Query URL: ""
+      Update URL: ""
+    description: null
+    enabled: true
+    id: step_configure_graph_store
+    label: Configure Graph Store
+    max_retries: 0
+    method: GraphStoreConfiguration
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to:
+      - step_166101
+      - step_mock_graph_generation
+    trigger: false
+    ui_config:
+      pos_x: 506
+      pos_y: 314
+  - complex_configs: {}
+    configs:
+      Class count:
+        '{"WindTurbine" : 1, "WindFarm" : 1, "OffshoreSubstation" : 1,
+        "DistributionSubstation" : 1, "OnshoreSubstation" : 1, "ArrayCable" :
+        1, "ExportCable" : 1, "Transmission" : 1, "DistributionLine" : 1, "Meter"
+        : 1, "ElectricCarCharger" : 1}'
+      Graph: source
+    description: null
+    enabled: true
+    id: step_mock_graph_generation
+    label: Generate Mock Graph
+    max_retries: 0
+    method: GraphFromMockData
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to: []
+    trigger: false
+    ui_config:
+      pos_x: 506
+      pos_y: 390
+  - complex_configs: {}
+    configs: {}
+    description: null
+    enabled: true
+    id: step_upload_rules
+    label: Upload Rules
+    max_retries: 0
+    method: null
+    params:
+      file_type: rules
+    retry_delay: 3
+    stype: file_uploader
+    system_component_id: null
+    transition_to:
+      - step_verify_rules
+    trigger: true
+    ui_config:
+      pos_x: 507
+      pos_y: 177
+system_components: []

cognite/neat/legacy/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml

@@ -0,0 +1,111 @@
+configs: []
+description: null
+implementation_module: null
+name: Visualize Semantic Data Model
+steps:
+-   complex_configs: {}
+    configs: {}
+    description: null
+    enabled: true
+    id: step_upload_excel_rules
+    label: Upload Excel Rules
+    max_retries: 0
+    method: null
+    params:
+        file_type: rules
+    retry_delay: 3
+    stype: file_uploader
+    system_component_id: null
+    transition_to:
+    - step_421523
+    - step_excel_to_rules
+    trigger: true
+    ui_config:
+        pos_x: 498
+        pos_y: 149
+-   complex_configs: {}
+    configs:
+        File name: ''
+        Report formatter: BasicHTML
+        Role: infer
+    description: null
+    enabled: true
+    id: step_excel_to_rules
+    label: Import and Validate Rules
+    max_retries: 0
+    method: ExcelToRules
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to:
+    - step_rules_to_ontology
+    trigger: false
+    ui_config:
+        pos_x: 497
+        pos_y: 234
+-   complex_configs: {}
+    configs:
+        File path: staging/semantic-data-model.ttl
+    description: null
+    enabled: true
+    id: step_rules_to_ontology
+    label: Rules to Ontology
+    max_retries: 0
+    method: RulesToSemanticDataModel
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to:
+    - step_configure_graph_store
+    trigger: false
+    ui_config:
+        pos_x: 497
+        pos_y: 323
+-   complex_configs: {}
+    configs:
+        Disk storage directory: semantic-data-model
+        Graph: source
+        Graph store type: oxigraph
+        GraphDB API root URL: ''
+        Init procedure: reset
+        Query URL: ''
+        Update URL: ''
+    description: null
+    enabled: true
+    id: step_configure_graph_store
+    label: Configure Graph Store
+    max_retries: 0
+    method: GraphStoreConfiguration
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to:
+    - step_load_semantic_data_model
+    trigger: false
+    ui_config:
+        pos_x: 495
+        pos_y: 399
+-   complex_configs: {}
+    configs:
+        Add base URI: 'False'
+        File path: staging/semantic-data-model.ttl
+        MIME type: text/turtle
+    description: null
+    enabled: true
+    id: step_load_semantic_data_model
+    label: Load Semantic Data Model
+    max_retries: 0
+    method: GraphFromRdfFile
+    params: {}
+    retry_delay: 3
+    stype: stdstep
+    system_component_id: null
+    transition_to: []
+    trigger: false
+    ui_config:
+        pos_x: 495
+        pos_y: 476
+system_components: []

cognite/neat/rules/exporters/_models.py

@@ -2,6 +2,8 @@ from abc import ABC
 from dataclasses import dataclass, field
 from functools import total_ordering
 
+from cognite.neat.rules.issues import IssueList
+
 
 @total_ordering
 @dataclass
@@ -32,6 +34,7 @@ class UploadResult(UploadResultCore):
     failed_changed: int = 0
     failed_deleted: int = 0
     error_messages: list[str] = field(default_factory=list)
+    issues: IssueList = field(default_factory=IssueList)
 
     @property
     def total(self) -> int:

cognite/neat/rules/exporters/_rules2dms.py

@@ -4,10 +4,13 @@ from pathlib import Path
 from typing import Literal, TypeAlias, cast
 
 from cognite.client import CogniteClient
-from cognite.client.data_classes._base import CogniteResourceList
+from cognite.client.data_classes._base import CogniteResource, CogniteResourceList
+from cognite.client.data_classes.data_modeling import DataModelApply, DataModelId
 from cognite.client.exceptions import CogniteAPIError
 
+from cognite.neat.rules import issues
 from cognite.neat.rules._shared import Rules
+from cognite.neat.rules.issues import IssueList
 from cognite.neat.rules.models.rules import InformationRules
 from cognite.neat.rules.models.rules._base import ExtensionCategory, SheetList
 from cognite.neat.rules.models.rules._dms_architect_rules import DMSContainer, DMSRules
@@ -43,6 +46,8 @@ class DMSExporter(CDFExporter[DMSSchema]):
         export_pipeline (bool, optional): Whether to export the pipeline. Defaults to False. This means setting
             up transformations, RAW databases and tables to populate the data model.
         instance_space (str, optional): The space to use for the instance. Defaults to None.
+        suppress_warnings (bool, optional): Suppress warnings. Defaults to False.
+
     ... note::
 
         - "fail": If any component already exists, the export will fail.
@@ -59,12 +64,14 @@ class DMSExporter(CDFExporter[DMSSchema]):
         existing_handling: Literal["fail", "skip", "update", "force"] = "update",
         export_pipeline: bool = False,
         instance_space: str | None = None,
+        suppress_warnings: bool = False,
     ):
         self.export_components = {export_components} if isinstance(export_components, str) else set(export_components)
         self.include_space = include_space
         self.existing_handling = existing_handling
         self.export_pipeline = export_pipeline
         self.instance_space = instance_space
+        self.suppress_warnings = suppress_warnings
         self._schema: DMSSchema | None = None
 
     def export_to_file(self, rules: Rules, filepath: Path) -> None:
@@ -115,11 +122,11 @@ class DMSExporter(CDFExporter[DMSSchema]):
         )
         is_new_model = dms_rules.reference is None
         if is_new_model or is_solution_model:
-            return dms_rules.as_schema(self.export_pipeline, self.instance_space)
+            return dms_rules.as_schema(False, self.export_pipeline, self.instance_space)
 
         # This is an extension of an existing model.
         reference_rules = cast(DMSRules, dms_rules.reference).model_copy(deep=True)
-        reference_schema = reference_rules.as_schema(self.export_pipeline)
+        reference_schema = reference_rules.as_schema(include_ref=False, include_pipeline=self.export_pipeline)
 
         # Todo Move this to an appropriate location
         # Merging Reference with User Rules
@@ -142,7 +149,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
                 property_.reference = None
                 combined_rules.properties.append(property_)
 
-        schema = combined_rules.as_schema(self.export_pipeline, self.instance_space)
+        schema = combined_rules.as_schema(True, self.export_pipeline, self.instance_space)
 
         if dms_rules.metadata.extension in (ExtensionCategory.addition, ExtensionCategory.reshape):
             # We do not freeze views as they might be changed, even for addition,
@@ -206,6 +213,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
         redeploy_data_model = False
 
         for all_items, loader in to_export:
+            issue_list = IssueList()
             all_item_ids = loader.get_ids(all_items)
             skipped = sum(1 for item_id in all_item_ids if item_id in schema.frozen_ids)
             item_ids = [item_id for item_id in all_item_ids if item_id not in schema.frozen_ids]
@@ -249,6 +257,9 @@ class DMSExporter(CDFExporter[DMSSchema]):
             else:
                 raise ValueError(f"Unsupported existing_handling {self.existing_handling}")
 
+            warning_list = self._validate(loader, items)
+            issue_list.extend(warning_list)
+
             error_messages: list[str] = []
             if not dry_run:
                 if to_delete:
@@ -284,6 +295,7 @@ class DMSExporter(CDFExporter[DMSSchema]):
                 failed_created=failed_created,
                 failed_changed=failed_changed,
                 error_messages=error_messages,
+                issues=issue_list,
             )
 
             if loader.resource_name == "views" and (created or changed) and not redeploy_data_model:
@@ -307,3 +319,33 @@ class DMSExporter(CDFExporter[DMSSchema]):
             to_export.append((schema.raw_tables, RawTableLoader(client)))
             to_export.append((schema.transformations, TransformationLoader(client)))
         return schema, to_export
+
+    def _validate(self, loader: ResourceLoader, items: list[CogniteResource]) -> IssueList:
+        issue_list = IssueList()
+        if isinstance(loader, DataModelLoader):
+            models = cast(list[DataModelApply], items)
+            if other_models := self._exist_other_data_models(loader, models):
+                warning = issues.dms.OtherDataModelsInSpaceWarning(models[0].space, other_models)
+                if not self.suppress_warnings:
+                    warnings.warn(warning, stacklevel=2)
+                issue_list.append(warning)
+
+        return issue_list
+
+    @classmethod
+    def _exist_other_data_models(cls, loader: DataModelLoader, models: list[DataModelApply]) -> list[DataModelId]:
+        if not models:
+            return []
+        space = models[0].space
+        external_id = models[0].external_id
+        try:
+            data_models = loader.client.data_modeling.data_models.list(space=space, limit=25, all_versions=False)
+        except CogniteAPIError as e:
+            warnings.warn(issues.importing.APIWarning(str(e)), stacklevel=2)
+            return []
+        else:
+            return [
+                data_model.as_id()
+                for data_model in data_models
+                if (data_model.space, data_model.external_id) != (space, external_id)
+            ]

cognite/neat/rules/exporters/_rules2excel.py

@@ -120,13 +120,6 @@ class ExcelExporter(BaseExporter[Workbook]):
             else:
                 sheet = workbook.create_sheet(sheet_name)
 
-            # Reorder such that the first column is class + the first field of the subclass
-            # of sheet entity. This is to make the properties/classes/views/containers sheet more readable.
-            # For example, for the properties these that means class, property, name, description
-            # instead of class, name, description, property
-            move = len(SheetEntity.model_fields) - 1  # -1 is for the class field
-            headers = headers[:1] + headers[move : move + 1] + headers[1:move] + headers[move + 1 :]
-
             main_header = self._main_header_by_sheet_name[sheet_name]
             sheet.append([main_header] + [""] * (len(headers) - 1))
             sheet.merge_cells(start_row=1, start_column=1, end_row=1, end_column=len(headers))
@@ -150,8 +143,6 @@ class ExcelExporter(BaseExporter[Workbook]):
                         cell.border = Border(left=side, right=side, top=side, bottom=side)
                     fill_color = next(fill_colors)
 
-                # Need to do the same reordering as for the headers above
-                row = row[:1] + row[move : move + 1] + row[1:move] + row[move + 1 :]
                 sheet.append(row)
                 if self._styling_level > 2 and is_properties:
                     for cell in sheet[sheet.max_row]:
@@ -205,8 +196,8 @@ class ExcelExporter(BaseExporter[Workbook]):
 
     @classmethod
     def _adjust_column_widths(cls, workbook: Workbook) -> None:
-        for sheet in workbook:
-            sheet = cast(Worksheet, sheet)
+        for sheet_ in workbook:
+            sheet = cast(Worksheet, sheet_)
             for column_cells in sheet.columns:
                 try:
                     max_length = max(len(str(cell.value)) for cell in column_cells if cell.value is not None)

cognite/neat/rules/exporters/_validation.py

@@ -9,13 +9,13 @@ from cognite.neat.rules.models.rules._types._base import DMS_PROPERTY_ID_COMPLIA
 
 
 @overload
-def are_entity_names_dms_compliant(rules: InformationRules, return_report: Literal[True]) -> tuple[bool, list[dict]]:
-    ...
+def are_entity_names_dms_compliant(
+    rules: InformationRules, return_report: Literal[True]
+) -> tuple[bool, list[dict]]: ...
 
 
 @overload
-def are_entity_names_dms_compliant(rules: InformationRules, return_report: Literal[False] = False) -> bool:
-    ...
+def are_entity_names_dms_compliant(rules: InformationRules, return_report: Literal[False] = False) -> bool: ...
 
 
 def are_entity_names_dms_compliant(
@@ -66,13 +66,11 @@ def are_entity_names_dms_compliant(
 
 
 @overload
-def are_properties_redefined(rules: InformationRules, return_report: Literal[True]) -> tuple[bool, list[dict]]:
-    ...
+def are_properties_redefined(rules: InformationRules, return_report: Literal[True]) -> tuple[bool, list[dict]]: ...
 
 
 @overload
-def are_properties_redefined(rules: InformationRules, return_report: Literal[False] = False) -> bool:
-    ...
+def are_properties_redefined(rules: InformationRules, return_report: Literal[False] = False) -> bool: ...
 
 
 def are_properties_redefined(rules: InformationRules, return_report: bool = False) -> bool | tuple[bool, list[dict]]:

cognite/neat/rules/importers/_base.py

@@ -20,14 +20,12 @@ class BaseImporter(ABC):
     """
 
     @overload
-    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules:
-        ...
+    def to_rules(self, errors: Literal["raise"], role: RoleTypes | None = None) -> Rules: ...
 
     @overload
     def to_rules(
         self, errors: Literal["continue"] = "continue", role: RoleTypes | None = None
-    ) -> tuple[Rules | None, IssueList]:
-        ...
+    ) -> tuple[Rules | None, IssueList]: ...
 
     @abstractmethod
     def to_rules(
@@ -62,6 +60,12 @@ class BaseImporter(ABC):
         else:
             return output, issues
 
+    @classmethod
+    def _return_or_raise(cls, issue_list: IssueList, errors: Literal["raise", "continue"]) -> tuple[None, IssueList]:
+        if errors == "raise":
+            raise issue_list.as_errors()
+        return None, issue_list
+
     def _default_metadata(self):
         return {
             "prefix": "neat",