cognite-neat 0.87.4__py3-none-any.whl → 0.88.0__py3-none-any.whl

cognite/neat/legacy/workflows/examples/Visualize_Semantic_Data_Model/workflow.yaml

@@ -1,111 +0,0 @@
-configs: []
-description: null
-implementation_module: null
-name: Visualize Semantic Data Model
-steps:
--   complex_configs: {}
-    configs: {}
-    description: null
-    enabled: true
-    id: step_upload_excel_rules
-    label: Upload Excel Rules
-    max_retries: 0
-    method: null
-    params:
-        file_type: rules
-    retry_delay: 3
-    stype: file_uploader
-    system_component_id: null
-    transition_to:
-    - step_421523
-    - step_excel_to_rules
-    trigger: true
-    ui_config:
-        pos_x: 498
-        pos_y: 149
--   complex_configs: {}
-    configs:
-        File name: ''
-        Report formatter: BasicHTML
-        Role: infer
-    description: null
-    enabled: true
-    id: step_excel_to_rules
-    label: Import and Validate Rules
-    max_retries: 0
-    method: ExcelToRules
-    params: {}
-    retry_delay: 3
-    stype: stdstep
-    system_component_id: null
-    transition_to:
-    - step_rules_to_ontology
-    trigger: false
-    ui_config:
-        pos_x: 497
-        pos_y: 234
--   complex_configs: {}
-    configs:
-        File path: staging/semantic-data-model.ttl
-    description: null
-    enabled: true
-    id: step_rules_to_ontology
-    label: Rules to Ontology
-    max_retries: 0
-    method: RulesToSemanticDataModel
-    params: {}
-    retry_delay: 3
-    stype: stdstep
-    system_component_id: null
-    transition_to:
-    - step_configure_graph_store
-    trigger: false
-    ui_config:
-        pos_x: 497
-        pos_y: 323
--   complex_configs: {}
-    configs:
-        Disk storage directory: semantic-data-model
-        Graph: source
-        Graph store type: oxigraph
-        GraphDB API root URL: ''
-        Init procedure: reset
-        Query URL: ''
-        Update URL: ''
-    description: null
-    enabled: true
-    id: step_configure_graph_store
-    label: Configure Graph Store
-    max_retries: 0
-    method: GraphStoreConfiguration
-    params: {}
-    retry_delay: 3
-    stype: stdstep
-    system_component_id: null
-    transition_to:
-    - step_load_semantic_data_model
-    trigger: false
-    ui_config:
-        pos_x: 495
-        pos_y: 399
--   complex_configs: {}
-    configs:
-        Add base URI: 'False'
-        File path: staging/semantic-data-model.ttl
-        MIME type: text/turtle
-    description: null
-    enabled: true
-    id: step_load_semantic_data_model
-    label: Load Semantic Data Model
-    max_retries: 0
-    method: GraphFromRdfFile
-    params: {}
-    retry_delay: 3
-    stype: stdstep
-    system_component_id: null
-    transition_to: []
-    trigger: false
-    ui_config:
-        pos_x: 495
-        pos_y: 476
-system_components: []

cognite/neat/workflows/examples/Extract_RDF_Graph_and_Generate_Assets/workflow.yaml

@@ -1,270 +0,0 @@
-configs: []
-description: null
-implementation_module: null
-name: Extract RDF Graph and Generate Assets
-steps:
-  - complex_configs: {}
-    configs: {}
-    description: null
-    enabled: true
-    id: step_http_trigger
-    label: Process trigger
-    max_retries: 0
-    method: null
-    params:
-      workflow_start_method: persistent_blocking
-    retry_delay: 3
-    stype: http_trigger
-    system_component_id: null
-    transition_to:
-      - step_rules_loader
-    trigger: true
-    ui_config:
-      pos_x: 454
-      pos_y: 77
-  - complex_configs: {}
-    configs:
-      file_name: Rules-Nordic44.xlsx
-      validation_report_file: rules_validation_report.txt
-      validation_report_storage_dir: rules_validation_report
-      version: ""
-    description: null
-    enabled: true
-    id: step_rules_loader
-    label: Rules loader
-    max_retries: 0
-    method: ImportExcelToRules
-    params: {}
-    retry_delay: 3
-    stype: stdstep
-    system_component_id: null
-    transition_to:
-      - step_234135
-      - step_configure_source_graph_store
-    trigger: false
-    ui_config:
-      pos_x: 452
-      pos_y: 167
-  - complex_configs: {}
-    configs:
-      add_base_iri: "True"
-      file_path: source-graphs/Knowledge-Graph-Nordic44-dirty.xml
-      mime_type: application/rdf+xml
-    description: null
-    enabled: true
-    id: step_graph_loader
-    label: Extract Graph from RDF File
-    max_retries: 0
-    method: ExtractGraphFromRdfFile
-    params: {}
-    retry_delay: 3
-    stype: stdstep
-    system_component_id: null
-    transition_to:
-      - step_transform
-    trigger: false
-    ui_config:
-      pos_x: 453
-      pos_y: 418
-  - complex_configs: {}
-    configs:
-      cdf_lookup_database: ""
-    description: null
-    enabled: true
-    id: step_transform
-    label: Transform graph
-    max_retries: 0
-    method: TransformSourceToSolutionGraph
-    params: {}
-    retry_delay: 3
-    stype: stdstep
-    system_component_id: null
-    transition_to: []
-    trigger: false
-    ui_config:
-      pos_x: 453
-      pos_y: 500
-  - complex_configs: {}
-    configs:
-      asset_external_id_prefix: ""
-      assets_cleanup_type: full
-      data_set_id: "2626756768281823"
-    description: null
-    enabled: true
-    id: step_generate_assets
-    label: Generate cdf assets
-    max_retries: 0
-    method: GenerateAssetsFromGraph
-    params: {}
-    retry_delay: 3
-    stype: stdstep
-    system_component_id: null
-    transition_to:
-      - step_generate_relationships
-    trigger: false
-    ui_config:
-      pos_x: 455
-      pos_y: 571
-  - complex_configs: {}
-    configs:
-      data_set_id: "2626756768281823"
-      relationship_external_id_prefix: ""
-    description: null
-    enabled: true
-    id: step_generate_relationships
-    label: Generate relationships
-    max_retries: 0
-    method: GenerateRelationshipsFromGraph
-    params: {}
-    retry_delay: 3
-    stype: stdstep
-    system_component_id: null
-    transition_to: []
-    trigger: false
-    ui_config:
-      pos_x: 456
-      pos_y: 633
-  - complex_configs: {}
-    configs:
-      db_server_api_root_url: ""
-      disk_store_dir: nordic44-graph-store
-      graph_name: source
-      init_procedure: reset
-      sparql_query_url: ""
-      sparql_update_url: ""
-      store_type: oxigraph
-    description: null
-    enabled: true
-    id: step_configure_source_graph_store
-    label: Configure source graph store
-    max_retries: 0
-    method: ConfigureGraphStore
-    params: {}
-    retry_delay: 3
-    stype: stdstep
-    system_component_id: null
-    transition_to:
-      - step_configure_solution_graph_store
-    trigger: false
-    ui_config:
-      pos_x: 451
-      pos_y: 245
-  - complex_configs: {}
-    configs:
-      db_server_api_root_url: ""
-      disk_store_dir: solution-graph-store-4
-      graph_name: solution
-      init_procedure: reset
-      sparql_query_url: ""
-      sparql_update_url: ""
-      store_type: oxigraph
-    description: null
-    enabled: true
-    id: step_configure_solution_graph_store
-    label: Configure solution graph store
-    max_retries: 0
-    method: ConfigureGraphStore
-    params: {}
-    retry_delay: 3
-    stype: stdstep
-    system_component_id: null
-    transition_to:
-      - step_graph_loader
-    trigger: false
-    ui_config:
-      pos_x: 454
-      pos_y: 334
-system_components:
-  - description: null
-    id: grid_management_system
-    label: Grid management system
-    transition_to:
-      - rdf_xml_file
-    ui_config:
-      pos_x: 171
-      pos_y: 6
-  - description: null
-    id: rdf_xml_file
-    label: RDF XML File
-    transition_to:
-      - graph_db_store
-    ui_config:
-      pos_x: 170
-      pos_y: 103
-  - description: null
-    id: graph_db_store
-    label: OxiGraph source graph_store
-    transition_to:
-      - transformer
-    ui_config:
-      pos_x: 240
-      pos_y: 177
-  - description: null
-    id: in_memmory_store
-    label: In-memory source graph_store
-    transition_to: []
-    ui_config:
-      pos_x: 73
-      pos_y: 175
-  - description: null
-    id: transformation_rules
-    label: Transformation rules
-    transition_to:
-      - transformer
-    ui_config:
-      pos_x: 413
-      pos_y: 195
-  - description: null
-    id: cdf_raw_table
-    label: Cdf raw lookup table
-    transition_to:
-      - transformer
-    ui_config:
-      pos_x: 586
-      pos_y: 195
-  - description: null
-    id: transformer
-    label: Transformer
-    transition_to:
-      - solution_graph
-    ui_config:
-      pos_x: 237
-      pos_y: 329
-  - description: null
-    id: solution_graph
-    label: Solution graph
-    transition_to:
-      - cdf_classic_exporter
-    ui_config:
-      pos_x: 237
-      pos_y: 405
-  - description: null
-    id: cdf_classic_exporter
-    label: CDF Classic exporter
-    transition_to:
-      - cdf_classic
-    ui_config:
-      pos_x: 181
-      pos_y: 498
-  - description: null
-    id: cdf_fdm_exporter
-    label: CDF FDM exporter
-    transition_to:
-      - cdf_fdm
-    ui_config:
-      pos_x: 345
-      pos_y: 498
-  - description: null
-    id: cdf_classic
-    label: CDF Classic (Asset,Relationships)
-    transition_to: []
-    ui_config:
-      pos_x: 182
-      pos_y: 586
-  - description: null
-    id: cdf_fdm
-    label: CDF FDM
-    transition_to: []
-    ui_config:
-      pos_x: 347
-      pos_y: 586

cognite/neat/workflows/migration/steps.py

@@ -1,91 +0,0 @@
-from pathlib import Path
-
-import yaml
-
-# Migration mapping
-
-# LoadTransformationRules to ImportExcelToRules
-# InstancesFromRdfFileToSourceGraph to ExtractGraphFromRdfFile
-# InstancesFromRulesToSolutionGraph to ExtractGraphFromRulesInstanceSheet
-# GraphCapturingSheetToGraph to ExtractGraphFromGraphCapturingSheet
-# GenerateMockGraph to ExtractGraphFromMockGraph
-# InstancesFromJsonToGraph to ExtractGraphFromJsonFile
-# InstancesFromAvevaPiAF to ExtractGraphFromAvevaPiAssetFramework
-# DexpiToGraph to ExtractGraphFromDexpiFile
-# GenerateCDFAssetsFromGraph to GenerateAssetsFromGraph
-# GenerateCDFRelationshipsFromGraph to GenerateRelationshipsFromGraph
-# GenerateCDFNodesAndEdgesFromGraph to GenerateNodesAndEdgesFromGraph
-# UploadCDFAssets to LoadAssetsToCDF
-# UploadCDFRelationships to LoadRelationshipsToCDF
-# UploadCDFNodes to LoadNodesToCDF
-# UploadCDFEdges to LoadEdgesToCDF
-# CreateCDFLabels to LoadLabelsToCDF
-# OpenApiToRules to `ImportOpenApiToRules
-# ArbitraryJsonYamlToRules to ImportArbitraryJsonYamlToRules
-# GraphToRules to ImportGraphToRules
-# OntologyToRules to ImportOntologyToRules
-# GraphQLSchemaFromRules to ExportGraphQLSchemaFromRules
-# OntologyFromRules to ExportOntologyFromRules
-# SHACLFromRules to ExportSHACLFromRules
-# GraphCaptureSpreadsheetFromRules to ExportRulesToGraphCapturingSheet
-# ExcelFromRules to ExportRulesToExcel
-
-# Define a dictionary that maps old step names to new ones
-step_rename_mapping = {
-    "LoadTransformationRules": "ImportExcelToRules",
-    "InstancesFromRdfFileToSourceGraph": "ExtractGraphFromRdfFile",
-    "InstancesFromRulesToSolutionGraph": "ExtractGraphFromRulesInstanceSheet",
-    "GraphCapturingSheetToGraph": "ExtractGraphFromGraphCapturingSheet",
-    "GenerateMockGraph": "ExtractGraphFromMockGraph",
-    "InstancesFromJsonToGraph": "ExtractGraphFromJsonFile",
-    "InstancesFromAvevaPiAF": "ExtractGraphFromAvevaPiAssetFramework",
-    "DexpiToGraph": "ExtractGraphFromDexpiFile",
-    "GenerateCDFAssetsFromGraph": "GenerateAssetsFromGraph",
-    "GenerateCDFRelationshipsFromGraph": "GenerateRelationshipsFromGraph",
-    "GenerateCDFNodesAndEdgesFromGraph": "GenerateNodesAndEdgesFromGraph",
-    "UploadCDFAssets": "LoadAssetsToCDF",
-    "UploadCDFRelationships": "LoadRelationshipsToCDF",
-    "UploadCDFNodes": "LoadNodesToCDF",
-    "UploadCDFEdges": "LoadEdgesToCDF",
-    "CreateCDFLabels": "LoadLabelsToCDF",
-    "OpenApiToRules": "ImportOpenApiToRules",
-    "ArbitraryJsonYamlToRules": "ImportArbitraryJsonYamlToRules",
-    "GraphToRules": "ImportGraphToRules",
-    "OntologyToRules": "ImportOntologyToRules",
-    "GraphQLSchemaFromRules": "ExportGraphQLSchemaFromRules",
-    "OntologyFromRules": "ExportOntologyFromRules",
-    "SHACLFromRules": "ExportSHACLFromRules",
-    "GraphCaptureSpreadsheetFromRules": "ExportRulesToGraphCapturingSheet",
-    "ExcelFromRules": "ExportRulesToExcel",
-}
-
-
-def rename_workflow_steps(workflow_path: Path, dry_run: bool) -> str:
-    # Load the YAML file
-    data = yaml.safe_load(workflow_path.read_text())
-    # Replace old step names with new ones
-    for step in data["steps"]:
-        if step["method"] in step_rename_mapping:
-            print(f"Renaming step {step['method']} to {step_rename_mapping[step['method']]}")
-            step["method"] = step_rename_mapping[step["method"]]
-
-    # Save the updated YAML file
-    with workflow_path.open("w") as file:
-        yaml.safe_dump(data, file)
-    return "ok"
-
-
-def migrate_all_workflow_names(workflows_path: str, dry_run: bool) -> str:
-    # Get all workflow files
-    workflow_files = Path(workflows_path).rglob("workflow.yaml")
-    # Migrate each workflow file
-    for workflow_file in workflow_files:
-        print(f"Migrating {workflow_file}")
-        rename_workflow_steps(workflow_file, dry_run)
-    return "ok"
-
-
-if __name__ == "__main__":
-    # print current directory
-    print("The current directory is", Path.cwd())
-    migrate_all_workflow_names("data/workflows", True)

cognite/neat/workflows/migration/wf_manifests.py

@@ -1,33 +0,0 @@
-import logging
-import os
-from pathlib import Path
-
-import yaml
-
-
-def migrate_wf_manifest(wf_store_path: Path):
-    """Migrate workflow manifests . Changed name of element from groups -> system_components"""
-    migrated_files = []
-    wf_store_path = wf_store_path / "workflows"
-    for wf_module_name in os.listdir(wf_store_path):
-        wf_module_full_path = wf_store_path / wf_module_name
-
-        if wf_module_full_path.is_dir():
-            metadata_file = wf_store_path / wf_module_name / "workflow.yaml"
-            metadata_file_migrated = wf_store_path / wf_module_name / "workflow.yaml"
-            logging.info(f"Loading workflow {wf_module_name} metadata from {metadata_file}")
-            if metadata_file.exists():
-                with metadata_file.open() as f:
-                    manifest_yaml = yaml.safe_load(f)
-                    logging.info(f"Loaded workflow {wf_module_name} metadata from {metadata_file}")
-                if "groups" in manifest_yaml:
-                    logging.info(f"Found groups in {metadata_file}, migrating to system_components")
-                    manifest_yaml["system_components"] = manifest_yaml["groups"]
-                    del manifest_yaml["groups"]
-                    with metadata_file_migrated.open("w") as f:
-                        yaml.dump(manifest_yaml, f, indent=4)
-                        migrated_files.append(metadata_file_migrated)
-            else:
-                logging.info(f"Metadata file {metadata_file} not found, skipping")
-                continue
-    return migrated_files

cognite/neat/workflows/steps/lib/legacy/__init__.py

@@ -1,7 +0,0 @@
-from .graph_extractor import *  # noqa
-from .graph_transformer import *  # noqa
-from .graph_loader import *  # noqa
-from .graph_store import *  # noqa
-from .rules_importer import *  # noqa
-from .rules_exporter import *  # noqa
-from .graph_contextualization import *  # noqa

cognite/neat/workflows/steps/lib/legacy/graph_contextualization.py

@@ -1,82 +0,0 @@
-from typing import ClassVar, cast
-
-from cognite.neat.legacy.graph.transformations.entity_matcher import simple_entity_matcher
-from cognite.neat.workflows._exceptions import StepNotInitialized
-from cognite.neat.workflows.model import FlowMessage
-from cognite.neat.workflows.steps.data_contracts import SolutionGraph, SourceGraph
-from cognite.neat.workflows.steps.step_model import Configurable, Step
-
-__all__ = ["SimpleGraphEntityMatcher"]
-CATEGORY = __name__.split(".")[-1].replace("_", " ").title() + " [LEGACY]"
-
-
-class SimpleGraphEntityMatcher(Step):
-    version = "legacy"
-    description = "The step matches entities in the graph and creates links based on provided configurations"
-    category = CATEGORY
-    configurables: ClassVar[list[Configurable]] = [
-        Configurable(name="source_class", value="", label="Name of the source class"),
-        Configurable(name="source_property", value="", label="Name of the source property"),
-        Configurable(
-            name="source_value_type",
-            value="single_value_str",
-            label="Type of the value in the source property.Propery can have single value \
-              or multiple values separated by comma.",
-            options=["single_value_str", "multi_value_str"],
-        ),
-        Configurable(name="target_class", value="", label="Name of the target class"),
-        Configurable(name="target_property", value="", label="Name of the target property"),
-        Configurable(name="relationship_name", value="link", label="Label of the relationship to be created"),
-        Configurable(
-            name="link_direction",
-            value="target_to_source",
-            label="Direction of the relationship.",
-            options=["target_to_source", "source_to_target"],
-        ),
-        Configurable(
-            name="matching_method",
-            value="regexp",
-            label="Method to be used for matching. Supported options .",
-            options=["exact_match", "regexp"],
-        ),
-        Configurable(
-            name="graph_name",
-            value="source",
-            label="The name of the graph to be used for matching.",
-            options=["source", "solution"],
-        ),
-        Configurable(
-            name="link_namespace",
-            value="http://purl.org/cognite/neat#",
-            label="The namespace of the link to be created",
-        ),
-    ]
-
-    def run(self, graph_store: SolutionGraph | SourceGraph) -> FlowMessage:  # type: ignore[override, syntax]
-        # We can't use the graph_store to get the graph as input parameter directly,
-        # resolver might resolve the wrong graph
-        # if both are present in the flow context
-        if self.configs is None:
-            raise StepNotInitialized(type(self).__name__)
-        configs = self.configs
-        if self.configs["graph_name"] == "solution":
-            # Todo Anders: Why is the graph fetched from context when it is passed as an argument?
-            graph_store = cast(SourceGraph | SolutionGraph, self.flow_context["SolutionGraph"])
-        else:
-            graph_store = cast(SourceGraph | SolutionGraph, self.flow_context["SourceGraph"])
-
-        self.graph_store = graph_store.graph
-        new_links_counter = simple_entity_matcher(
-            graph_store=self.graph_store,
-            source_class=configs["source_class"],
-            source_property=configs["source_property"],
-            source_value_type=configs["source_value_type"],
-            target_class=configs["target_class"],
-            target_property=configs["target_property"],
-            relationship_name=configs["relationship_name"],
-            link_direction=configs["link_direction"],
-            matching_method=configs["matching_method"],
-            link_namespace=configs["link_namespace"],
-        )
-        output_text = f"Matcher has created {new_links_counter} links"
-        return FlowMessage(output_text=output_text)