cognite-neat 0.123.2__py3-none-any.whl → 0.127.30__py3-none-any.whl

cognite/neat/core/_data_model/importers/_rdf/_shared.py

@@ -1,168 +0,0 @@
-from typing import cast
-
-from rdflib import BNode, Graph
-from rdflib.plugins.sparql import prepareQuery
-from rdflib.query import ResultRow
-
-from cognite.neat.core._issues._base import IssueList
-from cognite.neat.core._issues.errors._general import NeatValueError
-from cognite.neat.core._issues.warnings._resources import (
-    ResourceRedefinedWarning,
-    ResourceRetrievalWarning,
-)
-from cognite.neat.core._utils.rdf_ import convert_rdflib_content
-
-
-def parse_concepts(graph: Graph, query: str, language: str, issue_list: IssueList) -> tuple[dict, IssueList]:
-    """Parse concepts from graph
-
-    Args:
-        graph: Graph containing concept definitions
-        language: Language to use for parsing, by default "en"
-
-    Returns:
-        Dataframe containing owl classes
-    """
-
-    concepts: dict[str, dict] = {}
-
-    query = prepareQuery(query.format(language=language), initNs={k: v for k, v in graph.namespaces()})
-    expected_keys = [str(v) for v in query.algebra._vars]
-
-    for raw in graph.query(query):
-        res: dict = convert_rdflib_content(cast(ResultRow, raw).asdict(), True)
-        res = {key: res.get(key, None) for key in expected_keys}
-
-        concept_id = res["concept"]
-
-        # Safeguarding against incomplete semantic definitions
-        if res["implements"] and isinstance(res["implements"], BNode):
-            issue_list.append(
-                ResourceRetrievalWarning(
-                    concept_id,
-                    "implements",
-                    error=("Unable to determine concept that is being implemented"),
-                )
-            )
-            continue
-
-        if concept_id not in concepts:
-            concepts[concept_id] = res
-        else:
-            # Handling implements
-            if concepts[concept_id]["implements"] and isinstance(concepts[concept_id]["implements"], list):
-                if res["implements"] not in concepts[concept_id]["implements"]:
-                    concepts[concept_id]["implements"].append(res["implements"])
-
-            elif concepts[concept_id]["implements"] and isinstance(concepts[concept_id]["implements"], str):
-                concepts[concept_id]["implements"] = [concepts[concept_id]["implements"]]
-
-                if res["implements"] not in concepts[concept_id]["implements"]:
-                    concepts[concept_id]["implements"].append(res["implements"])
-            elif res["implements"]:
-                concepts[concept_id]["implements"] = [res["implements"]]
-
-            handle_meta("concept", concepts, concept_id, res, "name", issue_list)
-            handle_meta("concept", concepts, concept_id, res, "description", issue_list)
-
-    if not concepts:
-        issue_list.append(NeatValueError("Unable to parse concepts"))
-
-    return concepts, issue_list
-
-
-def parse_properties(graph: Graph, query: str, language: str, issue_list: IssueList) -> tuple[dict, IssueList]:
-    """Parse properties from graph
-
-    Args:
-        graph: Graph containing owl classes
-        language: Language to use for parsing, by default "en"
-
-    Returns:
-        Dataframe containing owl classes
-    """
-
-    properties: dict[str, dict] = {}
-
-    query = prepareQuery(query.format(language=language), initNs={k: v for k, v in graph.namespaces()})
-    expected_keys = [str(v) for v in query.algebra._vars]
-
-    for raw in graph.query(query):
-        res: dict = convert_rdflib_content(cast(ResultRow, raw).asdict(), True)
-        res = {key: res.get(key, None) for key in expected_keys}
-
-        property_id = res["property_"]
-
-        # Safeguarding against incomplete semantic definitions
-        if not res["concept"] or isinstance(res["concept"], BNode):
-            issue_list.append(
-                ResourceRetrievalWarning(
-                    property_id,
-                    "property",
-                    error=("Unable to determine to what concept property is being defined"),
-                )
-            )
-            continue
-
-        # Safeguarding against incomplete semantic definitions
-        if not res["value_type"] or isinstance(res["value_type"], BNode):
-            issue_list.append(
-                ResourceRetrievalWarning(
-                    property_id,
-                    "property",
-                    error=("Unable to determine value type of property"),
-                )
-            )
-            continue
-
-        id_ = f"{res['concept']}.{res['property_']}"
-
-        if id_ not in properties:
-            properties[id_] = res
-            properties[id_]["value_type"] = [properties[id_]["value_type"]]
-        else:
-            handle_meta("property", properties, id_, res, "name", issue_list)
-            handle_meta(
-                "property",
-                properties,
-                id_,
-                res,
-                "description",
-                issue_list,
-            )
-
-            # Handling multi-value types
-            if res["value_type"] not in properties[id_]["value_type"]:
-                properties[id_]["value_type"].append(res["value_type"])
-
-    for prop in properties.values():
-        prop["value_type"] = ", ".join(prop["value_type"])
-
-    if not properties:
-        issue_list.append(NeatValueError("Unable to parse properties"))
-
-    return properties, issue_list
-
-
-def handle_meta(
-    resource_type: str,
-    resources: dict[str, dict],
-    resource_id: str,
-    res: dict,
-    feature: str,
-    issue_list: IssueList,
-) -> None:
-    if not resources[resource_id][feature] and res[feature]:
-        resources[resource_id][feature] = res[feature]
-
-    # RAISE warning only if the feature is being redefined
-    elif resources[resource_id][feature] and res[feature]:
-        issue_list.append(
-            ResourceRedefinedWarning(
-                identifier=resource_id,
-                resource_type=resource_type,
-                feature=feature,
-                current_value=resources[resource_id][feature],
-                new_value=res[feature],
-            )
-        )

cognite/neat/core/_data_model/models/conceptual/_validation.py

@@ -1,294 +0,0 @@
-import itertools
-from collections import Counter, defaultdict
-from collections.abc import Iterable
-
-from cognite.neat.core._data_model._constants import PATTERNS, EntityTypes
-from cognite.neat.core._data_model.models.entities import ConceptEntity, UnknownEntity
-from cognite.neat.core._data_model.models.entities._multi_value import MultiValueTypeInfo
-from cognite.neat.core._issues import IssueList
-from cognite.neat.core._issues.errors import NeatValueError
-from cognite.neat.core._issues.errors._resources import (
-    ResourceDuplicatedError,
-    ResourceNotDefinedError,
-)
-from cognite.neat.core._issues.warnings._models import UndefinedConceptWarning
-from cognite.neat.core._issues.warnings._resources import (
-    ResourceNotDefinedWarning,
-    ResourceRegexViolationWarning,
-)
-from cognite.neat.core._utils.spreadsheet import SpreadsheetRead
-from cognite.neat.core._utils.text import humanize_collection
-
-from ._verified import ConceptualDataModel, ConceptualProperty
-
-
-class ConceptualValidation:
-    """This class does all the validation of the conceptual data model that have dependencies
-    between components."""
-
-    def __init__(
-        self,
-        data_model: ConceptualDataModel,
-        read_info_by_spreadsheet: dict[str, SpreadsheetRead] | None = None,
-    ):
-        self.data_model = data_model
-        self._read_info_by_spreadsheet = read_info_by_spreadsheet or {}
-        self._metadata = data_model.metadata
-        self._properties = data_model.properties
-        self._concepts = data_model.concepts
-        self.issue_list = IssueList()
-
-    def validate(self) -> IssueList:
-        self._duplicated_resources()
-        self._namespaces_reassigned()
-        self._classes_without_properties()
-        self._undefined_classes()
-        self._parent_concept_defined()
-        self._referenced_classes_exist()
-        self._referenced_value_types_exist()
-        self._regex_compliance_with_dms()
-
-        return self.issue_list
-
-    def _duplicated_resources(self) -> None:
-        properties_sheet = self._read_info_by_spreadsheet.get("Properties")
-        concepts_sheet = self._read_info_by_spreadsheet.get("Concepts")
-
-        visited = defaultdict(list)
-        for row_no, property_ in enumerate(self._properties):
-            visited[property_._identifier()].append(
-                properties_sheet.adjusted_row_number(row_no) if properties_sheet else row_no + 1
-            )
-
-        for identifier, rows in visited.items():
-            if len(rows) == 1:
-                continue
-            self.issue_list.append(
-                ResourceDuplicatedError(
-                    identifier[1],
-                    "property",
-                    (
-                        "the Properties sheet at row "
-                        f"{humanize_collection(rows)}"
-                        " if data model is read from a spreadsheet."
-                    ),
-                )
-            )
-
-        visited = defaultdict(list)
-        for row_no, concept in enumerate(self._concepts):
-            visited[concept._identifier()].append(
-                concepts_sheet.adjusted_row_number(row_no) if concepts_sheet else row_no + 1
-            )
-
-        for identifier, rows in visited.items():
-            if len(rows) == 1:
-                continue
-            self.issue_list.append(
-                ResourceDuplicatedError(
-                    identifier[0],
-                    "concept",
-                    (f"the Classes sheet at row {humanize_collection(rows)} if data model is read from a spreadsheet."),
-                )
-            )
-
-    def _classes_without_properties(self) -> None:
-        defined_concepts = {concept.concept for concept in self._concepts}
-        referred_classes = {property_.concept for property_ in self._properties}
-        concept_parent_pairs = self._concept_parent_pairs()
-
-        if concepts_without_properties := defined_concepts.difference(referred_classes):
-            for concept in concepts_without_properties:
-                # USE CASE: class has no direct properties and no parents with properties
-                # and it is a class in the prefix of data model, as long as it is in the
-                # same prefix, meaning same space
-                if not concept_parent_pairs[concept] and concept.prefix == self._metadata.prefix:
-                    self.issue_list.append(
-                        ResourceNotDefinedWarning(
-                            resource_type="concept",
-                            identifier=concept,
-                            location="Properties sheet",
-                        )
-                    )
-
-    def _undefined_classes(self) -> None:
-        defined_concept = {concept.concept for concept in self._concepts}
-        referred_concepts = {property_.concept for property_ in self._properties}
-
-        if undefined_concepts := referred_concepts.difference(defined_concept):
-            for concept in undefined_concepts:
-                self.issue_list.append(
-                    ResourceNotDefinedError(
-                        identifier=concept,
-                        resource_type="concept",
-                        location="Concepts sheet",
-                    )
-                )
-
-    def _parent_concept_defined(self) -> None:
-        """This is a validation to check if the parent concept is defined."""
-        concept_parent_pairs = self._concept_parent_pairs()
-        concepts = set(concept_parent_pairs.keys())
-        parents = set(itertools.chain.from_iterable(concept_parent_pairs.values()))
-
-        if undefined_parents := parents.difference(concepts):
-            for parent in undefined_parents:
-                if parent.prefix != self._metadata.prefix:
-                    self.issue_list.append(UndefinedConceptWarning(concept_id=str(parent)))
-                else:
-                    self.issue_list.append(
-                        ResourceNotDefinedWarning(
-                            resource_type="concept",
-                            identifier=parent,
-                            location="Concepts sheet",
-                        )
-                    )
-
-    def _referenced_classes_exist(self) -> None:
-        # needs to be complete for this validation to pass
-        defined_concept = {concept.concept for concept in self._concepts}
-        classes_with_explicit_properties = {property_.concept for property_ in self._properties}
-
-        # USE CASE: models are complete
-        if missing_classes := classes_with_explicit_properties.difference(defined_concept):
-            for concept in missing_classes:
-                self.issue_list.append(
-                    ResourceNotDefinedWarning(
-                        resource_type="concept",
-                        identifier=concept,
-                        location="Concepts sheet",
-                    )
-                )
-
-    def _referenced_value_types_exist(self) -> None:
-        # adding UnknownEntity to the set of defined classes to handle the case where a property references an unknown
-        defined_classes = {concept.concept for concept in self._concepts} | {UnknownEntity()}
-        referred_object_types = {
-            property_.value_type
-            for property_ in self.data_model.properties
-            if property_.type_ == EntityTypes.object_property
-        }
-
-        if missing_value_types := referred_object_types.difference(defined_classes):
-            # Todo: include row and column number
-            for missing in missing_value_types:
-                self.issue_list.append(
-                    ResourceNotDefinedWarning(
-                        resource_type="concept",
-                        identifier=missing,
-                        location="Concepts sheet",
-                    )
-                )
-
-    def _regex_compliance_with_dms(self) -> None:
-        """Check regex compliance with DMS of properties, classes and value types."""
-
-        for prop_ in self._properties:
-            if not PATTERNS.physical_property_id_compliance.match(prop_.property_):
-                self.issue_list.append(
-                    ResourceRegexViolationWarning(
-                        prop_.property_,
-                        "Property",
-                        "Properties sheet, Property column",
-                        PATTERNS.physical_property_id_compliance.pattern,
-                    )
-                )
-            if not PATTERNS.view_id_compliance.match(prop_.concept.suffix):
-                self.issue_list.append(
-                    ResourceRegexViolationWarning(
-                        prop_.concept,
-                        "Concept",
-                        "Properties sheet, Concept column",
-                        PATTERNS.view_id_compliance.pattern,
-                    )
-                )
-
-            # Handling Value Type
-            if (
-                isinstance(prop_.value_type, ConceptEntity)
-                and prop_.value_type != UnknownEntity()
-                and not PATTERNS.view_id_compliance.match(prop_.value_type.suffix)
-            ):
-                self.issue_list.append(
-                    ResourceRegexViolationWarning(
-                        prop_.value_type,
-                        "Value Type",
-                        "Properties sheet, Value Type column",
-                        PATTERNS.view_id_compliance.pattern,
-                    )
-                )
-            if isinstance(prop_.value_type, MultiValueTypeInfo):
-                for value_type in prop_.value_type.types:
-                    if (
-                        isinstance(prop_.value_type, ConceptEntity)
-                        and prop_.value_type != UnknownEntity()
-                        and not PATTERNS.view_id_compliance.match(value_type.suffix)
-                    ):
-                        self.issue_list.append(
-                            ResourceRegexViolationWarning(
-                                value_type,
-                                "Value Type",
-                                "Properties sheet, Value Type column",
-                                PATTERNS.view_id_compliance.pattern,
-                            )
-                        )
-
-        for concepts in self._concepts:
-            if not PATTERNS.view_id_compliance.match(concepts.concept.suffix):
-                self.issue_list.append(
-                    ResourceRegexViolationWarning(
-                        concepts.concept,
-                        "Concept",
-                        "Concepts sheet, Class column",
-                        PATTERNS.view_id_compliance.pattern,
-                    )
-                )
-
-            if concepts.implements:
-                for parent in concepts.implements:
-                    if not PATTERNS.view_id_compliance.match(parent.suffix):
-                        self.issue_list.append(
-                            ResourceRegexViolationWarning(
-                                parent,
-                                "Concept",
-                                "Concepts sheet, Implements column",
-                                PATTERNS.view_id_compliance.pattern,
-                            )
-                        )
-
-    def _concept_parent_pairs(self) -> dict[ConceptEntity, list[ConceptEntity]]:
-        concept_parent_pairs: dict[ConceptEntity, list[ConceptEntity]] = {}
-        concepts = self.data_model.model_copy(deep=True).concepts
-
-        for concept in concepts:
-            concept_parent_pairs[concept.concept] = []
-            if concept.implements is None:
-                continue
-            concept_parent_pairs[concept.concept].extend(concept.implements)
-
-        return concept_parent_pairs
-
-    def _namespaces_reassigned(self) -> None:
-        prefixes = self.data_model.prefixes.copy()
-        prefixes[self.data_model.metadata.namespace.prefix] = self.data_model.metadata.namespace
-
-        if len(set(prefixes.values())) != len(prefixes):
-            reused_namespaces = [value for value, count in Counter(prefixes.values()).items() if count > 1]
-            impacted_prefixes = [key for key, value in prefixes.items() if value in reused_namespaces]
-            self.issue_list.append(
-                NeatValueError(
-                    "Namespace collision detected. The following prefixes "
-                    f"are assigned to the same namespace: {impacted_prefixes}"
-                    f"\nImpacted namespaces: {reused_namespaces}"
-                    "\nMake sure that each unique namespace is assigned to a unique prefix"
-                )
-            )
-
-
-def duplicated_properties(
-    properties: Iterable[ConceptualProperty],
-) -> dict[tuple[ConceptEntity, str], list[tuple[int, ConceptualProperty]]]:
-    concept_properties_by_id: dict[tuple[ConceptEntity, str], list[tuple[int, ConceptualProperty]]] = defaultdict(list)
-    for prop_no, prop in enumerate(properties):
-        concept_properties_by_id[(prop.concept, prop.property_)].append((prop_no, prop))
-    return {k: v for k, v in concept_properties_by_id.items() if len(v) > 1}

cognite/neat/core/_data_model/models/entities/_loaders.py

@@ -1,75 +0,0 @@
-from typing import Literal
-
-from cognite.neat.core._data_model.models.data_types import DataType
-from cognite.neat.core._issues.errors import NeatTypeError
-
-from ._multi_value import MultiValueTypeInfo
-from ._single_value import (
-    ConceptEntity,
-    EdgeEntity,
-    PhysicalUnknownEntity,
-    ReverseConnectionEntity,
-    Unknown,
-    UnknownEntity,
-    ViewEntity,
-)
-
-
-def load_value_type(
-    raw: str | MultiValueTypeInfo | DataType | ConceptEntity | UnknownEntity,
-    default_prefix: str,
-) -> MultiValueTypeInfo | DataType | ConceptEntity | UnknownEntity:
-    if isinstance(raw, MultiValueTypeInfo | DataType | ConceptEntity | UnknownEntity):
-        return raw
-    elif isinstance(raw, str):
-        # property holding xsd data type
-        # check if it is multi value type
-        if "," in raw:
-            value_type = MultiValueTypeInfo.load(raw)
-            value_type.set_default_prefix(default_prefix)
-            return value_type
-        elif DataType.is_data_type(raw):
-            return DataType.load(raw)
-
-        # unknown value type
-        elif raw == str(Unknown):
-            return UnknownEntity()
-
-        # property holding link to class
-        else:
-            return ConceptEntity.load(raw, prefix=default_prefix)
-    else:
-        raise NeatTypeError(f"Invalid value type: {type(raw)}")
-
-
-def load_dms_value_type(
-    raw: str | DataType | ViewEntity | PhysicalUnknownEntity,
-    default_space: str,
-    default_version: str,
-) -> DataType | ViewEntity | PhysicalUnknownEntity:
-    if isinstance(raw, DataType | ViewEntity | PhysicalUnknownEntity):
-        return raw
-    elif isinstance(raw, str):
-        if DataType.is_data_type(raw):
-            return DataType.load(raw)
-        elif raw == str(Unknown):
-            return PhysicalUnknownEntity()
-        else:
-            return ViewEntity.load(raw, space=default_space, version=default_version)
-    raise NeatTypeError(f"Invalid value type: {type(raw)}")
-
-
-def load_connection(
-    raw: Literal["direct"] | ReverseConnectionEntity | EdgeEntity | str | None,
-    default_space: str,
-    default_version: str,
-) -> Literal["direct"] | ReverseConnectionEntity | EdgeEntity | None:
-    if isinstance(raw, str) and raw.lower() == "direct":
-        return "direct"  # type: ignore[return-value]
-    elif isinstance(raw, EdgeEntity | ReverseConnectionEntity) or raw is None:
-        return raw  # type: ignore[return-value]
-    elif isinstance(raw, str) and raw.startswith("edge"):
-        return EdgeEntity.load(raw, space=default_space, version=default_version)  # type: ignore[return-value]
-    elif isinstance(raw, str) and raw.startswith("reverse"):
-        return ReverseConnectionEntity.load(raw)  # type: ignore[return-value]
-    raise NeatTypeError(f"Invalid connection: {type(raw)}")

cognite/neat/plugins/__init__.py

@@ -1,3 +0,0 @@
-from ._manager import get_plugin_manager
-
-__all__ = ["get_plugin_manager"]

cognite/neat/plugins/data_model/importers/__init__.py

@@ -1,5 +0,0 @@
-from ._base import DataModelImporterPlugin
-
-__all__ = [
-    "DataModelImporterPlugin",
-]

cognite/neat/plugins/data_model/importers/_base.py

@@ -1,28 +0,0 @@
-from pathlib import Path
-from typing import Any
-
-from cognite.neat.core._data_model.importers._base import BaseImporter
-
-
-class DataModelImporterPlugin:
-    """This class is used an interface for data model import plugins.
-    Any plugin that is used for importing data models should inherit from this class.
-    It is expected to implement the `configure` method which returns a configured importer.
-    """
-
-    def configure(self, io: str | Path | None, **kwargs: Any) -> BaseImporter:
-        """Return a configure plugin for data model import.
-
-        Args:
-            io (str | Path | None): The input/output interface for the plugin.
-            **kwargs (Any): Additional keyword arguments for configuration.
-
-        Returns:
-            BaseImporter: A configured instance of the BaseImporter.
-
-        !!! note "Returns"
-            The method must return an instance of `BaseImporter` or its subclasses
-            meaning it must implement the `BaseImporter` interface.
-        """
-
-        raise NotImplementedError()