cognite-neat 0.88.0__py3-none-any.whl → 0.88.2__py3-none-any.whl

cognite/neat/graph/extractors/_classic_cdf/_timeseries.py

@@ -1,39 +1,37 @@
-import json
-from collections.abc import Iterable
+from collections.abc import Callable, Set
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import cast
 
 from cognite.client import CogniteClient
-from cognite.client.data_classes import TimeSeries, TimeSeriesList
+from cognite.client.data_classes import TimeSeries, TimeSeriesFilter, TimeSeriesList
 from pydantic import AnyHttpUrl, ValidationError
 from rdflib import RDF, Literal, Namespace, URIRef
 
-from cognite.neat.constants import DEFAULT_NAMESPACE
-from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.auxiliary import string_to_ideal_type
 
+from ._base import DEFAULT_SKIP_METADATA_VALUES, ClassicCDFExtractor
 
-class TimeSeriesExtractor(BaseExtractor):
+
+class TimeSeriesExtractor(ClassicCDFExtractor[TimeSeries]):
     """Extract data from Cognite Data Fusions TimeSeries into Neat.
 
     Args:
-        timeseries (Iterable[TimeSeries]): An iterable of timeseries.
+        items (Iterable[TimeSeries]): An iterable of items.
         namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
+        to_type (Callable[[TimeSeries], str | None], optional): A function to convert an item to a type.
+            Defaults to None. If None or if the function returns None, the asset will be set to the default type.
+        total (int, optional): The total number of items to load. If passed, you will get a progress bar if rich
+            is installed. Defaults to None.
+        limit (int, optional): The maximal number of items to load. Defaults to None. This is typically used for
+            testing setup of the extractor. For example, if you are extracting 100 000 assets, you might want to
+            limit the extraction to 1000 assets to test the setup.
         unpack_metadata (bool, optional): Whether to unpack metadata. Defaults to False, which yields the metadata as
             a JSON string.
+        skip_metadata_values (set[str] | frozenset[str] | None, optional): If you are unpacking metadata, then
+           values in this set will be skipped.
     """
 
-    def __init__(
-        self,
-        timeseries: Iterable[TimeSeries],
-        namespace: Namespace | None = None,
-        unpack_metadata: bool = True,
-    ):
-        self.namespace = namespace or DEFAULT_NAMESPACE
-        self.timeseries = timeseries
-        self.unpack_metadata = unpack_metadata
+    _default_rdf_type = "TimeSeries"
 
     @classmethod
     def from_dataset(
@@ -41,15 +39,23 @@ class TimeSeriesExtractor(BaseExtractor):
         client: CogniteClient,
         data_set_external_id: str,
         namespace: Namespace | None = None,
+        to_type: Callable[[TimeSeries], str | None] | None = None,
+        limit: int | None = None,
         unpack_metadata: bool = True,
+        skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
     ):
+        total = client.time_series.aggregate_count(
+            filter=TimeSeriesFilter(data_set_ids=[{"externalId": data_set_external_id}])
+        )
+
         return cls(
-            cast(
-                Iterable[TimeSeries],
-                client.time_series(data_set_external_ids=data_set_external_id),
-            ),
-            namespace,
-            unpack_metadata,
+            client.time_series(data_set_external_ids=data_set_external_id),
+            total=total,
+            namespace=namespace,
+            to_type=to_type,
+            limit=limit,
+            unpack_metadata=unpack_metadata,
+            skip_metadata_values=skip_metadata_values,
         )
 
     @classmethod
@@ -57,23 +63,30 @@ class TimeSeriesExtractor(BaseExtractor):
         cls,
         file_path: str,
         namespace: Namespace | None = None,
+        to_type: Callable[[TimeSeries], str | None] | None = None,
+        limit: int | None = None,
         unpack_metadata: bool = True,
+        skip_metadata_values: Set[str] | None = DEFAULT_SKIP_METADATA_VALUES,
     ):
-        return cls(TimeSeriesList.load(Path(file_path).read_text()), namespace, unpack_metadata)
-
-    def extract(self) -> Iterable[Triple]:
-        """Extract timeseries as triples."""
-        for timeseries in self.timeseries:
-            yield from self._timeseries2triples(timeseries)
+        timeseries = TimeSeriesList.load(Path(file_path).read_text())
+        return cls(
+            timeseries,
+            total=len(timeseries),
+            namespace=namespace,
+            to_type=to_type,
+            limit=limit,
+            unpack_metadata=unpack_metadata,
+            skip_metadata_values=skip_metadata_values,
+        )
 
-    def _timeseries2triples(self, timeseries: TimeSeries) -> list[Triple]:
+    def _item2triples(self, timeseries: TimeSeries) -> list[Triple]:
         id_ = self.namespace[f"TimeSeries_{timeseries.id}"]
 
         # Set rdf type
-        triples: list[Triple] = [(id_, RDF.type, self.namespace.TimeSeries)]
+        type_ = self._get_rdf_type(timeseries)
+        triples: list[Triple] = [(id_, RDF.type, self.namespace[type_])]
 
         # Create attributes
-
         if timeseries.external_id:
             triples.append((id_, self.namespace.external_id, Literal(timeseries.external_id)))
 
@@ -84,22 +97,7 @@ class TimeSeriesExtractor(BaseExtractor):
             triples.append((id_, self.namespace.is_string, Literal(timeseries.is_string)))
 
         if timeseries.metadata:
-            if self.unpack_metadata:
-                for key, value in timeseries.metadata.items():
-                    if value:
-                        type_aware_value = string_to_ideal_type(value)
-                        try:
-                            triples.append((id_, self.namespace[key], URIRef(str(AnyHttpUrl(type_aware_value)))))  # type: ignore
-                        except ValidationError:
-                            triples.append((id_, self.namespace[key], Literal(type_aware_value)))
-            else:
-                triples.append(
-                    (
-                        id_,
-                        self.namespace.metadata,
-                        Literal(json.dumps(timeseries.metadata)),
-                    )
-                )
+            triples.extend(self._metadata_to_triples(id_, timeseries.metadata))
 
         if timeseries.unit:
             triples.append((id_, self.namespace.unit, Literal(timeseries.unit)))

cognite/neat/graph/loaders/_base.py

@@ -7,8 +7,8 @@ from cognite.client import CogniteClient
 from cognite.client.data_classes.capabilities import Capability
 
 from cognite.neat.graph import NeatGraphStore
-from cognite.neat.graph.issues.loader import FailedAuthorizationError
-from cognite.neat.issues import NeatIssue, NeatIssueList
+from cognite.neat.issues import IssueList, NeatIssue, NeatIssueList
+from cognite.neat.issues.errors.external import FailedAuthorizationError
 from cognite.neat.utils.auxiliary import class_html_doc
 from cognite.neat.utils.upload import UploadResult, UploadResultList
 
@@ -66,7 +66,7 @@ class CDFLoader(BaseLoader[T_Output]):
                 yield upload_result
                 return
 
-        issues = NeatIssueList[NeatIssue]()
+        issues = IssueList()
         items: list[T_Output] = []
         for result in self._load(stop_on_exception=False):
             if isinstance(result, NeatIssue):
@@ -79,7 +79,7 @@ class CDFLoader(BaseLoader[T_Output]):
 
             if len(items) >= self._UPLOAD_BATCH_SIZE or result is _END_OF_CLASS:
                 yield from self._upload_to_cdf(client, items, dry_run, issues)
-                issues = NeatIssueList[NeatIssue]()
+                issues = IssueList()
                 items = []
         if items:
             yield from self._upload_to_cdf(client, items, dry_run, issues)

cognite/neat/graph/loaders/_rdf2asset.py

@@ -19,9 +19,9 @@ from cognite.client.exceptions import CogniteAPIError, CogniteDuplicatedError
 
 from cognite.neat.graph._tracking.base import Tracker
 from cognite.neat.graph._tracking.log import LogTracker
-from cognite.neat.graph.issues import loader as loader_issues
 from cognite.neat.graph.stores import NeatGraphStore
-from cognite.neat.issues import NeatIssue, NeatIssueList
+from cognite.neat.issues import IssueList, NeatIssue, NeatIssueList
+from cognite.neat.issues.errors.resources import InvalidResourceError
 from cognite.neat.rules.analysis._asset import AssetAnalysis
 from cognite.neat.rules.models import AssetRules
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
@@ -80,7 +80,7 @@ class AssetLoader(CDFLoader[AssetWrite]):
         self.external_id_prefix = external_id_prefix
 
         self.processed_assets: set[str] = set()
-        self._issues = NeatIssueList[NeatIssue](create_issues or [])
+        self._issues = IssueList(create_issues or [])
         self._tracker: type[Tracker] = tracker or LogTracker
 
     def _load(self, stop_on_exception: bool = False) -> Iterable[AssetWrite | NeatIssue | type[_END_OF_CLASS]]:
@@ -143,8 +143,8 @@ class AssetLoader(CDFLoader[AssetWrite]):
 
                 # check on parent
                 if "parentExternalId" in fields and fields["parentExternalId"] not in self.processed_assets:
-                    error = loader_issues.InvalidInstanceError(
-                        type_=EntityTypes.asset,
+                    error = InvalidResourceError(
+                        resource_type=EntityTypes.asset,
                         identifier=identifier,
                         reason=(
                             f"Parent asset {fields['parentExternalId']} does not exist or failed creation"
@@ -171,9 +171,7 @@ class AssetLoader(CDFLoader[AssetWrite]):
                     yield AssetWrite.load(fields)
                     self.processed_assets.add(identifier)
                 except KeyError as e:
-                    error = loader_issues.InvalidInstanceError(
-                        type_=EntityTypes.asset, identifier=identifier, reason=str(e)
-                    )
+                    error = InvalidResourceError(resource_type=EntityTypes.asset, identifier=identifier, reason=str(e))
                     tracker.issue(error)
                     if stop_on_exception:
                         raise error.as_exception() from e
@@ -203,8 +201,8 @@ class AssetLoader(CDFLoader[AssetWrite]):
 
                 # check if source asset exists
                 if source_external_id not in self.processed_assets:
-                    error = loader_issues.InvalidInstanceError(
-                        type_=EntityTypes.relationship,
+                    error = InvalidResourceError(
+                        resource_type=EntityTypes.relationship,
                         identifier=source_external_id,
                         reason=(
                             f"Asset {source_external_id} does not exist! "
@@ -223,8 +221,8 @@ class AssetLoader(CDFLoader[AssetWrite]):
                         target_external_id = f"{self.external_id_prefix or ''}{target_external_id}"
                         # check if source asset exists
                         if target_external_id not in self.processed_assets:
-                            error = loader_issues.InvalidInstanceError(
-                                type_=EntityTypes.relationship,
+                            error = InvalidResourceError(
+                                resource_type=EntityTypes.relationship,
                                 identifier=target_external_id,
                                 reason=(
                                     f"Asset {target_external_id} does not exist! "
@@ -250,8 +248,8 @@ class AssetLoader(CDFLoader[AssetWrite]):
                                 labels=[label] if self.use_labels else None,
                             )
                         except KeyError as e:
-                            error = loader_issues.InvalidInstanceError(
-                                type_=EntityTypes.relationship,
+                            error = InvalidResourceError(
+                                resource_type=EntityTypes.relationship,
                                 identifier=external_id,
                                 reason=str(e),
                             )

cognite/neat/graph/loaders/_rdf2dms.py

@@ -14,10 +14,12 @@ from cognite.client.data_classes.data_modeling.views import SingleEdgeConnection
 from cognite.client.exceptions import CogniteAPIError
 from pydantic import BaseModel, ValidationInfo, create_model, field_validator
 
+import cognite.neat.issues.errors.resources
 from cognite.neat.graph._tracking import LogTracker, Tracker
-from cognite.neat.graph.issues import loader as loader_issues
 from cognite.neat.graph.stores import NeatGraphStore
-from cognite.neat.issues import NeatIssue, NeatIssueList
+from cognite.neat.issues import IssueList, NeatIssue, NeatIssueList
+from cognite.neat.issues.errors.resources import FailedConvertError, InvalidResourceError, ResourceNotFoundError
+from cognite.neat.issues.neat_warnings.models import InvalidClassWarning
 from cognite.neat.rules.models import DMSRules
 from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE, Json
 from cognite.neat.utils.auxiliary import create_sha256_hash
@@ -51,7 +53,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         self.data_model = data_model
         self.instance_space = instance_space
         self.class_by_view_id = class_by_view_id or {}
-        self._issues = NeatIssueList[NeatIssue](create_issues or [])
+        self._issues = IssueList(create_issues or [])
         self._tracker: type[Tracker] = tracker or LogTracker
 
     @classmethod
@@ -67,7 +69,9 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
         try:
             data_model = client.data_modeling.data_models.retrieve(data_model_id, inline_views=True).latest_version()
         except Exception as e:
-            issues.append(loader_issues.MissingDataModelError(identifier=repr(data_model_id), reason=str(e)))
+            issues.append(
+                ResourceNotFoundError(identifier=repr(data_model_id), resource_type="Data Model", reason=str(e))
+            )
 
         return cls(graph_store, data_model, instance_space, {}, issues)
 
@@ -79,7 +83,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
             data_model = rules.as_schema().as_read_model()
         except Exception as e:
             issues.append(
-                loader_issues.FailedConvertError(
+                FailedConvertError(
                     identifier=rules.metadata.as_identifier(),
                     target_format="read DMS model",
                     reason=str(e),
@@ -110,7 +114,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                 try:
                     yield self._create_node(identifier, properties, pydantic_cls, view_id)
                 except ValueError as e:
-                    error = loader_issues.InvalidInstanceError(type_="node", identifier=identifier, reason=str(e))
+                    error = InvalidResourceError(resource_type="node", identifier=identifier, reason=str(e))
                     tracker.issue(error)
                     if stop_on_exception:
                         raise error.as_exception() from e
@@ -141,7 +145,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
     def _create_validation_classes(
         self, view: dm.View
     ) -> tuple[type[BaseModel], dict[str, dm.EdgeConnection], NeatIssueList]:
-        issues = NeatIssueList[NeatIssue]()
+        issues = IssueList()
         field_definitions: dict[str, tuple[type, Any]] = {}
         edge_by_property: dict[str, dm.EdgeConnection] = {}
         validators: dict[str, classmethod] = {}
@@ -158,7 +162,7 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                     data_type = _DATA_TYPE_BY_DMS_TYPE.get(prop.type._type)
                     if not data_type:
                         issues.append(
-                            loader_issues.InvalidClassWarning(
+                            InvalidClassWarning(
                                 class_name=repr(view.as_id()),
                                 reason=f"Unknown data type for property {prop_name}: {prop.type._type}",
                             )
@@ -247,8 +251,8 @@ class DMSLoader(CDFLoader[dm.InstanceApply]):
                 continue
             edge = edge_by_properties[prop]
             if isinstance(edge, SingleEdgeConnection) and len(values) > 1:
-                error = loader_issues.InvalidInstanceError(
-                    type_="edge",
+                error = cognite.neat.issues.errors.resources.InvalidResourceError(
+                    resource_type="edge",
                     identifier=identifier,
                     reason=f"Multiple values for single edge {edge}. Expected only one.",
                 )

cognite/neat/graph/queries/_base.py

@@ -98,47 +98,40 @@ class Queries:
         self,
         instance_id: URIRef,
         property_renaming_config: dict | None = None,
-    ) -> tuple[str, dict[str, list[str]]]:
+    ) -> tuple[str, dict[str, list[str]]] | None:
         """DESCRIBE instance for a given class from the graph store
 
         Args:
             instance_id: Instance id for which we want to generate query
-            property_rename_config: Dictionary to rename properties, default None
+            property_renaming_config: Dictionary to rename properties, default None
 
         Returns:
             Dictionary of instance properties
         """
-
         property_values: dict[str, list[str]] = defaultdict(list)
-
-        for subject, predicate, object_ in cast(list[ResultRow], self.graph.query(f"DESCRIBE <{instance_id}>")):
-            if object_.lower() not in [
+        identifier = remove_namespace_from_uri(instance_id, validation="prefix")
+        for _, predicate, object_ in cast(list[ResultRow], self.graph.query(f"DESCRIBE <{instance_id}>")):
+            if object_.lower() in [
                 "",
                 "none",
                 "nan",
                 "null",
             ]:
-                # we are skipping deep validation with Pydantic to remove namespace here
-                # as it reduces time to process triples by 10-15x
-                identifier, value = cast(  # type: ignore[misc]
-                    (str, str),
-                    remove_namespace_from_uri(*(subject, object_), validation="prefix"),
-                )  # type: ignore[misc, index]
-
-                # use-case: calling describe without renaming properties
-                # losing the namespace from the predicate!
-                if not property_renaming_config and predicate != RDF.type:
-                    property_values[remove_namespace_from_uri(predicate, validation="prefix")].append(value)
-
-                # use-case: calling describe with renaming properties
-                # renaming the property to the new name, if the property is defined
-                # in the RULES sheet
-                elif property_renaming_config and (property_ := property_renaming_config.get(predicate, None)):
-                    property_values[property_].append(value)
-
-                # use-case: skip the property if it is not defined in property_renaming_config
-                else:
-                    continue
+                continue
+            # we are skipping deep validation with Pydantic to remove namespace here
+            # as it reduces time to process triples by 10-15x
+            value = remove_namespace_from_uri(object_, validation="prefix")
+
+            # use-case: calling describe without renaming properties
+            # losing the namespace from the predicate!
+            if not property_renaming_config and predicate != RDF.type:
+                property_values[remove_namespace_from_uri(predicate, validation="prefix")].append(value)
+
+            # use-case: calling describe with renaming properties
+            # renaming the property to the new name, if the property is defined
+            # in the RULES sheet
+            elif property_renaming_config and (property_ := property_renaming_config.get(predicate, None)):
+                property_values[property_].append(value)
 
         if property_values:
             return (
@@ -146,7 +139,7 @@ class Queries:
                 property_values,
             )
         else:
-            return ()  # type: ignore [return-value]
+            return None
 
     def construct_instances_of_class(
         self,
@@ -177,7 +170,7 @@ class Queries:
             result = self.graph.query(query)
 
             # We cannot include the RDF.type in case there is a neat:type property
-            return [remove_namespace_from_uri(*triple) for triple in result if triple[1] != RDF.type]  # type: ignore[misc, index]
+            return [remove_namespace_from_uri(cast(ResultRow, triple)) for triple in result if triple[1] != RDF.type]  # type: ignore[misc, index]
         else:
             warnings.warn(
                 "No rules found for the graph store, returning empty list.",

cognite/neat/graph/queries/_shared.py

@@ -160,7 +160,7 @@ def triples2dictionary(triples: Iterable[tuple[URIRef, URIRef, str | URIRef]]) -
         value: str
         uri: URIRef
 
-        id_, property_, value = remove_namespace_from_uri(*triple)  # type: ignore[misc]
+        id_, property_, value = remove_namespace_from_uri(triple)  # type: ignore[misc]
         uri = triple[0]
 
         if uri not in dictionary:

cognite/neat/graph/stores/_base.py

@@ -108,7 +108,7 @@ class NeatGraphStore:
 
     @classmethod
     def from_memory_store(cls, rules: InformationRules | None = None) -> "Self":
-        return cls(Graph(), rules)
+        return cls(Graph(identifier=DEFAULT_NAMESPACE), rules)
 
     @classmethod
     def from_sparql_store(
@@ -126,17 +126,17 @@ class NeatGraphStore:
             postAsEncoded=False,
             autocommit=False,
         )
-        graph = Graph(store=store)
+        graph = Graph(store=store, identifier=DEFAULT_NAMESPACE)
         return cls(graph, rules)
 
     @classmethod
     def from_oxi_store(cls, storage_dir: Path | None = None, rules: InformationRules | None = None) -> "Self":
         """Creates a NeatGraphStore from an Oxigraph store."""
         local_import("pyoxigraph", "oxi")
+        local_import("oxrdflib", "oxi")
+        import oxrdflib
         import pyoxigraph
 
-        from cognite.neat.graph.stores._oxrdflib import OxigraphStore
-
         # Adding support for both oxigraph in-memory and file-based storage
         for i in range(4):
             try:
@@ -149,8 +149,10 @@ class NeatGraphStore:
         else:
             raise Exception("Error initializing Oxigraph store")
 
-        graph = Graph(store=OxigraphStore(store=oxi_store))
-        graph.default_union = True
+        graph = Graph(
+            store=oxrdflib.OxigraphStore(store=oxi_store),
+            identifier=DEFAULT_NAMESPACE,
+        )
 
         return cls(graph, rules)
 
@@ -208,7 +210,8 @@ class NeatGraphStore:
         property_renaming_config = InformationAnalysis(self.rules).define_property_renaming_config(class_entity)
 
         for instance_id in instance_ids:
-            yield self.queries.describe(instance_id, property_renaming_config)
+            if res := self.queries.describe(instance_id, property_renaming_config):
+                yield res
 
     def _parse_file(
         self,
@@ -221,7 +224,7 @@ class NeatGraphStore:
         Args:
             filepath : File path to file containing graph data, by default None
             mime_type : MIME type of graph data, by default "application/rdf+xml"
-            add_base_iri : Add base IRI to graph, by default True
+            base_uri : Add base IRI to graph, by default True
         """
 
         # Oxigraph store, do not want to type hint this as it is an optional dependency
@@ -229,10 +232,15 @@ class NeatGraphStore:
 
             def parse_to_oxi_store():
                 local_import("pyoxigraph", "oxi")
-                from cognite.neat.graph.stores._oxrdflib import OxigraphStore
+                import pyoxigraph
 
-                cast(OxigraphStore, self.graph.store)._inner.bulk_load(str(filepath), mime_type, base_iri=base_uri)  # type: ignore[attr-defined]
-                cast(OxigraphStore, self.graph.store)._inner.optimize()  # type: ignore[attr-defined]
+                cast(pyoxigraph.Store, self.graph.store._store).bulk_load(
+                    str(filepath),
+                    mime_type,
+                    base_iri=base_uri,
+                    to_graph=pyoxigraph.NamedNode(self.graph.identifier),
+                )
+                cast(pyoxigraph.Store, self.graph.store._store).optimize()
 
             parse_to_oxi_store()
 

cognite/neat/graph/transformers/_rdfpath.py

@@ -1,4 +1,4 @@
-from rdflib import RDF, Graph
+from rdflib import Graph
 
 from cognite.neat.rules.analysis import InformationAnalysis
 from cognite.neat.rules.models._rdfpath import RDFPath, SingleProperty
@@ -17,6 +17,7 @@ class AddSelfReferenceProperty(BaseTransformer):
     description: str = "Adds property that contains id of reference to all references of given class in Rules"
     _use_only_once: bool = True
     _need_changes = frozenset({})
+    _ref_template: str = """SELECT ?s WHERE {{?s a <{type_}>}}"""
 
     def __init__(
         self,
@@ -32,7 +33,7 @@ class AddSelfReferenceProperty(BaseTransformer):
 
             namespace = self.rules.prefixes[prefix] if prefix in self.rules.prefixes else self.rules.metadata.namespace
 
-            for reference in graph.subjects(RDF.type, namespace[suffix]):
+            for (reference,) in graph.query(self._ref_template.format(type_=namespace[suffix])):  # type: ignore [misc]
                 graph.add(
                     (
                         reference,

cognite/neat/issues/__init__.py

@@ -0,0 +1,16 @@
+"""This is module contains all the Neat Exceptions (Errors) and Warnings as well
+as
+
+"""
+
+from ._base import DefaultWarning, IssueList, MultiValueError, NeatError, NeatIssue, NeatIssueList, NeatWarning
+
+__all__ = [
+    "NeatIssue",
+    "NeatError",
+    "NeatWarning",
+    "DefaultWarning",
+    "NeatIssueList",
+    "IssueList",
+    "MultiValueError",
+]

cognite/neat/{issues.py → issues/_base.py}

@@ -9,6 +9,7 @@ from typing import Any, ClassVar, TypeVar
 from warnings import WarningMessage
 
 import pandas as pd
+from pydantic_core import ErrorDetails, PydanticCustomError
 
 if sys.version_info < (3, 11):
     from exceptiongroup import ExceptionGroup
@@ -17,11 +18,24 @@ else:
     from typing import Self
 
 
+__all__ = [
+    "NeatIssue",
+    "NeatError",
+    "NeatWarning",
+    "DefaultWarning",
+    "NeatIssueList",
+    "MultiValueError",
+]
+
+
 @total_ordering
 @dataclass(frozen=True)
 class NeatIssue(ABC):
+    """This is the base class for all exceptions and warnings (issues) used in Neat."""
+
     description: ClassVar[str]
-    fix: ClassVar[str]
+    extra: ClassVar[str | None] = None
+    fix: ClassVar[str | None] = None
 
     def message(self) -> str:
         """Return a human-readable message for the issue.
@@ -30,7 +44,7 @@ class NeatIssue(ABC):
         It is recommended to override this method in subclasses with a more
         specific message.
         """
-        return self.description
+        return self.__doc__ or "Missing"
 
     @abstractmethod
     def dump(self) -> dict[str, Any]:
@@ -56,6 +70,65 @@ class NeatError(NeatIssue, ABC):
     def as_exception(self) -> ValueError:
         return ValueError(self.message())
 
+    def as_pydantic_exception(self) -> PydanticCustomError:
+        return PydanticCustomError(
+            type(self).__name__,
+            self.message(),
+            dict(description=self.__doc__, fix=self.fix),
+        )
+
+    @classmethod
+    def from_pydantic_errors(cls, errors: list[ErrorDetails], **kwargs) -> "list[NeatError]":
+        """Convert a list of pydantic errors to a list of Error instances.
+
+        This is intended to be overridden in subclasses to handle specific error types.
+        """
+        all_errors: list[NeatError] = []
+        for error in errors:
+            if isinstance(ctx := error.get("ctx"), dict) and isinstance(
+                multi_error := ctx.get("error"), MultiValueError
+            ):
+                all_errors.extend(multi_error.errors)  # type: ignore[arg-type]
+            else:
+                all_errors.append(DefaultPydanticError.from_pydantic_error(error))
+        return all_errors
+
+
+@dataclass(frozen=True)
+class DefaultPydanticError(NeatError):
+    type: str
+    loc: tuple[int | str, ...]
+    msg: str
+    input: Any
+    ctx: dict[str, Any] | None
+
+    @classmethod
+    def from_pydantic_error(cls, error: ErrorDetails) -> "DefaultPydanticError":
+        return cls(
+            type=error["type"],
+            loc=error["loc"],
+            msg=error["msg"],
+            input=error.get("input"),
+            ctx=error.get("ctx"),
+        )
+
+    def dump(self) -> dict[str, Any]:
+        output = super().dump()
+        output["type"] = self.type
+        output["loc"] = self.loc
+        output["msg"] = self.msg
+        output["input"] = self.input
+        output["ctx"] = self.ctx
+        return output
+
+    def message(self) -> str:
+        if self.loc and len(self.loc) == 1:
+            return f"{self.loc[0]} sheet: {self.msg}"
+        elif self.loc and len(self.loc) == 2:
+            return f"{self.loc[0]} sheet field/column <{self.loc[1]}>: {self.msg}"
+        else:
+            return self.msg
+
 
 @dataclass(frozen=True)
 class NeatWarning(NeatIssue, ABC, UserWarning):
@@ -148,3 +221,6 @@ class MultiValueError(ValueError):
 
     def __init__(self, errors: Sequence[T_NeatIssue]):
         self.errors = list(errors)
+
+
+class IssueList(NeatIssueList[NeatIssue]): ...