schemapack 1.0.0__py3-none-any.whl

schemapack/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A specification (plus tooling) for describing linked data models based on JSON schema."""
+
+from importlib.metadata import version
+
+__version__ = version(__package__)

schemapack/__main__.py

@@ -0,0 +1,16 @@
+# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Entrypoint of the package"""

schemapack/exceptions.py

@@ -0,0 +1,202 @@
+# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Collection of package-specific exceptions"""
+
+from abc import ABC
+from dataclasses import dataclass
+from typing import Optional
+
+import pydantic_core
+
+
+class BaseError(ABC, Exception):
+    """Base class for all schemapack errors."""
+
+
+class SpecError(BaseError, ValueError):
+    """An error indicating that an instance was not valid against the specification."""
+
+    def __init__(self, message: str, *, details: list[pydantic_core.ErrorDetails]):
+        """Initiate a SpecError.
+
+        Args:
+            message: A human-readable message describing the error.
+            details: Details on which fields of the instance are invalid.
+        """
+        super().__init__(message)
+        self.message = message
+        self.details = details
+
+
+class SchemaPackSpecError(SpecError):
+    """An error indicating that the provided data is not valid against the schemapack
+    specification.
+    """
+
+    def __init__(self, message: str, *, details: list[pydantic_core.ErrorDetails]):
+        """Initiate a SchemaPackSpecError.
+
+        Args:
+            message: A human-readable message describing the error.
+            details: Details on which fields of the instance are invalid.
+        """
+        message = (
+            "The provided object was not compatible with the schemapack"
+            + f" specification: {message}"
+        )
+        super().__init__(message, details=details)
+
+
+class DataPackSpecError(SpecError):
+    """An error indicating that the provided data is not valid against the datapack
+    specification (this is independent of validation done using a schemapack).
+    """
+
+    def __init__(self, message: str, *, details: list[pydantic_core.ErrorDetails]):
+        """Initiate a SchemaPackSpecError.
+
+        Args:
+            message: A human-readable message describing the error.
+            details: Details on which fields of the instance are invalid.
+        """
+        message = (
+            "The provided object was not compatible with the datapack"
+            + f" specification: {message}"
+        )
+        super().__init__(message, details=details)
+
+
+class ValidationPluginError(BaseError, ValueError):
+    """Raised by a ValidationPlugin."""
+
+    def __init__(
+        self, *, type_: str, message: str, details: Optional[dict[str, object]] = None
+    ):
+        """Initiate a ValidationPluginError.
+
+        Args:
+            type: A preferably short type label.
+            message: A human-readable message describing the error.
+            details: A dictionary for transporting additional machine-readable details.
+        """
+        super().__init__(message)
+        self.type_ = type_
+        self.message = message
+        self.details = details if details else {}
+
+
+@dataclass
+class ValidationErrorRecord:
+    """A record of an Error occuring during validation on a specific context
+    (e.g. a specific resource) regarding a specific validation aspect.
+
+    Attributes:
+        subject_class:
+            If relevant, the name of the subject class as per the schemapack.
+        subject_resource:
+            If relevant, the ID or the subject resource of the subject class.
+        type:
+            A preferably short type label.
+        message:
+            A human-readable message describing the error.
+        details:
+            A dictionary for transporting additional machine-readable details.
+    """
+
+    subject_class: Optional[str]
+    subject_resource: Optional[str]
+    type: str
+    message: str
+    details: dict[str, object]
+
+
+def _val_record_to_str(record: ValidationErrorRecord) -> str:
+    """Translate a ValidationErrorRecords into a human-readable message
+    to be displayed to the user on the terminal.
+    """
+    if record.subject_class:
+        if record.subject_resource:
+            context = f"resource '{record.subject_class}.{record.subject_resource}'"
+        else:
+            context = "class '{record.subject_class}'"
+    else:
+        context = "global datapack"
+
+    return (
+        f"Error in {context}:"
+        + f"\n\tType: {record.type}"
+        + f"\n\tMessage: {record.message}"
+    )
+
+
+def _sorted_val_records(
+    records: list[ValidationErrorRecord],
+) -> list[ValidationErrorRecord]:
+    """Sort a set of ValidationErrorRecords."""
+    return sorted(records, key=lambda r: (r.subject_class, r.subject_resource, r.type))
+
+
+class ValidationError(BaseError, ValueError):
+    """A collection of ValidationErrorRecords raised when a datapack fails validation
+    against a schemapack.
+    """
+
+    def __init__(self, records: list[ValidationErrorRecord]):
+        """Initiate a ValidationError.
+
+        Args:
+            records: A list of ValidationErrorRecords.
+        """
+        if not records:
+            raise ValueError("ValidationError must be raised with at least one record.")
+
+        self.records = _sorted_val_records(records)
+        n_records = len(self.records)
+        record_messages = "\n".join(_val_record_to_str(r) for r in self.records)
+        message = f"Validation failed with {n_records} issue(s):\n{record_messages}"
+
+        super().__init__(message)
+
+
+class ValidationAssumptionError(BaseError, RuntimeError):
+    """This is raised when a unit of code assumes to work with a datapack that has
+    already been validated against a specific schemapack, however, it became apparent
+    that this assumption was not correct. The error should not contain the details on
+    which aspect is invalid. These details can be retrieved by performing a validation
+    and receiving a ValidationError.
+    """
+
+    def __init__(self, *, context: str):
+        """Initiate a ValidationAssumptionError.
+
+        Args:
+            context: Some context to be included in the error message.
+        """
+        message = (
+            "It was assumed that the provided datapack has already been validated"
+            + " against the provided schemapack, however, this assumption failed in"
+            + f" the context of {context}. Please validate the datapack against the"
+            + " schemapack to get further details."
+        )
+        super().__init__(message)
+        self.message = message
+
+
+class CircularRelationError(BaseError, ValueError):
+    """Raised when a circular relation between resources is detected, but the requested
+    operation cannot be performed on datapacks with circular relations.
+    """

schemapack/integrate.py

@@ -0,0 +1,131 @@
+# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Integrate rooted datapacks into nested json objects."""
+
+from collections import defaultdict
+from collections.abc import Mapping
+from typing import Optional
+
+from typing_extensions import TypeAlias
+
+from schemapack.exceptions import CircularRelationError, ValidationAssumptionError
+from schemapack.spec.datapack import ClassName, DataPack, ResourceId
+from schemapack.spec.schemapack import SchemaPack
+
+JsonObjectCompatible: TypeAlias = dict[str, object]
+
+
+def integrate(  # noqa: PLR0912,C901
+    *,
+    datapack: DataPack,
+    schemapack: SchemaPack,
+    _resource_blacklist: Optional[Mapping[ClassName, set[ResourceId]]] = None,
+    _alt_root_class_name: Optional[ClassName] = None,
+    _alt_root_resource_id: Optional[ResourceId] = None,
+) -> JsonObjectCompatible:
+    """Integrate a rooted datapack into a nested json object-compatible data structure.
+    It is assumed that the provided datapack has already been validated against the
+    provided schemapack.
+
+    Args:
+        datapack:
+            The datapack to be integrated. Must be rooted.
+        schemapack:
+            The schemapack to be used for looking up the classes of relations.
+        _resource_blacklist:
+            An optional blacklist of resources (a mapping with resource ids as values
+            and class names as keys) that cause an error to be raised if they are
+            encountered when assembling the json compatible object. This is only used
+            internally to prevent circular relations.
+        _alt_root_class_name:
+            An optional alternative root class name to be used instead of the class
+            name specified as root in the datapack. This is only used internally for
+            recursive calls.
+        _alt_root_resource_id:
+            An optional alternative root resource id to be used instead of the resource
+            id specified as root in the datapack. This is only used internally for
+            recursive calls.
+
+    Raises:
+        ValueError:
+            If the datapack is not rooted.
+        schemapack.exceptions.ValidationAssumptionError:
+            If the datapack is not valid against the schemapack.
+        schemapack.exceptions.CircularRelationError:
+            If a circular relation is detected.
+    """
+    if not datapack.root:
+        raise ValueError("Datapack must be rooted.")
+
+    root_class_name = _alt_root_class_name or datapack.root.class_name
+    root_resource_id = _alt_root_resource_id or datapack.root.resource_id
+
+    resource_blacklist: dict[ClassName, set[ResourceId]] = defaultdict(set)
+    resource_blacklist[root_class_name].add(root_resource_id)
+    if _resource_blacklist:
+        for class_name, resource_ids in _resource_blacklist.items():
+            resource_blacklist[class_name].update(resource_ids)
+
+    root_resource = datapack.resources[root_class_name][root_resource_id]
+    try:
+        root_class_definition = schemapack.classes[root_class_name]
+    except KeyError as error:
+        raise ValidationAssumptionError(context="class lookup") from error
+
+    integrated_object: JsonObjectCompatible = {}
+    integrated_object.update(root_resource.content)
+
+    for relation_name, foreign_ids in root_resource.relations.items():
+        if isinstance(foreign_ids, str):
+            foreign_ids = [foreign_ids]
+
+        try:
+            relation_definition = root_class_definition.relations[relation_name]
+        except KeyError as error:
+            raise ValidationAssumptionError(context="relation resolution") from error
+
+        foreign_class_name = relation_definition.to
+
+        is_plural = relation_definition.cardinality.endswith("to_many")
+        if is_plural:
+            integrated_object[relation_name] = []
+
+        for foreign_id in foreign_ids:
+            if (
+                foreign_class_name in resource_blacklist
+                and foreign_id in resource_blacklist[foreign_class_name]
+            ):
+                raise CircularRelationError(
+                    "Cannot perform integration of datapack with circular relations."
+                    + " The circular relation involved the resource with id"
+                    + f" {foreign_id} of class {foreign_class_name}."
+                )
+
+            foreign_resource = integrate(
+                datapack=datapack,
+                schemapack=schemapack,
+                _resource_blacklist=resource_blacklist,
+                _alt_root_class_name=foreign_class_name,
+                _alt_root_resource_id=foreign_id,
+            )
+
+            if is_plural:
+                integrated_object[relation_name].append(foreign_resource)  # type: ignore
+            else:
+                integrated_object[relation_name] = foreign_resource
+
+    return integrated_object

schemapack/isolate.py

@@ -0,0 +1,185 @@
+# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Logic to isolate a resource from a non-rooted datapack to create a rooted datapack."""
+
+from collections import defaultdict
+from collections.abc import Mapping
+from typing import Optional
+
+from schemapack.exceptions import ValidationAssumptionError
+from schemapack.spec.datapack import (
+    ClassName,
+    DataPack,
+    Resource,
+    ResourceId,
+    RootResource,
+)
+from schemapack.spec.schemapack import SchemaPack
+
+
+def identify_dependencies(
+    *,
+    datapack: DataPack,
+    class_name: ClassName,
+    resource_id: ResourceId,
+    schemapack: SchemaPack,
+    include_target: bool = False,
+    _resource_blacklist: Optional[Mapping[ClassName, set[ResourceId]]] = None,
+) -> Mapping[ClassName, set[ResourceId]]:
+    """Identify all dependencies (recursively) for a given resource
+    of the given class in the given datapack. Please note that it is assumed that
+    the datapack has already been validated against the schemapack.
+
+    Args:
+        datapack:
+            Containing the target resource and all dependencies.
+        class_name:
+            The class of the target resource.
+        resource_id:
+            The id of the target resource.
+        schemapack:
+            The schemapack used for looking up the classes of relations.
+        include_target:
+            Set to true if the target dependency shall be included in the result.
+        _resource_blacklist:
+            A mapping containing resources (resource ids as values and class names as
+            keys) that shall be ignored during the dependency resolution and thus will
+            not appear in the result. This is only used internally for recursion.
+
+    Returns:
+        A mapping containing resource ids (values) of dependencies by class names
+        (keys).
+
+    Raises:
+        KeyError:
+            If the resource_class or the resource_id does not exist in the schemapack
+            or datapack.
+        schemapack.Exceptions.ValidationAssumptionError:
+            If it became apparent that the datapack was not already validated against
+            the schemapack.
+    """
+    target_resource = datapack.resources[class_name][resource_id]
+    target_class_definition = schemapack.classes[class_name]
+
+    # Define a blacklist of resources to avoid getting lost in infinity loop for
+    # circular dependencies:
+    resource_blacklist: dict[ClassName, set[ResourceId]] = defaultdict(set)
+    resource_blacklist[class_name].add(resource_id)
+    if _resource_blacklist:
+        for class_name, resource_ids in _resource_blacklist.items():
+            resource_blacklist[class_name].update(resource_ids)
+
+    dependencies_by_class: dict[ClassName, set[ResourceId]] = defaultdict(set)
+
+    for relation_name, foreign_ids in target_resource.relations.items():
+        if isinstance(foreign_ids, str):
+            foreign_ids = [foreign_ids]
+
+        try:
+            foreign_class_name = target_class_definition.relations[relation_name].to
+        except KeyError as error:
+            raise ValidationAssumptionError(context="relation resolution") from error
+
+        for foreign_id in foreign_ids:
+            if (
+                foreign_class_name in resource_blacklist
+                and foreign_id in resource_blacklist[foreign_class_name]
+            ):
+                continue
+
+            dependencies_by_class[foreign_class_name].add(foreign_id)
+
+            # Recursively add dependencies of this foreign resource:
+            nested_dependencies = identify_dependencies(
+                datapack=datapack,
+                class_name=foreign_class_name,
+                resource_id=foreign_id,
+                schemapack=schemapack,
+                _resource_blacklist=resource_blacklist,
+            )
+            for nested_class_name, nested_ids in nested_dependencies.items():
+                dependencies_by_class[nested_class_name].update(nested_ids)
+
+    if include_target:
+        dependencies_by_class[class_name].add(resource_id)
+
+    return dependencies_by_class
+
+
+def downscope_datapack(
+    datapack: DataPack,
+    resource_map: Mapping[ClassName, set[ResourceId]],
+    ignore_non_existing: bool = False,
+) -> DataPack:
+    """Downscope a datapack to only contain the given resources.
+
+    Args:
+        ignore_non_existing:
+            Controls how to handle resources from the resource_map that are not in the
+            datapack. If set to `True`, these resources will be ignored. If set to
+            `False` (default), a KeyError will be raised.
+
+    Raises:
+        KeyError:
+            If a resource from the resource_map is not in the datapack and
+            ignore_non_existing is set to `False`.
+
+    """
+    resources: dict[ClassName, dict[ResourceId, Resource]] = defaultdict(dict)
+
+    for class_name, resource_ids in resource_map.items():
+        try:
+            existing_resources = datapack.resources[class_name]
+        except KeyError as error:
+            if ignore_non_existing:
+                continue
+            raise error
+
+        for resource_id in resource_ids:
+            try:
+                existing_resource = existing_resources[resource_id]
+            except KeyError as error:
+                if ignore_non_existing:
+                    continue
+                raise error
+
+            resources[class_name][resource_id] = existing_resource
+
+    return datapack.model_copy(update={"resources": resources})
+
+
+def isolate(
+    *,
+    datapack: DataPack,
+    class_name: ClassName,
+    resource_id: ResourceId,
+    schemapack: SchemaPack,
+) -> DataPack:
+    """Isolate a resource from a non-rooted datapack to created a rooted datapack. I.e.
+    the resulting datapack will only contain resources referenced by the root resource
+    as well as the root resource itself.
+    """
+    dependency_map = identify_dependencies(
+        datapack=datapack,
+        class_name=class_name,
+        resource_id=resource_id,
+        schemapack=schemapack,
+        include_target=True,
+    )
+    rooted_datapack = downscope_datapack(datapack=datapack, resource_map=dependency_map)
+    rooted_datapack.root = RootResource(class_name=class_name, resource_id=resource_id)
+    return rooted_datapack

schemapack/load.py

@@ -0,0 +1,49 @@
+# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Loading schemapack and datapack definitions."""
+
+from pathlib import Path
+
+import pydantic
+
+from schemapack.exceptions import DataPackSpecError, SchemaPackSpecError
+from schemapack.spec.datapack import DataPack
+from schemapack.spec.schemapack import SchemaPack
+from schemapack.utils import read_json_or_yaml, transient_directory_change
+
+
+def load_schemapack(path: Path):
+    """Load a schemapack definition from a file."""
+    schemapack_dict = read_json_or_yaml(path)
+
+    with transient_directory_change(path.parent):
+        try:
+            return SchemaPack.model_validate(schemapack_dict)
+        except pydantic.ValidationError as error:
+            raise SchemaPackSpecError(
+                message=str(error), details=error.errors()
+            ) from error
+
+
+def load_datapack(path: Path):
+    """Load a datapack definition from a file."""
+    datapack_dict = read_json_or_yaml(path)
+
+    try:
+        return DataPack.model_validate(datapack_dict)
+    except pydantic.ValidationError as error:
+        raise DataPackSpecError(message=str(error), details=error.errors()) from error

schemapack/main.py

@@ -0,0 +1,45 @@
+# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""High-level convenience functions."""
+
+from pathlib import Path
+
+from schemapack.load import load_datapack, load_schemapack
+from schemapack.spec.datapack import DataPack
+from schemapack.spec.schemapack import SchemaPack
+from schemapack.validation import SchemaPackValidator
+
+
+def load_and_validate(
+    *, schemapack_path: Path, datapack_path: Path
+) -> tuple[SchemaPack, DataPack]:
+    """Load and validate a datapack with respect to a schemapack using the default
+    validation plugins.
+
+    Raises:
+        schemapack.exceptions.SchemaPackFormatError:
+            If the schemapack has an invalid format.
+        schemapack.exceptions.DataPackFormatError:
+            If the datapack has an invalid format.
+        schemapack.exceptions.ValidationError:
+            If the datapack is not valid against the schemapack.
+    """
+    schemapack = load_schemapack(schemapack_path)
+    datapack = load_datapack(datapack_path)
+    validator = SchemaPackValidator(schemapack=schemapack)
+    validator.validate(datapack=datapack)
+    return schemapack, datapack

schemapack/spec/__init__.py

@@ -0,0 +1,17 @@
+# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Models for SchemaPack and DataPack definitions."""