fairspec-metadata 0.0.0.dev0__py3-none-any.whl

fairspec_metadata/actions/column/create_spec.py

@@ -0,0 +1,83 @@
+from .create import create_column_from_property
+
+
+class TestCreateColumnFromProperty:
+    def test_creates_string_column(self):
+        column = create_column_from_property("name", {"type": "string"})
+        assert column.type == "string"
+        assert column.nullable is None
+
+    def test_creates_nullable_column_for_type_null(self):
+        column = create_column_from_property("name", {"type": ["string", "null"]})
+        assert column.type == "string"
+        assert column.nullable is True
+
+    def test_creates_nullable_column_for_null_type(self):
+        column = create_column_from_property("name", {"type": ["null", "string"]})
+        assert column.type == "string"
+        assert column.nullable is True
+
+    def test_creates_nullable_date_column(self):
+        column = create_column_from_property(
+            "created", {"type": ["string", "null"], "format": "date"}
+        )
+        assert column.type == "date"
+        assert column.nullable is True
+
+    def test_creates_integer_column(self):
+        column = create_column_from_property("id", {"type": "integer"})
+        assert column.type == "integer"
+
+    def test_creates_boolean_column(self):
+        column = create_column_from_property("flag", {"type": "boolean"})
+        assert column.type == "boolean"
+
+    def test_creates_number_column(self):
+        column = create_column_from_property("value", {"type": "number"})
+        assert column.type == "number"
+
+    def test_creates_array_column(self):
+        column = create_column_from_property("items", {"type": "array"})
+        assert column.type == "array"
+
+    def test_creates_object_column(self):
+        column = create_column_from_property("meta", {"type": "object"})
+        assert column.type == "object"
+
+    def test_creates_geojson_column(self):
+        column = create_column_from_property(
+            "geo", {"type": "object", "format": "geojson"}
+        )
+        assert column.type == "geojson"
+
+    def test_creates_topojson_column(self):
+        column = create_column_from_property(
+            "topo", {"type": "object", "format": "topojson"}
+        )
+        assert column.type == "topojson"
+
+    def test_creates_categorical_column_from_string(self):
+        column = create_column_from_property(
+            "cat", {"type": "string", "format": "categorical"}
+        )
+        assert column.type == "categorical"
+
+    def test_creates_categorical_column_from_integer(self):
+        column = create_column_from_property(
+            "cat", {"type": "integer", "format": "categorical"}
+        )
+        assert column.type == "categorical"
+
+    def test_creates_unknown_column_for_none_type(self):
+        column = create_column_from_property("x", {})
+        assert column.type == "unknown"
+
+    def test_creates_email_column(self):
+        column = create_column_from_property(
+            "email", {"type": "string", "format": "email"}
+        )
+        assert column.type == "email"
+
+    def test_creates_url_column(self):
+        column = create_column_from_property("url", {"type": "string", "format": "url"})
+        assert column.type == "url"

fairspec_metadata/actions/column/property.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from fairspec_metadata.models.column.column import Column, ColumnProperty
+
+
+def get_base_property_type(type: str | Sequence[str] | None) -> str | None:
+    if type is None:
+        return None
+    if isinstance(type, str):
+        return type
+    for t in type:
+        if t != "null":
+            return t
+    return "null"
+
+
+def get_is_nullable_property_type(type: str | Sequence[str] | None) -> bool:
+    if type is None:
+        return False
+    if isinstance(type, str):
+        return False
+    return "null" in type
+
+
+def get_column_properties(columns: list[Column]) -> dict[str, ColumnProperty]:
+    return {column.name: column.property for column in columns}

fairspec_metadata/actions/column/property_spec.py

@@ -0,0 +1,29 @@
+from .property import get_base_property_type, get_is_nullable_property_type
+
+
+class TestGetBasePropertyType:
+    def test_returns_type_for_string(self):
+        assert get_base_property_type("string") == "string"
+
+    def test_returns_base_type_for_type_null(self):
+        assert get_base_property_type(["string", "null"]) == "string"
+
+    def test_returns_base_type_for_null_type(self):
+        assert get_base_property_type(["null", "string"]) == "string"
+
+    def test_returns_none_for_none(self):
+        assert get_base_property_type(None) is None
+
+
+class TestGetIsNullablePropertyType:
+    def test_returns_false_for_string(self):
+        assert get_is_nullable_property_type("string") is False
+
+    def test_returns_true_for_type_null(self):
+        assert get_is_nullable_property_type(["string", "null"]) is True
+
+    def test_returns_true_for_null_type(self):
+        assert get_is_nullable_property_type(["null", "string"]) is True
+
+    def test_returns_false_for_none(self):
+        assert get_is_nullable_property_type(None) is False

fairspec_metadata/actions/data_schema/assert_.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from fairspec_metadata.models.data_schema import DataSchema
+from fairspec_metadata.models.descriptor import Descriptor
+from fairspec_metadata.models.exception import FairspecException
+
+from .validate import validate_data_schema
+
+
+def assert_data_schema(source: Descriptor) -> DataSchema:
+    result = validate_data_schema(source)
+
+    if not result.data_schema:
+        raise FairspecException("Invalid Data Schema", report=result)
+
+    return result.data_schema

fairspec_metadata/actions/data_schema/load.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from fairspec_metadata.actions.descriptor.load import load_descriptor
+from fairspec_metadata.models.data_schema import DataSchema
+
+from .assert_ import assert_data_schema
+
+
+def load_data_schema(path: str) -> DataSchema:
+    descriptor = load_descriptor(path)
+    return assert_data_schema(descriptor)

fairspec_metadata/actions/data_schema/resolve.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from fairspec_metadata.actions.json_schema.load import load_json_schema
+from fairspec_metadata.models.data_schema import DataSchema
+
+
+def resolve_data_schema(
+    data_schema: DataSchema | str | None = None,
+) -> DataSchema | None:
+    if data_schema is None:
+        return None
+
+    if not isinstance(data_schema, str):
+        return data_schema
+
+    return load_json_schema(data_schema)

fairspec_metadata/actions/data_schema/save.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from fairspec_metadata.actions.descriptor.copy import copy_descriptor
+from fairspec_metadata.actions.descriptor.save import save_descriptor
+from fairspec_metadata.models.data_schema import DataSchema
+from fairspec_metadata.settings import FAIRSPEC_VERSION
+
+
+def save_data_schema(
+    data_schema: DataSchema,
+    *,
+    path: str,
+    overwrite: bool = False,
+) -> None:
+    descriptor = copy_descriptor(data_schema)
+
+    if "$schema" not in descriptor:
+        descriptor["$schema"] = (
+            f"https://fairspec.org/profiles/{FAIRSPEC_VERSION}/data-schema.json"
+        )
+
+    save_descriptor(descriptor, path=path, overwrite=overwrite)

fairspec_metadata/actions/data_schema/validate.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from fairspec_metadata.actions.descriptor.load import load_descriptor
+from fairspec_metadata.actions.descriptor.validate import validate_descriptor
+from fairspec_metadata.actions.profile.load import load_profile
+from fairspec_metadata.models.data_schema import DataSchema
+from fairspec_metadata.models.descriptor import Descriptor
+from fairspec_metadata.models.profile import ProfileType
+from fairspec_metadata.models.report import Report
+
+
+class DataSchemaValidationResult(Report):
+    data_schema: DataSchema | None
+
+
+def validate_data_schema(
+    source: Descriptor | str,
+    *,
+    root_json_pointer: str | None = None,
+) -> DataSchemaValidationResult:
+    descriptor = load_descriptor(source) if isinstance(source, str) else source
+
+    schema = descriptor.get("$schema")
+    schema_url = (
+        schema
+        if isinstance(schema, str)
+        else "https://fairspec.org/profiles/latest/data-schema.json"
+    )
+
+    profile = load_profile(schema_url, profile_type=ProfileType.data_schema)
+
+    report = validate_descriptor(
+        descriptor,
+        profile=profile,
+        root_json_pointer=root_json_pointer,
+    )
+
+    data_schema: DataSchema | None = None
+    if report.valid:
+        data_schema = descriptor
+
+    return DataSchemaValidationResult(
+        valid=report.valid,
+        errors=report.errors,
+        data_schema=data_schema,
+    )

fairspec_metadata/actions/dataset/assert_.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from fairspec_metadata.models.dataset import Dataset
+from fairspec_metadata.models.descriptor import Descriptor
+from fairspec_metadata.models.exception import FairspecException
+
+from .validate import validate_dataset_descriptor
+
+
+def assert_dataset(source: Descriptor, *, basepath: str | None = None) -> Dataset:
+    result = validate_dataset_descriptor(source, basepath=basepath)
+
+    if not result.dataset:
+        raise FairspecException("Invalid Dataset", report=result)
+
+    return result.dataset

fairspec_metadata/actions/dataset/denormalize.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from fairspec_metadata.actions.resource.denormalize import denormalize_resource
+from fairspec_metadata.models.dataset import Dataset
+
+
+def denormalize_dataset(dataset: Dataset, *, basepath: str | None = None) -> Dataset:
+    result = dataset.model_dump(by_alias=True, exclude_none=True)
+
+    if dataset.resources:
+        result["resources"] = [
+            denormalize_resource(resource, basepath=basepath)
+            for resource in dataset.resources
+        ]
+
+    return Dataset(**result)

fairspec_metadata/actions/dataset/fixtures/dataset-invalid.json

@@ -0,0 +1,8 @@
+{
+  "resources": [
+    {
+      "name": 1,
+      "data": "table.csv"
+    }
+  ]
+}

fairspec_metadata/actions/dataset/fixtures/dataset.json

@@ -0,0 +1,66 @@
+{
+  "$schema": "https://fairspec.org/profiles/latest/dataset.json",
+  "titles": [
+    {
+      "title": "title"
+    }
+  ],
+  "creators": [
+    {
+      "name": "title"
+    }
+  ],
+  "publisher": {
+    "name": "publisher"
+  },
+  "publicationYear": "2017",
+  "subjects": [
+    {
+      "subject": "keyword1"
+    },
+    {
+      "subject": "keyword2"
+    }
+  ],
+  "contributors": [
+    {
+      "name": "title",
+      "contributorType": "ContactPerson"
+    }
+  ],
+  "dates": [
+    {
+      "date": "2017-01-01",
+      "dateType": "Created"
+    }
+  ],
+  "version": "1.0",
+  "rightsList": [
+    {
+      "rights": "MIT"
+    }
+  ],
+  "descriptions": [
+    {
+      "description": "description",
+      "descriptionType": "Abstract"
+    }
+  ],
+  "relatedIdentifiers": [
+    {
+      "relatedIdentifier": "http://example.com",
+      "relatedIdentifierType": "URL",
+      "relationType": "IsDescribedBy"
+    }
+  ],
+  "resources": [
+    {
+      "name": "name",
+      "data": "table.csv",
+      "fileDialect": {
+        "format": "csv"
+      },
+      "tableSchema": "schema.json"
+    }
+  ]
+}

fairspec_metadata/actions/dataset/fixtures/schema.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "https://fairspec.org/profiles/latest/table-schema.json",
+  "required": ["id", "name"],
+  "properties": {
+    "id": {
+      "type": "integer"
+    },
+    "name": {
+      "type": "string"
+    }
+  }
+}

fairspec_metadata/actions/dataset/fixtures/table.csv

@@ -0,0 +1,3 @@
+id,name
+1,english
+2,中文

fairspec_metadata/actions/dataset/load.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from fairspec_metadata.actions.descriptor.load import load_descriptor
+from fairspec_metadata.actions.path.basepath import resolve_basepath
+from fairspec_metadata.models.dataset import Dataset
+
+from .assert_ import assert_dataset
+
+
+def load_dataset_descriptor(path: str) -> Dataset:
+    basepath = resolve_basepath(path)
+    descriptor = load_descriptor(path)
+    return assert_dataset(descriptor, basepath=basepath)

fairspec_metadata/actions/dataset/normalize.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from fairspec_metadata.actions.resource.normalize import normalize_resource
+from fairspec_metadata.models.dataset import Dataset
+
+
+def normalize_dataset(dataset: Dataset, *, basepath: str | None = None) -> Dataset:
+    result = dataset.model_dump(by_alias=True, exclude_none=True)
+
+    if dataset.resources:
+        result["resources"] = [
+            normalize_resource(resource, basepath=basepath)
+            for resource in dataset.resources
+        ]
+
+    return Dataset(**result)

fairspec_metadata/actions/dataset/save.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from fairspec_metadata.actions.descriptor.save import save_descriptor
+from fairspec_metadata.actions.path.basepath import get_basepath
+from fairspec_metadata.settings import FAIRSPEC_VERSION
+
+from .denormalize import denormalize_dataset
+
+if TYPE_CHECKING:
+    from fairspec_metadata.models.dataset import Dataset
+
+
+def save_dataset_descriptor(
+    dataset: Dataset,
+    *,
+    path: str,
+    overwrite: bool = False,
+) -> None:
+    basepath = get_basepath(path)
+    denormalized = denormalize_dataset(dataset, basepath=basepath)
+    descriptor = denormalized.model_dump(by_alias=True, exclude_none=True)
+
+    if "$schema" not in descriptor:
+        descriptor["$schema"] = (
+            f"https://fairspec.org/profiles/{FAIRSPEC_VERSION}/dataset.json"
+        )
+
+    save_descriptor(descriptor, path=path, overwrite=overwrite)

fairspec_metadata/actions/dataset/save_spec.py

@@ -0,0 +1,102 @@
+import json
+
+import pytest
+
+from fairspec_metadata.models.datacite.creator import Creator
+from fairspec_metadata.models.datacite.title import Title
+from fairspec_metadata.models.dataset import Dataset
+from fairspec_metadata.models.resource import Resource
+from fairspec_metadata.settings import FAIRSPEC_VERSION
+
+from .save import save_dataset_descriptor
+
+
+class TestSaveDatasetDescriptor:
+    def test_saves_dataset(self, tmp_path):
+        path = str(tmp_path / "dataset.json")
+        dataset = Dataset(
+            creators=[Creator(name="Test Creator")],
+            titles=[Title(title="Test Dataset")],
+            resources=[
+                Resource(name="test_resource", data=str(tmp_path / "data.csv")),
+            ],
+        )
+        save_dataset_descriptor(dataset, path=path)
+        with open(path, encoding="utf-8") as f:
+            content = json.load(f)
+        assert content["$schema"].endswith("dataset.json")
+        assert content["creators"][0]["name"] == "Test Creator"
+        assert content["resources"][0]["name"] == "test_resource"
+
+    def test_sets_default_schema(self, tmp_path):
+        path = str(tmp_path / "dataset.json")
+        dataset = Dataset(
+            resources=[
+                Resource(data=str(tmp_path / "data.csv")),
+            ],
+        )
+        save_dataset_descriptor(dataset, path=path)
+        with open(path, encoding="utf-8") as f:
+            content = json.load(f)
+        expected = f"https://fairspec.org/profiles/{FAIRSPEC_VERSION}/dataset.json"
+        assert content["$schema"] == expected
+
+    def test_preserves_custom_schema(self, tmp_path):
+        path = str(tmp_path / "dataset.json")
+        dataset = Dataset(
+            profile="https://custom.schema.url/dataset.json",
+            resources=[
+                Resource(data=str(tmp_path / "data.csv")),
+            ],
+        )
+        save_dataset_descriptor(dataset, path=path)
+        with open(path, encoding="utf-8") as f:
+            content = json.load(f)
+        assert content["$schema"] == "https://custom.schema.url/dataset.json"
+
+    def test_throws_when_file_exists(self, tmp_path):
+        path = str(tmp_path / "dataset.json")
+        dataset = Dataset(resources=[Resource(data=str(tmp_path / "data.csv"))])
+        save_dataset_descriptor(dataset, path=path)
+        with pytest.raises(FileExistsError):
+            save_dataset_descriptor(dataset, path=path)
+
+    def test_overwrites_when_flag_set(self, tmp_path):
+        path = str(tmp_path / "dataset.json")
+        dataset1 = Dataset(
+            creators=[Creator(name="Initial")],
+            resources=[Resource(data=str(tmp_path / "data.csv"))],
+        )
+        dataset2 = Dataset(
+            creators=[Creator(name="Updated")],
+            resources=[Resource(data=str(tmp_path / "data.csv"))],
+        )
+        save_dataset_descriptor(dataset1, path=path)
+        save_dataset_descriptor(dataset2, path=path, overwrite=True)
+        with open(path, encoding="utf-8") as f:
+            content = json.load(f)
+        assert content["creators"][0]["name"] == "Updated"
+
+    def test_saves_to_nested_directory(self, tmp_path):
+        path = str(tmp_path / "nested" / "dir" / "dataset.json")
+        dataset = Dataset(
+            resources=[
+                Resource(data=str(tmp_path / "nested" / "dir" / "data.csv")),
+            ],
+        )
+        save_dataset_descriptor(dataset, path=path)
+        with open(path, encoding="utf-8") as f:
+            content = json.load(f)
+        assert "resources" in content
+
+    def test_denormalizes_resource_paths(self, tmp_path):
+        path = str(tmp_path / "dataset.json")
+        dataset = Dataset(
+            resources=[
+                Resource(name="test", data=str(tmp_path / "data.csv")),
+            ],
+        )
+        save_dataset_descriptor(dataset, path=path)
+        with open(path, encoding="utf-8") as f:
+            content = json.load(f)
+        assert content["resources"][0]["data"] == "data.csv"

fairspec_metadata/actions/dataset/validate.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from fairspec_metadata.actions.data_schema.validate import validate_data_schema
+from fairspec_metadata.actions.descriptor.load import load_descriptor
+from fairspec_metadata.actions.descriptor.validate import validate_descriptor
+from fairspec_metadata.actions.file_dialect.validate import validate_file_dialect
+from fairspec_metadata.actions.profile.load import load_profile
+from fairspec_metadata.actions.table_schema.validate import validate_table_schema
+from fairspec_metadata.models.dataset import Dataset
+from fairspec_metadata.models.descriptor import Descriptor
+from fairspec_metadata.models.profile import ProfileType
+from fairspec_metadata.models.report import Report
+
+from .normalize import normalize_dataset
+
+
+class DatasetValidationResult(Report):
+    dataset: Dataset | None = None
+
+
+def validate_dataset_descriptor(
+    source: Descriptor | str,
+    *,
+    basepath: str | None = None,
+) -> DatasetValidationResult:
+    descriptor = load_descriptor(source) if isinstance(source, str) else source
+
+    schema = descriptor.get("$schema")
+    schema_url = (
+        schema
+        if isinstance(schema, str)
+        else "https://fairspec.org/profiles/latest/dataset.json"
+    )
+
+    profile = load_profile(schema_url, profile_type=ProfileType.dataset)
+
+    report = validate_descriptor(descriptor, profile=profile)
+
+    normalized: Dataset | None = None
+    if report.valid:
+        # Valid -> we can cast
+        normalized = normalize_dataset(Dataset(**descriptor), basepath=basepath)
+
+    if normalized:
+        for index, resource in enumerate(normalized.resources or []):
+            root_json_pointer = f"/resources/{index}"
+
+            if isinstance(resource.fileDialect, str):
+                file_dialect_result = validate_file_dialect(
+                    resource.fileDialect,
+                    root_json_pointer=root_json_pointer,
+                )
+                report.errors.extend(file_dialect_result.errors)
+
+            if isinstance(resource.dataSchema, str):
+                data_schema_result = validate_data_schema(
+                    resource.dataSchema,
+                    root_json_pointer=root_json_pointer,
+                )
+                report.errors.extend(data_schema_result.errors)
+
+            if isinstance(resource.tableSchema, str):
+                table_schema_result = validate_table_schema(
+                    resource.tableSchema,
+                    root_json_pointer=root_json_pointer,
+                )
+                report.errors.extend(table_schema_result.errors)
+
+        if report.errors:
+            normalized = None
+            report.valid = False
+
+    return DatasetValidationResult(
+        valid=report.valid,
+        errors=report.errors,
+        dataset=normalized,
+    )

fairspec_metadata/actions/dataset/validate_spec.py

@@ -0,0 +1,30 @@
+from .validate import validate_dataset_descriptor
+
+
+class TestValidateDatasetDescriptor:
+    def test_valid_dataset(self):
+        dataset = {"resources": [{"data": "data.csv"}]}
+        result = validate_dataset_descriptor(dataset)
+        assert result.valid is True
+        assert result.errors == []
+
+    def test_invalid_dataset(self):
+        dataset = {"resources": "not-an-array"}
+        result = validate_dataset_descriptor(dataset)
+        assert result.valid is False
+        assert len(result.errors) > 0
+
+    def test_missing_schema_is_valid(self):
+        dataset = {"resources": [{"data": "data.csv"}]}
+        result = validate_dataset_descriptor(dataset)
+        assert result.valid is True
+
+    def test_dataset_with_datacite(self):
+        dataset = {
+            "creators": [{"name": "John Doe"}],
+            "titles": [{"title": "Example Dataset"}],
+            "resources": [{"data": "data.csv"}],
+        }
+        result = validate_dataset_descriptor(dataset)
+        assert result.valid is True
+        assert result.errors == []

fairspec_metadata/actions/descriptor/copy.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+import copy
+
+from fairspec_metadata.models.descriptor import Descriptor
+
+
+def copy_descriptor(descriptor: Descriptor) -> Descriptor:
+    return copy.deepcopy(descriptor)

fairspec_metadata/actions/descriptor/fixtures/schema.json

@@ -0,0 +1 @@
+{"fields": [{"name": "id", "type": "integer"}, {"name": "name", "type": "string"}]}