fairspec-library 0.0.0.dev0__py3-none-any.whl

fairspec_library/__init__.py

@@ -0,0 +1,29 @@
+from fairspec_dataset import *  # noqa: F403
+from fairspec_metadata import *  # noqa: F403
+from fairspec_table import *  # noqa: F403
+
+from .actions.data.load import load_data as load_data
+from .actions.data.validate import validate_data as validate_data
+from .actions.data_schema.infer import infer_data_schema as infer_data_schema
+from .actions.data_schema.render import render_data_schema_as as render_data_schema_as
+from .actions.dataset.foreign_key import (
+    validate_dataset_foreign_keys as validate_dataset_foreign_keys,
+)
+from .actions.dataset.infer import infer_dataset as infer_dataset
+from .actions.dataset.load import load_dataset as load_dataset
+from .actions.dataset.render import render_dataset_as as render_dataset_as
+from .actions.dataset.save import save_dataset as save_dataset
+from .actions.dataset.validate import validate_dataset as validate_dataset
+from .actions.file_dialect.infer import infer_file_dialect as infer_file_dialect
+from .actions.resource.infer import infer_resource as infer_resource
+from .actions.resource.validate import validate_resource as validate_resource
+from .actions.table.infer import infer_table as infer_table
+from .actions.table.load import load_table as load_table
+from .actions.table.save import save_table as save_table
+from .actions.table.validate import validate_table as validate_table
+from .actions.table_schema.infer import infer_table_schema as infer_table_schema
+from .actions.table_schema.render import render_table_schema_as as render_table_schema_as
+from .models.table import ValidateTableOptions as ValidateTableOptions
+from .plugin import Plugin as Plugin
+from .system import System as System
+from .system import system as system

fairspec_library/actions/data/load.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from fairspec_metadata import (
+    get_data_first_path,
+    get_data_value,
+    get_supported_file_dialect,
+    load_descriptor,
+)
+
+if TYPE_CHECKING:
+    from fairspec_metadata import Resource
+
+
+def load_data(resource: Resource) -> object | None:
+    data_value = get_data_value(resource)
+    if data_value:
+        return data_value
+
+    first_path = get_data_first_path(resource)
+    if first_path:
+        dialect = get_supported_file_dialect(resource, ["json"])
+        if dialect:
+            return load_descriptor(first_path)
+
+    return None

fairspec_library/actions/data/load_spec.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import json
+
+from fairspec_dataset import write_temp_file
+from fairspec_metadata import Resource
+
+from .load import load_data
+
+
+class TestLoadData:
+    def test_should_return_inline_data(self):
+        resource = Resource(data=[{"id": 1}, {"id": 2}])
+        result = load_data(resource)
+        assert result == [{"id": 1}, {"id": 2}]
+
+    def test_should_return_inline_object(self):
+        resource = Resource(data={"key": "value"})
+        result = load_data(resource)
+        assert result == {"key": "value"}
+
+    def test_should_load_json_file(self):
+        data = {"key": "value"}
+        path = write_temp_file(json.dumps(data), format="json")
+        resource = Resource(data=path)
+        result = load_data(resource)
+        assert result == data
+
+    def test_should_return_none_for_non_json_file(self):
+        path = write_temp_file("id,name\n1,english", format="csv")
+        resource = Resource(data=path)
+        result = load_data(resource)
+        assert result is None
+
+    def test_should_return_none_for_empty_resource(self):
+        resource = Resource()
+        result = load_data(resource)
+        assert result is None

fairspec_library/actions/data/validate.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from fairspec_metadata import (
+    DataError,
+    FairspecError,
+    Report,
+    create_report,
+    inspect_json,
+    resolve_data_schema,
+)
+
+from .load import load_data
+
+if TYPE_CHECKING:
+    from fairspec_metadata import Resource
+
+
+def validate_data(resource: Resource) -> Report:
+    errors: list[FairspecError] = []
+
+    data_schema = resolve_data_schema(resource.dataSchema)
+    if not data_schema:
+        return create_report()
+
+    data = load_data(resource)
+    if data is None:
+        return create_report()
+
+    notes = inspect_json(data, json_schema=data_schema)
+    for note in notes:
+        errors.append(
+            DataError(
+                type="data",
+                message=note["message"],
+                jsonPointer=note["jsonPointer"],
+            )
+        )
+
+    return create_report(errors)

fairspec_library/actions/data/validate_spec.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from fairspec_metadata import Resource
+
+from .validate import validate_data
+
+
+class TestValidateData:
+    def test_should_return_valid_when_no_schema(self):
+        resource = Resource(data=[{"id": 1}])
+        report = validate_data(resource)
+        assert report.valid is True
+
+    def test_should_validate_inline_data(self):
+        resource = Resource(
+            data={"name": "test", "age": 25},
+            dataSchema={
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "age": {"type": "integer"},
+                },
+                "required": ["name", "age"],
+            },
+        )
+        report = validate_data(resource)
+        assert report.valid is True
+
+    def test_should_detect_invalid_data(self):
+        resource = Resource(
+            data={"name": 123},
+            dataSchema={
+                "type": "object",
+                "properties": {"name": {"type": "string"}},
+            },
+        )
+        report = validate_data(resource)
+        assert report.valid is False
+        assert len(report.errors) > 0
+
+    def test_should_return_valid_for_no_data(self):
+        resource = Resource(
+            dataSchema={"type": "object"},
+        )
+        report = validate_data(resource)
+        assert report.valid is True

fairspec_library/actions/data_schema/infer.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from genson import SchemaBuilder
+
+from fairspec_library.actions.data.load import load_data
+
+if TYPE_CHECKING:
+    from fairspec_metadata import JsonSchema, Resource
+
+
+def infer_data_schema(resource: Resource) -> JsonSchema | None:
+    data = load_data(resource)
+    if not data:
+        return None
+
+    try:
+        builder = SchemaBuilder()
+        builder.add_object(data)
+        return builder.to_schema()
+    except Exception:
+        return None

fairspec_library/actions/data_schema/infer_spec.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import json
+
+from fairspec_dataset import write_temp_file
+from fairspec_metadata import Resource
+
+from .infer import infer_data_schema
+
+
+class TestInferDataSchema:
+    def test_should_infer_schema_from_inline_data(self):
+        resource = Resource(data={"name": "test", "age": 25})
+        schema = infer_data_schema(resource)
+        assert schema is not None
+        assert schema.get("type") == "object"
+
+    def test_should_infer_schema_from_inline_array(self):
+        resource = Resource(data=[{"id": 1}, {"id": 2}])
+        schema = infer_data_schema(resource)
+        assert schema is not None
+
+    def test_should_infer_schema_from_json_file(self):
+        data = {"name": "test", "value": 42}
+        path = write_temp_file(json.dumps(data), format="json")
+        resource = Resource(data=path)
+        schema = infer_data_schema(resource)
+        assert schema is not None
+        assert schema.get("type") == "object"
+
+    def test_should_return_none_for_no_data(self):
+        resource = Resource()
+        schema = infer_data_schema(resource)
+        assert schema is None
+
+    def test_should_return_none_for_csv_file(self):
+        path = write_temp_file("id,name\n1,english", format="csv")
+        resource = Resource(data=path)
+        schema = infer_data_schema(resource)
+        assert schema is None

fairspec_library/actions/data_schema/render.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from fairspec_library.system import system
+
+if TYPE_CHECKING:
+    from fairspec_metadata import DataSchema, RenderDataSchemaOptions
+
+
+def render_data_schema_as(
+    data_schema: DataSchema, options: RenderDataSchemaOptions
+) -> str | None:
+    for plugin in system.plugins:
+        result = plugin.render_data_schema_as(data_schema, options)
+        if result is not None:
+            return result
+
+    return None

fairspec_library/actions/dataset/foreign_key.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Unpack
+
+import polars as pl
+from fairspec_metadata import (
+    FairspecError,
+    ForeignKey,
+    ForeignKeyError,
+    Report,
+    create_report,
+    resolve_table_schema,
+)
+
+from fairspec_library.actions.table.load import load_table
+from fairspec_library.models.table import ValidateTableOptions
+
+if TYPE_CHECKING:
+    from fairspec_metadata import Dataset, Resource
+
+
+def validate_dataset_foreign_keys(
+    dataset: Dataset, **options: Unpack[ValidateTableOptions]
+) -> Report:
+    errors: list[FairspecError] = []
+
+    for resource in dataset.resources or []:
+        table_schema = resolve_table_schema(resource.tableSchema)
+        if not table_schema:
+            continue
+
+        for foreign_key in table_schema.foreignKeys or []:
+            fk_errors = _validate_foreign_key(
+                resource, foreign_key, dataset, **options
+            )
+            errors.extend(fk_errors)
+
+    return create_report(errors)
+
+
+def _validate_foreign_key(
+    resource: Resource,
+    foreign_key: ForeignKey,
+    dataset: Dataset,
+    **options: Unpack[ValidateTableOptions],
+) -> list[ForeignKeyError]:
+    reference = foreign_key.reference
+    columns = foreign_key.columns
+    ref_columns = reference.columns
+
+    if not columns or not ref_columns or len(columns) != len(ref_columns):
+        return []
+
+    ref_resource = _find_resource(dataset, reference.resource)
+    if not ref_resource:
+        return []
+
+    table = load_table(resource, denormalized=True, **options)
+    if table is None:
+        return []
+
+    ref_table = load_table(ref_resource, denormalized=True, **options)
+    if ref_table is None:
+        return []
+
+    rename_mapping = dict(zip(ref_columns, columns))
+    ref_selected = ref_table.select(
+        [pl.col(name).alias(rename_mapping[name]) for name in ref_columns]
+    ).unique()
+
+    violations: pl.DataFrame = table.select(columns).join(  # ty: ignore[invalid-assignment] https://github.com/astral-sh/ty/issues/2278
+        ref_selected, on=columns, how="anti"
+    ).unique().collect()
+
+    errors: list[ForeignKeyError] = []
+    for row in violations.to_dicts():
+        cells = [str(row[c]) for c in columns]
+        errors.append(
+            ForeignKeyError(
+                type="foreignKey",
+                resourceName=resource.name,
+                foreignKey=foreign_key,
+                cells=cells,
+            )
+        )
+
+    return errors
+
+
+def _find_resource(dataset: Dataset, name: str | None) -> Resource | None:
+    if not name:
+        return None
+
+    for resource in dataset.resources or []:
+        if resource.name == name:
+            return resource
+
+    return None

fairspec_library/actions/dataset/foreign_key_spec.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from fairspec_dataset import write_temp_file
+from fairspec_metadata import (
+    Dataset,
+    ForeignKey,
+    ForeignKeyReference,
+    IntegerColumnProperty,
+    Resource,
+    StringColumnProperty,
+    TableSchema,
+)
+
+from .foreign_key import validate_dataset_foreign_keys
+
+
+class TestValidateDatasetForeignKeys:
+    def test_should_validate_valid_foreign_keys(self):
+        path1 = write_temp_file("id,name\n1,english\n2,中文", format="csv")
+        path2 = write_temp_file("id,name_id\n1,1\n2,2", format="csv")
+        dataset = Dataset(
+            resources=[
+                Resource(
+                    data=path1,
+                    name="names",
+                    tableSchema=TableSchema(
+                        properties={
+                            "id": IntegerColumnProperty(),
+                            "name": StringColumnProperty(),
+                        }
+                    ),
+                ),
+                Resource(
+                    data=path2,
+                    name="refs",
+                    tableSchema=TableSchema(
+                        properties={
+                            "id": IntegerColumnProperty(),
+                            "name_id": IntegerColumnProperty(),
+                        },
+                        foreignKeys=[
+                            ForeignKey(
+                                columns=["name_id"],
+                                reference=ForeignKeyReference(
+                                    resource="names",
+                                    columns=["id"],
+                                ),
+                            )
+                        ],
+                    ),
+                ),
+            ]
+        )
+        report = validate_dataset_foreign_keys(dataset)
+        assert report.valid is True
+
+    def test_should_detect_foreign_key_violation(self):
+        path1 = write_temp_file("id,name\n1,english\n2,中文", format="csv")
+        path2 = write_temp_file("id,name_id\n1,1\n2,999", format="csv")
+        dataset = Dataset(
+            resources=[
+                Resource(
+                    data=path1,
+                    name="names",
+                    tableSchema=TableSchema(
+                        properties={
+                            "id": IntegerColumnProperty(),
+                            "name": StringColumnProperty(),
+                        }
+                    ),
+                ),
+                Resource(
+                    data=path2,
+                    name="refs",
+                    tableSchema=TableSchema(
+                        properties={
+                            "id": IntegerColumnProperty(),
+                            "name_id": IntegerColumnProperty(),
+                        },
+                        foreignKeys=[
+                            ForeignKey(
+                                columns=["name_id"],
+                                reference=ForeignKeyReference(
+                                    resource="names",
+                                    columns=["id"],
+                                ),
+                            )
+                        ],
+                    ),
+                ),
+            ]
+        )
+        report = validate_dataset_foreign_keys(dataset)
+        assert report.valid is False
+        assert len(report.errors) > 0
+
+    def test_should_handle_no_foreign_keys(self):
+        path = write_temp_file("id,name\n1,english", format="csv")
+        dataset = Dataset(
+            resources=[
+                Resource(
+                    data=path,
+                    name="test",
+                    tableSchema=TableSchema(
+                        properties={
+                            "id": IntegerColumnProperty(),
+                            "name": StringColumnProperty(),
+                        }
+                    ),
+                )
+            ]
+        )
+        report = validate_dataset_foreign_keys(dataset)
+        assert report.valid is True

fairspec_library/actions/dataset/infer.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from fairspec_metadata import Dataset
+
+from fairspec_library.actions.resource.infer import infer_resource
+
+
+def infer_dataset(dataset: Dataset) -> Dataset:
+    dataset = dataset.model_copy(deep=True)
+
+    if dataset.resources:
+        for index, resource in enumerate(dataset.resources):
+            dataset.resources[index] = infer_resource(
+                resource, resource_number=index + 1
+            )
+
+    return dataset

fairspec_library/actions/dataset/infer_spec.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from fairspec_dataset import write_temp_file
+from fairspec_metadata import Dataset, Resource
+
+from .infer import infer_dataset
+
+
+class TestInferDataset:
+    def test_should_infer_resource_names(self):
+        path = write_temp_file("id,name\n1,english", format="csv")
+        dataset = Dataset(resources=[Resource(data=path)])
+        result = infer_dataset(dataset)
+        assert result.resources is not None
+        assert len(result.resources) == 1
+        assert result.resources[0].name is not None
+
+    def test_should_not_mutate_original(self):
+        path = write_temp_file("id,name\n1,english", format="csv")
+        dataset = Dataset(resources=[Resource(data=path)])
+        result = infer_dataset(dataset)
+        assert result is not dataset
+
+    def test_should_handle_empty_resources(self):
+        dataset = Dataset(resources=[])
+        result = infer_dataset(dataset)
+        assert result.resources == []
+
+    def test_should_handle_no_resources(self):
+        dataset = Dataset()
+        result = infer_dataset(dataset)
+        assert result.resources is None

fairspec_library/actions/dataset/load.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from fairspec_dataset import DatasetPlugin
+
+from fairspec_library.system import system
+
+if TYPE_CHECKING:
+    from fairspec_metadata import Descriptor
+
+
+def load_dataset(source: str) -> Descriptor | None:
+    for plugin in system.plugins:
+        if isinstance(plugin, DatasetPlugin):
+            result = plugin.load_dataset(source)
+            if result is not None:
+                return result
+
+    return None

fairspec_library/actions/dataset/load_spec.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import json
+import os
+
+from fairspec_dataset import get_temp_folder_path
+
+from .load import load_dataset
+
+
+class TestLoadDataset:
+    def test_should_load_dataset_from_descriptor_path(self):
+        folder = get_temp_folder_path()
+        descriptor = {
+            "resources": [{"data": "data.csv", "name": "data"}]
+        }
+        path = os.path.join(folder, "datapackage.json")
+        with open(path, "w") as f:
+            json.dump(descriptor, f)
+        result = load_dataset(path)
+        assert result is not None
+        assert "resources" in result
+
+    def test_should_return_none_for_unsupported_source(self):
+        result = load_dataset("nonexistent-source")
+        assert result is None

fairspec_library/actions/dataset/render.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from fairspec_library.system import system
+
+if TYPE_CHECKING:
+    from fairspec_metadata import Dataset, RenderDatasetOptions
+
+
+def render_dataset_as(dataset: Dataset, options: RenderDatasetOptions) -> str | None:
+    for plugin in system.plugins:
+        result = plugin.render_dataset_as(dataset, options)
+        if result is not None:
+            return result
+
+    return None

fairspec_library/actions/dataset/save.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from typing import Unpack
+
+from fairspec_dataset import DatasetPlugin, SaveDatasetOptions, SaveDatasetResult
+from fairspec_metadata import Dataset
+
+from fairspec_library.system import system
+
+
+def save_dataset(
+    dataset: Dataset, **options: Unpack[SaveDatasetOptions]
+) -> SaveDatasetResult | None:
+    for plugin in system.plugins:
+        if isinstance(plugin, DatasetPlugin):
+            result = plugin.save_dataset(dataset, **options)
+            if result is not None:
+                return result
+
+    return None

fairspec_library/actions/dataset/save_spec.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from fairspec_dataset import get_temp_file_path
+from fairspec_metadata import Dataset, Resource
+
+from .save import save_dataset
+
+
+class TestSaveDataset:
+    def test_should_save_dataset_to_zip(self):
+        path = get_temp_file_path(format="zip")
+        dataset = Dataset(resources=[Resource(data=[{"id": 1}], name="data")])
+        result = save_dataset(dataset, target=path)
+        assert result is not None
+
+    def test_should_return_none_for_unsupported_target(self):
+        dataset = Dataset(resources=[Resource(data=[{"id": 1}], name="data")])
+        result = save_dataset(dataset, target="/tmp/unknown.xyz")
+        assert result is None