fairspec-library 0.0.0.dev0__py3-none-any.whl

fairspec_library/actions/dataset/validate.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Unpack
+
+from fairspec_metadata import (
+    Dataset,
+    FairspecException,
+    Report,
+    create_report,
+    infer_resource_name,
+    load_dataset_descriptor,
+    normalize_dataset,
+)
+
+from fairspec_library.actions.dataset.foreign_key import validate_dataset_foreign_keys
+from fairspec_library.actions.resource.validate import validate_resource
+from fairspec_library.models.table import ValidateTableOptions
+
+if TYPE_CHECKING:
+    from fairspec_metadata import FairspecError
+
+
+def validate_dataset(
+    source: Dataset | str, **options: Unpack[ValidateTableOptions]
+) -> Report:
+    if isinstance(source, str):
+        try:
+            descriptor = load_dataset_descriptor(source)
+            source = Dataset.model_validate(descriptor)
+        except FairspecException as exception:
+            if exception.report:
+                return exception.report
+            return create_report()
+
+    dataset = normalize_dataset(source)
+    errors = _validate_dataset_resources(dataset, **options)
+    fk_report = validate_dataset_foreign_keys(dataset, **options)
+    errors.extend(fk_report.errors)
+
+    return create_report(errors)
+
+
+def _validate_dataset_resources(
+    dataset: Dataset, **options: Unpack[ValidateTableOptions]
+) -> list[FairspecError]:
+    errors: list[FairspecError] = []
+
+    for index, resource in enumerate(dataset.resources or []):
+        if not resource.name:
+            resource.name = infer_resource_name(resource, resource_number=index + 1)
+
+        report = validate_resource(resource, **options)
+        for error in report.errors:
+            error.resourceName = resource.name
+        errors.extend(report.errors)
+
+    return errors

fairspec_library/actions/dataset/validate_spec.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from fairspec_metadata import (
+    Dataset,
+    IntegerColumnProperty,
+    Resource,
+    StringColumnProperty,
+    TableSchema,
+)
+
+from .validate import validate_dataset
+
+
+class TestValidateDataset:
+    def test_should_validate_valid_dataset(self):
+        dataset = Dataset(
+            resources=[
+                Resource(
+                    data=[{"id": 1, "name": "english"}, {"id": 2, "name": "中文"}],
+                    name="test",
+                    tableSchema=TableSchema(
+                        properties={
+                            "id": IntegerColumnProperty(),
+                            "name": StringColumnProperty(),
+                        }
+                    ),
+                )
+            ]
+        )
+        report = validate_dataset(dataset)
+        assert report.valid is True
+
+    def test_should_detect_invalid_resource(self):
+        dataset = Dataset(
+            resources=[
+                Resource(
+                    data=[{"id": "BAD", "name": "english"}],
+                    name="test",
+                    tableSchema=TableSchema(
+                        properties={
+                            "id": IntegerColumnProperty(),
+                            "name": StringColumnProperty(),
+                        }
+                    ),
+                )
+            ]
+        )
+        report = validate_dataset(dataset)
+        assert report.valid is False
+
+    def test_should_handle_empty_dataset(self):
+        dataset = Dataset(resources=[])
+        report = validate_dataset(dataset)
+        assert report.valid is True

fairspec_library/actions/file_dialect/infer.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Unpack
+
+from fairspec_dataset import DatasetPlugin, InferFileDialectOptions
+
+from fairspec_library.system import system
+
+if TYPE_CHECKING:
+    from fairspec_metadata import FileDialect, Resource
+
+
+def infer_file_dialect(
+    resource: Resource, **options: Unpack[InferFileDialectOptions]
+) -> FileDialect | None:
+    for plugin in system.plugins:
+        if isinstance(plugin, DatasetPlugin):
+            result = plugin.infer_file_dialect(resource, **options)
+            if result is not None:
+                return result
+
+    return None

fairspec_library/actions/file_dialect/infer_spec.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from fairspec_dataset import write_temp_file
+from fairspec_metadata import Resource
+
+from .infer import infer_file_dialect
+
+
+class TestInferFileDialect:
+    def test_should_infer_csv_dialect(self):
+        path = write_temp_file("id,name\n1,english\n2,中文", format="csv")
+        resource = Resource(data=path)
+        dialect = infer_file_dialect(resource)
+        assert dialect is not None
+
+    def test_should_return_none_for_json_file(self):
+        path = write_temp_file('[{"id": 1}]', format="json")
+        resource = Resource(data=path)
+        dialect = infer_file_dialect(resource)
+        assert dialect is None
+
+    def test_should_infer_xlsx_dialect(self):
+        resource = Resource(data="test.xlsx")
+        dialect = infer_file_dialect(resource)
+        assert dialect is not None
+
+    def test_should_return_none_for_unknown_format(self):
+        resource = Resource(data="test.unknown")
+        dialect = infer_file_dialect(resource)
+        assert dialect is None
+
+    def test_should_return_none_for_no_data(self):
+        resource = Resource()
+        dialect = infer_file_dialect(resource)
+        assert dialect is None

fairspec_library/actions/resource/infer.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from fairspec_metadata import (
+    Resource,
+    infer_resource_name,
+    resolve_file_dialect,
+)
+from fairspec_dataset import infer_integrity, infer_textual
+
+from fairspec_library.actions.data_schema.infer import infer_data_schema
+from fairspec_library.actions.file_dialect.infer import infer_file_dialect
+from fairspec_library.actions.table_schema.infer import infer_table_schema
+
+
+def infer_resource(
+    resource: Resource, *, resource_number: int | None = None
+) -> Resource:
+    resource = resource.model_copy(deep=True)
+
+    if not resource.name:
+        resource.name = infer_resource_name(
+            resource, resource_number=resource_number
+        )
+
+    if not resource.fileDialect:
+        resource.fileDialect = infer_file_dialect(resource)
+
+    if resource.textual is None:
+        resolved_dialect = resolve_file_dialect(resource.fileDialect)
+        if resolved_dialect:
+            resource.textual = infer_textual(resource)
+
+    if not resource.integrity:
+        resource.integrity = infer_integrity(resource)
+
+    if not resource.dataSchema:
+        resource.dataSchema = infer_data_schema(resource)
+
+    if not resource.tableSchema:
+        resource.tableSchema = infer_table_schema(resource)
+
+    return resource

fairspec_library/actions/resource/infer_spec.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from fairspec_dataset import write_temp_file
+from fairspec_metadata import Resource
+
+from .infer import infer_resource
+
+
+class TestInferResource:
+    def test_should_infer_name(self):
+        path = write_temp_file("id,name\n1,english", format="csv")
+        resource = Resource(data=path)
+        result = infer_resource(resource)
+        assert result.name is not None
+
+    def test_should_infer_file_dialect(self):
+        path = write_temp_file("id,name\n1,english", format="csv")
+        resource = Resource(data=path)
+        result = infer_resource(resource)
+        assert result.fileDialect is not None
+
+    def test_should_not_overwrite_existing_name(self):
+        path = write_temp_file("id,name\n1,english", format="csv")
+        resource = Resource(data=path, name="custom")
+        result = infer_resource(resource)
+        assert result.name == "custom"
+
+    def test_should_not_mutate_original(self):
+        path = write_temp_file("id,name\n1,english", format="csv")
+        resource = Resource(data=path)
+        result = infer_resource(resource)
+        assert result is not resource
+
+    def test_should_handle_resource_number(self):
+        resource = Resource(data=[{"id": 1}])
+        result = infer_resource(resource, resource_number=5)
+        assert result.name == "resource5"

fairspec_library/actions/resource/validate.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Unpack
+
+from fairspec_dataset import validate_file
+from fairspec_metadata import Report, create_report
+
+from fairspec_library.actions.data.validate import validate_data
+from fairspec_library.actions.table.validate import validate_table
+from fairspec_library.models.table import ValidateTableOptions
+
+if TYPE_CHECKING:
+    from fairspec_metadata import FairspecError, Resource
+
+
+def validate_resource(
+    resource: Resource, **options: Unpack[ValidateTableOptions]
+) -> Report:
+    errors: list[FairspecError] = []
+
+    file_report = validate_file(resource)
+    errors.extend(file_report.errors)
+    if not file_report.valid:
+        return create_report(errors)
+
+    data_report = validate_data(resource)
+    errors.extend(data_report.errors)
+    if not data_report.valid:
+        return create_report(errors)
+
+    table_report = validate_table(resource, **options)
+    errors.extend(table_report.errors)
+
+    return create_report(errors)

fairspec_library/actions/resource/validate_spec.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from fairspec_dataset import write_temp_file
+from fairspec_metadata import IntegerColumnProperty, Resource, StringColumnProperty, TableSchema
+
+from .validate import validate_resource
+
+
+class TestValidateResource:
+    def test_should_validate_valid_resource(self):
+        path = write_temp_file("id,name\n1,english\n2,中文", format="csv")
+        resource = Resource(
+            data=path,
+            tableSchema=TableSchema(
+                properties={
+                    "id": IntegerColumnProperty(),
+                    "name": StringColumnProperty(),
+                }
+            ),
+        )
+        report = validate_resource(resource)
+        assert report.valid is True
+
+    def test_should_detect_type_error(self):
+        path = write_temp_file("id,name\nBAD,english", format="csv")
+        resource = Resource(
+            data=path,
+            tableSchema=TableSchema(
+                properties={
+                    "id": IntegerColumnProperty(),
+                    "name": StringColumnProperty(),
+                }
+            ),
+        )
+        report = validate_resource(resource)
+        assert report.valid is False
+
+    def test_should_validate_resource_without_schema(self):
+        path = write_temp_file("id,name\n1,english", format="csv")
+        resource = Resource(data=path)
+        report = validate_resource(resource)
+        assert report.valid is True

fairspec_library/actions/table/infer.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Unpack
+
+from fairspec_metadata import Resource, resolve_table_schema
+from fairspec_table import denormalize_table
+from fairspec_table.models.table import LoadTableOptions
+
+from fairspec_library.actions.file_dialect.infer import infer_file_dialect
+from fairspec_library.actions.table.load import load_table
+from fairspec_library.actions.table_schema.infer import infer_table_schema
+
+if TYPE_CHECKING:
+    from fairspec_table import Table
+
+
+def infer_table(
+    resource: Resource, **options: Unpack[LoadTableOptions]
+) -> Table | None:
+    resource = resource.model_copy(deep=True)
+
+    if not resource.fileDialect:
+        resource.fileDialect = infer_file_dialect(resource)
+
+    table = load_table(resource, denormalized=True, **options)
+    if table is None:
+        return None
+
+    table_schema = resolve_table_schema(resource.tableSchema)
+    if not table_schema:
+        table_schema = infer_table_schema(resource, **options)
+
+    if table_schema:
+        table = denormalize_table(table, table_schema)
+
+    return table

fairspec_library/actions/table/infer_spec.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+import polars as pl
+from fairspec_dataset import write_temp_file
+from fairspec_metadata import Resource
+
+from .infer import infer_table
+
+
+class TestInferTable:
+    def test_should_infer_and_load_table(self):
+        path = write_temp_file("id,name\n1,english\n2,中文", format="csv")
+        resource = Resource(data=path)
+        table = infer_table(resource)
+        assert table is not None
+        frame: pl.DataFrame = table.collect()  # ty: ignore[invalid-assignment] https://github.com/astral-sh/ty/issues/2278
+        assert len(frame) == 2
+
+    def test_should_return_none_for_empty_resource(self):
+        resource = Resource()
+        table = infer_table(resource)
+        assert table is None

fairspec_library/actions/table/load.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Unpack
+
+from fairspec_table import TablePlugin
+from fairspec_table.models.table import LoadTableOptions
+
+from fairspec_library.system import system
+
+if TYPE_CHECKING:
+    from fairspec_metadata import Resource
+    from fairspec_table import Table
+
+
+def load_table(
+    resource: Resource, **options: Unpack[LoadTableOptions]
+) -> Table | None:
+    for plugin in system.plugins:
+        if isinstance(plugin, TablePlugin):
+            result = plugin.load_table(resource, **options)
+            if result is not None:
+                return result
+
+    return None

fairspec_library/actions/table/load_spec.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import polars as pl
+from fairspec_dataset import write_temp_file
+from fairspec_metadata import Resource
+
+from .load import load_table
+
+
+class TestLoadTable:
+    def test_should_load_csv_table(self):
+        path = write_temp_file("id,name\n1,english\n2,中文", format="csv")
+        resource = Resource(data=path)
+        table = load_table(resource)
+        assert table is not None
+        frame: pl.DataFrame = table.collect()  # ty: ignore[invalid-assignment] https://github.com/astral-sh/ty/issues/2278
+        assert frame.to_dicts() == [
+            {"id": 1, "name": "english"},
+            {"id": 2, "name": "中文"},
+        ]
+
+    def test_should_load_inline_table(self):
+        resource = Resource(data=[{"id": 1, "name": "english"}, {"id": 2, "name": "中文"}])
+        table = load_table(resource)
+        assert table is not None
+        frame: pl.DataFrame = table.collect()  # ty: ignore[invalid-assignment] https://github.com/astral-sh/ty/issues/2278
+        assert frame.to_dicts() == [
+            {"id": 1, "name": "english"},
+            {"id": 2, "name": "中文"},
+        ]
+
+    def test_should_return_none_for_empty_resource(self):
+        resource = Resource()
+        table = load_table(resource)
+        assert table is None
+
+    def test_should_load_json_table(self):
+        path = write_temp_file('[{"id": 1, "name": "english"}]', format="json")
+        resource = Resource(data=path)
+        table = load_table(resource)
+        assert table is not None

fairspec_library/actions/table/save.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from typing import Unpack
+
+from fairspec_table import TablePlugin
+from fairspec_table.models.table import SaveTableOptions, Table
+
+from fairspec_library.system import system
+
+
+def save_table(table: Table, **options: Unpack[SaveTableOptions]) -> str | None:
+    for plugin in system.plugins:
+        if isinstance(plugin, TablePlugin):
+            result = plugin.save_table(table, **options)
+            if result is not None:
+                return result
+
+    return None

fairspec_library/actions/table/save_spec.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import polars as pl
+from fairspec_dataset import get_temp_file_path
+
+from .save import save_table
+
+
+class TestSaveTable:
+    def test_should_save_table_to_csv(self):
+        path = get_temp_file_path(format="csv")
+        table = pl.DataFrame({"id": [1, 2], "name": ["english", "中文"]}).lazy()
+        result = save_table(table, path=path)
+        assert result is not None
+
+    def test_should_save_table_to_json(self):
+        path = get_temp_file_path(format="json")
+        table = pl.DataFrame({"id": [1, 2], "name": ["english", "中文"]}).lazy()
+        result = save_table(table, path=path)
+        assert result is not None
+
+    def test_should_return_none_for_unknown_format(self):
+        path = get_temp_file_path(format="unknown")
+        table = pl.DataFrame({"id": [1, 2]}).lazy()
+        result = save_table(table, path=path)
+        assert result is None

fairspec_library/actions/table/validate.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Unpack
+
+from fairspec_metadata import Report, create_report, resolve_table_schema
+from fairspec_table import inspect_table
+
+from fairspec_library.actions.file_dialect.infer import infer_file_dialect
+from fairspec_library.actions.table.load import load_table
+from fairspec_library.actions.table_schema.infer import infer_table_schema
+from fairspec_library.models.table import ValidateTableOptions
+
+if TYPE_CHECKING:
+    from fairspec_metadata import Resource
+
+
+def validate_table(
+    resource: Resource, **options: Unpack[ValidateTableOptions]
+) -> Report:
+    resource = resource.model_copy(deep=True)
+
+    if not resource.fileDialect:
+        resource.fileDialect = infer_file_dialect(resource)
+
+    no_infer = options.get("noInfer", False)
+    max_errors = options.get("maxErrors", 1000)
+
+    table_schema = resolve_table_schema(resource.tableSchema)
+    if not table_schema and not no_infer:
+        table_schema = infer_table_schema(resource, **options)
+
+    table = load_table(resource, denormalized=True, **options)
+    if table is None:
+        return create_report()
+
+    errors = inspect_table(table, table_schema=table_schema, max_errors=max_errors)
+    return create_report(list(errors))

fairspec_library/actions/table/validate_spec.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from fairspec_dataset import write_temp_file
+from fairspec_metadata import IntegerColumnProperty, Resource, StringColumnProperty, TableSchema
+
+from .validate import validate_table
+
+
+class TestValidateTable:
+    def test_should_validate_valid_table(self):
+        path = write_temp_file("id,name\n1,english\n2,中文", format="csv")
+        resource = Resource(
+            data=path,
+            tableSchema=TableSchema(
+                properties={
+                    "id": IntegerColumnProperty(),
+                    "name": StringColumnProperty(),
+                }
+            ),
+        )
+        report = validate_table(resource)
+        assert report.valid is True
+
+    def test_should_detect_type_error(self):
+        path = write_temp_file("id,name\nBAD,english", format="csv")
+        resource = Resource(
+            data=path,
+            tableSchema=TableSchema(
+                properties={
+                    "id": IntegerColumnProperty(),
+                    "name": StringColumnProperty(),
+                }
+            ),
+        )
+        report = validate_table(resource)
+        assert report.valid is False
+        assert len(report.errors) > 0
+
+    def test_should_validate_without_schema(self):
+        path = write_temp_file("id,name\n1,english", format="csv")
+        resource = Resource(data=path)
+        report = validate_table(resource)
+        assert report.valid is True

fairspec_library/actions/table_schema/infer.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Unpack
+
+from fairspec_table import TablePlugin, infer_table_schema_from_table
+from fairspec_table.models.schema import InferTableSchemaOptions
+
+from fairspec_library.actions.table.load import load_table
+from fairspec_library.system import system
+
+if TYPE_CHECKING:
+    from fairspec_metadata import Resource, TableSchema
+
+
+def infer_table_schema(
+    resource: Resource, **options: Unpack[InferTableSchemaOptions]
+) -> TableSchema | None:
+    for plugin in system.plugins:
+        if isinstance(plugin, TablePlugin):
+            result = plugin.infer_table_schema(resource, **options)
+            if result is not None:
+                return result
+
+    table = load_table(resource, denormalized=True)
+    if table is None:
+        return None
+
+    return infer_table_schema_from_table(table, **options)

fairspec_library/actions/table_schema/infer_spec.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from fairspec_dataset import write_temp_file
+from fairspec_metadata import Resource
+
+from .infer import infer_table_schema
+
+
+class TestInferTableSchema:
+    def test_should_infer_schema_from_csv(self):
+        path = write_temp_file("id,name\n1,english\n2,中文", format="csv")
+        resource = Resource(data=path)
+        schema = infer_table_schema(resource)
+        assert schema is not None
+        assert schema.properties is not None
+
+    def test_should_return_none_for_empty_resource(self):
+        resource = Resource()
+        schema = infer_table_schema(resource)
+        assert schema is None
+
+    def test_should_infer_schema_from_inline_data(self):
+        resource = Resource(data=[{"id": 1, "name": "english"}, {"id": 2, "name": "中文"}])
+        schema = infer_table_schema(resource)
+        assert schema is not None

fairspec_library/actions/table_schema/render.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from fairspec_library.system import system
+
+if TYPE_CHECKING:
+    from fairspec_metadata import RenderTableSchemaOptions, TableSchema
+
+
+def render_table_schema_as(
+    table_schema: TableSchema, options: RenderTableSchemaOptions
+) -> str | None:
+    for plugin in system.plugins:
+        result = plugin.render_table_schema_as(table_schema, options)
+        if result is not None:
+            return result
+
+    return None