fairspec-dataset 0.0.0.dev0__tar.gz

fairspec_dataset-0.0.0.dev0/.gitignore

@@ -0,0 +1,229 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer,
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml
+
+# Node
+node_modules/
+jspm_packages/
+.lock-wscript
+build/Release
+.node_repl_history
+*.tgz
+.npm
+*.so
+
+# User
+/.claude/settings.local.json

fairspec_dataset-0.0.0.dev0/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: fairspec-dataset
+Version: 0.0.0.dev0
+Summary: Fairspec Python is a fast data management framework built on top of the Fairspec standard and Polars DataFrames
+Project-URL: homepage, https://github.com/fairspec/fairspec-python
+Project-URL: repository, https://github.com/fairspec/fairspec-python
+Author: Evgeny Karev
+License-Expression: MIT
+Keywords: ckan,data,dataframe,datahub,dataset,fair,fairspec,jsonschema,metadata,polars,python,quality,tableschema,validation
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.12
+Requires-Dist: charset-normalizer>=3.0
+Requires-Dist: fairspec-metadata
+Description-Content-Type: text/markdown
+
+# fairspec-dataset
+
+Fairspec Python is a fast data management framework built on top of the Fairspec standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please read the [project's documentation](https://python.fairspec.org).

fairspec_dataset-0.0.0.dev0/README.md

@@ -0,0 +1,3 @@
+# fairspec-dataset
+
+Fairspec Python is a fast data management framework built on top of the Fairspec standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please read the [project's documentation](https://python.fairspec.org).

fairspec_dataset-0.0.0.dev0/fairspec_dataset/__init__.py

@@ -0,0 +1,92 @@
+from .actions.dataset.basepath import get_common_local_basepath, get_dataset_basepath
+from .actions.dataset.merge import merge_datasets
+from .actions.file.copy import copy_file
+from .actions.file.describe import describe_file
+from .models.file import FileDescription
+from .actions.file.infer import infer_bytes, infer_hash, infer_integrity, infer_textual
+from .actions.file.load import load_file
+from .actions.file.path import assert_local_path_vacant, get_is_local_path_exist
+from .actions.file.prefetch import prefetch_file, prefetch_files
+from .actions.file.save import save_file
+from .actions.file.temp import get_temp_file_path, write_temp_file
+from .actions.file.validate import validate_file
+from .actions.folder.create import create_folder
+from .actions.folder.temp import get_temp_folder_path
+from .actions.resource.save import SaveFileCallback, SaveFileProps, save_resource_files
+from .actions.stream.concat import concat_file_streams
+from .actions.stream.load import load_file_stream
+from .actions.stream.save import save_file_stream
+from .models.dataset import SaveDatasetOptions
+from .models.file_dialect import InferFileDialectOptions
+from .models.dataset import SaveDatasetResult
+from .plugin import DatasetPlugin
+from .plugins.descriptor import DescriptorPlugin
+from .plugins.ckan import CkanPlugin, load_dataset_from_ckan, save_dataset_to_ckan
+from .plugins.folder import (
+    FolderPlugin,
+    load_dataset_from_folder,
+    save_dataset_to_folder,
+)
+from .plugins.github import (
+    GithubPlugin,
+    load_dataset_from_github,
+    save_dataset_to_github,
+)
+from .plugins.zenodo import (
+    ZenodoPlugin,
+    load_dataset_from_zenodo,
+    save_dataset_to_zenodo,
+)
+from .plugins.zip import ZipPlugin, load_dataset_from_zip, save_dataset_to_zip
+from fairspec_metadata.plugin import MetadataPlugin
+
+__all__ = [
+    "CkanPlugin",
+    "DatasetPlugin",
+    "DescriptorPlugin",
+    "FileDescription",
+    "FolderPlugin",
+    "GithubPlugin",
+    "InferFileDialectOptions",
+    "MetadataPlugin",
+    "SaveDatasetOptions",
+    "SaveDatasetResult",
+    "SaveFileCallback",
+    "SaveFileProps",
+    "ZenodoPlugin",
+    "ZipPlugin",
+    "assert_local_path_vacant",
+    "concat_file_streams",
+    "copy_file",
+    "create_folder",
+    "describe_file",
+    "get_common_local_basepath",
+    "get_dataset_basepath",
+    "get_is_local_path_exist",
+    "get_temp_file_path",
+    "get_temp_folder_path",
+    "infer_bytes",
+    "infer_hash",
+    "infer_integrity",
+    "infer_textual",
+    "load_dataset_from_ckan",
+    "load_dataset_from_folder",
+    "load_dataset_from_github",
+    "load_dataset_from_zenodo",
+    "load_dataset_from_zip",
+    "load_file",
+    "load_file_stream",
+    "merge_datasets",
+    "prefetch_file",
+    "prefetch_files",
+    "save_dataset_to_ckan",
+    "save_dataset_to_folder",
+    "save_dataset_to_github",
+    "save_dataset_to_zenodo",
+    "save_dataset_to_zip",
+    "save_file",
+    "save_file_stream",
+    "save_resource_files",
+    "validate_file",
+    "write_temp_file",
+]

fairspec_dataset-0.0.0.dev0/fairspec_dataset/actions/dataset/basepath.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING
+
+from fairspec_metadata import get_basepath, get_data_paths, get_is_remote_path
+from fairspec_metadata.actions.path.general import safe_relpath
+
+if TYPE_CHECKING:
+    from fairspec_metadata import Dataset
+
+
+def get_dataset_basepath(dataset: Dataset) -> str | None:
+    paths: list[str] = []
+
+    for resource in dataset.resources or []:
+        resource_paths = get_data_paths(resource)
+        paths.extend(resource_paths)
+
+    return get_common_local_basepath(paths)
+
+
+def get_common_local_basepath(paths: list[str]) -> str | None:
+    absolute_basepaths = [
+        os.path.abspath(get_basepath(path))
+        for path in paths
+        if not get_is_remote_path(path)
+    ]
+
+    if not absolute_basepaths:
+        return None
+
+    segment_table = [
+        [segment or "/" for segment in path.split(os.sep)]
+        for path in absolute_basepaths
+    ]
+
+    column = 0
+    segments: list[str] = []
+
+    while True:
+        segment_column = [
+            row[column] if column < len(row) else None for row in segment_table
+        ]
+        unique_segments = set(segment_column)
+
+        if len(unique_segments) != 1:
+            break
+        if segment_column[0] is None:
+            break
+
+        column += 1
+        segments.append(segment_column[0])
+
+    if not segments:
+        raise ValueError("Cannot find common basepath")
+
+    if segments[0].endswith(":"):
+        segments[0] += os.sep
+
+    basepath = safe_relpath(os.path.join(*segments))
+    return "" if basepath == "." else basepath

fairspec_dataset-0.0.0.dev0/fairspec_dataset/actions/dataset/basepath_spec.py

@@ -0,0 +1,60 @@
+import pytest
+
+from .basepath import get_common_local_basepath
+
+
+class TestGetCommonLocalBasepath:
+    @pytest.mark.parametrize(
+        "description, paths, expected",
+        [
+            (
+                "same directory different files",
+                ["data/table1.csv", "data/table2.csv"],
+                "data",
+            ),
+            (
+                "nested directories",
+                ["data/nested/file1.csv", "data/nested/file2.csv", "data/file3.csv"],
+                "data",
+            ),
+            (
+                "single path",
+                ["data/file.csv"],
+                "data",
+            ),
+            (
+                "root level files",
+                ["file1.csv", "file2.csv"],
+                "",
+            ),
+            (
+                "different top-level directories",
+                ["data1/file1.csv", "data2/file2.csv"],
+                "",
+            ),
+            (
+                "empty paths array",
+                [],
+                None,
+            ),
+            (
+                "some paths are remote",
+                ["https://example.com/table.csv", "data/table.csv"],
+                "data",
+            ),
+            (
+                "all paths are remote",
+                [
+                    "https://example.com/table1.csv",
+                    "https://example.com/table2.csv",
+                ],
+                None,
+            ),
+        ],
+    )
+    def test_get_common_local_basepath(self, description, paths, expected):
+        result = get_common_local_basepath(paths)
+        if expected is None:
+            assert result is None
+        else:
+            assert result == expected

fairspec_dataset-0.0.0.dev0/fairspec_dataset/actions/dataset/merge.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from fairspec_metadata import load_dataset_descriptor
+from fairspec_metadata import Dataset
+
+
+def merge_datasets(
+    *,
+    system_dataset: Dataset,
+    user_dataset_path: str | None = None,
+) -> Dataset:
+    system = system_dataset.model_dump(by_alias=True, exclude_none=True)
+
+    user_dataset = (
+        load_dataset_descriptor(user_dataset_path).model_dump(
+            by_alias=True, exclude_none=True
+        )
+        if user_dataset_path
+        else None
+    )
+
+    merged = {**system, **user_dataset} if user_dataset else {**system}
+    return Dataset(**merged)

fairspec_dataset-0.0.0.dev0/fairspec_dataset/actions/file/copy.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from fairspec_dataset.actions.stream.load import load_file_stream
+from fairspec_dataset.actions.stream.save import save_file_stream
+
+
+def copy_file(
+    *,
+    source_path: str,
+    target_path: str,
+    max_bytes: int | None = None,
+) -> None:
+    stream = load_file_stream(source_path, max_bytes=max_bytes)
+    save_file_stream(stream, path=target_path)

fairspec_dataset-0.0.0.dev0/fairspec_dataset/actions/file/copy_spec.py

@@ -0,0 +1,69 @@
+import os
+
+from fairspec_dataset.actions.file.temp import write_temp_file
+
+from .copy import copy_file
+
+
+class TestCopyFile:
+    def test_copies_file(self, tmp_path):
+        source = write_temp_file("test content")
+        target = str(tmp_path / "target.txt")
+        copy_file(source_path=source, target_path=target)
+        assert os.path.exists(target)
+        with open(target, encoding="utf-8") as f:
+            assert f.read() == "test content"
+
+    def test_copies_exact_content(self, tmp_path):
+        content = "Hello, World! This is a test file."
+        source = write_temp_file(content)
+        target = str(tmp_path / "copy.txt")
+        copy_file(source_path=source, target_path=target)
+        with open(target, encoding="utf-8") as f:
+            assert f.read() == content
+
+    def test_copies_binary_file(self, tmp_path):
+        binary_data = bytes([0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10])
+        source = write_temp_file(binary_data)
+        target = str(tmp_path / "binary.bin")
+        copy_file(source_path=source, target_path=target)
+        with open(target, "rb") as f:
+            assert f.read() == binary_data
+
+    def test_copies_empty_file(self, tmp_path):
+        source = write_temp_file("")
+        target = str(tmp_path / "empty.txt")
+        copy_file(source_path=source, target_path=target)
+        with open(target, encoding="utf-8") as f:
+            assert f.read() == ""
+
+    def test_copies_large_file(self, tmp_path):
+        content = "x" * 100000
+        source = write_temp_file(content)
+        target = str(tmp_path / "large.txt")
+        copy_file(source_path=source, target_path=target)
+        with open(target, encoding="utf-8") as f:
+            assert f.read() == content
+
+    def test_copies_special_characters(self, tmp_path):
+        content = "Special characters: é, ñ, ü, ö, à, 中文, 日本語"
+        source = write_temp_file(content)
+        target = str(tmp_path / "special.txt")
+        copy_file(source_path=source, target_path=target)
+        with open(target, encoding="utf-8") as f:
+            assert f.read() == content
+
+    def test_copies_to_nested_directory(self, tmp_path):
+        source = write_temp_file("nested content")
+        target = str(tmp_path / "nested" / "dir" / "file.txt")
+        copy_file(source_path=source, target_path=target)
+        with open(target, encoding="utf-8") as f:
+            assert f.read() == "nested content"
+
+    def test_copies_with_newlines(self, tmp_path):
+        content = "Line 1\nLine 2\nLine 3\n"
+        source = write_temp_file(content)
+        target = str(tmp_path / "multiline.txt")
+        copy_file(source_path=source, target_path=target)
+        with open(target, encoding="utf-8") as f:
+            assert f.read() == content

fairspec_dataset-0.0.0.dev0/fairspec_dataset/actions/file/describe.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from fairspec_metadata import Resource
+
+from fairspec_dataset.models.file import FileDescription
+
+from .infer import infer_bytes, infer_integrity, infer_textual
+from .prefetch import prefetch_file
+
+
+def describe_file(
+    path: str,
+    *,
+    hash_type: str = "sha256",
+) -> FileDescription:
+    local_path = prefetch_file(path)
+    resource = Resource(data=local_path)
+
+    return FileDescription(
+        bytes=infer_bytes(resource),
+        textual=infer_textual(resource),
+        integrity=infer_integrity(resource, hash_type=hash_type),
+    )

fairspec_dataset-0.0.0.dev0/fairspec_dataset/actions/file/fixtures/generated/TestPrefetchFiles.test_prefetches_remote_file.yaml

@@ -0,0 +1,68 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Connection:
+      - close
+      Host:
+      - raw.githubusercontent.com
+      User-Agent:
+      - Python-urllib/3.12
+    method: GET
+    uri: https://raw.githubusercontent.com/fairspec/fairspec-typescript/refs/heads/main/table/plugins/csv/actions/table/fixtures/table.csv
+  response:
+    body:
+      string: "id,name\n1,english\n2,\u4E2D\u6587\n"
+    headers:
+      Accept-Ranges:
+      - bytes
+      Access-Control-Allow-Origin:
+      - '*'
+      Cache-Control:
+      - max-age=300
+      Connection:
+      - close
+      Content-Length:
+      - '27'
+      Content-Security-Policy:
+      - default-src 'none'; style-src 'unsafe-inline'; sandbox
+      Content-Type:
+      - text/plain; charset=utf-8
+      Cross-Origin-Resource-Policy:
+      - cross-origin
+      Date:
+      - Tue, 10 Feb 2026 15:25:06 GMT
+      ETag:
+      - '"f32e41a0928646326054d6fac4ec63789daf58505efb50237c672db52692e86e"'
+      Expires:
+      - Tue, 10 Feb 2026 15:30:06 GMT
+      Source-Age:
+      - '0'
+      Strict-Transport-Security:
+      - max-age=31536000
+      Vary:
+      - Authorization,Accept-Encoding
+      Via:
+      - 1.1 varnish
+      X-Cache:
+      - MISS
+      X-Cache-Hits:
+      - '0'
+      X-Content-Type-Options:
+      - nosniff
+      X-Fastly-Request-ID:
+      - 53bfbb3d5d4a7589325b5a5e917af77c465ca0d7
+      X-Frame-Options:
+      - deny
+      X-GitHub-Request-Id:
+      - 9622:3B543:C6F20F:DB1A96:698B4DD1
+      X-Served-By:
+      - cache-lis1490031-LIS
+      X-Timer:
+      - S1770737106.006396,VS0,VE190
+      X-XSS-Protection:
+      - 1; mode=block
+    status:
+      code: 200
+      message: OK
+version: 1