iden 0.0.1a0__tar.gz

iden-0.0.1a0/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2024, Thibaut Durand
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

iden-0.0.1a0/PKG-INFO

@@ -0,0 +1,145 @@
+Metadata-Version: 2.1
+Name: iden
+Version: 0.0.1a0
+Summary: 
+Home-page: https://github.com/durandtibo/iden
+License: BSD-3-Clause
+Keywords: dataset,shard
+Author: Thibaut Durand
+Author-email: durand.tibo+gh@gmail.com
+Requires-Python: >=3.9,<3.13
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries
+Provides-Extra: all
+Requires-Dist: coola (>=0.4,<1.0)
+Requires-Dist: numpy (>=1.22,<2.0) ; extra == "all"
+Requires-Dist: objectory (>=0.1,<1.0)
+Requires-Dist: safetensors (>=0.4,<1.0) ; extra == "all"
+Requires-Dist: torch (>=1.11,<3.0) ; extra == "all"
+Project-URL: Repository, https://github.com/durandtibo/iden
+Description-Content-Type: text/markdown
+
+# iden
+
+<p align="center">
+    <a href="https://github.com/durandtibo/iden/actions">
+        <img alt="CI" src="https://github.com/durandtibo/iden/workflows/CI/badge.svg">
+    </a>
+    <a href="https://durandtibo.github.io/iden/">
+        <img alt="Documentation" src="https://github.com/durandtibo/iden/workflows/Documentation%20(stable)/badge.svg">
+    </a>
+    <a href="https://github.com/durandtibo/iden/actions">
+        <img alt="Nightly Tests" src="https://github.com/durandtibo/iden/workflows/Nightly%20Tests/badge.svg">
+    </a>
+    <a href="https://github.com/durandtibo/iden/actions">
+        <img alt="Nightly Package Tests" src="https://github.com/durandtibo/iden/workflows/Nightly%20Package%20Tests/badge.svg">
+    </a>
+    <br/>
+    <a href="https://codecov.io/gh/durandtibo/iden">
+        <img alt="Codecov" src="https://codecov.io/gh/durandtibo/iden/branch/main/graph/badge.svg">
+    </a>
+    <a href="https://codeclimate.com/github/durandtibo/iden/maintainability">
+        <img src="https://api.codeclimate.com/v1/badges/e1d6a491463b53dde654/maintainability" />
+    </a>
+    <a href="https://codeclimate.com/github/durandtibo/iden/test_coverage">
+        <img src="https://api.codeclimate.com/v1/badges/e1d6a491463b53dde654/test_coverage" />
+    </a>
+    <br/>
+    <a href="https://github.com/psf/black">
+        <img  alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg">
+    </a>
+    <a href="https://google.github.io/styleguide/pyguide.html#s3.8-comments-and-docstrings">
+        <img  alt="Doc style: google" src="https://img.shields.io/badge/%20style-google-3666d6.svg">
+    </a>
+    <a href="https://github.com/astral-sh/ruff">
+        <img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json" alt="Ruff" style="max-width:100%;">
+    </a>
+    <a href="https://github.com/guilatrova/tryceratops">
+        <img  alt="Doc style: google" src="https://img.shields.io/badge/try%2Fexcept%20style-tryceratops%20%F0%9F%A6%96%E2%9C%A8-black">
+    </a>
+    <br/>
+    <a href="https://pypi.org/project/iden/">
+        <img alt="PYPI version" src="https://img.shields.io/pypi/v/iden">
+    </a>
+    <a href="https://pypi.org/project/iden/">
+        <img alt="Python" src="https://img.shields.io/pypi/pyversions/iden.svg">
+    </a>
+    <a href="https://opensource.org/licenses/BSD-3-Clause">
+        <img alt="BSD-3-Clause" src="https://img.shields.io/pypi/l/iden">
+    </a>
+    <br/>
+    <a href="https://pepy.tech/project/iden">
+        <img  alt="Downloads" src="https://static.pepy.tech/badge/iden">
+    </a>
+    <a href="https://pepy.tech/project/iden">
+        <img  alt="Monthly downloads" src="https://static.pepy.tech/badge/iden/month">
+    </a>
+    <br/>
+</p>
+
+## Overview
+
+`iden` is a simple Python library to manage a dataset of shards when training a machine learning
+model.
+`iden` uses a lazy loading approach to load the shard's data, so it is easy to manage shards without
+loading their data.
+`iden` supports different format to store shards on disk.
+
+## Installation
+
+We highly recommend installing
+a [virtual environment](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/).
+`iden` can be installed from pip using the following command:
+
+```shell
+pip install iden
+```
+
+To make the package as slim as possible, only the minimal packages required to use `iden` are
+installed.
+To include all the dependencies, you can use the following command:
+
+```shell
+pip install iden[all]
+```
+
+Please check the [get started page](https://durandtibo.github.io/iden/get_started) to see how to
+install only some specific dependencies or other alternatives to install the library.
+The following is the corresponding `iden` versions and tested dependencies.
+
+## Contributing
+
+Please check the instructions in [CONTRIBUTING.md](.github/CONTRIBUTING.md).
+
+## Suggestions and Communication
+
+Everyone is welcome to contribute to the community.
+If you have any questions or suggestions, you can
+submit [Github Issues](https://github.com/durandtibo/iden/issues).
+We will reply to you as soon as possible. Thank you very much.
+
+## API stability
+
+:warning: While `iden` is in development stage, no API is guaranteed to be stable from one
+release to the next.
+In fact, it is very likely that the API will change multiple times before a stable 1.0.0 release.
+In practice, this means that upgrading `iden` to a new version will possibly break any code that
+was using the old version of `iden`.
+
+## License
+
+`iden` is licensed under BSD 3-Clause "New" or "Revised" license available in [LICENSE](LICENSE)
+file.
+

iden-0.0.1a0/README.md

@@ -0,0 +1,111 @@
+# iden
+
+<p align="center">
+    <a href="https://github.com/durandtibo/iden/actions">
+        <img alt="CI" src="https://github.com/durandtibo/iden/workflows/CI/badge.svg">
+    </a>
+    <a href="https://durandtibo.github.io/iden/">
+        <img alt="Documentation" src="https://github.com/durandtibo/iden/workflows/Documentation%20(stable)/badge.svg">
+    </a>
+    <a href="https://github.com/durandtibo/iden/actions">
+        <img alt="Nightly Tests" src="https://github.com/durandtibo/iden/workflows/Nightly%20Tests/badge.svg">
+    </a>
+    <a href="https://github.com/durandtibo/iden/actions">
+        <img alt="Nightly Package Tests" src="https://github.com/durandtibo/iden/workflows/Nightly%20Package%20Tests/badge.svg">
+    </a>
+    <br/>
+    <a href="https://codecov.io/gh/durandtibo/iden">
+        <img alt="Codecov" src="https://codecov.io/gh/durandtibo/iden/branch/main/graph/badge.svg">
+    </a>
+    <a href="https://codeclimate.com/github/durandtibo/iden/maintainability">
+        <img src="https://api.codeclimate.com/v1/badges/e1d6a491463b53dde654/maintainability" />
+    </a>
+    <a href="https://codeclimate.com/github/durandtibo/iden/test_coverage">
+        <img src="https://api.codeclimate.com/v1/badges/e1d6a491463b53dde654/test_coverage" />
+    </a>
+    <br/>
+    <a href="https://github.com/psf/black">
+        <img  alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg">
+    </a>
+    <a href="https://google.github.io/styleguide/pyguide.html#s3.8-comments-and-docstrings">
+        <img  alt="Doc style: google" src="https://img.shields.io/badge/%20style-google-3666d6.svg">
+    </a>
+    <a href="https://github.com/astral-sh/ruff">
+        <img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json" alt="Ruff" style="max-width:100%;">
+    </a>
+    <a href="https://github.com/guilatrova/tryceratops">
+        <img  alt="Doc style: google" src="https://img.shields.io/badge/try%2Fexcept%20style-tryceratops%20%F0%9F%A6%96%E2%9C%A8-black">
+    </a>
+    <br/>
+    <a href="https://pypi.org/project/iden/">
+        <img alt="PYPI version" src="https://img.shields.io/pypi/v/iden">
+    </a>
+    <a href="https://pypi.org/project/iden/">
+        <img alt="Python" src="https://img.shields.io/pypi/pyversions/iden.svg">
+    </a>
+    <a href="https://opensource.org/licenses/BSD-3-Clause">
+        <img alt="BSD-3-Clause" src="https://img.shields.io/pypi/l/iden">
+    </a>
+    <br/>
+    <a href="https://pepy.tech/project/iden">
+        <img  alt="Downloads" src="https://static.pepy.tech/badge/iden">
+    </a>
+    <a href="https://pepy.tech/project/iden">
+        <img  alt="Monthly downloads" src="https://static.pepy.tech/badge/iden/month">
+    </a>
+    <br/>
+</p>
+
+## Overview
+
+`iden` is a simple Python library to manage a dataset of shards when training a machine learning
+model.
+`iden` uses a lazy loading approach to load the shard's data, so it is easy to manage shards without
+loading their data.
+`iden` supports different format to store shards on disk.
+
+## Installation
+
+We highly recommend installing
+a [virtual environment](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/).
+`iden` can be installed from pip using the following command:
+
+```shell
+pip install iden
+```
+
+To make the package as slim as possible, only the minimal packages required to use `iden` are
+installed.
+To include all the dependencies, you can use the following command:
+
+```shell
+pip install iden[all]
+```
+
+Please check the [get started page](https://durandtibo.github.io/iden/get_started) to see how to
+install only some specific dependencies or other alternatives to install the library.
+The following is the corresponding `iden` versions and tested dependencies.
+
+## Contributing
+
+Please check the instructions in [CONTRIBUTING.md](.github/CONTRIBUTING.md).
+
+## Suggestions and Communication
+
+Everyone is welcome to contribute to the community.
+If you have any questions or suggestions, you can
+submit [Github Issues](https://github.com/durandtibo/iden/issues).
+We will reply to you as soon as possible. Thank you very much.
+
+## API stability
+
+:warning: While `iden` is in development stage, no API is guaranteed to be stable from one
+release to the next.
+In fact, it is very likely that the API will change multiple times before a stable 1.0.0 release.
+In practice, this means that upgrading `iden` to a new version will possibly break any code that
+was using the old version of `iden`.
+
+## License
+
+`iden` is licensed under BSD 3-Clause "New" or "Revised" license available in [LICENSE](LICENSE)
+file.

iden-0.0.1a0/pyproject.toml

@@ -0,0 +1,263 @@
+[tool.poetry]
+name = "iden"
+version = "0.0.1a0"
+description = ""
+readme = "README.md"
+authors = ["Thibaut Durand <durand.tibo+gh@gmail.com>"]
+homepage = "https://github.com/durandtibo/iden"
+repository = "https://github.com/durandtibo/iden"
+keywords = ["dataset", "shard"]
+license = "BSD-3-Clause"
+
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Information Technology",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: BSD License",
+    "Operating System :: POSIX :: Linux",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+packages = [
+    { include = "iden", from = "src" },
+]
+
+[tool.poetry.dependencies]
+# Core dependencies
+coola = ">=0.4,<1.0"
+objectory = ">=0.1,<1.0"
+python = ">=3.9,<3.13"
+
+# Optional dependencies
+numpy = ">=1.22,<2.0"
+safetensors = { version = ">=0.4,<1.0", optional = true }
+torch = { version = ">=1.11,<3.0", optional = true }
+
+[tool.poetry.extras]
+all = ["numpy", "safetensors", "torch"]
+
+[tool.poetry.group.docs]
+optional = true
+
+[tool.poetry.group.docs.dependencies]
+mike = "^2.0"
+mkdocs-material = "^9.5"
+mkdocstrings = { extras = ["python"], version = "^0.24" }
+
+[tool.poetry.group.dev.dependencies]
+black = ">=24.1"
+coverage = { extras = ["toml"], version = "^7.4" }
+docformatter = { extras = ["tomli"], version = "^1.7" }
+pre-commit = "^3.6"
+pygments = "^2.17"
+pytest = "^8.0"
+pytest-cov = "^4.1"
+pytest-timeout = "^2.2"
+ruff = ">=0.3.0,<1.0"
+xdoctest = "^1.1"
+
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.coverage.paths]
+source = ["src", "*/site-packages"]
+
+[tool.coverage.run]
+branch = true
+source = ["iden"]
+
+[tool.coverage.report]
+show_missing = true
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:"
+]
+
+[tool.pytest.ini_options]
+testpaths = "tests/"
+log_format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"
+log_level = "DEBUG"
+addopts = "--color yes --durations 10 -rf"
+# Configuration of the short test summary info
+# https://docs.pytest.org/en/stable/usage.html#detailed-summary-report
+
+[tool.black]
+line-length = 100
+target-version = ["py39", "py310", "py311", "py312"]
+include = '\.pyi?$'
+
+[tool.pylint.FORMAT]
+max-line-length = 100
+
+[tool.isort]
+profile = "black"
+
+[tool.docformatter]
+recursive = true
+wrap-summaries = 72
+wrap-descriptions = 72
+syntax = "google"
+
+[tool.ruff]
+# List of rules: https://docs.astral.sh/ruff/rules/
+lint.select = [
+    "A", # builtins
+    "ANN", # annotations
+    "ARG", # flake8-unused-arguments
+    "B", # bugbear
+    "BLE", # flake8-blind-except
+    "C4", # flake8-comprehensions
+    "D", # pydocstyle
+    "E", # pycodestyle (Error)
+    "EM", # flake8-errmsg
+    "F", # pyflakes
+    "FA", # flake8-future-annotations
+    "FURB", # refurb
+    "ICN", # flake8-import-conventions
+    "INP", # flake8-no-pep420
+    "ISC", # flake8-implicit-str-concat
+    "LOG", # logging
+    "N", # naming
+    "PD", # pandas-vet
+    "PERF", # Perflint
+    "PGH", # pygrep-hooks
+    "PIE", # flake8-pie
+    "PL", # Pylint
+    "PT", # flake8-pytest-style
+    "PTH", # pathlib
+    "PYI", # flake8-pyi
+    "Q", # flake8-quotes
+    "RET", # flake8-return
+    "RSE", # flake8-raise
+    "S", # flake8-bandit
+    "SIM", # flake8-simplify
+    "T20", # flake8-print
+    "TCH", # flake8-type-checking
+    "TD", # flake8-todos
+    "TID", # flake8-tidy-imports
+    "TRY", # tryceratops
+    "UP", # pyupgrade
+    "W", # pycodestyle (Warning)
+]
+lint.ignore = [
+    "A003", # Class attribute `{name}` is shadowing a python builtin
+    "ANN101", # Missing type annotation for `self` in method
+    "ANN102", # Missing type annotation for `cls` in classmethod
+    "ANN401", # Dynamically typed expressions (typing.Any) are disallowed.
+    "B905", # `zip()` without an explicit strict= parameter set. The `strict=` argument was added in Python 3.10
+    "E501", # Line lengths are recommended to be no greater than 79 characters.
+    "W505", # Doc line too long
+    "PLR0913", # Too many arguments in function definition (8 > 5)
+    "PLR2004", # Magic value used in comparison, consider replacing 0.0 with a constant variable
+
+    # The following rules are ignored because they conflict with another library like docformatter.
+    "D102", # Missing docstring in public method
+    "D105", # Missing docstring in magic method
+    "D107", # Missing docstring in `__init__`
+    "D205", # 1 blank line required between summary line and description
+    "D209", # Multi-line docstring closing quotes should be on a separate line
+]
+
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+lint.fixable = [
+    "A",
+    "ANN",
+    "ARG",
+    "B",
+    "BLE",
+    "C4",
+    "D",
+    "E",
+    "EM",
+    "F",
+    "FA",
+    "FURB",
+    "ICN",
+    "INP",
+    "ISC",
+    "LOG",
+    "PD",
+    "PERF",
+    "PGH",
+    "PIE",
+    "PL",
+    "PT",
+    "PTH",
+    "PYI",
+    "Q",
+    "RET",
+    "RSE",
+    "S",
+    "SIM",
+    "T20",
+    "TCH",
+    "TD",
+    "TID",
+    "TRY",
+    "W",
+]
+lint.unfixable = []
+
+lint.exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+]
+
+# Enable on top of the Google convention.
+lint.extend-select = ["D400", "D401", "D404"]
+
+line-length = 100
+target-version = "py39"
+src = ["src"]
+
+[tool.ruff.lint.per-file-ignores]
+# Ignore all directories named `tests`.
+"tests/**" = [
+    "D", # pydocstyle
+    "PL", # Pylint
+    "S101", # flake8-bandit
+]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 10
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.pycodestyle]
+max-doc-length = 72
+
+[tool.ruff.lint.isort]
+force-sort-within-sections = true
+combine-as-imports = true
+known-first-party = ["src"]
+
+[tool.ruff.lint.flake8-import-conventions]
+[tool.ruff.lint.flake8-import-conventions.aliases]
+numpy = "np"

iden-0.0.1a0/src/iden/__init__.py

@@ -0,0 +1 @@
+r"""Root package."""

iden-0.0.1a0/src/iden/constants.py

@@ -0,0 +1,10 @@
+r"""Contain the main constants."""
+
+from __future__ import annotations
+
+__all__ = ["ASSETS", "KWARGS", "LOADER", "SHARDS"]
+
+ASSETS = "assets"
+KWARGS = "kwargs"
+LOADER = "loader"
+SHARDS = "shards"

iden-0.0.1a0/src/iden/dataset/__init__.py

@@ -0,0 +1,9 @@
+r"""Contain dataset implementations."""
+
+from __future__ import annotations
+
+__all__ = ["BaseDataset", "VanillaDataset", "create_vanilla_dataset", "load_from_uri"]
+
+from iden.dataset.base import BaseDataset
+from iden.dataset.loading import load_from_uri
+from iden.dataset.vanilla import VanillaDataset, create_vanilla_dataset