pyrosome 0.1.11__tar.gz

pyrosome-0.1.11/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ian Green
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyrosome-0.1.11/PKG-INFO

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: pyrosome
+Version: 0.1.11
+Summary: Versioned, append-only storage for polars data frames, safe for concurrent processes
+Author: ian
+Author-email: ian <dev@iangreen.dev>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Dist: polars>=2.0.0
+Requires-Dist: pandas>=3.1.0 ; extra == 'pandas'
+Requires-Dist: pyarrow>=26.0.0 ; extra == 'pandas'
+Requires-Python: >=3.14
+Provides-Extra: pandas
+Description-Content-Type: text/markdown
+
+# pyrosome
+
+Versioned, append-only storage for [polars](https://pola.rs) data frames.
+
+pyrosome stores data frames on disk as parquet (or CSV) files, keeps every version you write, and reads them back as lazy frames. Several processes can read, write and append to the same data at once without overwriting each other or seeing half-written data.
+
+## Purpose
+
+Saving data frames to files is easy. Keeping those files trustworthy is harder:
+
+- overwriting a file loses the previous data, and a crash halfway through can leave it corrupt;
+- two processes writing at once can clobber each other;
+- a reader can pick up a file that is still being written.
+
+pyrosome handles this with a small, predictable layout on the filesystem. There is no database or server to run.
+
+## Project summary
+
+- **Versioned**: every `write` creates a new version (`v0`, `v1`, …). Older versions stay on disk until you remove them, so rollbacks and audits are simple.
+- **Append-only**: `append` adds a new file to the latest version and never rewrites existing files. Columns are reordered and cast to match the stored data.
+- **Safe with several processes**: versions are claimed with an atomic `mkdir`, appends claim their index by exclusively creating a marker file so they never overwrite each other, and the manifest naming the latest version is replaced in one step.
+- **Lazy**: `read` returns a `polars.LazyFrame`, so filters and column selections are pushed down into the file scan.
+- **Parquet or CSV**: `ParquetDal` stores parquet; `CsvDal` has the same API and stores CSV for people and tools that need to open the files directly.
+
+Data is organised into *profiles*, named by one or more keys:
+
+```text
+data/
+└── sales/
+    └── orders/                # profile for keys ("sales", "orders")
+        ├── manifest.json      # {"latest_data_dir": "v1"}
+        ├── v0/
+        │   ├── 0.parquet      # written by write()
+        │   └── 1.parquet      # added by append()
+        └── v1/
+            └── 0.parquet
+```
+
+## Platform support
+
+pyrosome supports Windows, Linux and macOS, with the data directory on a local disk or an NFS share (NFSv3 or later).
+
+SMB shares are supported as a lower priority. pyrosome aims to work on them, but they get less attention and testing, and some SMB setups have limitations. For example, they may ignore the read-only permission pyrosome sets on finished data files.
+
+## Install
+
+pyrosome requires Python 3.14 or newer.
+
+Add it to a uv project:
+
+```console
+uv add pyrosome
+```
+
+Or install it into the current environment with pip:
+
+```console
+pip install pyrosome
+```
+
+### Development setup
+
+```console
+uv sync --all-groups
+```
+
+This installs the development tools (pytest, ruff, ty, mypy, pylint) and the documentation tools (Sphinx, furo).
+
+| Task | Command |
+| --- | --- |
+| Run tests, including every example in the docs and this README | `uv run pytest` |
+| Lint and format | `uv run ruff check` and `uv run ruff format` |
+| Type check | `uv run ty check` and `uv run mypy src` |
+| Pylint | `uv run pylint src` |
+| Build docs | `uv run sphinx-build -W -b html docs docs/_build/html` |
+
+## Quickstart
+
+`write` creates a new version and returns its directory:
+
+```pycon
+>>> from pathlib import Path
+>>> import polars as pl
+>>> from pyrosome import ParquetDal
+>>> dal = ParquetDal(Path("data"))
+>>> orders = pl.DataFrame(
+...     {"order_id": [1, 2, 3], "region": ["eu", "eu", "us"], "total": [9.5, 20.0, 7.25]}
+... )
+>>> dal.write(orders, "sales", "orders").as_posix()
+'data/sales/orders/v0'
+
+```
+
+`append` adds a file to the latest version. Column order doesn't matter:
+
+```pycon
+>>> more = pl.DataFrame({"total": [12.0], "order_id": [4], "region": ["us"]})
+>>> dal.append(more, "sales", "orders").as_posix()
+'data/sales/orders/v0/1.parquet'
+
+```
+
+`read` returns a `LazyFrame` over the latest version:
+
+```pycon
+>>> (
+...     dal.read("sales", "orders")
+...     .filter(pl.col("region") == "eu")
+...     .select(pl.col("total").sum())
+...     .collect()
+...     .item()
+... )
+29.5
+
+```
+
+Writing again creates `v1`, and `read` now returns only the new data. The earlier version stays on disk:
+
+```pycon
+>>> _ = dal.write(pl.DataFrame({"order_id": [10], "status": ["open"]}), "sales", "orders")
+>>> dal.read("sales", "orders").collect().columns
+['order_id', 'status']
+>>> sorted(path.as_posix() for path in Path("data").rglob("[!.]*.*"))
+['data/sales/orders/manifest.json', 'data/sales/orders/v0/0.parquet', 'data/sales/orders/v0/1.parquet', 'data/sales/orders/v1/0.parquet']
+
+```
+
+To store CSV instead, use `CsvDal` with the same methods. Keep each format in its own data directory, because a CSV write to a parquet profile hides the parquet data from `ParquetDal`:
+
+```pycon
+>>> from pyrosome import CsvDal
+>>> CsvDal(Path("exports")).write(orders, "sales", "orders").as_posix()
+'exports/sales/orders/v0'
+
+```
+
+`read` raises `FileNotFoundError` for a profile with no data, and `append` raises `ValueError` if the columns don't match the stored data:
+
+```pycon
+>>> dal.read("sales", "returns")
+Traceback (most recent call last):
+...
+FileNotFoundError: Cannot find files for ('sales', 'returns')
+
+```
+
+## Documentation
+
+The full documentation is in `docs/` and covers concepts, 14 recipes (compaction, rollbacks, pruning old versions, multi-process ingest and more), and the API reference. Build it with:
+
+```console
+uv run sphinx-build -W -b html docs docs/_build/html
+```
+
+Then open `docs/_build/html/index.html`.
+
+## License
+
+MIT. See [LICENSE](LICENSE).

pyrosome-0.1.11/README.md

@@ -0,0 +1,159 @@
+# pyrosome
+
+Versioned, append-only storage for [polars](https://pola.rs) data frames.
+
+pyrosome stores data frames on disk as parquet (or CSV) files, keeps every version you write, and reads them back as lazy frames. Several processes can read, write and append to the same data at once without overwriting each other or seeing half-written data.
+
+## Purpose
+
+Saving data frames to files is easy. Keeping those files trustworthy is harder:
+
+- overwriting a file loses the previous data, and a crash halfway through can leave it corrupt;
+- two processes writing at once can clobber each other;
+- a reader can pick up a file that is still being written.
+
+pyrosome handles this with a small, predictable layout on the filesystem. There is no database or server to run.
+
+## Project summary
+
+- **Versioned**: every `write` creates a new version (`v0`, `v1`, …). Older versions stay on disk until you remove them, so rollbacks and audits are simple.
+- **Append-only**: `append` adds a new file to the latest version and never rewrites existing files. Columns are reordered and cast to match the stored data.
+- **Safe with several processes**: versions are claimed with an atomic `mkdir`, appends claim their index by exclusively creating a marker file so they never overwrite each other, and the manifest naming the latest version is replaced in one step.
+- **Lazy**: `read` returns a `polars.LazyFrame`, so filters and column selections are pushed down into the file scan.
+- **Parquet or CSV**: `ParquetDal` stores parquet; `CsvDal` has the same API and stores CSV for people and tools that need to open the files directly.
+
+Data is organised into *profiles*, named by one or more keys:
+
+```text
+data/
+└── sales/
+    └── orders/                # profile for keys ("sales", "orders")
+        ├── manifest.json      # {"latest_data_dir": "v1"}
+        ├── v0/
+        │   ├── 0.parquet      # written by write()
+        │   └── 1.parquet      # added by append()
+        └── v1/
+            └── 0.parquet
+```
+
+## Platform support
+
+pyrosome supports Windows, Linux and macOS, with the data directory on a local disk or an NFS share (NFSv3 or later).
+
+SMB shares are supported as a lower priority. pyrosome aims to work on them, but they get less attention and testing, and some SMB setups have limitations. For example, they may ignore the read-only permission pyrosome sets on finished data files.
+
+## Install
+
+pyrosome requires Python 3.14 or newer.
+
+Add it to a uv project:
+
+```console
+uv add pyrosome
+```
+
+Or install it into the current environment with pip:
+
+```console
+pip install pyrosome
+```
+
+### Development setup
+
+```console
+uv sync --all-groups
+```
+
+This installs the development tools (pytest, ruff, ty, mypy, pylint) and the documentation tools (Sphinx, furo).
+
+| Task | Command |
+| --- | --- |
+| Run tests, including every example in the docs and this README | `uv run pytest` |
+| Lint and format | `uv run ruff check` and `uv run ruff format` |
+| Type check | `uv run ty check` and `uv run mypy src` |
+| Pylint | `uv run pylint src` |
+| Build docs | `uv run sphinx-build -W -b html docs docs/_build/html` |
+
+## Quickstart
+
+`write` creates a new version and returns its directory:
+
+```pycon
+>>> from pathlib import Path
+>>> import polars as pl
+>>> from pyrosome import ParquetDal
+>>> dal = ParquetDal(Path("data"))
+>>> orders = pl.DataFrame(
+...     {"order_id": [1, 2, 3], "region": ["eu", "eu", "us"], "total": [9.5, 20.0, 7.25]}
+... )
+>>> dal.write(orders, "sales", "orders").as_posix()
+'data/sales/orders/v0'
+
+```
+
+`append` adds a file to the latest version. Column order doesn't matter:
+
+```pycon
+>>> more = pl.DataFrame({"total": [12.0], "order_id": [4], "region": ["us"]})
+>>> dal.append(more, "sales", "orders").as_posix()
+'data/sales/orders/v0/1.parquet'
+
+```
+
+`read` returns a `LazyFrame` over the latest version:
+
+```pycon
+>>> (
+...     dal.read("sales", "orders")
+...     .filter(pl.col("region") == "eu")
+...     .select(pl.col("total").sum())
+...     .collect()
+...     .item()
+... )
+29.5
+
+```
+
+Writing again creates `v1`, and `read` now returns only the new data. The earlier version stays on disk:
+
+```pycon
+>>> _ = dal.write(pl.DataFrame({"order_id": [10], "status": ["open"]}), "sales", "orders")
+>>> dal.read("sales", "orders").collect().columns
+['order_id', 'status']
+>>> sorted(path.as_posix() for path in Path("data").rglob("[!.]*.*"))
+['data/sales/orders/manifest.json', 'data/sales/orders/v0/0.parquet', 'data/sales/orders/v0/1.parquet', 'data/sales/orders/v1/0.parquet']
+
+```
+
+To store CSV instead, use `CsvDal` with the same methods. Keep each format in its own data directory, because a CSV write to a parquet profile hides the parquet data from `ParquetDal`:
+
+```pycon
+>>> from pyrosome import CsvDal
+>>> CsvDal(Path("exports")).write(orders, "sales", "orders").as_posix()
+'exports/sales/orders/v0'
+
+```
+
+`read` raises `FileNotFoundError` for a profile with no data, and `append` raises `ValueError` if the columns don't match the stored data:
+
+```pycon
+>>> dal.read("sales", "returns")
+Traceback (most recent call last):
+...
+FileNotFoundError: Cannot find files for ('sales', 'returns')
+
+```
+
+## Documentation
+
+The full documentation is in `docs/` and covers concepts, 14 recipes (compaction, rollbacks, pruning old versions, multi-process ingest and more), and the API reference. Build it with:
+
+```console
+uv run sphinx-build -W -b html docs docs/_build/html
+```
+
+Then open `docs/_build/html/index.html`.
+
+## License
+
+MIT. See [LICENSE](LICENSE).

pyrosome-0.1.11/pyproject.toml

@@ -0,0 +1,161 @@
+[project]
+name = "pyrosome"
+version = "0.1.11"
+description = "Versioned, append-only storage for polars data frames, safe for concurrent processes"
+license = "MIT"
+license-files = ["LICENSE"]
+readme = "README.md"
+requires-python = ">=3.14"
+dependencies = ["polars>=2.0.0"]
+
+[[project.authors]]
+name = "ian"
+email = "dev@iangreen.dev"
+
+[project.optional-dependencies]
+pandas = [
+    "pandas>=3.1.0",
+    "pyarrow>=26.0.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.13.5,<0.14.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "mypy>=2.4.0",
+    "pandas>=3.1.0",
+    "pandas-stubs>=3.0.5.260914",
+    "pyarrow>=26.0.0",
+    "pylint>=4.1.3",
+    "pytest>=9.1.1",
+    "ruff>=0.17.2",
+    "ty>=0.0.91",
+]
+docs = [
+    "furo>=2025.12.19",
+    "sphinx>=9.2.0",
+]
+
+[tool.ruff]
+target-version = "py314"
+preview = true
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "missing-trailing-comma",
+    "missing-copyright-notice",
+    "docstring-extraneous-returns",
+    "docstring-extraneous-exception",
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["assert"]
+"docs/conf.py" = ["implicit-namespace-package"]
+"docs/examples/**" = [
+    "implicit-namespace-package",
+    "print",
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.flake8-type-checking]
+strict = true
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ty.rules]
+all = "error"
+
+[tool.ty.terminal]
+error-on-warning = true
+
+[tool.mypy]
+python_version = "3.14"
+strict = true
+warn_unreachable = true
+local_partial_types = true
+strict_bytes = true
+disallow_any_unimported = true
+disallow_any_expr = true
+disallow_any_decorated = true
+disallow_any_explicit = true
+enable_error_code = [
+    "deprecated",
+    "exhaustive-match",
+    "explicit-override",
+    "ignore-without-code",
+    "mutable-override",
+    "possibly-undefined",
+    "redundant-expr",
+    "redundant-self",
+    "truthy-bool",
+    "truthy-iterable",
+    "unimported-reveal",
+    "unused-awaitable",
+    "unused-ignore",
+]
+
+[tool.pylint.main]
+py-version = "3.14"
+fail-on = ["all"]
+load-plugins = [
+    "pylint.extensions.bad_builtin",
+    "pylint.extensions.broad_try_clause",
+    "pylint.extensions.check_elif",
+    "pylint.extensions.code_style",
+    "pylint.extensions.comparison_placement",
+    "pylint.extensions.confusing_elif",
+    "pylint.extensions.consider_refactoring_into_while_condition",
+    "pylint.extensions.consider_ternary_expression",
+    "pylint.extensions.dict_init_mutate",
+    "pylint.extensions.docparams",
+    "pylint.extensions.docstyle",
+    "pylint.extensions.dunder",
+    "pylint.extensions.empty_comment",
+    "pylint.extensions.eq_without_hash",
+    "pylint.extensions.for_any_all",
+    "pylint.extensions.magic_value",
+    "pylint.extensions.mccabe",
+    "pylint.extensions.no_self_use",
+    "pylint.extensions.overlapping_exceptions",
+    "pylint.extensions.private_import",
+    "pylint.extensions.redefined_loop_name",
+    "pylint.extensions.redefined_variable_type",
+    "pylint.extensions.set_membership",
+    "pylint.extensions.typing",
+    "pylint.extensions.while_used",
+]
+
+[tool.pylint."messages control"]
+enable = ["all"]
+disable = ["redundant-returns-doc"]
+
+[tool.pylint.parameter_documentation]
+accept-no-param-doc = false
+accept-no-raise-doc = false
+accept-no-return-doc = false
+accept-no-yields-doc = false
+default-docstring-type = "google"
+
+[tool.pylint.typing]
+runtime-typing = false
+
+[tool.pytest]
+testpaths = [
+    "tests",
+    "src",
+    "docs",
+    "README.md",
+]
+addopts = [
+    "--doctest-modules",
+    "--doctest-glob=*.rst",
+    "--doctest-glob=*.md",
+]
+doctest_optionflags = ["ELLIPSIS"]
+usefixtures = ["run_in_tmp_path"]

pyrosome-0.1.11/pyproject.toml.orig

@@ -0,0 +1,160 @@
+[project]
+name = "pyrosome"
+version = "0.1.11"
+description = "Versioned, append-only storage for polars data frames, safe for concurrent processes"
+license = "MIT"
+license-files = ["LICENSE"]
+readme = "README.md"
+authors = [
+    { name = "ian", email = "dev@iangreen.dev" }
+]
+requires-python = ">=3.14"
+dependencies = [
+    "polars>=2.0.0",
+]
+
+[project.optional-dependencies]
+pandas = [
+    "pandas>=3.1.0",
+    "pyarrow>=26.0.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.13.5,<0.14.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "mypy>=2.4.0",
+    "pandas>=3.1.0",
+    "pandas-stubs>=3.0.5.260914",
+    "pyarrow>=26.0.0",
+    "pylint>=4.1.3",
+    "pytest>=9.1.1",
+    "ruff>=0.17.2",
+    "ty>=0.0.91",
+]
+docs = [
+    "furo>=2025.12.19",
+    "sphinx>=9.2.0",
+]
+
+[tool.ruff]
+target-version = "py314"
+preview = true
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    # conflicts with `ruff format`
+    "missing-trailing-comma",
+    # no copyright headers in this project
+    "missing-copyright-notice",
+    # every docstring documents its returns and raises, including None and
+    # exceptions raised by functions it calls
+    "docstring-extraneous-returns",
+    "docstring-extraneous-exception",
+]
+
+[tool.ruff.lint.per-file-ignores]
+# pytest relies on bare asserts
+"tests/**" = ["assert"]
+# Sphinx requires conf.py to sit in docs/ without an __init__.py
+"docs/conf.py" = ["implicit-namespace-package"]
+# standalone scripts included in the docs, which print their results
+"docs/examples/**" = ["implicit-namespace-package", "print"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.flake8-type-checking]
+strict = true
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ty.rules]
+all = "error"
+
+[tool.ty.terminal]
+error-on-warning = true
+
+[tool.mypy]
+python_version = "3.14"
+strict = true
+warn_unreachable = true
+local_partial_types = true
+strict_bytes = true
+disallow_any_unimported = true
+disallow_any_expr = true
+disallow_any_decorated = true
+disallow_any_explicit = true
+enable_error_code = [
+    "deprecated",
+    "exhaustive-match",
+    "explicit-override",
+    "ignore-without-code",
+    "mutable-override",
+    "possibly-undefined",
+    "redundant-expr",
+    "redundant-self",
+    "truthy-bool",
+    "truthy-iterable",
+    "unimported-reveal",
+    "unused-awaitable",
+    "unused-ignore",
+]
+
+[tool.pylint.main]
+py-version = "3.14"
+fail-on = ["all"]
+load-plugins = [
+    "pylint.extensions.bad_builtin",
+    "pylint.extensions.broad_try_clause",
+    "pylint.extensions.check_elif",
+    "pylint.extensions.code_style",
+    "pylint.extensions.comparison_placement",
+    "pylint.extensions.confusing_elif",
+    "pylint.extensions.consider_refactoring_into_while_condition",
+    "pylint.extensions.consider_ternary_expression",
+    "pylint.extensions.dict_init_mutate",
+    "pylint.extensions.docparams",
+    "pylint.extensions.docstyle",
+    "pylint.extensions.dunder",
+    "pylint.extensions.empty_comment",
+    "pylint.extensions.eq_without_hash",
+    "pylint.extensions.for_any_all",
+    "pylint.extensions.magic_value",
+    "pylint.extensions.mccabe",
+    "pylint.extensions.no_self_use",
+    "pylint.extensions.overlapping_exceptions",
+    "pylint.extensions.private_import",
+    "pylint.extensions.redefined_loop_name",
+    "pylint.extensions.redefined_variable_type",
+    "pylint.extensions.set_membership",
+    "pylint.extensions.typing",
+    "pylint.extensions.while_used",
+]
+
+[tool.pylint."messages control"]
+enable = ["all"]
+# every docstring documents its returns, including None
+disable = ["redundant-returns-doc"]
+
+[tool.pylint.parameter_documentation]
+accept-no-param-doc = false
+accept-no-raise-doc = false
+accept-no-return-doc = false
+accept-no-yields-doc = false
+default-docstring-type = "google"
+
+[tool.pylint.typing]
+runtime-typing = false
+
+[tool.pytest]
+testpaths = ["tests", "src", "docs", "README.md"]
+# run the examples in docstrings, docs pages and the README as doctests
+addopts = ["--doctest-modules", "--doctest-glob=*.rst", "--doctest-glob=*.md"]
+doctest_optionflags = ["ELLIPSIS"]
+# defined in conftest.py
+usefixtures = ["run_in_tmp_path"]

pyrosome-0.1.11/src/pyrosome/__init__.py

@@ -0,0 +1,14 @@
+"""File IO layer."""
+
+from pyrosome.api import SimpleIo
+from pyrosome.dal import CsvDal, Dal, ParquetDal, get_accessor
+from pyrosome.datasets import Dataset
+
+__all__ = [
+    "CsvDal",
+    "Dal",
+    "Dataset",
+    "ParquetDal",
+    "SimpleIo",
+    "get_accessor",
+]