batchcorder 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. batchcorder-0.1.0/.github/workflows/build-wheels.yaml +91 -0
  2. batchcorder-0.1.0/.github/workflows/ci-lint.yaml +79 -0
  3. batchcorder-0.1.0/.github/workflows/ci-pre-release.yaml +36 -0
  4. batchcorder-0.1.0/.github/workflows/ci-release.yaml +37 -0
  5. batchcorder-0.1.0/.github/workflows/ci-test.yaml +74 -0
  6. batchcorder-0.1.0/.gitignore +90 -0
  7. batchcorder-0.1.0/.pre-commit-config.yaml +116 -0
  8. batchcorder-0.1.0/.readthedocs.yaml +14 -0
  9. batchcorder-0.1.0/.yamllint.yaml +8 -0
  10. batchcorder-0.1.0/Cargo.lock +2091 -0
  11. batchcorder-0.1.0/Cargo.toml +39 -0
  12. batchcorder-0.1.0/LICENSE +201 -0
  13. batchcorder-0.1.0/Makefile +23 -0
  14. batchcorder-0.1.0/PKG-INFO +130 -0
  15. batchcorder-0.1.0/README.md +106 -0
  16. batchcorder-0.1.0/_typos.toml +5 -0
  17. batchcorder-0.1.0/docs/quarto/.gitignore +2 -0
  18. batchcorder-0.1.0/docs/quarto/_quarto.yml +13 -0
  19. batchcorder-0.1.0/docs/quarto/how-to/cache-config.qmd +111 -0
  20. batchcorder-0.1.0/docs/quarto/how-to/duckdb.qmd +114 -0
  21. batchcorder-0.1.0/docs/quarto/how-to/eviction.qmd +105 -0
  22. batchcorder-0.1.0/docs/quarto/reference/api-overview.qmd +65 -0
  23. batchcorder-0.1.0/docs/quarto/tutorials/getting-started.qmd +126 -0
  24. batchcorder-0.1.0/docs/requirements.txt +110 -0
  25. batchcorder-0.1.0/docs/source/api.rst +15 -0
  26. batchcorder-0.1.0/docs/source/conf.py +125 -0
  27. batchcorder-0.1.0/docs/source/index.rst +33 -0
  28. batchcorder-0.1.0/pyproject.toml +90 -0
  29. batchcorder-0.1.0/python/batchcorder/__init__.py +483 -0
  30. batchcorder-0.1.0/python/batchcorder/_batchcorder.pyi +55 -0
  31. batchcorder-0.1.0/python/batchcorder/py.typed +0 -0
  32. batchcorder-0.1.0/scripts/build-docs.sh +47 -0
  33. batchcorder-0.1.0/src/bin/stub_gen.rs +31 -0
  34. batchcorder-0.1.0/src/cached_dataset.rs +739 -0
  35. batchcorder-0.1.0/src/lib.rs +20 -0
  36. batchcorder-0.1.0/tests/test_as_record_batch_reader.py +179 -0
  37. batchcorder-0.1.0/tests/test_cast.py +448 -0
  38. batchcorder-0.1.0/tests/test_docstrings.py +14 -0
  39. batchcorder-0.1.0/tests/test_duckdb.py +552 -0
  40. batchcorder-0.1.0/tests/test_errors.py +148 -0
  41. batchcorder-0.1.0/tests/test_stream_cache.py +534 -0
  42. batchcorder-0.1.0/uv.lock +1643 -0
@@ -0,0 +1,91 @@
1
+ name: build-wheels
2
+
3
+ on:
4
+ workflow_call:
5
+
6
+ permissions:
7
+ contents: read
8
+
9
+ jobs:
10
+ linux:
11
+ runs-on: ubuntu-latest
12
+ strategy:
13
+ matrix:
14
+ target: [x86_64, aarch64]
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ - uses: actions/setup-python@v5
18
+ with:
19
+ python-version: '3.11'
20
+ - name: Build wheels
21
+ uses: PyO3/maturin-action@v1
22
+ with:
23
+ target: ${{ matrix.target }}
24
+ args: --release --out dist --find-interpreter
25
+ sccache: 'true'
26
+ manylinux: ${{ matrix.target == 'aarch64' && '2_28' || 'auto' }}
27
+ - name: Upload wheels
28
+ uses: actions/upload-artifact@v4
29
+ with:
30
+ name: wheel-ubuntu-${{ matrix.target }}
31
+ path: dist
32
+
33
+ windows:
34
+ runs-on: windows-latest
35
+ strategy:
36
+ matrix:
37
+ target: [x64, x86]
38
+ steps:
39
+ - uses: actions/checkout@v4
40
+ - uses: actions/setup-python@v5
41
+ with:
42
+ python-version: '3.11'
43
+ architecture: ${{ matrix.target }}
44
+ - name: Build wheels
45
+ uses: PyO3/maturin-action@v1
46
+ with:
47
+ target: ${{ matrix.target }}
48
+ args: --release --out dist --find-interpreter
49
+ sccache: 'true'
50
+ - name: Upload wheels
51
+ uses: actions/upload-artifact@v4
52
+ with:
53
+ name: wheel-windows-${{ matrix.target }}
54
+ path: dist
55
+
56
+ macos:
57
+ runs-on: macos-latest
58
+ strategy:
59
+ matrix:
60
+ target: [x86_64, aarch64]
61
+ steps:
62
+ - uses: actions/checkout@v4
63
+ - uses: actions/setup-python@v5
64
+ with:
65
+ python-version: '3.11'
66
+ - name: Build wheels
67
+ uses: PyO3/maturin-action@v1
68
+ with:
69
+ target: ${{ matrix.target }}
70
+ args: --release --out dist --find-interpreter
71
+ sccache: 'true'
72
+ - name: Upload wheels
73
+ uses: actions/upload-artifact@v4
74
+ with:
75
+ name: wheel-macos-${{ matrix.target }}
76
+ path: dist
77
+
78
+ sdist:
79
+ runs-on: ubuntu-latest
80
+ steps:
81
+ - uses: actions/checkout@v4
82
+ - name: Build sdist
83
+ uses: PyO3/maturin-action@v1
84
+ with:
85
+ command: sdist
86
+ args: --out dist
87
+ - name: Upload sdist
88
+ uses: actions/upload-artifact@v4
89
+ with:
90
+ name: sdist
91
+ path: dist
@@ -0,0 +1,79 @@
1
+ name: ci-lint
2
+
3
+ on:
4
+ push:
5
+ paths-ignore:
6
+ - "docs/**"
7
+ - "**/*.md"
8
+ - "*.md"
9
+ - ".envrc"
10
+ branches:
11
+ - main
12
+ tags:
13
+ - '*'
14
+ pull_request:
15
+ workflow_dispatch:
16
+
17
+ permissions:
18
+ contents: read
19
+
20
+ jobs:
21
+ lint:
22
+ runs-on: ubuntu-latest
23
+ # sccache-action sets RUSTC_WRAPPER=sccache; all cargo sub-processes
24
+ # (maturin develop, cargo clippy, cargo doc in pre-commit) pick it up.
25
+ env:
26
+ SCCACHE_GHA_ENABLED: "true"
27
+ RUSTC_WRAPPER: "sccache"
28
+ UV_NO_SYNC: "1"
29
+ steps:
30
+ - uses: actions/checkout@v4
31
+
32
+ - uses: actions/setup-python@v5
33
+ with:
34
+ python-version: "3.13"
35
+
36
+ - uses: astral-sh/setup-uv@v5
37
+ with:
38
+ enable-cache: true
39
+
40
+ # Nightly is required by the `cargo +nightly fmt` pre-commit hook.
41
+ - name: Install Rust nightly
42
+ run: rustup toolchain install nightly --profile minimal --component rustfmt --no-self-update
43
+
44
+ - name: Set up sccache
45
+ uses: mozilla-actions/sccache-action@v0.0.9
46
+
47
+ - name: Cargo cache
48
+ uses: actions/cache@v4
49
+ with:
50
+ path: |
51
+ ~/.cargo/registry/index/
52
+ ~/.cargo/registry/cache/
53
+ ~/.cargo/git/db/
54
+ ./target/
55
+ key: cargo-lint-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}
56
+ restore-keys: cargo-lint-
57
+
58
+ - name: Install Python dependencies
59
+ run: uv sync --locked --group dev --no-install-project
60
+
61
+ # The pyright, griffe, and mypy-stubtest pre-commit hooks import the
62
+ # compiled extension at runtime, so we must build it first.
63
+ - name: Build extension
64
+ uses: PyO3/maturin-action@v1
65
+ with:
66
+ command: develop
67
+ args: --uv
68
+ sccache: 'true'
69
+ manylinux: 'off'
70
+
71
+ - name: Cache pre-commit environments
72
+ uses: actions/cache@v4
73
+ with:
74
+ path: ~/.cache/pre-commit
75
+ key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
76
+ restore-keys: pre-commit-
77
+
78
+ - name: Run pre-commit
79
+ run: uv run pre-commit run --all-files
@@ -0,0 +1,36 @@
1
+ name: ci-pre-release
2
+
3
+ on:
4
+ workflow_dispatch:
5
+
6
+ # we do not want more than one release workflow executing at the same time, ever
7
+ concurrency:
8
+ group: release
9
+ # cancelling in the middle of a release would create incomplete releases
10
+ # so cancel-in-progress is false
11
+ cancel-in-progress: false
12
+
13
+ permissions:
14
+ contents: read
15
+
16
+ jobs:
17
+ build:
18
+ uses: ./.github/workflows/build-wheels.yaml
19
+
20
+ release:
21
+ name: Release
22
+ runs-on: ubuntu-latest
23
+ needs: build
24
+ steps:
25
+ - uses: actions/download-artifact@v4
26
+ with:
27
+ path: dist
28
+ merge-multiple: true
29
+ - name: Publish to TestPyPI
30
+ uses: PyO3/maturin-action@v1
31
+ env:
32
+ MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
33
+ MATURIN_REPOSITORY: "testpypi"
34
+ with:
35
+ command: upload
36
+ args: --non-interactive --skip-existing ./dist/*
@@ -0,0 +1,37 @@
1
+ name: ci-release
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ release:
6
+ types: [published]
7
+
8
+ # we do not want more than one release workflow executing at the same time, ever
9
+ concurrency:
10
+ group: release
11
+ # cancelling in the middle of a release would create incomplete releases
12
+ # so cancel-in-progress is false
13
+ cancel-in-progress: false
14
+
15
+ permissions:
16
+ contents: read
17
+
18
+ jobs:
19
+ build:
20
+ uses: ./.github/workflows/build-wheels.yaml
21
+
22
+ release:
23
+ name: Release
24
+ runs-on: ubuntu-latest
25
+ needs: build
26
+ steps:
27
+ - uses: actions/download-artifact@v4
28
+ with:
29
+ path: dist
30
+ merge-multiple: true
31
+ - name: Publish to PyPI
32
+ uses: PyO3/maturin-action@v1
33
+ env:
34
+ MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
35
+ with:
36
+ command: upload
37
+ args: --non-interactive --skip-existing ./dist/*
@@ -0,0 +1,74 @@
1
+ name: ci-test
2
+
3
+ on:
4
+ push:
5
+ paths-ignore:
6
+ - "docs/**"
7
+ - "**/*.md"
8
+ - "*.md"
9
+ - ".envrc"
10
+ branches:
11
+ - main
12
+ tags:
13
+ - '*'
14
+ pull_request:
15
+ workflow_dispatch:
16
+
17
+ permissions:
18
+ contents: read
19
+
20
+ jobs:
21
+ linux:
22
+ runs-on: ubuntu-latest
23
+ env:
24
+ UV_NO_SYNC: "1"
25
+ strategy:
26
+ matrix:
27
+ # requires-python = ">=3.11"; test the floor and the latest stable.
28
+ python-version: ["3.11", "3.13"]
29
+ steps:
30
+ - uses: actions/checkout@v4
31
+
32
+ - name: Update Rust stable
33
+ run: rustup update stable
34
+
35
+ - uses: actions/setup-python@v5
36
+ with:
37
+ python-version: ${{ matrix.python-version }}
38
+
39
+ - uses: astral-sh/setup-uv@v5
40
+ with:
41
+ enable-cache: true
42
+
43
+ - name: Cargo cache
44
+ uses: actions/cache@v4
45
+ with:
46
+ path: |
47
+ ~/.cargo/registry/index/
48
+ ~/.cargo/registry/cache/
49
+ ~/.cargo/git/db/
50
+ ./target/
51
+ # abi3-py311 produces a single binary regardless of Python version,
52
+ # so the cache key does not include python-version.
53
+ key: cargo-test-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}
54
+ restore-keys: cargo-test-
55
+
56
+ # --no-install-project: maturin develop handles the batchcorder install.
57
+ - name: Install Python dependencies
58
+ run: uv sync --locked --group dev --no-install-project
59
+
60
+ - name: Build extension
61
+ uses: PyO3/maturin-action@v1
62
+ with:
63
+ command: develop
64
+ args: --release --strip --uv
65
+ sccache: 'true'
66
+ manylinux: 'off'
67
+
68
+ - name: Run tests
69
+ run: uv run pytest --import-mode=importlib --cov --cov-report=xml
70
+
71
+ - name: Upload coverage to Codecov
72
+ uses: codecov/codecov-action@v5.4.0
73
+ with:
74
+ token: ${{ secrets.CODECOV_TOKEN }}
@@ -0,0 +1,90 @@
1
+ /target
2
+
3
+ # Byte-compiled / optimized / DLL files
4
+ __pycache__/
5
+ .pytest_cache/
6
+ *.py[cod]
7
+
8
+ # C extensions
9
+ *.so
10
+
11
+ # Distribution / packaging
12
+ .Python
13
+ .venv/
14
+ env/
15
+ bin/
16
+ build/
17
+ develop-eggs/
18
+ dist/
19
+ eggs/
20
+ lib/
21
+ lib64/
22
+ parts/
23
+ sdist/
24
+ var/
25
+ include/
26
+ man/
27
+ venv/
28
+ *.egg-info/
29
+ .installed.cfg
30
+ *.egg
31
+
32
+ # Installer logs
33
+ pip-log.txt
34
+ pip-delete-this-directory.txt
35
+ pip-selfcheck.json
36
+
37
+ # Unit test / coverage reports
38
+ htmlcov/
39
+ .tox/
40
+ .coverage
41
+ .cache
42
+ nosetests.xml
43
+ coverage.xml
44
+
45
+ # Translations
46
+ *.mo
47
+
48
+ # Mr Developer
49
+ .mr.developer.cfg
50
+ .project
51
+ .pydevproject
52
+
53
+ # Rope
54
+ .ropeproject
55
+
56
+ # Django stuff:
57
+ *.log
58
+ *.pot
59
+
60
+ .DS_Store
61
+
62
+ # Sphinx documentation
63
+ docs/_build/
64
+ docs/build/
65
+
66
+ # Quarto-generated Markdown files (build artifacts — source is in docs/quarto/)
67
+ docs/source/**/*.md
68
+
69
+ # Quarto project cache
70
+ docs/quarto/.quarto/
71
+
72
+ # sphinx-autoapi generated RST files
73
+ docs/source/_autoapi/
74
+
75
+ # PyCharm
76
+ .idea/
77
+
78
+ # VSCode
79
+ .vscode/
80
+
81
+ # Pyenv
82
+ .python-version
83
+
84
+ .direnv/
85
+
86
+ .envrc
87
+
88
+ !src/bin
89
+
90
+ site
@@ -0,0 +1,116 @@
1
+ default_language_version:
2
+ python: python3
3
+
4
+ repos:
5
+ - repo: https://github.com/pre-commit/pre-commit-hooks
6
+ rev: v6.0.0
7
+ hooks:
8
+ - id: check-merge-conflict
9
+ - id: trailing-whitespace
10
+ - id: end-of-file-fixer
11
+ - id: check-added-large-files
12
+ args: ["--maxkb=500"]
13
+ - id: name-tests-test
14
+ args: ["--pytest-test-first"]
15
+
16
+ - repo: https://github.com/abravalheri/validate-pyproject
17
+ rev: v0.25
18
+ hooks:
19
+ - id: validate-pyproject
20
+
21
+ # typos: fast, code-aware typo checker (understands identifiers/tokens)
22
+ - repo: https://github.com/adhtruong/mirrors-typos
23
+ rev: v1.44.0
24
+ hooks:
25
+ - id: typos
26
+ args: ["--config", "pyproject.toml"]
27
+
28
+ # codespell: natural-language spell checker for comments and docs
29
+ - repo: https://github.com/codespell-project/codespell
30
+ rev: v2.4.2
31
+ hooks:
32
+ - id: codespell
33
+ args: [--toml, "pyproject.toml"]
34
+ additional_dependencies:
35
+ - tomli
36
+
37
+ - repo: https://github.com/astral-sh/ruff-pre-commit
38
+ rev: v0.15.6
39
+ hooks:
40
+ - id: ruff-check
41
+ args: ["--output-format=full", "--fix"]
42
+ - id: ruff-format
43
+
44
+ - repo: https://github.com/rhysd/actionlint
45
+ rev: v1.7.11
46
+ hooks:
47
+ - id: actionlint
48
+
49
+ - repo: https://github.com/adrienverge/yamllint
50
+ rev: v1.37.1
51
+ hooks:
52
+ - id: yamllint
53
+ args: [--config-file, .yamllint.yaml]
54
+
55
+ - repo: https://github.com/astral-sh/uv-pre-commit
56
+ rev: 0.10.10
57
+ hooks:
58
+ - id: uv-lock
59
+
60
+ - repo: https://github.com/ComPWA/taplo-pre-commit
61
+ rev: v0.9.3
62
+ hooks:
63
+ - id: taplo-format
64
+
65
+ - repo: https://github.com/shellcheck-py/shellcheck-py
66
+ rev: v0.11.0.1
67
+ hooks:
68
+ - id: shellcheck
69
+ args: ["scripts/build-docs.sh"]
70
+
71
+ - repo: local
72
+ hooks:
73
+ - id: rust-fmt
74
+ name: Rust fmt
75
+ description: Run cargo fmt on files included in the commit.
76
+ entry: cargo +nightly fmt --all --
77
+ language: system
78
+ pass_filenames: true
79
+ types: [file, rust]
80
+
81
+ - id: rust-clippy
82
+ name: Rust clippy
83
+ description: Run cargo clippy — deny all warnings.
84
+ entry: cargo clippy --all-targets --all-features -- -Dclippy::all -D warnings -Aclippy::redundant_closure
85
+ language: system
86
+ pass_filenames: false
87
+ types: [file, rust]
88
+
89
+ - id: pyright
90
+ name: pyright
91
+ description: Static type-checking for Python code and stubs.
92
+ entry: .venv/bin/pyright
93
+ language: system
94
+ pass_filenames: false
95
+ types_or: [python, pyi]
96
+
97
+ # griffe validates that NumPy-style docstrings match function signatures.
98
+ # Dynamically imports the installed batchcorder extension.
99
+ - id: griffe
100
+ name: griffe (docstring validation)
101
+ description: Validate NumPy-style docstrings are complete and consistent.
102
+ entry: .venv/bin/griffe dump -d numpy -LWARNING -o /dev/null batchcorder
103
+ language: system
104
+ pass_filenames: false
105
+ types_or: [python, pyi]
106
+
107
+ # mypy.stubtest imports the compiled extension and checks that every
108
+ # name, arity, and type annotation in batchcorder.pyi matches runtime.
109
+ # Requires: maturin develop --uv (compiled .so must be present)
110
+ - id: mypy-stubtest
111
+ name: mypy stubtest
112
+ description: Validate _batchcorder.pyi matches the compiled extension at runtime.
113
+ entry: .venv/bin/python -m mypy.stubtest batchcorder._batchcorder
114
+ language: system
115
+ pass_filenames: false
116
+ types_or: [python, pyi]
@@ -0,0 +1,14 @@
1
+ version: 2
2
+
3
+ build:
4
+ os: ubuntu-24.04
5
+ tools:
6
+ python: "3.12"
7
+ rust: "1.91"
8
+ commands:
9
+ # Run the build script with error handling and security checks
10
+ - ./scripts/build-docs.sh
11
+ python:
12
+ install:
13
+ - method: pip
14
+ path: .
@@ -0,0 +1,8 @@
1
+ extends: default
2
+
3
+ rules:
4
+ line-length:
5
+ max: 120
6
+ level: warning
7
+ document-start: disable
8
+ truthy: disable