polars-learn 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polars_learn-0.1.0/.github/workflows/create-release.yml +99 -0
- polars_learn-0.1.0/.github/workflows/python-publish.yml +87 -0
- polars_learn-0.1.0/.github/workflows/tests.yml +34 -0
- polars_learn-0.1.0/.gitignore +44 -0
- polars_learn-0.1.0/.python-version +1 -0
- polars_learn-0.1.0/LICENSE +21 -0
- polars_learn-0.1.0/PKG-INFO +183 -0
- polars_learn-0.1.0/README.md +167 -0
- polars_learn-0.1.0/pyproject.toml +44 -0
- polars_learn-0.1.0/src/polars_learn/__init__.py +23 -0
- polars_learn-0.1.0/src/polars_learn/_utils.py +143 -0
- polars_learn-0.1.0/src/polars_learn/exceptions.py +17 -0
- polars_learn-0.1.0/src/polars_learn/feature_engineering.py +223 -0
- polars_learn-0.1.0/src/polars_learn/feature_spec.py +122 -0
- polars_learn-0.1.0/src/polars_learn/py.typed +0 -0
- polars_learn-0.1.0/src/polars_learn/schema_validation.py +280 -0
- polars_learn-0.1.0/src/polars_learn/types.py +15 -0
- polars_learn-0.1.0/tests/conftest.py +7 -0
- polars_learn-0.1.0/tests/test_feature_engineering.py +129 -0
- polars_learn-0.1.0/tests/test_feature_spec.py +58 -0
- polars_learn-0.1.0/tests/test_pandas_interop.py +89 -0
- polars_learn-0.1.0/tests/test_schema_validation.py +152 -0
- polars_learn-0.1.0/tests/test_sklearn_integration.py +95 -0
- polars_learn-0.1.0/uv.lock +797 -0
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
name: Create GitHub Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
|
|
6
|
+
permissions:
|
|
7
|
+
contents: read
|
|
8
|
+
|
|
9
|
+
concurrency:
|
|
10
|
+
group: create-release
|
|
11
|
+
cancel-in-progress: false
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
create-release:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
permissions:
|
|
17
|
+
contents: write
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
- uses: astral-sh/setup-uv@v5
|
|
21
|
+
- name: Install Python
|
|
22
|
+
run: uv python install 3.11
|
|
23
|
+
- name: Install locked dependencies
|
|
24
|
+
run: uv sync --locked --extra pandas --python 3.11
|
|
25
|
+
- name: Validate package
|
|
26
|
+
run: |
|
|
27
|
+
uv run --locked --extra pandas pytest --doctest-modules src tests
|
|
28
|
+
uv run --locked --extra pandas ruff check .
|
|
29
|
+
uv run --locked --extra pandas ruff format --check .
|
|
30
|
+
uv run --locked --extra pandas pyright
|
|
31
|
+
- name: Read package version
|
|
32
|
+
id: version
|
|
33
|
+
run: |
|
|
34
|
+
uv run --locked --extra pandas python - <<'PY'
|
|
35
|
+
import os
|
|
36
|
+
import re
|
|
37
|
+
import tomllib
|
|
38
|
+
|
|
39
|
+
with open("pyproject.toml", "rb") as source:
|
|
40
|
+
version = tomllib.load(source)["project"]["version"]
|
|
41
|
+
from polars_learn import __version__
|
|
42
|
+
|
|
43
|
+
if __version__ != version:
|
|
44
|
+
raise SystemExit("Package __version__ must match project.version")
|
|
45
|
+
if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version):
|
|
46
|
+
raise SystemExit("Release version must have the form X.Y.Z")
|
|
47
|
+
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
|
48
|
+
output.write(f"tag=v{version}\n")
|
|
49
|
+
PY
|
|
50
|
+
- name: Reject an existing release tag
|
|
51
|
+
env:
|
|
52
|
+
GH_TOKEN: ${{ github.token }}
|
|
53
|
+
GH_REPO: ${{ github.repository }}
|
|
54
|
+
RELEASE_TAG: ${{ steps.version.outputs.tag }}
|
|
55
|
+
run: |
|
|
56
|
+
gh api "repos/$GH_REPO/git/matching-refs/tags/$RELEASE_TAG" > /tmp/release-refs.json
|
|
57
|
+
uv run --locked --extra pandas python - <<'PY'
|
|
58
|
+
import json
|
|
59
|
+
import tomllib
|
|
60
|
+
|
|
61
|
+
with open("pyproject.toml", "rb") as source:
|
|
62
|
+
tag = "v" + tomllib.load(source)["project"]["version"]
|
|
63
|
+
with open("/tmp/release-refs.json") as source:
|
|
64
|
+
refs = json.load(source)
|
|
65
|
+
if any(ref["ref"] == f"refs/tags/{tag}" for ref in refs):
|
|
66
|
+
raise SystemExit(f"{tag} already exists; bump project.version before releasing")
|
|
67
|
+
PY
|
|
68
|
+
- name: Build and validate wheel and source archive
|
|
69
|
+
run: |
|
|
70
|
+
uv build
|
|
71
|
+
uvx twine check --strict dist/*
|
|
72
|
+
- name: Save distributions for PyPI publishing
|
|
73
|
+
uses: actions/upload-artifact@v4
|
|
74
|
+
with:
|
|
75
|
+
name: created-release-dists
|
|
76
|
+
path: |
|
|
77
|
+
dist/*.whl
|
|
78
|
+
dist/*.tar.gz
|
|
79
|
+
if-no-files-found: error
|
|
80
|
+
- name: Create release with distributions
|
|
81
|
+
env:
|
|
82
|
+
GH_TOKEN: ${{ github.token }}
|
|
83
|
+
GH_REPO: ${{ github.repository }}
|
|
84
|
+
RELEASE_TAG: ${{ steps.version.outputs.tag }}
|
|
85
|
+
run: |
|
|
86
|
+
gh release create "$RELEASE_TAG" dist/*.whl dist/*.tar.gz \
|
|
87
|
+
--target "$GITHUB_SHA" \
|
|
88
|
+
--title "$RELEASE_TAG" \
|
|
89
|
+
--generate-notes
|
|
90
|
+
|
|
91
|
+
publish:
|
|
92
|
+
needs: create-release
|
|
93
|
+
permissions:
|
|
94
|
+
contents: read
|
|
95
|
+
id-token: write
|
|
96
|
+
uses: ./.github/workflows/python-publish.yml
|
|
97
|
+
with:
|
|
98
|
+
artifact-name: created-release-dists
|
|
99
|
+
secrets: inherit
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
name: Upload Python Package
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
workflow_call:
|
|
7
|
+
inputs:
|
|
8
|
+
artifact-name:
|
|
9
|
+
description: Existing distribution artifact from the calling workflow
|
|
10
|
+
required: true
|
|
11
|
+
type: string
|
|
12
|
+
|
|
13
|
+
permissions:
|
|
14
|
+
contents: read
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
release-build:
|
|
18
|
+
runs-on: ubuntu-latest
|
|
19
|
+
steps:
|
|
20
|
+
- name: Use distributions supplied by caller
|
|
21
|
+
if: inputs.artifact-name != ''
|
|
22
|
+
run: echo "Using the calling workflow's distribution artifact"
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
if: inputs.artifact-name == ''
|
|
25
|
+
- uses: astral-sh/setup-uv@v5
|
|
26
|
+
if: inputs.artifact-name == ''
|
|
27
|
+
- name: Install Python
|
|
28
|
+
if: inputs.artifact-name == ''
|
|
29
|
+
run: uv python install 3.11
|
|
30
|
+
- name: Install locked dependencies
|
|
31
|
+
if: inputs.artifact-name == ''
|
|
32
|
+
run: uv sync --locked --extra pandas --python 3.11
|
|
33
|
+
- name: Check release version and package
|
|
34
|
+
if: inputs.artifact-name == ''
|
|
35
|
+
env:
|
|
36
|
+
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
|
37
|
+
run: |
|
|
38
|
+
uv run --locked --extra pandas python - <<'PY'
|
|
39
|
+
import os
|
|
40
|
+
import tomllib
|
|
41
|
+
|
|
42
|
+
with open("pyproject.toml", "rb") as source:
|
|
43
|
+
version = tomllib.load(source)["project"]["version"]
|
|
44
|
+
from polars_learn import __version__
|
|
45
|
+
|
|
46
|
+
if __version__ != version:
|
|
47
|
+
raise SystemExit("Package __version__ must match project.version")
|
|
48
|
+
expected = f"v{version}"
|
|
49
|
+
if os.environ["RELEASE_TAG"] != expected:
|
|
50
|
+
raise SystemExit(f"Release tag must match project.version: {expected}")
|
|
51
|
+
PY
|
|
52
|
+
uv run --locked --extra pandas pytest --doctest-modules src tests
|
|
53
|
+
uv run --locked --extra pandas ruff check .
|
|
54
|
+
uv run --locked --extra pandas ruff format --check .
|
|
55
|
+
uv run --locked --extra pandas pyright
|
|
56
|
+
- name: Build release distributions
|
|
57
|
+
if: inputs.artifact-name == ''
|
|
58
|
+
run: |
|
|
59
|
+
uv build
|
|
60
|
+
uvx twine check --strict dist/*
|
|
61
|
+
- name: Upload distributions
|
|
62
|
+
if: inputs.artifact-name == ''
|
|
63
|
+
uses: actions/upload-artifact@v4
|
|
64
|
+
with:
|
|
65
|
+
name: release-dists
|
|
66
|
+
path: dist/
|
|
67
|
+
if-no-files-found: error
|
|
68
|
+
|
|
69
|
+
pypi-publish:
|
|
70
|
+
runs-on: ubuntu-latest
|
|
71
|
+
needs: release-build
|
|
72
|
+
permissions:
|
|
73
|
+
id-token: write
|
|
74
|
+
environment:
|
|
75
|
+
name: pypi
|
|
76
|
+
url: https://pypi.org/project/polars-learn/
|
|
77
|
+
steps:
|
|
78
|
+
- name: Retrieve release distributions
|
|
79
|
+
uses: actions/download-artifact@v4
|
|
80
|
+
with:
|
|
81
|
+
name: ${{ inputs.artifact-name || 'release-dists' }}
|
|
82
|
+
path: dist/
|
|
83
|
+
- name: Publish release distributions to PyPI
|
|
84
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
85
|
+
with:
|
|
86
|
+
packages-dir: dist/
|
|
87
|
+
password: ${{ secrets.PYPI_API_TOKEN }}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
name: Tests
|
|
2
|
+
|
|
3
|
+
on: [push, pull_request]
|
|
4
|
+
|
|
5
|
+
permissions:
|
|
6
|
+
contents: read
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ['3.11', '3.12']
|
|
15
|
+
pandas: [false, true]
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
- uses: astral-sh/setup-uv@v5
|
|
19
|
+
- name: Install Python
|
|
20
|
+
run: uv python install ${{ matrix.python-version }}
|
|
21
|
+
- name: Install locked core dependencies
|
|
22
|
+
if: '!matrix.pandas'
|
|
23
|
+
run: uv sync --locked --python ${{ matrix.python-version }}
|
|
24
|
+
- name: Install locked pandas dependencies
|
|
25
|
+
if: matrix.pandas
|
|
26
|
+
run: uv sync --locked --extra pandas --python ${{ matrix.python-version }}
|
|
27
|
+
- name: Test
|
|
28
|
+
run: uv run --locked pytest --doctest-modules src tests
|
|
29
|
+
- name: Lint
|
|
30
|
+
run: uv run --locked ruff check .
|
|
31
|
+
- name: Check formatting
|
|
32
|
+
run: uv run --locked ruff format --check .
|
|
33
|
+
- name: Check types
|
|
34
|
+
run: uv run --locked pyright
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Python bytecode and caches
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# Package builds
|
|
7
|
+
/build/
|
|
8
|
+
/dist/
|
|
9
|
+
/wheels/
|
|
10
|
+
*.egg-info/
|
|
11
|
+
.eggs/
|
|
12
|
+
|
|
13
|
+
# Virtual environments and local tool caches
|
|
14
|
+
/.venv/
|
|
15
|
+
/venv/
|
|
16
|
+
/env/
|
|
17
|
+
.uv-cache/
|
|
18
|
+
|
|
19
|
+
# Test, lint, type-checking, and coverage output
|
|
20
|
+
.pytest_cache/
|
|
21
|
+
.ruff_cache/
|
|
22
|
+
.pyright/
|
|
23
|
+
.mypy_cache/
|
|
24
|
+
.hypothesis/
|
|
25
|
+
.coverage
|
|
26
|
+
.coverage.*
|
|
27
|
+
coverage.xml
|
|
28
|
+
htmlcov/
|
|
29
|
+
junit.xml
|
|
30
|
+
|
|
31
|
+
# Local credentials and environment configuration
|
|
32
|
+
.env
|
|
33
|
+
.env.*
|
|
34
|
+
!.env.example
|
|
35
|
+
/.aws/
|
|
36
|
+
|
|
37
|
+
# Editor and operating-system files
|
|
38
|
+
.idea/
|
|
39
|
+
.vscode/
|
|
40
|
+
*.swp
|
|
41
|
+
*.swo
|
|
42
|
+
*~
|
|
43
|
+
.DS_Store
|
|
44
|
+
Thumbs.db
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.11
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jack
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: polars-learn
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Native Polars feature engineering and schema validation for scikit-learn
|
|
5
|
+
Author: Jack
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Requires-Dist: numpy>=1.24
|
|
10
|
+
Requires-Dist: polars>=1.20
|
|
11
|
+
Requires-Dist: scikit-learn>=1.6
|
|
12
|
+
Provides-Extra: pandas
|
|
13
|
+
Requires-Dist: pandas>=2; extra == 'pandas'
|
|
14
|
+
Requires-Dist: pyarrow>=14; extra == 'pandas'
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# polars-learn
|
|
18
|
+
|
|
19
|
+
High-performance feature engineering for scikit-learn, powered by Polars.
|
|
20
|
+
|
|
21
|
+
## Features
|
|
22
|
+
|
|
23
|
+
- Composable native Polars expressions, evaluated in dependency order.
|
|
24
|
+
- Learned schemas with strict, semi-strict, and permissive validation.
|
|
25
|
+
- scikit-learn estimators with cloning, parameters, and feature names.
|
|
26
|
+
- Native Polars output and optional, explicit pandas conversion.
|
|
27
|
+
- Python 3.11+, Polars 1.20+, and scikit-learn 1.6+.
|
|
28
|
+
|
|
29
|
+
## Installation
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
uv add polars-learn
|
|
33
|
+
# From a local checkout:
|
|
34
|
+
uv pip install .
|
|
35
|
+
# Optional pandas and PyArrow support:
|
|
36
|
+
uv pip install '.[pandas]'
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Quick Start
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
import polars as pl
|
|
43
|
+
from polars_learn import FeatureEngineer, FeatureSpec, SchemaValidator
|
|
44
|
+
|
|
45
|
+
X = pl.DataFrame({"price": [10.0, 20.0, 30.0], "quantity": [2, 3, 4]})
|
|
46
|
+
validator = SchemaValidator(mode="strict").fit(X)
|
|
47
|
+
validated = validator.transform(X)
|
|
48
|
+
|
|
49
|
+
engineer = FeatureEngineer(
|
|
50
|
+
[
|
|
51
|
+
FeatureSpec(pl.col("price") * pl.col("quantity"), alias="notional"),
|
|
52
|
+
FeatureSpec(
|
|
53
|
+
pl.col("notional").log1p(),
|
|
54
|
+
alias="log_notional",
|
|
55
|
+
drop=["quantity"],
|
|
56
|
+
),
|
|
57
|
+
]
|
|
58
|
+
)
|
|
59
|
+
result = engineer.fit_transform(validated)
|
|
60
|
+
print(result)
|
|
61
|
+
print(engineer.get_feature_names_out())
|
|
62
|
+
# ['price' 'notional' 'log_notional']
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Feature Engineering
|
|
66
|
+
|
|
67
|
+
`FeatureSpec` accepts exactly one of `expression` (one `pl.Expr`) or `expressions` (a nonempty sequence of expressions). It stores a single canonical expression tuple. Names follow Polars conventions: an existing alias or a native output name needs no additional alias. An explicit `alias` overrides the name of one output; it is rejected for multiple outputs. Selectors and expression expansion are supported, with their output names resolved against the frame.
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
spec = FeatureSpec(
|
|
71
|
+
expressions=[
|
|
72
|
+
(pl.col("ask") - pl.col("bid")).alias("spread"),
|
|
73
|
+
((pl.col("ask") + pl.col("bid")) / 2).alias("mid"),
|
|
74
|
+
],
|
|
75
|
+
drop=["ask", "bid"],
|
|
76
|
+
name="quote_features",
|
|
77
|
+
)
|
|
78
|
+
quotes = pl.DataFrame({"ask": [11.0], "bid": [9.0]})
|
|
79
|
+
print(FeatureEngineer([spec]).fit_transform(quotes))
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Expressions within one specification are evaluated together and cannot reference new columns created by siblings. Separate specifications execute sequentially; later ones can use earlier outputs. Drops happen after each specification. Missing expression inputs, missing drop columns, and duplicate output names raise `FeatureEngineeringError` (duplicate names detectable at construction raise `ValueError`). Overwriting existing columns is allowed and keeps Polars' column position. For example, `FeatureSpec(pl.col("price") + 1)` replaces `price`.
|
|
83
|
+
|
|
84
|
+
`fit` executes the training transformation to validate expressions and learn output names and dtypes. `transform` requires the fitted input column names and order and the exact fitted output schema. Input dtype differences are acceptable only when the resulting schema remains identical; use `SchemaValidator` first for explicit input type policies. Empty feature lists are identity transforms. The public API accepts eager DataFrames; internally, lazy stages preserve sequential semantics and collect once per execution. Expressions are stateless; training transformations do not learn statistics. Use deterministic expressions to keep training and inference consistent.
|
|
85
|
+
|
|
86
|
+
## Schema Validation
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
validator = SchemaValidator(
|
|
90
|
+
mode="semi-strict",
|
|
91
|
+
allow_extra_columns=False,
|
|
92
|
+
enforce_column_order=True,
|
|
93
|
+
)
|
|
94
|
+
validator.fit(X) # Always learns the raw input schema
|
|
95
|
+
print(validator.schema_)
|
|
96
|
+
validated = validator.transform(X) # Always validates against that schema
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
| Mode | Policy for differing dtypes |
|
|
100
|
+
| ------------------ | ------------------------------------------------------- |
|
|
101
|
+
| `strict` (default) | Reject every dtype difference. |
|
|
102
|
+
| `semi-strict` | Allow only the explicit lossless widening policy below. |
|
|
103
|
+
| `permissive` | Use Polars `cast(strict=True)` and report failures. |
|
|
104
|
+
|
|
105
|
+
Semi-strict allows integer widening when the entire source dtype's range fits inside the target integer dtype (including compatible unsigned-to-signed widening), Float32 → Float64, and integer → Float64 only when **every non-null value round-trips exactly**. For example, Int64 `2**53` and `2**54` are accepted, but `2**53 + 1` is rejected. It rejects narrowing, strings to numbers, and Null dtype to a different dtype. Null values in already compatible columns are preserved; nullability constraints are not part of this version.
|
|
106
|
+
|
|
107
|
+
Permissive casts may be lossy according to Polars semantics, such as floats to integers. Failed casts never silently replace invalid values with nulls. Missing required columns always fail; multiple schema mismatches are reported in `SchemaValidationError`. Strict mode does not cast or rename columns.
|
|
108
|
+
|
|
109
|
+
Extra columns fail by default. Set `allow_extra_columns=True` to preserve them, or also set `extra_columns="drop"` to explicitly remove them. Column order checks compare the relative order of required columns; permitted extras may appear anywhere. With `enforce_column_order=False`, output is normalised to fitted column order, followed by preserved extras in their input order. `get_feature_names_out()` describes fitted required columns; inference-only extras are not included. Avoid preserving inference-only extras before a fixed-width downstream estimator or `FeatureEngineer`.
|
|
110
|
+
|
|
111
|
+
The shorthand `strict=True` means strict mode, and `strict=False` means permissive mode. Conflicting `strict` and `mode` arguments are rejected. Schemas learned by the validator and feature engineer are independent.
|
|
112
|
+
|
|
113
|
+
## scikit-learn Pipelines
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
import numpy as np
|
|
117
|
+
from sklearn.ensemble import RandomForestRegressor
|
|
118
|
+
from sklearn.pipeline import Pipeline
|
|
119
|
+
from sklearn.preprocessing import StandardScaler
|
|
120
|
+
|
|
121
|
+
pipeline = Pipeline(
|
|
122
|
+
[
|
|
123
|
+
("schema", SchemaValidator(mode="semi-strict")),
|
|
124
|
+
("features", engineer),
|
|
125
|
+
("scale", StandardScaler().set_output(transform="polars")),
|
|
126
|
+
("model", RandomForestRegressor(n_estimators=20, random_state=0)),
|
|
127
|
+
]
|
|
128
|
+
)
|
|
129
|
+
pipeline.fit(X, np.array([1.0, 2.0, 3.0]))
|
|
130
|
+
print(pipeline.predict(X))
|
|
131
|
+
print(pipeline[:-1].get_feature_names_out())
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
These transformers implement `fit`, `transform`, `fit_transform`, `get_params`, `set_params`, and `get_feature_names_out`. Fitted attributes include `feature_names_in_`, `n_features_in_`, and `output_features_`. Cloning discards fitted state. Supplied `input_features` must exactly match fitted input names.
|
|
135
|
+
|
|
136
|
+
Data stays in Polars between our transformers. Both support `set_output(transform="default")` and `set_output(transform="polars")`, which return native Polars frames. They reject `"pandas"` output and deliberately bypass scikit-learn's automatic output wrapping, including global output settings, to preserve native names and allowed extras. No automatic NumPy conversion occurs. Downstream estimator support for Polars depends on the estimator and installed scikit-learn version; arbitrary string columns do not work with random forests.
|
|
137
|
+
|
|
138
|
+
Tests run applicable scikit-learn constructor, parameter, and cloning checks. Generic checks requiring NumPy array input do not apply to this deliberately DataFrame-only API. No LazyFrame, sparse-array, or NumPy input is accepted.
|
|
139
|
+
|
|
140
|
+
## Optional Pandas Support
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
uv sync --extra pandas
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
# Requires the pandas extra; both transformers expose these options.
|
|
148
|
+
import pandas as pd
|
|
149
|
+
|
|
150
|
+
pandas_X = pd.DataFrame({"price": [10.0, 20.0], "quantity": [2, 3]})
|
|
151
|
+
validator = SchemaValidator(convert_pandas=True, warn_on_conversion=False)
|
|
152
|
+
polars_X = validator.fit_transform(pandas_X)
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Pandas input is rejected unless `convert_pandas=True`. Conversion returns Polars and emits `PandasConversionWarning` about memory and performance overhead unless `warn_on_conversion=False`. Imports are lazy; core functionality works without pandas or PyArrow. The extra includes PyArrow because nullable, extension, and other non-NumPy-backed pandas dtypes can require it for `pl.from_pandas`. Conversion follows Polars conventions (including NaN-to-null and ignoring the pandas index). Unsupported conversion dtypes raise rather than falling back to row-wise operations.
|
|
156
|
+
|
|
157
|
+
## Development
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
uv sync
|
|
161
|
+
uv sync --extra pandas # Optional interoperability test coverage
|
|
162
|
+
uv run pytest
|
|
163
|
+
uv run ruff check .
|
|
164
|
+
uv run ruff format --check .
|
|
165
|
+
uv run pyright
|
|
166
|
+
uv build
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
`uv.lock` pins reproducible development dependencies. Pyright includes its Node runtime through the development extra, so no system Node installation is needed. Pandas tests use `pytest.importorskip("pandas")`; core tests also run without the optional extra. The package ships `py.typed` for downstream typing tools.
|
|
170
|
+
|
|
171
|
+
Library requirements use minimum versions without speculative upper bounds. Add an upper bound or exclude a release only when a known incompatibility requires it; the lockfile pins the development environment without constraining downstream applications.
|
|
172
|
+
|
|
173
|
+
### Releases
|
|
174
|
+
|
|
175
|
+
GitHub Actions follows the same release setup as Conformity. Configure the repository or `pypi` environment secret `PYPI_API_TOKEN` with a PyPI upload token for `polars-learn`. The publishing job uses the `pypi` environment. No API token belongs in this repository.
|
|
176
|
+
|
|
177
|
+
Update `project.version` in `pyproject.toml` and `__version__` in `src/polars_learn/__init__.py`, run `uv lock`, and push the changes. Then run **Create GitHub Release** from the Actions tab. It checks tests, linting, formatting, and types; rejects an existing `vX.Y.Z` tag; builds and validates distributions; creates a GitHub release with those files; and passes the same artifacts directly to **Upload Python Package** for PyPI publishing. The reusable workflow call is necessary because a release created with GitHub's workflow token does not trigger another release workflow.
|
|
178
|
+
|
|
179
|
+
Alternatively, publish a GitHub release tagged `vX.Y.Z` matching `project.version`. **Upload Python Package** checks the package, builds distributions, and publishes them automatically. Pushes and pull requests run **Tests** on Python 3.11 and 3.12, both with and without the pandas extra. Creating or publishing a release uploads that version to PyPI; ordinary pushes do not publish packages.
|
|
180
|
+
|
|
181
|
+
## License
|
|
182
|
+
|
|
183
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# polars-learn
|
|
2
|
+
|
|
3
|
+
High-performance feature engineering for scikit-learn, powered by Polars.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- Composable native Polars expressions, evaluated in dependency order.
|
|
8
|
+
- Learned schemas with strict, semi-strict, and permissive validation.
|
|
9
|
+
- scikit-learn estimators with cloning, parameters, and feature names.
|
|
10
|
+
- Native Polars output and optional, explicit pandas conversion.
|
|
11
|
+
- Python 3.11+, Polars 1.20+, and scikit-learn 1.6+.
|
|
12
|
+
|
|
13
|
+
## Installation
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
uv add polars-learn
|
|
17
|
+
# From a local checkout:
|
|
18
|
+
uv pip install .
|
|
19
|
+
# Optional pandas and PyArrow support:
|
|
20
|
+
uv pip install '.[pandas]'
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Quick Start
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
import polars as pl
|
|
27
|
+
from polars_learn import FeatureEngineer, FeatureSpec, SchemaValidator
|
|
28
|
+
|
|
29
|
+
X = pl.DataFrame({"price": [10.0, 20.0, 30.0], "quantity": [2, 3, 4]})
|
|
30
|
+
validator = SchemaValidator(mode="strict").fit(X)
|
|
31
|
+
validated = validator.transform(X)
|
|
32
|
+
|
|
33
|
+
engineer = FeatureEngineer(
|
|
34
|
+
[
|
|
35
|
+
FeatureSpec(pl.col("price") * pl.col("quantity"), alias="notional"),
|
|
36
|
+
FeatureSpec(
|
|
37
|
+
pl.col("notional").log1p(),
|
|
38
|
+
alias="log_notional",
|
|
39
|
+
drop=["quantity"],
|
|
40
|
+
),
|
|
41
|
+
]
|
|
42
|
+
)
|
|
43
|
+
result = engineer.fit_transform(validated)
|
|
44
|
+
print(result)
|
|
45
|
+
print(engineer.get_feature_names_out())
|
|
46
|
+
# ['price' 'notional' 'log_notional']
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Feature Engineering
|
|
50
|
+
|
|
51
|
+
`FeatureSpec` accepts exactly one of `expression` (one `pl.Expr`) or `expressions` (a nonempty sequence of expressions). It stores a single canonical expression tuple. Names follow Polars conventions: an existing alias or a native output name needs no additional alias. An explicit `alias` overrides the name of one output; it is rejected for multiple outputs. Selectors and expression expansion are supported, with their output names resolved against the frame.
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
spec = FeatureSpec(
|
|
55
|
+
expressions=[
|
|
56
|
+
(pl.col("ask") - pl.col("bid")).alias("spread"),
|
|
57
|
+
((pl.col("ask") + pl.col("bid")) / 2).alias("mid"),
|
|
58
|
+
],
|
|
59
|
+
drop=["ask", "bid"],
|
|
60
|
+
name="quote_features",
|
|
61
|
+
)
|
|
62
|
+
quotes = pl.DataFrame({"ask": [11.0], "bid": [9.0]})
|
|
63
|
+
print(FeatureEngineer([spec]).fit_transform(quotes))
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Expressions within one specification are evaluated together and cannot reference new columns created by siblings. Separate specifications execute sequentially; later ones can use earlier outputs. Drops happen after each specification. Missing expression inputs, missing drop columns, and duplicate output names raise `FeatureEngineeringError` (duplicate names detectable at construction raise `ValueError`). Overwriting existing columns is allowed and keeps Polars' column position. For example, `FeatureSpec(pl.col("price") + 1)` replaces `price`.
|
|
67
|
+
|
|
68
|
+
`fit` executes the training transformation to validate expressions and learn output names and dtypes. `transform` requires the fitted input column names and order and the exact fitted output schema. Input dtype differences are acceptable only when the resulting schema remains identical; use `SchemaValidator` first for explicit input type policies. Empty feature lists are identity transforms. The public API accepts eager DataFrames; internally, lazy stages preserve sequential semantics and collect once per execution. Expressions are stateless; training transformations do not learn statistics. Use deterministic expressions to keep training and inference consistent.
|
|
69
|
+
|
|
70
|
+
## Schema Validation
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
validator = SchemaValidator(
|
|
74
|
+
mode="semi-strict",
|
|
75
|
+
allow_extra_columns=False,
|
|
76
|
+
enforce_column_order=True,
|
|
77
|
+
)
|
|
78
|
+
validator.fit(X) # Always learns the raw input schema
|
|
79
|
+
print(validator.schema_)
|
|
80
|
+
validated = validator.transform(X) # Always validates against that schema
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
| Mode | Policy for differing dtypes |
|
|
84
|
+
| ------------------ | ------------------------------------------------------- |
|
|
85
|
+
| `strict` (default) | Reject every dtype difference. |
|
|
86
|
+
| `semi-strict` | Allow only the explicit lossless widening policy below. |
|
|
87
|
+
| `permissive` | Use Polars `cast(strict=True)` and report failures. |
|
|
88
|
+
|
|
89
|
+
Semi-strict allows integer widening when the entire source dtype's range fits inside the target integer dtype (including compatible unsigned-to-signed widening), Float32 → Float64, and integer → Float64 only when **every non-null value round-trips exactly**. For example, Int64 `2**53` and `2**54` are accepted, but `2**53 + 1` is rejected. It rejects narrowing, strings to numbers, and Null dtype to a different dtype. Null values in already compatible columns are preserved; nullability constraints are not part of this version.
|
|
90
|
+
|
|
91
|
+
Permissive casts may be lossy according to Polars semantics, such as floats to integers. Failed casts never silently replace invalid values with nulls. Missing required columns always fail; multiple schema mismatches are reported in `SchemaValidationError`. Strict mode does not cast or rename columns.
|
|
92
|
+
|
|
93
|
+
Extra columns fail by default. Set `allow_extra_columns=True` to preserve them, or also set `extra_columns="drop"` to explicitly remove them. Column order checks compare the relative order of required columns; permitted extras may appear anywhere. With `enforce_column_order=False`, output is normalised to fitted column order, followed by preserved extras in their input order. `get_feature_names_out()` describes fitted required columns; inference-only extras are not included. Avoid preserving inference-only extras before a fixed-width downstream estimator or `FeatureEngineer`.
|
|
94
|
+
|
|
95
|
+
The shorthand `strict=True` means strict mode, and `strict=False` means permissive mode. Conflicting `strict` and `mode` arguments are rejected. Schemas learned by the validator and feature engineer are independent.
|
|
96
|
+
|
|
97
|
+
## scikit-learn Pipelines
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
import numpy as np
|
|
101
|
+
from sklearn.ensemble import RandomForestRegressor
|
|
102
|
+
from sklearn.pipeline import Pipeline
|
|
103
|
+
from sklearn.preprocessing import StandardScaler
|
|
104
|
+
|
|
105
|
+
pipeline = Pipeline(
|
|
106
|
+
[
|
|
107
|
+
("schema", SchemaValidator(mode="semi-strict")),
|
|
108
|
+
("features", engineer),
|
|
109
|
+
("scale", StandardScaler().set_output(transform="polars")),
|
|
110
|
+
("model", RandomForestRegressor(n_estimators=20, random_state=0)),
|
|
111
|
+
]
|
|
112
|
+
)
|
|
113
|
+
pipeline.fit(X, np.array([1.0, 2.0, 3.0]))
|
|
114
|
+
print(pipeline.predict(X))
|
|
115
|
+
print(pipeline[:-1].get_feature_names_out())
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
These transformers implement `fit`, `transform`, `fit_transform`, `get_params`, `set_params`, and `get_feature_names_out`. Fitted attributes include `feature_names_in_`, `n_features_in_`, and `output_features_`. Cloning discards fitted state. Supplied `input_features` must exactly match fitted input names.
|
|
119
|
+
|
|
120
|
+
Data stays in Polars between our transformers. Both support `set_output(transform="default")` and `set_output(transform="polars")`, which return native Polars frames. They reject `"pandas"` output and deliberately bypass scikit-learn's automatic output wrapping, including global output settings, to preserve native names and allowed extras. No automatic NumPy conversion occurs. Downstream estimator support for Polars depends on the estimator and installed scikit-learn version; arbitrary string columns do not work with random forests.
|
|
121
|
+
|
|
122
|
+
Tests run applicable scikit-learn constructor, parameter, and cloning checks. Generic checks requiring NumPy array input do not apply to this deliberately DataFrame-only API. No LazyFrame, sparse-array, or NumPy input is accepted.
|
|
123
|
+
|
|
124
|
+
## Optional Pandas Support
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
uv sync --extra pandas
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
# Requires the pandas extra; both transformers expose these options.
|
|
132
|
+
import pandas as pd
|
|
133
|
+
|
|
134
|
+
pandas_X = pd.DataFrame({"price": [10.0, 20.0], "quantity": [2, 3]})
|
|
135
|
+
validator = SchemaValidator(convert_pandas=True, warn_on_conversion=False)
|
|
136
|
+
polars_X = validator.fit_transform(pandas_X)
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Pandas input is rejected unless `convert_pandas=True`. Conversion returns Polars and emits `PandasConversionWarning` about memory and performance overhead unless `warn_on_conversion=False`. Imports are lazy; core functionality works without pandas or PyArrow. The extra includes PyArrow because nullable, extension, and other non-NumPy-backed pandas dtypes can require it for `pl.from_pandas`. Conversion follows Polars conventions (including NaN-to-null and ignoring the pandas index). Unsupported conversion dtypes raise rather than falling back to row-wise operations.
|
|
140
|
+
|
|
141
|
+
## Development
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
uv sync
|
|
145
|
+
uv sync --extra pandas # Optional interoperability test coverage
|
|
146
|
+
uv run pytest
|
|
147
|
+
uv run ruff check .
|
|
148
|
+
uv run ruff format --check .
|
|
149
|
+
uv run pyright
|
|
150
|
+
uv build
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
`uv.lock` pins reproducible development dependencies. Pyright includes its Node runtime through the development extra, so no system Node installation is needed. Pandas tests use `pytest.importorskip("pandas")`; core tests also run without the optional extra. The package ships `py.typed` for downstream typing tools.
|
|
154
|
+
|
|
155
|
+
Library requirements use minimum versions without speculative upper bounds. Add an upper bound or exclude a release only when a known incompatibility requires it; the lockfile pins the development environment without constraining downstream applications.
|
|
156
|
+
|
|
157
|
+
### Releases
|
|
158
|
+
|
|
159
|
+
GitHub Actions follows the same release setup as Conformity. Configure the repository or `pypi` environment secret `PYPI_API_TOKEN` with a PyPI upload token for `polars-learn`. The publishing job uses the `pypi` environment. No API token belongs in this repository.
|
|
160
|
+
|
|
161
|
+
Update `project.version` in `pyproject.toml` and `__version__` in `src/polars_learn/__init__.py`, run `uv lock`, and push the changes. Then run **Create GitHub Release** from the Actions tab. It checks tests, linting, formatting, and types; rejects an existing `vX.Y.Z` tag; builds and validates distributions; creates a GitHub release with those files; and passes the same artifacts directly to **Upload Python Package** for PyPI publishing. The reusable workflow call is necessary because a release created with GitHub's workflow token does not trigger another release workflow.
|
|
162
|
+
|
|
163
|
+
Alternatively, publish a GitHub release tagged `vX.Y.Z` matching `project.version`. **Upload Python Package** checks the package, builds distributions, and publishes them automatically. Pushes and pull requests run **Tests** on Python 3.11 and 3.12, both with and without the pandas extra. Creating or publishing a release uploads that version to PyPI; ordinary pushes do not publish packages.
|
|
164
|
+
|
|
165
|
+
## License
|
|
166
|
+
|
|
167
|
+
MIT. See [LICENSE](LICENSE).
|