sparkleframe 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.docker/docker-compose.yaml +5 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.github/workflows/deploy.yml +9 -5
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.github/workflows/predeploy.yml +2 -1
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.github/workflows/verification.yml +0 -2
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.gitignore +2 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/Makefile +27 -14
- sparkleframe-0.2.2/PKG-INFO +130 -0
- sparkleframe-0.2.2/README.md +114 -0
- sparkleframe-0.2.2/changelog.md +16 -0
- sparkleframe-0.2.2/docs/generate_supported_api.py +112 -0
- sparkleframe-0.2.2/docs/images/favicon.ico +0 -0
- sparkleframe-0.2.2/docs/images/logo.png +0 -0
- sparkleframe-0.2.2/docs/images/logo_dark.png +0 -0
- sparkleframe-0.2.2/docs/images/logo_white.png +0 -0
- sparkleframe-0.2.2/docs/index.md +9 -0
- sparkleframe-0.2.2/docs/overrides/main.html +8 -0
- sparkleframe-0.2.2/docs/stylesheets/extra.css +9 -0
- sparkleframe-0.2.2/docs/supported_api.md +5 -0
- sparkleframe-0.2.2/mkdocs.yml +75 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/requirements-dev.in +7 -0
- sparkleframe-0.2.2/requirements-dev.txt +93 -0
- sparkleframe-0.2.2/requirements-pkg.txt +21 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/generate_changelog.py +12 -10
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/generate_changelog_test.py +4 -10
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/utils.py +4 -4
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/gpt_utils.py +4 -4
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/__init__.py +2 -0
- sparkleframe-0.2.2/sparkleframe/tests/__init__.py +0 -0
- sparkleframe-0.2.2/sparkleframe/version.txt +1 -0
- sparkleframe-0.2.0/PKG-INFO +0 -22
- sparkleframe-0.2.0/README.md +0 -6
- sparkleframe-0.2.0/changelog.md +0 -9
- sparkleframe-0.2.0/requirements-dev.txt +0 -170
- sparkleframe-0.2.0/requirements-pkg.txt +0 -46
- sparkleframe-0.2.0/sparkleframe/version.txt +0 -1
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.coverage +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.docker/Dockerfile +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.github/actions/setup-dependencies/action.yml +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.github/dependabot.yml +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.github/hooks/prepare-commit-msg +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.pylintrc +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/LICENSE +0 -0
- /sparkleframe-0.2.0/scripts/__init__.py → /sparkleframe-0.2.2/docs/overrides/partials/footer.html +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/pyproject.toml +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/requirements-pkg.in +0 -0
- {sparkleframe-0.2.0/sparkleframe/base → sparkleframe-0.2.2/scripts}/__init__.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/calculate_version.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/calculate_version_test.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/set_version.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/utils_test.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/__init__.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/activate.py +0 -0
- {sparkleframe-0.2.0/sparkleframe/tests → sparkleframe-0.2.2/sparkleframe/base}/__init__.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/base/dataframe.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/conftest.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/context.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/engine.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/column.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/column_test.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/dataframe.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/dataframe_test.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/functions.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/functions_test.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/functions_utils.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/group.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/session.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/session_test.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/types.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/types_test.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/window.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/window_test.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/tests/pyspark_test.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/tests/spark.py +0 -0
- {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/tests/utils.py +0 -0
|
@@ -6,7 +6,12 @@ services:
|
|
|
6
6
|
container_name: sparkleframe
|
|
7
7
|
environment:
|
|
8
8
|
TIMEZONE: UTC
|
|
9
|
+
ports:
|
|
10
|
+
- "8000:8000"
|
|
9
11
|
volumes:
|
|
12
|
+
- ./../README.md:/README.md
|
|
13
|
+
- ./../docs:/docs
|
|
14
|
+
- ./../mkdocs.yml:/mkdocs.yml
|
|
10
15
|
- ./../sparkleframe:/sparkleframe
|
|
11
16
|
- ./../.coverage:/sparkleframe/.coverage
|
|
12
17
|
- ./../pyproject.toml:/pyproject.toml
|
|
@@ -13,7 +13,7 @@ jobs:
|
|
|
13
13
|
predeploy:
|
|
14
14
|
uses: ./.github/workflows/predeploy.yml
|
|
15
15
|
|
|
16
|
-
deploy
|
|
16
|
+
deploy:
|
|
17
17
|
needs: predeploy
|
|
18
18
|
runs-on: ubuntu-latest
|
|
19
19
|
timeout-minutes: 5
|
|
@@ -51,7 +51,11 @@ jobs:
|
|
|
51
51
|
flit publish
|
|
52
52
|
env:
|
|
53
53
|
FLIT_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
54
|
+
- name: Deploy docs
|
|
55
|
+
run: |
|
|
56
|
+
git config --global user.email "flypipe@dummy.com"
|
|
57
|
+
git config --global user.name "flypipe"
|
|
58
|
+
export PYTHONPATH=$PYTHONPATH:./sparkleframe
|
|
59
|
+
make docs-deploy version=${{ env.VERSION_NUMBER }}
|
|
60
|
+
env:
|
|
61
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
@@ -58,7 +58,8 @@ jobs:
|
|
|
58
58
|
uses: actions/upload-artifact@v4
|
|
59
59
|
with:
|
|
60
60
|
name: changelog.md
|
|
61
|
-
path: ./changelog.md
|
|
61
|
+
path: ./docs/changelog.md
|
|
62
|
+
|
|
62
63
|
# Build and upload the wheel so we can access the wheel without having to upload it to pypi
|
|
63
64
|
- name: Build wheel
|
|
64
65
|
run: flit build --no-use-vcs
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
SHELL :=/usr/bin/env bash
|
|
2
2
|
PYTEST_THREADS ?=$(shell echo $$((`getconf _NPROCESSORS_ONLN` / 3)))
|
|
3
3
|
LOCAL_DIR :=./.docker
|
|
4
|
-
MIN_COVERAGE =
|
|
4
|
+
MIN_COVERAGE = 79
|
|
5
|
+
version ?=
|
|
5
6
|
|
|
6
7
|
bash:
|
|
7
8
|
docker compose -f $(LOCAL_DIR)/docker-compose.yaml run --remove-orphans -it sparkleframe bash
|
|
@@ -23,15 +24,10 @@ lint:
|
|
|
23
24
|
docker compose -f $(LOCAL_DIR)/docker-compose.yaml run --remove-orphans sparkleframe sh -c "python -m ruff check --line-length 119 sparkleframe"
|
|
24
25
|
.PHONY: lint
|
|
25
26
|
|
|
26
|
-
pip-compile:
|
|
27
|
-
pip-compile requirements-pkg.in && pip-compile requirements-dev.in
|
|
28
|
-
.PHONY: pip-compile
|
|
29
|
-
|
|
30
27
|
coverage:
|
|
31
28
|
docker compose -f $(LOCAL_DIR)/docker-compose.yaml run --remove-orphans sparkleframe sh -c "pytest -n $(PYTEST_THREADS) -k '_test.py' --cov-config=sparkleframe/.coverage --cov=sparkleframe --no-cov-on-fail --cov-fail-under=$(MIN_COVERAGE) -v sparkleframe"
|
|
32
29
|
.PHONY: coverage
|
|
33
30
|
|
|
34
|
-
|
|
35
31
|
test:
|
|
36
32
|
docker compose -f $(LOCAL_DIR)/docker-compose.yaml run --remove-orphans sparkleframe sh -c "pytest -n $(PYTEST_THREADS) -k '_test.py' -vv $(f)"
|
|
37
33
|
.PHONY: test
|
|
@@ -52,18 +48,35 @@ githooks:
|
|
|
52
48
|
.PHONY: githooks
|
|
53
49
|
|
|
54
50
|
pip-compile:
|
|
55
|
-
pip-
|
|
56
|
-
pip-compile requirements-
|
|
57
|
-
|
|
58
|
-
make githooks
|
|
51
|
+
pip install -r requirements-pkg.in
|
|
52
|
+
pip-compile requirements-pkg.in --no-annotate --no-header
|
|
53
|
+
pip-compile requirements-dev.in --no-annotate --no-header
|
|
59
54
|
.PHONY: pip-compile
|
|
60
55
|
|
|
61
|
-
setup:
|
|
62
|
-
|
|
63
|
-
pip-compile requirements-pkg.in
|
|
64
|
-
pip-compile requirements-dev.in
|
|
56
|
+
setup: pip-compile
|
|
57
|
+
export PYTHONPATH=$PYTHONPATH:./sparkleframe
|
|
65
58
|
pip install -r requirements-dev.txt
|
|
66
59
|
make build
|
|
67
60
|
make githooks
|
|
68
61
|
.PHONY: setup
|
|
69
62
|
|
|
63
|
+
docs:
|
|
64
|
+
@if [ ! -f docs/changelog.md ]; then \
|
|
65
|
+
echo "docs/changelog.md does not exist, running command..."; \
|
|
66
|
+
python scripts/generate_changelog.py; \
|
|
67
|
+
fi
|
|
68
|
+
@if [ ! -f docs/supported_api_doc.md ]; then \
|
|
69
|
+
echo "docs/supported_api_doc.md does not exist, running command..."; \
|
|
70
|
+
python docs/generate_supported_api.py; \
|
|
71
|
+
fi
|
|
72
|
+
mkdocs serve
|
|
73
|
+
.PHONY: docs
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
docs-deploy:
|
|
77
|
+
@[ -n "$(version)" ] || (echo "ERROR: version is required"; exit 1)
|
|
78
|
+
python scripts/generate_changelog.py
|
|
79
|
+
python docs/generate_supported_api.py
|
|
80
|
+
mike deploy --allow-empty --push --update-aliases $(shell echo $(version) | awk -F. '{print $$1"."$$2}') latest
|
|
81
|
+
mike set-default --push latest
|
|
82
|
+
.PHONY: docs-deploy
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sparkleframe
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Sparkleframe
|
|
5
|
+
Author-email: Jose Helio de Brum Muller <josemuller.flypipe@gmail.com>
|
|
6
|
+
Requires-Python: >=3.8
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Information Technology
|
|
11
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: polars >=1.30
|
|
15
|
+
Project-URL: Home, https://github.com/flypipe/sparkleframe.git
|
|
16
|
+
|
|
17
|
+

|
|
18
|
+
|
|
19
|
+
SparkleFrame implements the PySpark DataFrame API in order to enable running transformation pipelines
|
|
20
|
+
directly on [Polars Dataframe](https://docs.pola.rs/api/python/stable/reference/index.html) - no Spark clusters or
|
|
21
|
+
dependencies required.
|
|
22
|
+
|
|
23
|
+
Apache Spark is designed for distributed, large-scale data processing, but it is not optimized for low-latency use
|
|
24
|
+
cases. There are scenarios, however, where you need to quickly re-compute certain data—for example,
|
|
25
|
+
regenerating features for a machine learning model in real time or near-real time.
|
|
26
|
+
|
|
27
|
+
SparkleFrame is great for:
|
|
28
|
+
|
|
29
|
+
* Users who want to run PySpark code quickly locally without the overhead of starting a Spark session
|
|
30
|
+
* Users who want to run PySpark DataFrame code without the complexity of using Spark for processing
|
|
31
|
+
* Useful for unit testing, feature prototyping, or serving small pipelines in microservices.
|
|
32
|
+
|
|
33
|
+
## Documentation
|
|
34
|
+
|
|
35
|
+
Full documentation is available at [https://flypipe.github.io/sparkleframe/](https://flypipe.github.io/sparkleframe/).
|
|
36
|
+
|
|
37
|
+
## Installation
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pip install sparkleframe
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
SparkleFrame can be used in two ways:
|
|
46
|
+
|
|
47
|
+
* Directly importing the `sparkleframe.polarsdf` package
|
|
48
|
+
* Using the `activate` function to allow for continuing to use `pyspark.sql` but have it use SparkleFrame behind the scenes.
|
|
49
|
+
|
|
50
|
+
### Directly importing
|
|
51
|
+
|
|
52
|
+
If converting a PySpark pipeline, all `pyspark.sql` should be replaced with `sparkleframe.polarsdf`.
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
# PySpark import
|
|
56
|
+
# from pyspark.sql import SparkSession
|
|
57
|
+
# from pyspark.sql import functions as F
|
|
58
|
+
# from pyspark.sql.dataframe import DataFrame
|
|
59
|
+
# SparkleFrame import
|
|
60
|
+
from sparkleframe.polarsdf.session import SparkSession
|
|
61
|
+
from sparkleframe.polarsdf import functions as F
|
|
62
|
+
from sparkleframe.polarsdf.dataframe import DataFrame
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Activating SparkleFrame
|
|
66
|
+
|
|
67
|
+
SparkleFrame can either replace pyspark imports or be used alongside them. To replace pyspark imports,
|
|
68
|
+
use the activate function to set the engine to use.
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from sparkleframe.activate import activate
|
|
72
|
+
|
|
73
|
+
# Activate SparkleFrame
|
|
74
|
+
activate()
|
|
75
|
+
|
|
76
|
+
from pyspark.sql import SparkSession
|
|
77
|
+
session = SparkSession.builder.getOrCreate()
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
`SparkSession` will now be a SparkleFrame Session object and everything will be run on Polars Dataframe directly.
|
|
81
|
+
|
|
82
|
+
SparkleFrame can also be directly imported which both maintains pyspark imports:
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
from sparkleframe.polarsdf.session import SparkSession
|
|
86
|
+
session = SparkSession.builder.getOrCreate()
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Example Usage
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
from sparkleframe.activate import activate
|
|
93
|
+
|
|
94
|
+
# Activate SparkleFrame
|
|
95
|
+
activate()
|
|
96
|
+
|
|
97
|
+
from pyspark.sql import SparkSession
|
|
98
|
+
from pyspark.sql import functions as F
|
|
99
|
+
|
|
100
|
+
session = SparkSession.builder.getOrCreate()
|
|
101
|
+
df = session.createDataFrame(data=[{"col1": 1, "col2": 2}])
|
|
102
|
+
df = df.withColumn("col3", F.col("col2") + F.col("col1"))
|
|
103
|
+
```
|
|
104
|
+
```python
|
|
105
|
+
>>> print(type(df))
|
|
106
|
+
<class 'sparkleframe.polarsdf.dataframe.DataFrame'>
|
|
107
|
+
```
|
|
108
|
+
```python
|
|
109
|
+
>>> df.show()
|
|
110
|
+
shape: (1, 3)
|
|
111
|
+
┌──────┬──────┬──────┐
|
|
112
|
+
│ col1 ┆ col2 ┆ col3 │
|
|
113
|
+
│ --- ┆ --- ┆ --- │
|
|
114
|
+
│ i64 ┆ i64 ┆ i64 │
|
|
115
|
+
╞══════╪══════╪══════╡
|
|
116
|
+
│ 1 ┆ 2 ┆ 3 │
|
|
117
|
+
└──────┴──────┴──────┘
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
!!! note
|
|
121
|
+
|
|
122
|
+
- If you encounter any transformation that is not implemented, please open an [issue on GitHub](https://github.com/flypipe/sparkleframe/issues) so it can be prioritized.
|
|
123
|
+
|
|
124
|
+
You can learn more about the design motivation behind Sparkleframe in this
|
|
125
|
+
[discussion thread](https://github.com/eakmanrq/sparkleframe/issues/409).
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
## Source Code
|
|
129
|
+
|
|
130
|
+
API code is available at [https://github.com/flypipe/sparkleframe](https://github.com/flypipe/sparkleframe).
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+

|
|
2
|
+
|
|
3
|
+
SparkleFrame implements the PySpark DataFrame API in order to enable running transformation pipelines
|
|
4
|
+
directly on [Polars Dataframe](https://docs.pola.rs/api/python/stable/reference/index.html) - no Spark clusters or
|
|
5
|
+
dependencies required.
|
|
6
|
+
|
|
7
|
+
Apache Spark is designed for distributed, large-scale data processing, but it is not optimized for low-latency use
|
|
8
|
+
cases. There are scenarios, however, where you need to quickly re-compute certain data—for example,
|
|
9
|
+
regenerating features for a machine learning model in real time or near-real time.
|
|
10
|
+
|
|
11
|
+
SparkleFrame is great for:
|
|
12
|
+
|
|
13
|
+
* Users who want to run PySpark code quickly locally without the overhead of starting a Spark session
|
|
14
|
+
* Users who want to run PySpark DataFrame code without the complexity of using Spark for processing
|
|
15
|
+
* Useful for unit testing, feature prototyping, or serving small pipelines in microservices.
|
|
16
|
+
|
|
17
|
+
## Documentation
|
|
18
|
+
|
|
19
|
+
Full documentation is available at [https://flypipe.github.io/sparkleframe/](https://flypipe.github.io/sparkleframe/).
|
|
20
|
+
|
|
21
|
+
## Installation
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install sparkleframe
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Usage
|
|
28
|
+
|
|
29
|
+
SparkleFrame can be used in two ways:
|
|
30
|
+
|
|
31
|
+
* Directly importing the `sparkleframe.polarsdf` package
|
|
32
|
+
* Using the `activate` function to allow for continuing to use `pyspark.sql` but have it use SparkleFrame behind the scenes.
|
|
33
|
+
|
|
34
|
+
### Directly importing
|
|
35
|
+
|
|
36
|
+
If converting a PySpark pipeline, all `pyspark.sql` should be replaced with `sparkleframe.polarsdf`.
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
# PySpark import
|
|
40
|
+
# from pyspark.sql import SparkSession
|
|
41
|
+
# from pyspark.sql import functions as F
|
|
42
|
+
# from pyspark.sql.dataframe import DataFrame
|
|
43
|
+
# SparkleFrame import
|
|
44
|
+
from sparkleframe.polarsdf.session import SparkSession
|
|
45
|
+
from sparkleframe.polarsdf import functions as F
|
|
46
|
+
from sparkleframe.polarsdf.dataframe import DataFrame
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Activating SparkleFrame
|
|
50
|
+
|
|
51
|
+
SparkleFrame can either replace pyspark imports or be used alongside them. To replace pyspark imports,
|
|
52
|
+
use the activate function to set the engine to use.
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from sparkleframe.activate import activate
|
|
56
|
+
|
|
57
|
+
# Activate SparkleFrame
|
|
58
|
+
activate()
|
|
59
|
+
|
|
60
|
+
from pyspark.sql import SparkSession
|
|
61
|
+
session = SparkSession.builder.getOrCreate()
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
`SparkSession` will now be a SparkleFrame Session object and everything will be run on Polars Dataframe directly.
|
|
65
|
+
|
|
66
|
+
SparkleFrame can also be directly imported which both maintains pyspark imports:
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from sparkleframe.polarsdf.session import SparkSession
|
|
70
|
+
session = SparkSession.builder.getOrCreate()
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Example Usage
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
from sparkleframe.activate import activate
|
|
77
|
+
|
|
78
|
+
# Activate SparkleFrame
|
|
79
|
+
activate()
|
|
80
|
+
|
|
81
|
+
from pyspark.sql import SparkSession
|
|
82
|
+
from pyspark.sql import functions as F
|
|
83
|
+
|
|
84
|
+
session = SparkSession.builder.getOrCreate()
|
|
85
|
+
df = session.createDataFrame(data=[{"col1": 1, "col2": 2}])
|
|
86
|
+
df = df.withColumn("col3", F.col("col2") + F.col("col1"))
|
|
87
|
+
```
|
|
88
|
+
```python
|
|
89
|
+
>>> print(type(df))
|
|
90
|
+
<class 'sparkleframe.polarsdf.dataframe.DataFrame'>
|
|
91
|
+
```
|
|
92
|
+
```python
|
|
93
|
+
>>> df.show()
|
|
94
|
+
shape: (1, 3)
|
|
95
|
+
┌──────┬──────┬──────┐
|
|
96
|
+
│ col1 ┆ col2 ┆ col3 │
|
|
97
|
+
│ --- ┆ --- ┆ --- │
|
|
98
|
+
│ i64 ┆ i64 ┆ i64 │
|
|
99
|
+
╞══════╪══════╪══════╡
|
|
100
|
+
│ 1 ┆ 2 ┆ 3 │
|
|
101
|
+
└──────┴──────┴──────┘
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
!!! note
|
|
105
|
+
|
|
106
|
+
- If you encounter any transformation that is not implemented, please open an [issue on GitHub](https://github.com/flypipe/sparkleframe/issues) so it can be prioritized.
|
|
107
|
+
|
|
108
|
+
You can learn more about the design motivation behind Sparkleframe in this
|
|
109
|
+
[discussion thread](https://github.com/eakmanrq/sparkleframe/issues/409).
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
## Source Code
|
|
113
|
+
|
|
114
|
+
API code is available at [https://github.com/flypipe/sparkleframe](https://github.com/flypipe/sparkleframe).
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
Changelog
|
|
2
|
+
=========
|
|
3
|
+
|
|
4
|
+
<h2><a href="https://github.com/flypipe/sparkleframe/tree/release/0.2.2" target="_blank" rel="noopener noreferrer">release/0.2.2</a><h2>
|
|
5
|
+
|
|
6
|
+
<br/>- <a href="https://github.com/flypipe/sparkleframe/issues/48" target="_blank" rel="noopener noreferrer">48 small fixes</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/46" target="_blank" rel="noopener noreferrer">46 fix mkdocs serve</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/44" target="_blank" rel="noopener noreferrer">44 do not run verification workflow on push</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/42" target="_blank" rel="noopener noreferrer">42 remove header from pip compile</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/39" target="_blank" rel="noopener noreferrer">39 GitAction upload changelog.md as artifact</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/38" target="_blank" rel="noopener noreferrer">38 fix change log artifact</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/31" target="_blank" rel="noopener noreferrer">31 Build Docs</a>
|
|
7
|
+
|
|
8
|
+
<h2><a href="https://github.com/flypipe/sparkleframe/tree/release/0.2.1" target="_blank" rel="noopener noreferrer">release/0.2.1</a><h2><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/33" target="_blank" rel="noopener noreferrer">33 Fix import error of functions</a>
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
<h2><a href="https://github.com/flypipe/sparkleframe/tree/release/0.2.0" target="_blank" rel="noopener noreferrer">release/0.2.0</a><h2><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/27" target="_blank" rel="noopener noreferrer">27 fix git workflow to create version and changelog from the current branch</a>
|
|
13
|
+
|
|
14
|
+
<h2><a href="https://github.com/flypipe/sparkleframe/tree/release/0.1.0" target="_blank" rel="noopener noreferrer">release/0.1.0</a><h2><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/9" target="_blank" rel="noopener noreferrer">9 Implement pyspark.sql.Window (and rank, dense_rank and row_number)</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/27" target="_blank" rel="noopener noreferrer">27 fix git workflow to create version and changelog from the current branch</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/24" target="_blank" rel="noopener noreferrer">24 add support to pyspark.sql.DataFrame.schema</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/22" target="_blank" rel="noopener noreferrer">22 add support to pyspark.pandas.DataFrame.dtypes</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/20" target="_blank" rel="noopener noreferrer">20 add support to pyspark.sql.functions.length</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/18" target="_blank" rel="noopener noreferrer">18 implement pyspark.sql.functions.to_timestamp</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/16" target="_blank" rel="noopener noreferrer">16 show issues in changelog in descending order on issue_id</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/14" target="_blank" rel="noopener noreferrer">14 Update changelog to get changelog from the previous release branch</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/12" target="_blank" rel="noopener noreferrer">12 Implement pyspark.sql.functions.regexp_replace</a>
|
|
15
|
+
|
|
16
|
+
<h2><a href="https://github.com/flypipe/sparkleframe/tree/release/0.0.2" target="_blank" rel="noopener noreferrer">release/0.0.2</a><h2><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/10" target="_blank" rel="noopener noreferrer">10 Implement simpleString attribute from data types</a>
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
|
|
2
|
+
import inspect
|
|
3
|
+
import os
|
|
4
|
+
import pathlib
|
|
5
|
+
import urllib
|
|
6
|
+
|
|
7
|
+
import requests
|
|
8
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
9
|
+
|
|
10
|
+
def get_data_types(T):
|
|
11
|
+
# Get all data type classes from the module
|
|
12
|
+
return sorted([
|
|
13
|
+
getattr(T, attr).__name__ for attr in dir(T)
|
|
14
|
+
if attr.endswith('Type') and isinstance(getattr(T, attr), type)
|
|
15
|
+
])
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get_functions(obj):
|
|
19
|
+
"""Extract all public functions from pyspark.sql.functions"""
|
|
20
|
+
all_names = dir(obj)
|
|
21
|
+
return sorted([
|
|
22
|
+
name for name in all_names
|
|
23
|
+
if not name.startswith("_") and inspect.isfunction(getattr(obj, name))
|
|
24
|
+
])
|
|
25
|
+
|
|
26
|
+
def check_doc_url(func_name, url_key):
|
|
27
|
+
"""Check if the function's API URL exists"""
|
|
28
|
+
url = f"https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.{url_key}.{func_name}.html"
|
|
29
|
+
try:
|
|
30
|
+
response = requests.head(url, timeout=5)
|
|
31
|
+
exists = response.status_code == 200
|
|
32
|
+
except requests.RequestException as e:
|
|
33
|
+
print(f"Error checking {url}: {e}")
|
|
34
|
+
exists = False
|
|
35
|
+
return url, exists, func_name
|
|
36
|
+
|
|
37
|
+
if __name__ == "__main__":
|
|
38
|
+
from pyspark.sql.catalog import Catalog as PYSPARK_CATALOG
|
|
39
|
+
SPARKLE_CATALOG = None
|
|
40
|
+
|
|
41
|
+
from pyspark.sql.column import Column as PYSPARK_COLUMN
|
|
42
|
+
from sparkleframe.polarsdf.column import Column as SPARKLE_COLUMN
|
|
43
|
+
|
|
44
|
+
from pyspark.sql.dataframe import DataFrame as PYSPARK_DATAFRAME
|
|
45
|
+
from sparkleframe.polarsdf.dataframe import DataFrame as SPARKLE_DATAFRAME
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
import pyspark.sql.functions as PYSPARK_FUNCTIONS
|
|
49
|
+
import sparkleframe.polarsdf.functions as SPARKLE_FUNCTIONS
|
|
50
|
+
|
|
51
|
+
from pyspark.sql.group import GroupedData as PYSPARK_GROUPED_DATA
|
|
52
|
+
from sparkleframe.polarsdf.group import GroupedData as SPARKLE_GROUPED_DATA
|
|
53
|
+
|
|
54
|
+
from pyspark.sql.window import Window as PYSPARK_WINDOW
|
|
55
|
+
from sparkleframe.polarsdf.window import Window as SPARKLE_WINDOW
|
|
56
|
+
|
|
57
|
+
from pyspark.sql.session import SparkSession as PYSPARK_SESSION
|
|
58
|
+
from sparkleframe.polarsdf.session import SparkSession as SPARKLE_SESSION
|
|
59
|
+
|
|
60
|
+
import pyspark.sql.types as PYSPARK_TYPES
|
|
61
|
+
import sparkleframe.polarsdf.types as SPARKLE_TYPES
|
|
62
|
+
|
|
63
|
+
MODULES = [
|
|
64
|
+
|
|
65
|
+
{"url_key": "types", "module_url_key": "data_types", "pyspark": PYSPARK_TYPES, "sparkleframe": SPARKLE_TYPES, "lambda": get_data_types},
|
|
66
|
+
{"url_key": "GroupedData", "module_url_key": "grouping", "pyspark": PYSPARK_GROUPED_DATA, "sparkleframe": SPARKLE_GROUPED_DATA, "lambda": get_functions},
|
|
67
|
+
{"url_key": "Window", "module_url_key": "window", "pyspark": PYSPARK_WINDOW, "sparkleframe": SPARKLE_WINDOW, "lambda": get_functions},
|
|
68
|
+
{"url_key": "DataFrame", "module_url_key": "dataframe", "pyspark": PYSPARK_DATAFRAME, "sparkleframe": SPARKLE_DATAFRAME, "lambda": get_functions},
|
|
69
|
+
{"url_key": "Catalog", "module_url_key": "catalog", "pyspark": PYSPARK_CATALOG, "sparkleframe": SPARKLE_CATALOG, "lambda": get_functions},
|
|
70
|
+
{"url_key": "SparkSession", "module_url_key": "spark_session", "pyspark": PYSPARK_SESSION, "sparkleframe": SPARKLE_SESSION, "lambda": get_functions},
|
|
71
|
+
{"url_key": "Column", "module_url_key": "column", "pyspark": PYSPARK_COLUMN, "sparkleframe": SPARKLE_COLUMN, "lambda": get_functions},
|
|
72
|
+
{"url_key": "functions", "module_url_key": "functions", "pyspark": PYSPARK_FUNCTIONS, "sparkleframe": SPARKLE_FUNCTIONS, "lambda": get_functions},
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
pyspark_functions = []
|
|
76
|
+
# 3️⃣ Number of threads: CPU cores minus 1, minimum 1
|
|
77
|
+
num_threads = max(os.cpu_count() - 1, 1)
|
|
78
|
+
for modules in sorted(MODULES, key=lambda module: module["module_url_key"]):
|
|
79
|
+
|
|
80
|
+
pyspark_functions_list = modules['lambda'](modules["pyspark"])
|
|
81
|
+
sparkleframe_functions = modules['lambda'](modules["sparkleframe"])
|
|
82
|
+
pyspark_functions.append(f'\n\n## <a href="https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/{modules["module_url_key"]}.html" target="_blank">pyspark.sql.{modules["url_key"]}</a>')
|
|
83
|
+
with ThreadPoolExecutor(max_workers=num_threads) as executor:
|
|
84
|
+
futures = [executor.submit(check_doc_url, pyspark_func_name, modules['url_key']) for pyspark_func_name in sorted(pyspark_functions_list)]
|
|
85
|
+
|
|
86
|
+
for future in as_completed(futures):
|
|
87
|
+
url, exists, pyspark_func_name = future.result()
|
|
88
|
+
|
|
89
|
+
if not exists:
|
|
90
|
+
# not found in pyspark documentation
|
|
91
|
+
continue
|
|
92
|
+
|
|
93
|
+
line = f'<a href="{url}" target="_blank">{pyspark_func_name}</a>'
|
|
94
|
+
if pyspark_func_name in sparkleframe_functions:
|
|
95
|
+
line = f'* ✅ ' + line
|
|
96
|
+
else:
|
|
97
|
+
params = {
|
|
98
|
+
"title": f"[PYSPARK_API_REQUEST] Implement pyspark.sql.{modules['url_key']}.{pyspark_func_name}",
|
|
99
|
+
"body": f"Please implement feature [pyspark.sql.{modules['url_key']}.{pyspark_func_name}]({url})"
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
line = f'* ❌ ' + line + f" (<a href='https://github.com/flypipe/sparkleframe/issues/new?{urllib.parse.urlencode(params)}' target='_blank'>request feature :simple-github:</a>)"
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
print(line)
|
|
106
|
+
pyspark_functions.append(line)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
path = os.path.join(pathlib.Path(__file__).resolve().parent, "supported_api_doc.md")
|
|
111
|
+
with open(path, "w") as file:
|
|
112
|
+
file.write("\n".join(pyspark_functions))
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
[data-md-color-scheme="green"] img[src$="#only-dark"],
|
|
2
|
+
[data-md-color-scheme="green"] img[src$="#gh-dark-mode-only"] {
|
|
3
|
+
display: none; /* Hide dark images in light mode */
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
[data-md-color-scheme="green"] img[src$="#only-light"],
|
|
7
|
+
[data-md-color-scheme="green"] img[src$="#gh-light-mode-only"] {
|
|
8
|
+
display: none; /* Hide dark images in light mode */
|
|
9
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
site_name: SparkleFrame
|
|
2
|
+
repo_url: https://github.com/flypipe/sparkleframe
|
|
3
|
+
repo_name: flypipe/sparkleframe
|
|
4
|
+
nav:
|
|
5
|
+
- index.md
|
|
6
|
+
- Supported Pyspark API Methods: supported_api.md
|
|
7
|
+
- Changelog: changelog.md
|
|
8
|
+
theme:
|
|
9
|
+
name: material
|
|
10
|
+
logo: images/logo.png
|
|
11
|
+
favicon: images/favicon.ico
|
|
12
|
+
custom_dir: docs/overrides
|
|
13
|
+
icon:
|
|
14
|
+
repo: fontawesome/brands/github
|
|
15
|
+
|
|
16
|
+
palette:
|
|
17
|
+
- media: "(prefers-color-scheme: light)"
|
|
18
|
+
scheme: default
|
|
19
|
+
primary: #091c79
|
|
20
|
+
accent: #cce8fe
|
|
21
|
+
toggle:
|
|
22
|
+
icon: material/weather-sunny
|
|
23
|
+
name: Switch to dark mode
|
|
24
|
+
# Palette toggle for dark mode
|
|
25
|
+
- media: "(prefers-color-scheme: dark)"
|
|
26
|
+
scheme: slate
|
|
27
|
+
primary: #cce8fe
|
|
28
|
+
accent: #091c79
|
|
29
|
+
toggle:
|
|
30
|
+
icon: material/weather-night
|
|
31
|
+
name: Switch to light mode
|
|
32
|
+
features:
|
|
33
|
+
- navigation.top
|
|
34
|
+
- navigation.path
|
|
35
|
+
- navigation.indexes
|
|
36
|
+
- search.suggest
|
|
37
|
+
- search.highlight
|
|
38
|
+
- content.code.copy
|
|
39
|
+
- content.code.select
|
|
40
|
+
plugins:
|
|
41
|
+
- include-markdown
|
|
42
|
+
- search
|
|
43
|
+
- mkdocstrings:
|
|
44
|
+
handlers:
|
|
45
|
+
python:
|
|
46
|
+
paths: [ sparkleframe ]
|
|
47
|
+
markdown_extensions:
|
|
48
|
+
- tables
|
|
49
|
+
- pymdownx.highlight:
|
|
50
|
+
anchor_linenums: true
|
|
51
|
+
- pymdownx.inlinehilite
|
|
52
|
+
- pymdownx.snippets
|
|
53
|
+
- pymdownx.superfences
|
|
54
|
+
- pymdownx.tabbed:
|
|
55
|
+
alternate_style: true
|
|
56
|
+
- admonition
|
|
57
|
+
- pymdownx.details
|
|
58
|
+
- pymdownx.superfences:
|
|
59
|
+
custom_fences:
|
|
60
|
+
- name: mermaid
|
|
61
|
+
class: mermaid
|
|
62
|
+
format: !!python/name:pymdownx.superfences.fence_code_format
|
|
63
|
+
- attr_list
|
|
64
|
+
- pymdownx.emoji:
|
|
65
|
+
emoji_index: !!python/name:material.extensions.emoji.twemoji
|
|
66
|
+
emoji_generator: !!python/name:material.extensions.emoji.to_svg
|
|
67
|
+
- md_in_html
|
|
68
|
+
- pymdownx.blocks.caption
|
|
69
|
+
extra_css:
|
|
70
|
+
- stylesheets/extra.css
|
|
71
|
+
extra:
|
|
72
|
+
version:
|
|
73
|
+
provider: mike
|
|
74
|
+
default: latest
|
|
75
|
+
alias: true
|
|
@@ -2,6 +2,13 @@
|
|
|
2
2
|
black==24.10.0
|
|
3
3
|
coverage==7.6.10
|
|
4
4
|
chispa==0.11.1
|
|
5
|
+
mike==2.1.3
|
|
6
|
+
mkdocs-git-committers-plugin-2==2.5.0
|
|
7
|
+
mkdocs-include-markdown-plugin==7.1.5
|
|
8
|
+
mkdocs==1.6.1
|
|
9
|
+
mkdocs-dracula-theme==1.0.7
|
|
10
|
+
mkdocs-material==9.6.14
|
|
11
|
+
mkdocstrings[python]==0.29.1
|
|
5
12
|
numpy==1.26.4
|
|
6
13
|
pandas==2.2.3
|
|
7
14
|
polars==1.30.0
|