sparkleframe 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.docker/docker-compose.yaml +5 -0
  2. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.github/workflows/deploy.yml +9 -5
  3. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.github/workflows/predeploy.yml +2 -1
  4. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.github/workflows/verification.yml +0 -2
  5. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.gitignore +2 -0
  6. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/Makefile +27 -14
  7. sparkleframe-0.2.2/PKG-INFO +130 -0
  8. sparkleframe-0.2.2/README.md +114 -0
  9. sparkleframe-0.2.2/changelog.md +16 -0
  10. sparkleframe-0.2.2/docs/generate_supported_api.py +112 -0
  11. sparkleframe-0.2.2/docs/images/favicon.ico +0 -0
  12. sparkleframe-0.2.2/docs/images/logo.png +0 -0
  13. sparkleframe-0.2.2/docs/images/logo_dark.png +0 -0
  14. sparkleframe-0.2.2/docs/images/logo_white.png +0 -0
  15. sparkleframe-0.2.2/docs/index.md +9 -0
  16. sparkleframe-0.2.2/docs/overrides/main.html +8 -0
  17. sparkleframe-0.2.2/docs/stylesheets/extra.css +9 -0
  18. sparkleframe-0.2.2/docs/supported_api.md +5 -0
  19. sparkleframe-0.2.2/mkdocs.yml +75 -0
  20. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/requirements-dev.in +7 -0
  21. sparkleframe-0.2.2/requirements-dev.txt +93 -0
  22. sparkleframe-0.2.2/requirements-pkg.txt +21 -0
  23. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/generate_changelog.py +12 -10
  24. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/generate_changelog_test.py +4 -10
  25. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/utils.py +4 -4
  26. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/gpt_utils.py +4 -4
  27. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/__init__.py +2 -0
  28. sparkleframe-0.2.2/sparkleframe/tests/__init__.py +0 -0
  29. sparkleframe-0.2.2/sparkleframe/version.txt +1 -0
  30. sparkleframe-0.2.0/PKG-INFO +0 -22
  31. sparkleframe-0.2.0/README.md +0 -6
  32. sparkleframe-0.2.0/changelog.md +0 -9
  33. sparkleframe-0.2.0/requirements-dev.txt +0 -170
  34. sparkleframe-0.2.0/requirements-pkg.txt +0 -46
  35. sparkleframe-0.2.0/sparkleframe/version.txt +0 -1
  36. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.coverage +0 -0
  37. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.docker/Dockerfile +0 -0
  38. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.github/actions/setup-dependencies/action.yml +0 -0
  39. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.github/dependabot.yml +0 -0
  40. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.github/hooks/prepare-commit-msg +0 -0
  41. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/.pylintrc +0 -0
  42. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/LICENSE +0 -0
  43. /sparkleframe-0.2.0/scripts/__init__.py → /sparkleframe-0.2.2/docs/overrides/partials/footer.html +0 -0
  44. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/pyproject.toml +0 -0
  45. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/requirements-pkg.in +0 -0
  46. {sparkleframe-0.2.0/sparkleframe/base → sparkleframe-0.2.2/scripts}/__init__.py +0 -0
  47. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/calculate_version.py +0 -0
  48. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/calculate_version_test.py +0 -0
  49. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/set_version.py +0 -0
  50. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/scripts/utils_test.py +0 -0
  51. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/__init__.py +0 -0
  52. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/activate.py +0 -0
  53. {sparkleframe-0.2.0/sparkleframe/tests → sparkleframe-0.2.2/sparkleframe/base}/__init__.py +0 -0
  54. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/base/dataframe.py +0 -0
  55. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/conftest.py +0 -0
  56. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/context.py +0 -0
  57. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/engine.py +0 -0
  58. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/column.py +0 -0
  59. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/column_test.py +0 -0
  60. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/dataframe.py +0 -0
  61. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/dataframe_test.py +0 -0
  62. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/functions.py +0 -0
  63. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/functions_test.py +0 -0
  64. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/functions_utils.py +0 -0
  65. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/group.py +0 -0
  66. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/session.py +0 -0
  67. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/session_test.py +0 -0
  68. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/types.py +0 -0
  69. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/types_test.py +0 -0
  70. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/window.py +0 -0
  71. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/polarsdf/window_test.py +0 -0
  72. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/tests/pyspark_test.py +0 -0
  73. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/tests/spark.py +0 -0
  74. {sparkleframe-0.2.0 → sparkleframe-0.2.2}/sparkleframe/tests/utils.py +0 -0
@@ -6,7 +6,12 @@ services:
6
6
  container_name: sparkleframe
7
7
  environment:
8
8
  TIMEZONE: UTC
9
+ ports:
10
+ - "8000:8000"
9
11
  volumes:
12
+ - ./../README.md:/README.md
13
+ - ./../docs:/docs
14
+ - ./../mkdocs.yml:/mkdocs.yml
10
15
  - ./../sparkleframe:/sparkleframe
11
16
  - ./../.coverage:/sparkleframe/.coverage
12
17
  - ./../pyproject.toml:/pyproject.toml
@@ -13,7 +13,7 @@ jobs:
13
13
  predeploy:
14
14
  uses: ./.github/workflows/predeploy.yml
15
15
 
16
- deploy-pypi:
16
+ deploy:
17
17
  needs: predeploy
18
18
  runs-on: ubuntu-latest
19
19
  timeout-minutes: 5
@@ -51,7 +51,11 @@ jobs:
51
51
  flit publish
52
52
  env:
53
53
  FLIT_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
54
-
55
- # build-docs:
56
- # needs: deploy-pypi
57
- # uses: ./.github/workflows/build-docs.yml
54
+ - name: Deploy docs
55
+ run: |
56
+ git config --global user.email "flypipe@dummy.com"
57
+ git config --global user.name "flypipe"
58
+ export PYTHONPATH=$PYTHONPATH:./sparkleframe
59
+ make docs-deploy version=${{ env.VERSION_NUMBER }}
60
+ env:
61
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -58,7 +58,8 @@ jobs:
58
58
  uses: actions/upload-artifact@v4
59
59
  with:
60
60
  name: changelog.md
61
- path: ./changelog.md
61
+ path: ./docs/changelog.md
62
+
62
63
  # Build and upload the wheel so we can access the wheel without having to upload it to pypi
63
64
  - name: Build wheel
64
65
  run: flit build --no-use-vcs
@@ -3,8 +3,6 @@ name: verification
3
3
  # Controls when the workflow will run
4
4
  on:
5
5
  # Triggers the workflow on push or pull request events but only for the "main" branch
6
- push:
7
- branches: [ "main" ]
8
6
  pull_request:
9
7
  branches: [ "**" ]
10
8
 
@@ -172,3 +172,5 @@ cython_debug/
172
172
 
173
173
  # PyPI configuration file
174
174
  .pypirc
175
+ /docs/supported_api_doc.md
176
+ /docs/changelog.md
@@ -1,7 +1,8 @@
1
1
  SHELL :=/usr/bin/env bash
2
2
  PYTEST_THREADS ?=$(shell echo $$((`getconf _NPROCESSORS_ONLN` / 3)))
3
3
  LOCAL_DIR :=./.docker
4
- MIN_COVERAGE = 77
4
+ MIN_COVERAGE = 79
5
+ version ?=
5
6
 
6
7
  bash:
7
8
  docker compose -f $(LOCAL_DIR)/docker-compose.yaml run --remove-orphans -it sparkleframe bash
@@ -23,15 +24,10 @@ lint:
23
24
  docker compose -f $(LOCAL_DIR)/docker-compose.yaml run --remove-orphans sparkleframe sh -c "python -m ruff check --line-length 119 sparkleframe"
24
25
  .PHONY: lint
25
26
 
26
- pip-compile:
27
- pip-compile requirements-pkg.in && pip-compile requirements-dev.in
28
- .PHONY: pip-compile
29
-
30
27
  coverage:
31
28
  docker compose -f $(LOCAL_DIR)/docker-compose.yaml run --remove-orphans sparkleframe sh -c "pytest -n $(PYTEST_THREADS) -k '_test.py' --cov-config=sparkleframe/.coverage --cov=sparkleframe --no-cov-on-fail --cov-fail-under=$(MIN_COVERAGE) -v sparkleframe"
32
29
  .PHONY: coverage
33
30
 
34
-
35
31
  test:
36
32
  docker compose -f $(LOCAL_DIR)/docker-compose.yaml run --remove-orphans sparkleframe sh -c "pytest -n $(PYTEST_THREADS) -k '_test.py' -vv $(f)"
37
33
  .PHONY: test
@@ -52,18 +48,35 @@ githooks:
52
48
  .PHONY: githooks
53
49
 
54
50
  pip-compile:
55
- pip-compile requirements-pkg.in
56
- pip-compile requirements-dev.in
57
- make build
58
- make githooks
51
+ pip install -r requirements-pkg.in
52
+ pip-compile requirements-pkg.in --no-annotate --no-header
53
+ pip-compile requirements-dev.in --no-annotate --no-header
59
54
  .PHONY: pip-compile
60
55
 
61
- setup:
62
- pip install -r requirements-pkg.in
63
- pip-compile requirements-pkg.in
64
- pip-compile requirements-dev.in
56
+ setup: pip-compile
57
+ export PYTHONPATH=$PYTHONPATH:./sparkleframe
65
58
  pip install -r requirements-dev.txt
66
59
  make build
67
60
  make githooks
68
61
  .PHONY: setup
69
62
 
63
+ docs:
64
+ @if [ ! -f docs/changelog.md ]; then \
65
+ echo "docs/changelog.md does not exist, running command..."; \
66
+ python scripts/generate_changelog.py; \
67
+ fi
68
+ @if [ ! -f docs/supported_api_doc.md ]; then \
69
+ echo "docs/supported_api_doc.md does not exist, running command..."; \
70
+ python docs/generate_supported_api.py; \
71
+ fi
72
+ mkdocs serve
73
+ .PHONY: docs
74
+
75
+
76
+ docs-deploy:
77
+ @[ -n "$(version)" ] || (echo "ERROR: version is required"; exit 1)
78
+ python scripts/generate_changelog.py
79
+ python docs/generate_supported_api.py
80
+ mike deploy --allow-empty --push --update-aliases $(shell echo $(version) | awk -F. '{print $$1"."$$2}') latest
81
+ mike set-default --push latest
82
+ .PHONY: docs-deploy
@@ -0,0 +1,130 @@
1
+ Metadata-Version: 2.4
2
+ Name: sparkleframe
3
+ Version: 0.2.2
4
+ Summary: Sparkleframe
5
+ Author-email: Jose Helio de Brum Muller <josemuller.flypipe@gmail.com>
6
+ Requires-Python: >=3.8
7
+ Description-Content-Type: text/markdown
8
+ Classifier: Development Status :: 3 - Alpha
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: Intended Audience :: Information Technology
11
+ Classifier: License :: OSI Approved :: Apache Software License
12
+ Classifier: Programming Language :: Python :: 3
13
+ License-File: LICENSE
14
+ Requires-Dist: polars >=1.30
15
+ Project-URL: Home, https://github.com/flypipe/sparkleframe.git
16
+
17
+ ![SparkleFrame](docs/images/logo_white.png#only-dark)
18
+
19
+ SparkleFrame implements the PySpark DataFrame API in order to enable running transformation pipelines
20
+ directly on [Polars Dataframe](https://docs.pola.rs/api/python/stable/reference/index.html) - no Spark clusters or
21
+ dependencies required.
22
+
23
+ Apache Spark is designed for distributed, large-scale data processing, but it is not optimized for low-latency use
24
+ cases. There are scenarios, however, where you need to quickly re-compute certain data—for example,
25
+ regenerating features for a machine learning model in real time or near-real time.
26
+
27
+ SparkleFrame is great for:
28
+
29
+ * Users who want to run PySpark code quickly locally without the overhead of starting a Spark session
30
+ * Users who want to run PySpark DataFrame code without the complexity of using Spark for processing
31
+ * Useful for unit testing, feature prototyping, or serving small pipelines in microservices.
32
+
33
+ ## Documentation
34
+
35
+ Full documentation is available at [https://flypipe.github.io/sparkleframe/](https://flypipe.github.io/sparkleframe/).
36
+
37
+ ## Installation
38
+
39
+ ```bash
40
+ pip install sparkleframe
41
+ ```
42
+
43
+ ## Usage
44
+
45
+ SparkleFrame can be used in two ways:
46
+
47
+ * Directly importing the `sparkleframe.polarsdf` package
48
+ * Using the `activate` function to allow for continuing to use `pyspark.sql` but have it use SparkleFrame behind the scenes.
49
+
50
+ ### Directly importing
51
+
52
+ If converting a PySpark pipeline, all `pyspark.sql` should be replaced with `sparkleframe.polarsdf`.
53
+
54
+ ```python
55
+ # PySpark import
56
+ # from pyspark.sql import SparkSession
57
+ # from pyspark.sql import functions as F
58
+ # from pyspark.sql.dataframe import DataFrame
59
+ # SparkleFrame import
60
+ from sparkleframe.polarsdf.session import SparkSession
61
+ from sparkleframe.polarsdf import functions as F
62
+ from sparkleframe.polarsdf.dataframe import DataFrame
63
+ ```
64
+
65
+ ## Activating SparkleFrame
66
+
67
+ SparkleFrame can either replace pyspark imports or be used alongside them. To replace pyspark imports,
68
+ use the activate function to set the engine to use.
69
+
70
+ ```python
71
+ from sparkleframe.activate import activate
72
+
73
+ # Activate SparkleFrame
74
+ activate()
75
+
76
+ from pyspark.sql import SparkSession
77
+ session = SparkSession.builder.getOrCreate()
78
+ ```
79
+
80
+ `SparkSession` will now be a SparkleFrame Session object and everything will be run on Polars Dataframe directly.
81
+
82
+ SparkleFrame can also be directly imported which both maintains pyspark imports:
83
+
84
+ ```python
85
+ from sparkleframe.polarsdf.session import SparkSession
86
+ session = SparkSession.builder.getOrCreate()
87
+ ```
88
+
89
+ ## Example Usage
90
+
91
+ ```python
92
+ from sparkleframe.activate import activate
93
+
94
+ # Activate SparkleFrame
95
+ activate()
96
+
97
+ from pyspark.sql import SparkSession
98
+ from pyspark.sql import functions as F
99
+
100
+ session = SparkSession.builder.getOrCreate()
101
+ df = session.createDataFrame(data=[{"col1": 1, "col2": 2}])
102
+ df = df.withColumn("col3", F.col("col2") + F.col("col1"))
103
+ ```
104
+ ```python
105
+ >>> print(type(df))
106
+ <class 'sparkleframe.polarsdf.dataframe.DataFrame'>
107
+ ```
108
+ ```python
109
+ >>> df.show()
110
+ shape: (1, 3)
111
+ ┌──────┬──────┬──────┐
112
+ │ col1 ┆ col2 ┆ col3 │
113
+ │ --- ┆ --- ┆ --- │
114
+ │ i64 ┆ i64 ┆ i64 │
115
+ ╞══════╪══════╪══════╡
116
+ │ 1 ┆ 2 ┆ 3 │
117
+ └──────┴──────┴──────┘
118
+ ```
119
+
120
+ !!! note
121
+
122
+ - If you encounter any transformation that is not implemented, please open an [issue on GitHub](https://github.com/flypipe/sparkleframe/issues) so it can be prioritized.
123
+
124
+ You can learn more about the design motivation behind Sparkleframe in this
125
+ [discussion thread](https://github.com/eakmanrq/sparkleframe/issues/409).
126
+
127
+
128
+ ## Source Code
129
+
130
+ API code is available at [https://github.com/flypipe/sparkleframe](https://github.com/flypipe/sparkleframe).
@@ -0,0 +1,114 @@
1
+ ![SparkleFrame](docs/images/logo_white.png#only-dark)
2
+
3
+ SparkleFrame implements the PySpark DataFrame API in order to enable running transformation pipelines
4
+ directly on [Polars Dataframe](https://docs.pola.rs/api/python/stable/reference/index.html) - no Spark clusters or
5
+ dependencies required.
6
+
7
+ Apache Spark is designed for distributed, large-scale data processing, but it is not optimized for low-latency use
8
+ cases. There are scenarios, however, where you need to quickly re-compute certain data—for example,
9
+ regenerating features for a machine learning model in real time or near-real time.
10
+
11
+ SparkleFrame is great for:
12
+
13
+ * Users who want to run PySpark code quickly locally without the overhead of starting a Spark session
14
+ * Users who want to run PySpark DataFrame code without the complexity of using Spark for processing
15
+ * Useful for unit testing, feature prototyping, or serving small pipelines in microservices.
16
+
17
+ ## Documentation
18
+
19
+ Full documentation is available at [https://flypipe.github.io/sparkleframe/](https://flypipe.github.io/sparkleframe/).
20
+
21
+ ## Installation
22
+
23
+ ```bash
24
+ pip install sparkleframe
25
+ ```
26
+
27
+ ## Usage
28
+
29
+ SparkleFrame can be used in two ways:
30
+
31
+ * Directly importing the `sparkleframe.polarsdf` package
32
+ * Using the `activate` function to allow for continuing to use `pyspark.sql` but have it use SparkleFrame behind the scenes.
33
+
34
+ ### Directly importing
35
+
36
+ If converting a PySpark pipeline, all `pyspark.sql` should be replaced with `sparkleframe.polarsdf`.
37
+
38
+ ```python
39
+ # PySpark import
40
+ # from pyspark.sql import SparkSession
41
+ # from pyspark.sql import functions as F
42
+ # from pyspark.sql.dataframe import DataFrame
43
+ # SparkleFrame import
44
+ from sparkleframe.polarsdf.session import SparkSession
45
+ from sparkleframe.polarsdf import functions as F
46
+ from sparkleframe.polarsdf.dataframe import DataFrame
47
+ ```
48
+
49
+ ## Activating SparkleFrame
50
+
51
+ SparkleFrame can either replace pyspark imports or be used alongside them. To replace pyspark imports,
52
+ use the activate function to set the engine to use.
53
+
54
+ ```python
55
+ from sparkleframe.activate import activate
56
+
57
+ # Activate SparkleFrame
58
+ activate()
59
+
60
+ from pyspark.sql import SparkSession
61
+ session = SparkSession.builder.getOrCreate()
62
+ ```
63
+
64
+ `SparkSession` will now be a SparkleFrame Session object and everything will be run on Polars Dataframe directly.
65
+
66
+ SparkleFrame can also be directly imported which both maintains pyspark imports:
67
+
68
+ ```python
69
+ from sparkleframe.polarsdf.session import SparkSession
70
+ session = SparkSession.builder.getOrCreate()
71
+ ```
72
+
73
+ ## Example Usage
74
+
75
+ ```python
76
+ from sparkleframe.activate import activate
77
+
78
+ # Activate SparkleFrame
79
+ activate()
80
+
81
+ from pyspark.sql import SparkSession
82
+ from pyspark.sql import functions as F
83
+
84
+ session = SparkSession.builder.getOrCreate()
85
+ df = session.createDataFrame(data=[{"col1": 1, "col2": 2}])
86
+ df = df.withColumn("col3", F.col("col2") + F.col("col1"))
87
+ ```
88
+ ```python
89
+ >>> print(type(df))
90
+ <class 'sparkleframe.polarsdf.dataframe.DataFrame'>
91
+ ```
92
+ ```python
93
+ >>> df.show()
94
+ shape: (1, 3)
95
+ ┌──────┬──────┬──────┐
96
+ │ col1 ┆ col2 ┆ col3 │
97
+ │ --- ┆ --- ┆ --- │
98
+ │ i64 ┆ i64 ┆ i64 │
99
+ ╞══════╪══════╪══════╡
100
+ │ 1 ┆ 2 ┆ 3 │
101
+ └──────┴──────┴──────┘
102
+ ```
103
+
104
+ !!! note
105
+
106
+ - If you encounter any transformation that is not implemented, please open an [issue on GitHub](https://github.com/flypipe/sparkleframe/issues) so it can be prioritized.
107
+
108
+ You can learn more about the design motivation behind Sparkleframe in this
109
+ [discussion thread](https://github.com/eakmanrq/sparkleframe/issues/409).
110
+
111
+
112
+ ## Source Code
113
+
114
+ API code is available at [https://github.com/flypipe/sparkleframe](https://github.com/flypipe/sparkleframe).
@@ -0,0 +1,16 @@
1
+ Changelog
2
+ =========
3
+
4
+ <h2><a href="https://github.com/flypipe/sparkleframe/tree/release/0.2.2" target="_blank" rel="noopener noreferrer">release/0.2.2</a><h2>
5
+
6
+ <br/>- <a href="https://github.com/flypipe/sparkleframe/issues/48" target="_blank" rel="noopener noreferrer">48 small fixes</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/46" target="_blank" rel="noopener noreferrer">46 fix mkdocs serve</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/44" target="_blank" rel="noopener noreferrer">44 do not run verification workflow on push</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/42" target="_blank" rel="noopener noreferrer">42 remove header from pip compile</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/39" target="_blank" rel="noopener noreferrer">39 GitAction upload changelog.md as artifact</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/38" target="_blank" rel="noopener noreferrer">38 fix change log artifact</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/31" target="_blank" rel="noopener noreferrer">31 Build Docs</a>
7
+
8
+ <h2><a href="https://github.com/flypipe/sparkleframe/tree/release/0.2.1" target="_blank" rel="noopener noreferrer">release/0.2.1</a><h2><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/33" target="_blank" rel="noopener noreferrer">33 Fix import error of functions</a>
9
+
10
+
11
+
12
+ <h2><a href="https://github.com/flypipe/sparkleframe/tree/release/0.2.0" target="_blank" rel="noopener noreferrer">release/0.2.0</a><h2><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/27" target="_blank" rel="noopener noreferrer">27 fix git workflow to create version and changelog from the current branch</a>
13
+
14
+ <h2><a href="https://github.com/flypipe/sparkleframe/tree/release/0.1.0" target="_blank" rel="noopener noreferrer">release/0.1.0</a><h2><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/9" target="_blank" rel="noopener noreferrer">9 Implement pyspark.sql.Window (and rank, dense_rank and row_number)</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/27" target="_blank" rel="noopener noreferrer">27 fix git workflow to create version and changelog from the current branch</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/24" target="_blank" rel="noopener noreferrer">24 add support to pyspark.sql.DataFrame.schema</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/22" target="_blank" rel="noopener noreferrer">22 add support to pyspark.pandas.DataFrame.dtypes</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/20" target="_blank" rel="noopener noreferrer">20 add support to pyspark.sql.functions.length</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/18" target="_blank" rel="noopener noreferrer">18 implement pyspark.sql.functions.to_timestamp</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/16" target="_blank" rel="noopener noreferrer">16 show issues in changelog in descending order on issue_id</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/14" target="_blank" rel="noopener noreferrer">14 Update changelog to get changelog from the previous release branch</a><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/12" target="_blank" rel="noopener noreferrer">12 Implement pyspark.sql.functions.regexp_replace</a>
15
+
16
+ <h2><a href="https://github.com/flypipe/sparkleframe/tree/release/0.0.2" target="_blank" rel="noopener noreferrer">release/0.0.2</a><h2><br/>- <a href="https://github.com/flypipe/sparkleframe/issues/10" target="_blank" rel="noopener noreferrer">10 Implement simpleString attribute from data types</a>
@@ -0,0 +1,112 @@
1
+
2
+ import inspect
3
+ import os
4
+ import pathlib
5
+ import urllib
6
+
7
+ import requests
8
+ from concurrent.futures import ThreadPoolExecutor, as_completed
9
+
10
+ def get_data_types(T):
11
+ # Get all data type classes from the module
12
+ return sorted([
13
+ getattr(T, attr).__name__ for attr in dir(T)
14
+ if attr.endswith('Type') and isinstance(getattr(T, attr), type)
15
+ ])
16
+
17
+
18
+ def get_functions(obj):
19
+ """Extract all public functions from pyspark.sql.functions"""
20
+ all_names = dir(obj)
21
+ return sorted([
22
+ name for name in all_names
23
+ if not name.startswith("_") and inspect.isfunction(getattr(obj, name))
24
+ ])
25
+
26
+ def check_doc_url(func_name, url_key):
27
+ """Check if the function's API URL exists"""
28
+ url = f"https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.{url_key}.{func_name}.html"
29
+ try:
30
+ response = requests.head(url, timeout=5)
31
+ exists = response.status_code == 200
32
+ except requests.RequestException as e:
33
+ print(f"Error checking {url}: {e}")
34
+ exists = False
35
+ return url, exists, func_name
36
+
37
+ if __name__ == "__main__":
38
+ from pyspark.sql.catalog import Catalog as PYSPARK_CATALOG
39
+ SPARKLE_CATALOG = None
40
+
41
+ from pyspark.sql.column import Column as PYSPARK_COLUMN
42
+ from sparkleframe.polarsdf.column import Column as SPARKLE_COLUMN
43
+
44
+ from pyspark.sql.dataframe import DataFrame as PYSPARK_DATAFRAME
45
+ from sparkleframe.polarsdf.dataframe import DataFrame as SPARKLE_DATAFRAME
46
+
47
+
48
+ import pyspark.sql.functions as PYSPARK_FUNCTIONS
49
+ import sparkleframe.polarsdf.functions as SPARKLE_FUNCTIONS
50
+
51
+ from pyspark.sql.group import GroupedData as PYSPARK_GROUPED_DATA
52
+ from sparkleframe.polarsdf.group import GroupedData as SPARKLE_GROUPED_DATA
53
+
54
+ from pyspark.sql.window import Window as PYSPARK_WINDOW
55
+ from sparkleframe.polarsdf.window import Window as SPARKLE_WINDOW
56
+
57
+ from pyspark.sql.session import SparkSession as PYSPARK_SESSION
58
+ from sparkleframe.polarsdf.session import SparkSession as SPARKLE_SESSION
59
+
60
+ import pyspark.sql.types as PYSPARK_TYPES
61
+ import sparkleframe.polarsdf.types as SPARKLE_TYPES
62
+
63
+ MODULES = [
64
+
65
+ {"url_key": "types", "module_url_key": "data_types", "pyspark": PYSPARK_TYPES, "sparkleframe": SPARKLE_TYPES, "lambda": get_data_types},
66
+ {"url_key": "GroupedData", "module_url_key": "grouping", "pyspark": PYSPARK_GROUPED_DATA, "sparkleframe": SPARKLE_GROUPED_DATA, "lambda": get_functions},
67
+ {"url_key": "Window", "module_url_key": "window", "pyspark": PYSPARK_WINDOW, "sparkleframe": SPARKLE_WINDOW, "lambda": get_functions},
68
+ {"url_key": "DataFrame", "module_url_key": "dataframe", "pyspark": PYSPARK_DATAFRAME, "sparkleframe": SPARKLE_DATAFRAME, "lambda": get_functions},
69
+ {"url_key": "Catalog", "module_url_key": "catalog", "pyspark": PYSPARK_CATALOG, "sparkleframe": SPARKLE_CATALOG, "lambda": get_functions},
70
+ {"url_key": "SparkSession", "module_url_key": "spark_session", "pyspark": PYSPARK_SESSION, "sparkleframe": SPARKLE_SESSION, "lambda": get_functions},
71
+ {"url_key": "Column", "module_url_key": "column", "pyspark": PYSPARK_COLUMN, "sparkleframe": SPARKLE_COLUMN, "lambda": get_functions},
72
+ {"url_key": "functions", "module_url_key": "functions", "pyspark": PYSPARK_FUNCTIONS, "sparkleframe": SPARKLE_FUNCTIONS, "lambda": get_functions},
73
+ ]
74
+
75
+ pyspark_functions = []
76
+ # 3️⃣ Number of threads: CPU cores minus 1, minimum 1
77
+ num_threads = max(os.cpu_count() - 1, 1)
78
+ for modules in sorted(MODULES, key=lambda module: module["module_url_key"]):
79
+
80
+ pyspark_functions_list = modules['lambda'](modules["pyspark"])
81
+ sparkleframe_functions = modules['lambda'](modules["sparkleframe"])
82
+ pyspark_functions.append(f'\n\n## <a href="https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/{modules["module_url_key"]}.html" target="_blank">pyspark.sql.{modules["url_key"]}</a>')
83
+ with ThreadPoolExecutor(max_workers=num_threads) as executor:
84
+ futures = [executor.submit(check_doc_url, pyspark_func_name, modules['url_key']) for pyspark_func_name in sorted(pyspark_functions_list)]
85
+
86
+ for future in as_completed(futures):
87
+ url, exists, pyspark_func_name = future.result()
88
+
89
+ if not exists:
90
+ # not found in pyspark documentation
91
+ continue
92
+
93
+ line = f'<a href="{url}" target="_blank">{pyspark_func_name}</a>'
94
+ if pyspark_func_name in sparkleframe_functions:
95
+ line = f'* ✅ ' + line
96
+ else:
97
+ params = {
98
+ "title": f"[PYSPARK_API_REQUEST] Implement pyspark.sql.{modules['url_key']}.{pyspark_func_name}",
99
+ "body": f"Please implement feature [pyspark.sql.{modules['url_key']}.{pyspark_func_name}]({url})"
100
+ }
101
+
102
+ line = f'* ❌ ' + line + f" (<a href='https://github.com/flypipe/sparkleframe/issues/new?{urllib.parse.urlencode(params)}' target='_blank'>request feature :simple-github:</a>)"
103
+
104
+
105
+ print(line)
106
+ pyspark_functions.append(line)
107
+
108
+
109
+
110
+ path = os.path.join(pathlib.Path(__file__).resolve().parent, "supported_api_doc.md")
111
+ with open(path, "w") as file:
112
+ file.write("\n".join(pyspark_functions))
Binary file
@@ -0,0 +1,9 @@
1
+ # Welcome
2
+
3
+ ![SparkleFrame](images/logo_dark.png#only-light)
4
+
5
+ {%
6
+ include-markdown "../README.md"
7
+ %}
8
+
9
+
@@ -0,0 +1,8 @@
1
+ {% extends "base.html" %}
2
+
3
+ {% block outdated %}
4
+ You're not viewing the latest version.
5
+ <a href="{{ '../' ~ base_url }}">
6
+ <strong>Click here to go to latest.</strong>
7
+ </a>
8
+ {% endblock %}
@@ -0,0 +1,9 @@
1
+ [data-md-color-scheme="green"] img[src$="#only-dark"],
2
+ [data-md-color-scheme="green"] img[src$="#gh-dark-mode-only"] {
3
+ display: none; /* Hide dark images in light mode */
4
+ }
5
+
6
+ [data-md-color-scheme="green"] img[src$="#only-light"],
7
+ [data-md-color-scheme="green"] img[src$="#gh-light-mode-only"] {
8
+ display: none; /* Hide dark images in light mode */
9
+ }
@@ -0,0 +1,5 @@
1
+ See something that you would like to see supported? [Open an issue](https://github.com/flypipe/sparkleframe/issues/new)!
2
+
3
+ {%
4
+ include-markdown "supported_api_doc.md"
5
+ %}
@@ -0,0 +1,75 @@
1
+ site_name: SparkleFrame
2
+ repo_url: https://github.com/flypipe/sparkleframe
3
+ repo_name: flypipe/sparkleframe
4
+ nav:
5
+ - index.md
6
+ - Supported Pyspark API Methods: supported_api.md
7
+ - Changelog: changelog.md
8
+ theme:
9
+ name: material
10
+ logo: images/logo.png
11
+ favicon: images/favicon.ico
12
+ custom_dir: docs/overrides
13
+ icon:
14
+ repo: fontawesome/brands/github
15
+
16
+ palette:
17
+ - media: "(prefers-color-scheme: light)"
18
+ scheme: default
19
+ primary: #091c79
20
+ accent: #cce8fe
21
+ toggle:
22
+ icon: material/weather-sunny
23
+ name: Switch to dark mode
24
+ # Palette toggle for dark mode
25
+ - media: "(prefers-color-scheme: dark)"
26
+ scheme: slate
27
+ primary: #cce8fe
28
+ accent: #091c79
29
+ toggle:
30
+ icon: material/weather-night
31
+ name: Switch to light mode
32
+ features:
33
+ - navigation.top
34
+ - navigation.path
35
+ - navigation.indexes
36
+ - search.suggest
37
+ - search.highlight
38
+ - content.code.copy
39
+ - content.code.select
40
+ plugins:
41
+ - include-markdown
42
+ - search
43
+ - mkdocstrings:
44
+ handlers:
45
+ python:
46
+ paths: [ sparkleframe ]
47
+ markdown_extensions:
48
+ - tables
49
+ - pymdownx.highlight:
50
+ anchor_linenums: true
51
+ - pymdownx.inlinehilite
52
+ - pymdownx.snippets
53
+ - pymdownx.superfences
54
+ - pymdownx.tabbed:
55
+ alternate_style: true
56
+ - admonition
57
+ - pymdownx.details
58
+ - pymdownx.superfences:
59
+ custom_fences:
60
+ - name: mermaid
61
+ class: mermaid
62
+ format: !!python/name:pymdownx.superfences.fence_code_format
63
+ - attr_list
64
+ - pymdownx.emoji:
65
+ emoji_index: !!python/name:material.extensions.emoji.twemoji
66
+ emoji_generator: !!python/name:material.extensions.emoji.to_svg
67
+ - md_in_html
68
+ - pymdownx.blocks.caption
69
+ extra_css:
70
+ - stylesheets/extra.css
71
+ extra:
72
+ version:
73
+ provider: mike
74
+ default: latest
75
+ alias: true
@@ -2,6 +2,13 @@
2
2
  black==24.10.0
3
3
  coverage==7.6.10
4
4
  chispa==0.11.1
5
+ mike==2.1.3
6
+ mkdocs-git-committers-plugin-2==2.5.0
7
+ mkdocs-include-markdown-plugin==7.1.5
8
+ mkdocs==1.6.1
9
+ mkdocs-dracula-theme==1.0.7
10
+ mkdocs-material==9.6.14
11
+ mkdocstrings[python]==0.29.1
5
12
  numpy==1.26.4
6
13
  pandas==2.2.3
7
14
  polars==1.30.0