dsff 1.2.2__tar.gz → 1.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dsff-1.2.4/.github/dependabot.yml +15 -0
- dsff-1.2.4/.github/workflows/python-package.yml +137 -0
- dsff-1.2.4/.github/workflows/security.yml +31 -0
- dsff-1.2.2/.readthedocs.yml → dsff-1.2.4/.readthedocs.yaml +2 -2
- {dsff-1.2.2/src/dsff.egg-info → dsff-1.2.4}/PKG-INFO +7 -5
- {dsff-1.2.2 → dsff-1.2.4}/README.md +6 -4
- dsff-1.2.4/SECURITY.md +17 -0
- {dsff-1.2.2 → dsff-1.2.4}/docs/coverage.svg +1 -1
- {dsff-1.2.2 → dsff-1.2.4}/docs/mkdocs.yml +20 -6
- dsff-1.2.4/docs/pages/api.md +116 -0
- dsff-1.2.4/docs/pages/contributing.md +47 -0
- dsff-1.2.4/docs/pages/index.md +41 -0
- dsff-1.2.4/docs/pages/installation.md +55 -0
- dsff-1.2.4/docs/pages/internals.md +47 -0
- {dsff-1.2.2 → dsff-1.2.4}/docs/pages/usage.md +2 -2
- {dsff-1.2.2 → dsff-1.2.4}/docs/requirements.txt +1 -1
- dsff-1.2.4/src/dsff/VERSION.txt +1 -0
- {dsff-1.2.2 → dsff-1.2.4}/src/dsff/formats/__common__.py +6 -3
- {dsff-1.2.2 → dsff-1.2.4}/src/dsff/formats/__init__.py +116 -52
- dsff-1.2.4/src/dsff/formats/arff.py +191 -0
- dsff-1.2.4/src/dsff/formats/csv.py +44 -0
- {dsff-1.2.2 → dsff-1.2.4}/src/dsff/formats/dataset.py +22 -8
- dsff-1.2.4/src/dsff/formats/db.py +107 -0
- dsff-1.2.4/src/dsff/formats/pa.py +49 -0
- {dsff-1.2.2 → dsff-1.2.4/src/dsff.egg-info}/PKG-INFO +7 -5
- {dsff-1.2.2 → dsff-1.2.4}/src/dsff.egg-info/SOURCES.txt +13 -5
- {dsff-1.2.2 → dsff-1.2.4}/tests/test_dsff.py +11 -5
- dsff-1.2.4/tests/test_regressions.py +184 -0
- dsff-1.2.2/.github/workflows/python-package.yml +0 -109
- dsff-1.2.2/docs/pages/index.md +0 -38
- dsff-1.2.2/src/dsff/VERSION.txt +0 -1
- dsff-1.2.2/src/dsff/formats/arff.py +0 -168
- dsff-1.2.2/src/dsff/formats/csv.py +0 -40
- dsff-1.2.2/src/dsff/formats/db.py +0 -106
- dsff-1.2.2/src/dsff/formats/pa.py +0 -36
- {dsff-1.2.2 → dsff-1.2.4}/.coveragerc +0 -0
- {dsff-1.2.2 → dsff-1.2.4}/.gitignore +0 -0
- {dsff-1.2.2 → dsff-1.2.4}/LICENSE +0 -0
- {dsff-1.2.2/docs/pages/imgs → dsff-1.2.4/docs/pages/img}/dsff-from.png +0 -0
- {dsff-1.2.2/docs/pages/imgs → dsff-1.2.4/docs/pages/img}/dsff-to.png +0 -0
- {dsff-1.2.2/docs/pages/imgs → dsff-1.2.4/docs/pages/img}/icon.png +0 -0
- {dsff-1.2.2/docs/pages/imgs → dsff-1.2.4/docs/pages/img}/logo.png +0 -0
- {dsff-1.2.2 → dsff-1.2.4}/pyproject.toml +0 -0
- {dsff-1.2.2 → dsff-1.2.4}/pytest.ini +0 -0
- {dsff-1.2.2 → dsff-1.2.4}/requirements.txt +0 -0
- {dsff-1.2.2 → dsff-1.2.4}/setup.cfg +0 -0
- {dsff-1.2.2 → dsff-1.2.4}/src/dsff/__info__.py +0 -0
- {dsff-1.2.2 → dsff-1.2.4}/src/dsff/__init__.py +0 -0
- {dsff-1.2.2 → dsff-1.2.4}/src/dsff.egg-info/dependency_links.txt +0 -0
- {dsff-1.2.2 → dsff-1.2.4}/src/dsff.egg-info/requires.txt +0 -0
- {dsff-1.2.2 → dsff-1.2.4}/src/dsff.egg-info/top_level.txt +0 -0
- {dsff-1.2.2 → dsff-1.2.4}/tests/__init__.py +0 -0
- {dsff-1.2.2 → dsff-1.2.4}/tests/utils.py +0 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
updates:
|
|
3
|
+
- package-ecosystem: github-actions
|
|
4
|
+
directory: /
|
|
5
|
+
schedule:
|
|
6
|
+
interval: monthly
|
|
7
|
+
groups:
|
|
8
|
+
actions:
|
|
9
|
+
patterns: ["*"]
|
|
10
|
+
- package-ecosystem: pip
|
|
11
|
+
directory: /
|
|
12
|
+
schedule:
|
|
13
|
+
interval: monthly
|
|
14
|
+
# dependencies are minimum versions (>=) ; only open security updates, not routine upgrades
|
|
15
|
+
open-pull-requests-limit: 0
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
|
|
2
|
+
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
|
|
3
|
+
|
|
4
|
+
name: build
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
branches: [ "main" ]
|
|
9
|
+
pull_request:
|
|
10
|
+
branches: [ "main" ]
|
|
11
|
+
|
|
12
|
+
permissions:
|
|
13
|
+
contents: read
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
prepare:
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
outputs:
|
|
19
|
+
package: ${{ steps.pkg.outputs.package }}
|
|
20
|
+
steps:
|
|
21
|
+
- name: Compute package name from the repository's
|
|
22
|
+
id: pkg
|
|
23
|
+
run: |
|
|
24
|
+
name="${GITHUB_REPOSITORY##*/}"
|
|
25
|
+
echo "package=${name#python-}" >> $GITHUB_OUTPUT
|
|
26
|
+
build:
|
|
27
|
+
needs: prepare
|
|
28
|
+
runs-on: ${{ matrix.os }}
|
|
29
|
+
strategy:
|
|
30
|
+
fail-fast: false
|
|
31
|
+
matrix:
|
|
32
|
+
os: [ubuntu-latest]
|
|
33
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
34
|
+
env:
|
|
35
|
+
package: ${{ needs.prepare.outputs.package }}
|
|
36
|
+
cov_python_version: "3.13"
|
|
37
|
+
steps:
|
|
38
|
+
- uses: actions/checkout@v7
|
|
39
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
40
|
+
uses: actions/setup-python@v7
|
|
41
|
+
with:
|
|
42
|
+
python-version: ${{ matrix.python-version }}
|
|
43
|
+
- name: Install ${{ env.package }}
|
|
44
|
+
run: |
|
|
45
|
+
python -m pip install --upgrade pip
|
|
46
|
+
python -m pip install pytest pytest-cov coverage
|
|
47
|
+
pip install -r requirements.txt
|
|
48
|
+
pip install .
|
|
49
|
+
- name: Test ${{ env.package }} with pytest
|
|
50
|
+
run: |
|
|
51
|
+
pytest --cov=$package --cov-report=xml
|
|
52
|
+
- name: Upload coverage report
|
|
53
|
+
if: matrix.python-version == env.cov_python_version
|
|
54
|
+
uses: actions/upload-artifact@v7
|
|
55
|
+
with:
|
|
56
|
+
name: coverage-report
|
|
57
|
+
path: coverage.xml
|
|
58
|
+
coverage:
|
|
59
|
+
# only publish the badge for commits on main (not for pull requests)
|
|
60
|
+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
|
61
|
+
needs: build
|
|
62
|
+
permissions:
|
|
63
|
+
contents: write
|
|
64
|
+
runs-on: ubuntu-latest
|
|
65
|
+
env:
|
|
66
|
+
cov_badge_path: docs/coverage.svg
|
|
67
|
+
steps:
|
|
68
|
+
- uses: actions/checkout@v7
|
|
69
|
+
- name: Set up Python
|
|
70
|
+
uses: actions/setup-python@v7
|
|
71
|
+
with:
|
|
72
|
+
python-version: "3.13"
|
|
73
|
+
- name: Download coverage report
|
|
74
|
+
uses: actions/download-artifact@v8
|
|
75
|
+
with:
|
|
76
|
+
name: coverage-report
|
|
77
|
+
- name: Make coverage badge
|
|
78
|
+
run: |
|
|
79
|
+
pip install genbadge[coverage]
|
|
80
|
+
genbadge coverage -i coverage.xml -o $cov_badge_path
|
|
81
|
+
- name: Push coverage badge
|
|
82
|
+
run: |
|
|
83
|
+
# keep the badge alone on an orphan branch so that main is never modified by the workflow
|
|
84
|
+
git config --local user.email "github-actions[bot]@users.noreply.github.com"
|
|
85
|
+
git config --local user.name "github-actions[bot]"
|
|
86
|
+
if git fetch --depth=1 origin coverage-badge 2>/dev/null && \
|
|
87
|
+
git show FETCH_HEAD:$cov_badge_path | cmp -s - $cov_badge_path; then
|
|
88
|
+
echo "Coverage badge unchanged"
|
|
89
|
+
exit 0
|
|
90
|
+
fi
|
|
91
|
+
git checkout --orphan coverage-badge
|
|
92
|
+
git rm -rqf --cached .
|
|
93
|
+
git add $cov_badge_path
|
|
94
|
+
git commit -m "Update coverage badge"
|
|
95
|
+
git push origin coverage-badge --force
|
|
96
|
+
deploy:
|
|
97
|
+
# only release from main, never from pull requests
|
|
98
|
+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
|
99
|
+
# must match the environment of the trusted publisher configured on PyPI
|
|
100
|
+
environment:
|
|
101
|
+
name: pypi
|
|
102
|
+
url: https://pypi.org/p/dsff
|
|
103
|
+
permissions:
|
|
104
|
+
contents: read
|
|
105
|
+
id-token: write # for PyPI trusted publishing (OIDC)
|
|
106
|
+
runs-on: ubuntu-latest
|
|
107
|
+
needs: [prepare, build]
|
|
108
|
+
steps:
|
|
109
|
+
- uses: actions/checkout@v7
|
|
110
|
+
- name: Check whether the version is already on PyPI
|
|
111
|
+
# comparing with PyPI (not with the previous commit) also releases a version whose push did not trigger a deploy
|
|
112
|
+
id: version
|
|
113
|
+
run: |
|
|
114
|
+
version=$(tr -d '[:space:]' < src/dsff/VERSION.txt)
|
|
115
|
+
status=$(curl -s -o /dev/null -w '%{http_code}' --retry 3 "https://pypi.org/pypi/dsff/$version/json")
|
|
116
|
+
case "$status" in
|
|
117
|
+
200) echo "Version $version is already on PyPI" ; echo "new=false" >> "$GITHUB_OUTPUT" ;;
|
|
118
|
+
404) echo "Version $version is not on PyPI yet" ; echo "new=true" >> "$GITHUB_OUTPUT" ;;
|
|
119
|
+
*) echo "::error::Unexpected HTTP status $status from PyPI" ; exit 1 ;;
|
|
120
|
+
esac
|
|
121
|
+
- if: steps.version.outputs.new == 'true'
|
|
122
|
+
name: Cleanup README
|
|
123
|
+
run: |
|
|
124
|
+
sed -ri 's/^(##*)\s*:.*:\s*/\1 /g' README.md
|
|
125
|
+
awk '{if (match($0,"## Supporters")) exit; print}' README.md > README
|
|
126
|
+
mv -f README README.md
|
|
127
|
+
- if: steps.version.outputs.new == 'true'
|
|
128
|
+
name: Set up Python
|
|
129
|
+
uses: actions/setup-python@v7
|
|
130
|
+
with:
|
|
131
|
+
python-version: "3.13"
|
|
132
|
+
- if: steps.version.outputs.new == 'true'
|
|
133
|
+
name: Build ${{ needs.prepare.outputs.package }} package
|
|
134
|
+
run: python3 -m pip install --upgrade build && python3 -m build
|
|
135
|
+
- if: steps.version.outputs.new == 'true'
|
|
136
|
+
name: Publish to PyPI
|
|
137
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# This workflow audits the package's dependencies for known vulnerabilities
|
|
2
|
+
# It is separate from python-package.yml so that a new advisory never blocks a release
|
|
3
|
+
|
|
4
|
+
name: security
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
branches: [ "main" ]
|
|
9
|
+
pull_request:
|
|
10
|
+
branches: [ "main" ]
|
|
11
|
+
schedule:
|
|
12
|
+
- cron: "0 6 * * 1" # weekly, catches new advisories between pushes
|
|
13
|
+
workflow_dispatch:
|
|
14
|
+
|
|
15
|
+
permissions:
|
|
16
|
+
contents: read
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
pip-audit:
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v7
|
|
23
|
+
- name: Set up Python
|
|
24
|
+
uses: actions/setup-python@v7
|
|
25
|
+
with:
|
|
26
|
+
python-version: "3.13"
|
|
27
|
+
- name: Install pip-audit
|
|
28
|
+
run: python -m pip install --upgrade pip pip-audit
|
|
29
|
+
- name: Audit dependencies
|
|
30
|
+
# audit the project (resolved in an isolated environment), not the runner's own packages
|
|
31
|
+
run: pip-audit .
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dsff
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.4
|
|
4
4
|
Summary: DataSet File Format (DSFF)
|
|
5
5
|
Author-email: Alexandre D'Hondt <alexandre.dhondt@gmail.com>
|
|
6
6
|
License: GNU GENERAL PUBLIC LICENSE
|
|
@@ -702,16 +702,16 @@ Provides-Extra: arrow
|
|
|
702
702
|
Requires-Dist: pyarrow; extra == "arrow"
|
|
703
703
|
Dynamic: license-file
|
|
704
704
|
|
|
705
|
-
<p align="center" id="top"><img src="https://github.com/packing-box/python-dsff/raw/main/docs/pages/
|
|
705
|
+
<p align="center" id="top"><img src="https://github.com/packing-box/python-dsff/raw/main/docs/pages/img/logo.png"></p>
|
|
706
706
|
<h1 align="center">DataSet File Format <a href="https://twitter.com/intent/tweet?text=DataSet%20File%20Format%20-%20XSLX-based%20format%20for%20handling%20datasets.%0D%0ATiny%20library%20for%20handling%20a%20dataset%20as%20an%20XSLX%20and%20for%20converting%20it%20to%20ARFF,%20CSV%20or%20a%20FilelessDataset%20structure%20as%20for%20the%20Packing%20Box.%0D%0Ahttps%3a%2f%2fgithub%2ecom%2fpacking-box%2fpython-dsff%0D%0A&hashtags=python,dsff,machinelearning"><img src="https://img.shields.io/badge/Tweet--lightgrey?logo=twitter&style=social" alt="Tweet" height="20"/></a></h1>
|
|
707
707
|
<h3 align="center">Store a dataset in XSLX-like format.</h3>
|
|
708
708
|
|
|
709
709
|
[](https://pypi.python.org/pypi/dsff/)
|
|
710
710
|
[](https://python-dsff.readthedocs.io/en/latest/?badge=latest)
|
|
711
|
-
[](https://github.com/
|
|
712
|
-
[](https://github.com/packing-box/python-dsff/actions/workflows/python-package.yml)
|
|
712
|
+
[](https://github.com/packing-box/python-dsff/actions/workflows/security.yml)
|
|
713
|
+
[](#)
|
|
713
714
|
[](https://pypi.python.org/pypi/dsff/)
|
|
714
|
-
[](https://snyk.io/test/github/packing-box/python-dsff?targetFile=requirements.txt)
|
|
715
715
|
[](https://pypi.python.org/pypi/dsff/)
|
|
716
716
|
|
|
717
717
|
|
|
@@ -782,3 +782,5 @@ You may also like these:
|
|
|
782
782
|
- [REMINDer](https://github.com/packing-box/reminder): Packing detector using a simple heuristic (inspired from [this paper](https://ieeexplore.ieee.org/document/5404211)).
|
|
783
783
|
|
|
784
784
|
|
|
785
|
+
<p align="center"><a href="#top"><img src="https://img.shields.io/badge/Back%20to%20top--lightgrey?style=social" alt="Back to top" height="20"/></a></p>
|
|
786
|
+
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
<p align="center" id="top"><img src="https://github.com/packing-box/python-dsff/raw/main/docs/pages/
|
|
1
|
+
<p align="center" id="top"><img src="https://github.com/packing-box/python-dsff/raw/main/docs/pages/img/logo.png"></p>
|
|
2
2
|
<h1 align="center">DataSet File Format <a href="https://twitter.com/intent/tweet?text=DataSet%20File%20Format%20-%20XSLX-based%20format%20for%20handling%20datasets.%0D%0ATiny%20library%20for%20handling%20a%20dataset%20as%20an%20XSLX%20and%20for%20converting%20it%20to%20ARFF,%20CSV%20or%20a%20FilelessDataset%20structure%20as%20for%20the%20Packing%20Box.%0D%0Ahttps%3a%2f%2fgithub%2ecom%2fpacking-box%2fpython-dsff%0D%0A&hashtags=python,dsff,machinelearning"><img src="https://img.shields.io/badge/Tweet--lightgrey?logo=twitter&style=social" alt="Tweet" height="20"/></a></h1>
|
|
3
3
|
<h3 align="center">Store a dataset in XSLX-like format.</h3>
|
|
4
4
|
|
|
5
5
|
[](https://pypi.python.org/pypi/dsff/)
|
|
6
6
|
[](https://python-dsff.readthedocs.io/en/latest/?badge=latest)
|
|
7
|
-
[](https://github.com/
|
|
8
|
-
[](https://github.com/packing-box/python-dsff/actions/workflows/python-package.yml)
|
|
8
|
+
[](https://github.com/packing-box/python-dsff/actions/workflows/security.yml)
|
|
9
|
+
[](#)
|
|
9
10
|
[](https://pypi.python.org/pypi/dsff/)
|
|
10
|
-
[](https://snyk.io/test/github/packing-box/python-dsff?targetFile=requirements.txt)
|
|
11
11
|
[](https://pypi.python.org/pypi/dsff/)
|
|
12
12
|
|
|
13
13
|
|
|
@@ -78,3 +78,5 @@ You may also like these:
|
|
|
78
78
|
- [REMINDer](https://github.com/packing-box/reminder): Packing detector using a simple heuristic (inspired from [this paper](https://ieeexplore.ieee.org/document/5404211)).
|
|
79
79
|
|
|
80
80
|
|
|
81
|
+
<p align="center"><a href="#top"><img src="https://img.shields.io/badge/Back%20to%20top--lightgrey?style=social" alt="Back to top" height="20"/></a></p>
|
|
82
|
+
|
dsff-1.2.4/SECURITY.md
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Security Policy
|
|
2
|
+
|
|
3
|
+
## Supported Versions
|
|
4
|
+
|
|
5
|
+
Only the latest release of DSFF (`dsff` on [PyPI](https://pypi.org/project/dsff/)) receives security fixes.
|
|
6
|
+
|
|
7
|
+
## Reporting a Vulnerability
|
|
8
|
+
|
|
9
|
+
Please do **not** report security vulnerabilities through public GitHub issues.
|
|
10
|
+
|
|
11
|
+
Instead, report them privately through [GitHub's private vulnerability reporting](https://github.com/packing-box/python-dsff/security/advisories/new) (the repository's *Security* tab, then *Report a vulnerability*). Please include:
|
|
12
|
+
|
|
13
|
+
- the DSFF, openpyxl and (if relevant) pyarrow versions;
|
|
14
|
+
- the steps to reproduce the issue, and a sample file if relevant;
|
|
15
|
+
- the impact you expect.
|
|
16
|
+
|
|
17
|
+
You will get an answer as soon as possible. Once the issue is confirmed, a fix is released and the advisory is published.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
<svg xmlns="http://www.w3.org/2000/svg" width="114" height="20" role="img" aria-label="coverage:
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="114" height="20" role="img" aria-label="coverage: 92.68%"><title>coverage: 92.68%</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="114" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="61" height="20" fill="#555"/><rect x="61" width="53" height="20" fill="#4c1"/><rect width="114" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="315" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="510">coverage</text><text x="315" y="140" transform="scale(.1)" fill="#fff" textLength="510">coverage</text><text aria-hidden="true" x="865" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">92.68%</text><text x="865" y="140" transform="scale(.1)" fill="#fff" textLength="430">92.68%</text></g></svg>
|
|
@@ -1,12 +1,18 @@
|
|
|
1
1
|
site_author: dhondta
|
|
2
2
|
site_name: DSFF - DataSet File Format
|
|
3
3
|
site_description: Dedicated file format for exchanging datasets
|
|
4
|
-
|
|
4
|
+
site_url: https://python-dsff.readthedocs.io
|
|
5
5
|
repo_url: https://github.com/packing-box/python-dsff
|
|
6
|
+
edit_uri: edit/main/docs/pages/
|
|
7
|
+
copyright: Copyright © 2021 Alexandre D'Hondt
|
|
6
8
|
docs_dir: pages
|
|
7
9
|
nav:
|
|
8
|
-
-
|
|
10
|
+
- Home: index.md
|
|
11
|
+
- Installation: installation.md
|
|
9
12
|
- Usage: usage.md
|
|
13
|
+
- Python API: api.md
|
|
14
|
+
- How it works: internals.md
|
|
15
|
+
- Contributing: contributing.md
|
|
10
16
|
extra:
|
|
11
17
|
generator: false
|
|
12
18
|
social:
|
|
@@ -24,22 +30,30 @@ extra:
|
|
|
24
30
|
name: Alex on Twitter
|
|
25
31
|
theme:
|
|
26
32
|
name: material
|
|
33
|
+
logo: img/logo.png
|
|
34
|
+
favicon: img/icon.png
|
|
27
35
|
palette:
|
|
28
36
|
- scheme: default
|
|
29
37
|
toggle:
|
|
30
|
-
icon: material/brightness-7
|
|
38
|
+
icon: material/brightness-7
|
|
31
39
|
name: Switch to dark mode
|
|
32
40
|
- scheme: slate
|
|
33
41
|
toggle:
|
|
34
42
|
icon: material/brightness-4
|
|
35
43
|
name: Switch to light mode
|
|
36
|
-
|
|
44
|
+
features:
|
|
45
|
+
- content.code.copy
|
|
46
|
+
- navigation.sections
|
|
47
|
+
- navigation.top
|
|
48
|
+
- toc.follow
|
|
37
49
|
use_directory_urls: false
|
|
38
50
|
markdown_extensions:
|
|
39
51
|
- admonition
|
|
40
|
-
-
|
|
41
|
-
|
|
52
|
+
- attr_list
|
|
53
|
+
- def_list
|
|
42
54
|
- pymdownx.details
|
|
55
|
+
- pymdownx.highlight
|
|
56
|
+
- pymdownx.inlinehilite
|
|
43
57
|
- pymdownx.superfences
|
|
44
58
|
- tables
|
|
45
59
|
- toc:
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# Python API
|
|
2
|
+
|
|
3
|
+
Everything is available from the top-level `dsff` package.
|
|
4
|
+
|
|
5
|
+
## `DSFF`
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
dsff.DSFF(path=None, mode=None, logger=None)
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
`path`
|
|
12
|
+
: Path to the `.dsff` file. The `.dsff` extension is appended when missing. Use `"<memory>"` (`dsff.formats.INMEMORY`) for a workbook that is never saved.
|
|
13
|
+
|
|
14
|
+
`mode`
|
|
15
|
+
: One of `r`, `r+`, `w` or `w+` (see [Modes](usage.md#modes)). It defaults to `r` when a path is given and to `w` otherwise.
|
|
16
|
+
|
|
17
|
+
`logger`
|
|
18
|
+
: A `logging.Logger`; defaults to the logger named `DSFF`.
|
|
19
|
+
|
|
20
|
+
Opening a file in `r` or `r+` mode checks that it is a valid DSFF and raises `FileNotFoundError` or `BadDsffFile` otherwise. Opening it in `w` or `w+` mode with a path removes an existing file at that path and starts an empty workbook.
|
|
21
|
+
|
|
22
|
+
The class is a context manager: leaving the context saves the workbook (except in `r` mode, in memory, or when an exception was raised) and closes it.
|
|
23
|
+
|
|
24
|
+
In `r` mode, `write`, `from_*` and setting an item raise `ValueError`.
|
|
25
|
+
|
|
26
|
+
### Methods
|
|
27
|
+
|
|
28
|
+
`write(data=None, features=None, metadata=None, missing="?")`
|
|
29
|
+
: Write any of the three parts of a dataset, then save:
|
|
30
|
+
|
|
31
|
+
- `data`: a matrix (list of rows, the first one holding the headers), the path to a `;`-delimited `data.csv`, or the path to a FilelessDataset folder (which also provides `features.json` and `metadata.json`);
|
|
32
|
+
- `features`: a dictionary of feature names and descriptions, or the path to a `features.json`;
|
|
33
|
+
- `metadata`: a dictionary, or the path to a `metadata.json`.
|
|
34
|
+
|
|
35
|
+
The worksheets that are written are cleared first. `None` values in `data` are written as `missing`. Booleans, integers and floats, including strings that hold one (such as `"1"` from a CSV), are stored as such; any other value is stored as a string. Invalid input raises `BadInputData`.
|
|
36
|
+
|
|
37
|
+
`save()`
|
|
38
|
+
: Save the workbook. Only defined in the `r+`, `w` and `w+` modes.
|
|
39
|
+
|
|
40
|
+
`close()`
|
|
41
|
+
: Close the workbook without saving.
|
|
42
|
+
|
|
43
|
+
`to_<format>(path=None, ...)`
|
|
44
|
+
: Convert the dataset to another format, where `<format>` is `arff`, `csv`, `dataset`, `db`, and with `pyarrow` installed, `feather`, `orc` or `parquet`. `path` defaults to the name of the DSFF. The extension of the format is appended unless `path` already ends with it (no extension for `dataset`, which is a folder). Except for `to_dataset`, `text=True` returns the content (as a string, or bytes for Arrow formats) instead of writing a file. An empty DSFF raises `EmptyDsffFile`.
|
|
45
|
+
|
|
46
|
+
`from_<format>(path=None, ...)`
|
|
47
|
+
: Populate the DSFF from a file in another format (same formats as above). `path` is used as is if it is a file, otherwise with the extension of the format appended.
|
|
48
|
+
|
|
49
|
+
Format-specific keyword arguments:
|
|
50
|
+
|
|
51
|
+
| Method | Argument | Default | Meaning |
|
|
52
|
+
|:--|:--|:--|:--|
|
|
53
|
+
| `to_arff`, `from_arff` | `target` | `"label"` | Name of the column holding the class |
|
|
54
|
+
| `to_arff`, `from_arff` | `missing` | `"?"` | Token for missing values |
|
|
55
|
+
| `to_arff`, `load_csv` | `exclude` | `("hash", "realpath", "format", "size", "ctime", "mtime")` | Columns that are not features |
|
|
56
|
+
| `to_db` | `primary_index` | `0` | Index of the column used as the primary key, if its values are unique and none is missing |
|
|
57
|
+
| `to_db` | `missing` | `"?"` | Token for missing values, stored as `NULL` |
|
|
58
|
+
|
|
59
|
+
### Items and properties
|
|
60
|
+
|
|
61
|
+
`f["data"]`, `f["features"]`
|
|
62
|
+
: The `openpyxl` worksheets.
|
|
63
|
+
|
|
64
|
+
`f[name]`, `f[name] = value`
|
|
65
|
+
: A standard XSLX document property (such as `creator` or `title`) if it exists, otherwise a key of the metadata. Setting a value also stores it in the metadata. `description` always refers to the metadata, as this property holds it.
|
|
66
|
+
|
|
67
|
+
`len(f)`
|
|
68
|
+
: Number of rows in the `data` worksheet, headers included.
|
|
69
|
+
|
|
70
|
+
| Property | Description |
|
|
71
|
+
|:--|:--|
|
|
72
|
+
| `data` | Rows of the `data` worksheet, with values evaluated as Python literals when possible |
|
|
73
|
+
| `features` | Dictionary of feature names and descriptions |
|
|
74
|
+
| `headers` | Names of the features |
|
|
75
|
+
| `metadata` | Dictionary of metadata |
|
|
76
|
+
| `name` | Name of the dataset (settable): the XSLX `title`, or the file name without extension |
|
|
77
|
+
| `path` | Path to the `.dsff` file |
|
|
78
|
+
| `mode` | Mode the file was opened with |
|
|
79
|
+
| `logger` | Logger in use |
|
|
80
|
+
|
|
81
|
+
### Static methods
|
|
82
|
+
|
|
83
|
+
`DSFF.format(text_or_path)`
|
|
84
|
+
: Return the name of the format (`arff`, `csv`, `db`, ...) of the given text, bytes or path, or `None` if none matches.
|
|
85
|
+
|
|
86
|
+
`DSFF.is_valid_format(text_or_path)`
|
|
87
|
+
: Whether `DSFF.format` recognizes the input.
|
|
88
|
+
|
|
89
|
+
## Module-level functions
|
|
90
|
+
|
|
91
|
+
For each format, `dsff` exposes:
|
|
92
|
+
|
|
93
|
+
`is_<format>(text_or_path, ...)`
|
|
94
|
+
: Whether the input (text, bytes or path; a path for `is_dataset`) is valid in this format. A string is taken as a path if it is a single line and points to an existing file or folder. `is_dsff` checks for a DSFF file.
|
|
95
|
+
|
|
96
|
+
`load_<format>(path, ...)`
|
|
97
|
+
: Load the file as a dictionary with the `data`, `features` and `metadata` keys, which can be passed to `DSFF.write(**...)`.
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
>>> import dsff
|
|
101
|
+
>>> dsff.is_csv("a;b\n1;2\n")
|
|
102
|
+
True
|
|
103
|
+
>>> d = dsff.load_csv("my-dataset.csv")
|
|
104
|
+
>>> with dsff.DSFF("my-dataset", "w") as f:
|
|
105
|
+
... f.write(**d)
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Exceptions
|
|
109
|
+
|
|
110
|
+
These exceptions are registered as builtins when `dsff` is imported:
|
|
111
|
+
|
|
112
|
+
| Exception | Base | Raised when |
|
|
113
|
+
|:--|:--|:--|
|
|
114
|
+
| `BadDsffFile` | `OSError` | The file is not a valid DSFF |
|
|
115
|
+
| `BadInputData` | `ValueError` | Input to `write` or `from_*` is invalid |
|
|
116
|
+
| `EmptyDsffFile` | `ValueError` | Converting a DSFF without data |
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Contributions are welcome, for instance support for new formats.
|
|
4
|
+
|
|
5
|
+
## Reporting issues
|
|
6
|
+
|
|
7
|
+
Open an issue on [GitHub](https://github.com/packing-box/python-dsff/issues) and include:
|
|
8
|
+
|
|
9
|
+
- the DSFF version (`python -c "import dsff; print(dsff.__version__)"`);
|
|
10
|
+
- the code that triggers the issue and its output;
|
|
11
|
+
- if possible, a sample file.
|
|
12
|
+
|
|
13
|
+
## Security
|
|
14
|
+
|
|
15
|
+
Please do **not** report vulnerabilities in public issues. See the [security policy](https://github.com/packing-box/python-dsff/blob/main/SECURITY.md): report them privately through the repository's *Security* tab (*Report a vulnerability*).
|
|
16
|
+
|
|
17
|
+
Dependencies are monitored by Dependabot and audited with [pip-audit](https://github.com/pypa/pip-audit) by the `security` workflow (`.github/workflows/security.yml`), on every push and pull request to `main` and weekly.
|
|
18
|
+
|
|
19
|
+
## Adding a format
|
|
20
|
+
|
|
21
|
+
Add a module to `src/dsff/formats/` that defines `from_<format>`, `is_<format>`, `load_<format>` and `to_<format>` (see [How it works](internals.md#format-conversions)), list them in its `__all__` and import it in `src/dsff/formats/__init__.py`. The `from_*` and `to_*` functions are then bound to `DSFF` automatically.
|
|
22
|
+
|
|
23
|
+
## Development setup
|
|
24
|
+
|
|
25
|
+
```console
|
|
26
|
+
$ git clone https://github.com/packing-box/python-dsff
|
|
27
|
+
$ cd python-dsff
|
|
28
|
+
$ pip install -e .[all]
|
|
29
|
+
$ pip install pytest pytest-cov
|
|
30
|
+
$ pytest --cov=dsff
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
CI (`.github/workflows/python-package.yml`) runs the tests on Python 3.10 to 3.13 and publishes the coverage badge from the Python 3.13 run.
|
|
34
|
+
|
|
35
|
+
## Releasing
|
|
36
|
+
|
|
37
|
+
The version is stored in `src/dsff/VERSION.txt`. On every push to `main`, CI checks whether that version is already on PyPI. If not, it builds the package and publishes it.
|
|
38
|
+
|
|
39
|
+
## Building this documentation
|
|
40
|
+
|
|
41
|
+
```console
|
|
42
|
+
$ pip install -r docs/requirements.txt
|
|
43
|
+
$ mkdocs serve -f docs/mkdocs.yml # live preview at http://127.0.0.1:8000
|
|
44
|
+
$ mkdocs build --strict -f docs/mkdocs.yml # what Read the Docs runs
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Read the Docs builds the site from `.readthedocs.yaml` at the root of the repository.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# DataSet File Format
|
|
2
|
+
|
|
3
|
+
<p align="center"><img src="img/logo.png" alt="DSFF logo" width="300"></p>
|
|
4
|
+
|
|
5
|
+
**DSFF** (DataSet File Format) is a tiny library relying on [`openpyxl`](https://pypi.org/project/openpyxl) that stores a dataset with its features, for use with machine learning, in an XSLX file whose structure is enforced. It makes it easy to store, edit and exchange a dataset.
|
|
6
|
+
|
|
7
|
+
[](https://pypi.python.org/pypi/dsff/)
|
|
8
|
+
[](https://pypi.python.org/pypi/dsff/)
|
|
9
|
+
[](https://github.com/packing-box/python-dsff/actions/workflows/python-package.yml)
|
|
10
|
+
[](https://github.com/packing-box/python-dsff/actions/workflows/security.yml)
|
|
11
|
+
[](https://pypi.python.org/pypi/dsff/)
|
|
12
|
+
|
|
13
|
+
It is used with the [Packing Box](https://github.com/packing-box/docker-packing-box) to export datasets in a convenient format.
|
|
14
|
+
|
|
15
|
+
-----
|
|
16
|
+
|
|
17
|
+
## Features
|
|
18
|
+
|
|
19
|
+
- A single `.dsff` file holds the data, the description of each feature and the metadata of the dataset.
|
|
20
|
+
- The file is a regular XSLX workbook: it opens in any spreadsheet editor.
|
|
21
|
+
- Conversion from and to [ARFF](https://www.cs.waikato.ac.nz/ml/weka/arff.html) (for [Weka](https://www.cs.waikato.ac.nz/ml/weka)), [CSV](https://www.rfc-editor.org/rfc/rfc4180), [SQLite](https://www.sqlite.org/fileformat.html) databases and [FilelessDataset](https://docker-packing-box.readthedocs.io/en/latest/usage/datasets.html) folders from the Packing Box.
|
|
22
|
+
- With [`pyarrow`](https://arrow.apache.org/docs/python/index.html), conversion from and to the [Apache Arrow](https://arrow.apache.org/) formats Feather, ORC and Parquet.
|
|
23
|
+
|
|
24
|
+
## Quick start
|
|
25
|
+
|
|
26
|
+
```console
|
|
27
|
+
$ pip install dsff
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
>>> import dsff
|
|
32
|
+
>>> with dsff.DSFF("my-dataset", "w") as f:
|
|
33
|
+
... f.write(data=[["name", "size", "label"], ["a.exe", 1024, "upx"], ["b.exe", 2048, "?"]],
|
|
34
|
+
... features={'size': "size of the executable"},
|
|
35
|
+
... metadata={'source': "my lab"})
|
|
36
|
+
... f.to_csv() # creates ./my-dataset.csv
|
|
37
|
+
...
|
|
38
|
+
# while leaving the context, ./my-dataset.dsff is created
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
See [Installation](installation.md) and [Usage](usage.md) to go further.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# Installation
|
|
2
|
+
|
|
3
|
+
## Requirements
|
|
4
|
+
|
|
5
|
+
- Python **3.10 or later**. CI runs on 3.10 to 3.13.
|
|
6
|
+
- [`openpyxl`](https://pypi.org/project/openpyxl/), which pip installs automatically.
|
|
7
|
+
- Optionally, [`pyarrow`](https://arrow.apache.org/docs/python/index.html) (with [`pandas`](https://pypi.org/project/pandas/)) for the [Apache Arrow](https://arrow.apache.org/) formats.
|
|
8
|
+
|
|
9
|
+
## From PyPI
|
|
10
|
+
|
|
11
|
+
```console
|
|
12
|
+
$ pip install dsff
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
To also handle the Apache Arrow formats (Feather, ORC and Parquet), install the `arrow` extra:
|
|
16
|
+
|
|
17
|
+
```console
|
|
18
|
+
$ pip install dsff[arrow]
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
The `all` extra installs every available extension:
|
|
22
|
+
|
|
23
|
+
```console
|
|
24
|
+
$ pip install dsff[all]
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## From source
|
|
28
|
+
|
|
29
|
+
```console
|
|
30
|
+
$ git clone https://github.com/packing-box/python-dsff
|
|
31
|
+
$ cd python-dsff
|
|
32
|
+
$ pip install .
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
For development, use an editable install:
|
|
36
|
+
|
|
37
|
+
```console
|
|
38
|
+
$ pip install -e .
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Checking the installation
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
>>> import dsff
|
|
45
|
+
>>> dsff.__version__
|
|
46
|
+
'1.2.3'
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
The `to_feather`, `to_orc` and `to_parquet` methods (and their `from_*` counterparts) only exist when `pyarrow` and `pandas` can be imported:
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
>>> from dsff.formats import PYARROW
|
|
53
|
+
>>> PYARROW
|
|
54
|
+
True
|
|
55
|
+
```
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# How it works
|
|
2
|
+
|
|
3
|
+
## File format
|
|
4
|
+
|
|
5
|
+
DSFF is straightforward and contains only the minimum for storing a dataset. A `.dsff` file is an XSLX workbook (a ZIP archive) with two and only two worksheets, in this order:
|
|
6
|
+
|
|
7
|
+
1. `data`: the matrix of the whole dataset, headers included. It may contain the samples' metadata (such as `hash` or `realpath`) but mostly holds the feature values. The class column is named `label`. Missing values are written as `?`.
|
|
8
|
+
2. `features`: the name-description pairs of each feature used in `data`, under two headers: `name` and `description`.
|
|
9
|
+
|
|
10
|
+
The following document properties of the XSLX format are used:
|
|
11
|
+
|
|
12
|
+
- `title`: the name of the dataset;
|
|
13
|
+
- `description`: the metadata of the dataset, serialized as a JSON dictionary;
|
|
14
|
+
- `creator`: the user who saved the file, unless already set.
|
|
15
|
+
|
|
16
|
+
Booleans, integers and floats are stored as such in the cells, and any other value as a string. When read, a string holding a boolean, an integer or a float (as in files saved by older versions, which stored every value as a string) is converted, so that `"1"` comes back as an integer and `"2.5"` as a float.
|
|
17
|
+
|
|
18
|
+
## Validation
|
|
19
|
+
|
|
20
|
+
When a file is opened in `r` or `r+` mode, DSFF:
|
|
21
|
+
|
|
22
|
+
1. loads it with `openpyxl` from an open file, as `openpyxl` rejects the `.dsff` extension of a path; a file it cannot load raises `BadDsffFile`;
|
|
23
|
+
2. checks that the worksheets are exactly `data` and `features`;
|
|
24
|
+
3. checks that the first row of `features` is `name`, `description`.
|
|
25
|
+
|
|
26
|
+
Note that XSLX limits a worksheet to 1,048,576 rows.
|
|
27
|
+
|
|
28
|
+
## Format conversions
|
|
29
|
+
|
|
30
|
+
Each format lives in a module of `dsff.formats` (`arff`, `csv`, `dataset`, `db` and `pa` for Apache Arrow) that defines four functions:
|
|
31
|
+
|
|
32
|
+
- `is_<format>`: check whether some input is in this format;
|
|
33
|
+
- `load_<format>`: parse a file into a `{'data': ..., 'features': ..., 'metadata': ...}` dictionary;
|
|
34
|
+
- `from_<format>`: `load_<format>`, then `DSFF.write`;
|
|
35
|
+
- `to_<format>`: write the DSFF in this format.
|
|
36
|
+
|
|
37
|
+
When a `DSFF` is created, every `from_*` and `to_*` function is bound to the instance as a method. The wrapper fixes the target path (default name, extension) and, for `to_*`, refuses to convert an empty dataset. The Arrow functions are generated for Feather, ORC and Parquet, and are only defined when `pyarrow` and `pandas` can be imported.
|
|
38
|
+
|
|
39
|
+
| Format | Data | Features | Metadata |
|
|
40
|
+
|:--|:-:|:-:|:-:|
|
|
41
|
+
| ARFF | yes | as comments | as a comment |
|
|
42
|
+
| CSV | yes | no (names only on import) | no |
|
|
43
|
+
| SQLite (`db`), missing values as `NULL` | `data` table | `features` table | `metadata` table |
|
|
44
|
+
| FilelessDataset | `data.csv` | `features.json` | `metadata.json` |
|
|
45
|
+
| Feather, ORC, Parquet | table | schema metadata | schema metadata (`__metadata__`) |
|
|
46
|
+
|
|
47
|
+
When exporting to ARFF, the columns listed in `exclude` are dropped, the `label` column (if any) becomes the last attribute, a nominal one named `class`, and each other column becomes a `NUMERIC` attribute if all its values are numbers, a `STRING` one otherwise. String values are quoted, with backslash escapes. When importing, unquoted names, comments before `@RELATION` and quoted values are supported, and only the comments named after an attribute become feature descriptions.
|