datapruning 1.0.2__tar.gz → 1.0.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datapruning-1.0.5/MANIFEST.in +2 -0
- {datapruning-1.0.2/datapruning.egg-info → datapruning-1.0.5}/PKG-INFO +73 -79
- datapruning-1.0.5/README.md +51 -0
- {datapruning-1.0.2 → datapruning-1.0.5}/build_src/_core.c +37974 -38400
- datapruning-1.0.5/build_src/_core.pyx +227 -0
- datapruning-1.0.5/datapruning/__init__.py +61 -0
- datapruning-1.0.5/datapruning/pipeline.py +188 -0
- {datapruning-1.0.2 → datapruning-1.0.5/datapruning.egg-info}/PKG-INFO +73 -79
- {datapruning-1.0.2 → datapruning-1.0.5}/datapruning.egg-info/SOURCES.txt +3 -3
- {datapruning-1.0.2 → datapruning-1.0.5}/pyproject.toml +4 -1
- {datapruning-1.0.2 → datapruning-1.0.5}/setup.cfg +4 -4
- datapruning-1.0.5/setup.py +43 -0
- datapruning-1.0.2/LICENSE +0 -21
- datapruning-1.0.2/MANIFEST.in +0 -1
- datapruning-1.0.2/README.md +0 -55
- datapruning-1.0.2/datapruning/__init__.py +0 -23
- datapruning-1.0.2/setup.py +0 -45
- datapruning-1.0.2/tests/test_simple.py +0 -48
- {datapruning-1.0.2 → datapruning-1.0.5}/datapruning.egg-info/dependency_links.txt +0 -0
- {datapruning-1.0.2 → datapruning-1.0.5}/datapruning.egg-info/requires.txt +0 -0
- {datapruning-1.0.2 → datapruning-1.0.5}/datapruning.egg-info/top_level.txt +0 -0
|
@@ -1,79 +1,73 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: datapruning
|
|
3
|
-
Version: 1.0.
|
|
4
|
-
Summary: Intelligent data pruning for ML datasets
|
|
5
|
-
License: Proprietary
|
|
6
|
-
Project-URL: Homepage, https://www.datapruning.com
|
|
7
|
-
Classifier: Development Status :: 4 - Beta
|
|
8
|
-
Classifier: Intended Audience :: Developers
|
|
9
|
-
Classifier: Intended Audience :: Science/Research
|
|
10
|
-
Classifier: Programming Language :: Python :: 3
|
|
11
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
-
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
-
Requires-Python: >=3.10
|
|
18
|
-
Description-Content-Type: text/markdown
|
|
19
|
-
|
|
20
|
-
Requires-Dist:
|
|
21
|
-
Requires-Dist:
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
##
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
Datasets exceeding 100,000 rows require an enterprise license.
|
|
75
|
-
Visit https://www.datapruning.com for details.
|
|
76
|
-
|
|
77
|
-
## License
|
|
78
|
-
|
|
79
|
-
Proprietary software. See LICENSE file.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: datapruning
|
|
3
|
+
Version: 1.0.5
|
|
4
|
+
Summary: Intelligent data pruning for ML datasets
|
|
5
|
+
License: Proprietary
|
|
6
|
+
Project-URL: Homepage, https://www.datapruning.com
|
|
7
|
+
Classifier: Development Status :: 4 - Beta
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: Intended Audience :: Science/Research
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Requires-Dist: numpy>=1.24.0
|
|
20
|
+
Requires-Dist: pandas>=2.0.0
|
|
21
|
+
Requires-Dist: torch>=2.0.0
|
|
22
|
+
|
|
23
|
+
# DataPruning
|
|
24
|
+
|
|
25
|
+
Intelligent dataset pruning for ML — reduces dataset size by selecting the most informative rows.
|
|
26
|
+
|
|
27
|
+
## Installation
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install datapruning
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
**Note:** PyTorch (~2 GB) is a required dependency.
|
|
34
|
+
|
|
35
|
+
## Requirements
|
|
36
|
+
|
|
37
|
+
- Python >= 3.10
|
|
38
|
+
- PyTorch 2.0+
|
|
39
|
+
- Pandas 2.0+
|
|
40
|
+
- NumPy 1.24+
|
|
41
|
+
|
|
42
|
+
## Limits
|
|
43
|
+
|
|
44
|
+
- Minimum: 1,000 rows per dataset
|
|
45
|
+
- Maximum: 300,000 rows per dataset
|
|
46
|
+
|
|
47
|
+
## Usage
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
import pandas as pd
|
|
51
|
+
from datapruning import Pruner
|
|
52
|
+
|
|
53
|
+
df = pd.read_csv("dataset.csv")
|
|
54
|
+
p = Pruner(df)
|
|
55
|
+
filtered = p.prune(target_col="label", keep_ratio=0.5)
|
|
56
|
+
|
|
57
|
+
print(f"Kept {len(filtered)} / {len(df)} rows")
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Features
|
|
61
|
+
|
|
62
|
+
- Runs locally, no data upload
|
|
63
|
+
- Pre-training dataset optimization
|
|
64
|
+
- Pandas DataFrame input / output
|
|
65
|
+
- Compiled processing module
|
|
66
|
+
|
|
67
|
+
## Links
|
|
68
|
+
|
|
69
|
+
- Website: [datapruning.com](https://www.datapruning.com)
|
|
70
|
+
|
|
71
|
+
## License
|
|
72
|
+
|
|
73
|
+
Proprietary. See LICENSE file.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# DataPruning
|
|
2
|
+
|
|
3
|
+
Intelligent dataset pruning for ML — reduces dataset size by selecting the most informative rows.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install datapruning
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
**Note:** PyTorch (~2 GB) is a required dependency.
|
|
12
|
+
|
|
13
|
+
## Requirements
|
|
14
|
+
|
|
15
|
+
- Python >= 3.10
|
|
16
|
+
- PyTorch 2.0+
|
|
17
|
+
- Pandas 2.0+
|
|
18
|
+
- NumPy 1.24+
|
|
19
|
+
|
|
20
|
+
## Limits
|
|
21
|
+
|
|
22
|
+
- Minimum: 1,000 rows per dataset
|
|
23
|
+
- Maximum: 300,000 rows per dataset
|
|
24
|
+
|
|
25
|
+
## Usage
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
import pandas as pd
|
|
29
|
+
from datapruning import Pruner
|
|
30
|
+
|
|
31
|
+
df = pd.read_csv("dataset.csv")
|
|
32
|
+
p = Pruner(df)
|
|
33
|
+
filtered = p.prune(target_col="label", keep_ratio=0.5)
|
|
34
|
+
|
|
35
|
+
print(f"Kept {len(filtered)} / {len(df)} rows")
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Features
|
|
39
|
+
|
|
40
|
+
- Runs locally, no data upload
|
|
41
|
+
- Pre-training dataset optimization
|
|
42
|
+
- Pandas DataFrame input / output
|
|
43
|
+
- Compiled processing module
|
|
44
|
+
|
|
45
|
+
## Links
|
|
46
|
+
|
|
47
|
+
- Website: [datapruning.com](https://www.datapruning.com)
|
|
48
|
+
|
|
49
|
+
## License
|
|
50
|
+
|
|
51
|
+
Proprietary. See LICENSE file.
|