ml-datasets 0.2.0a0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ml_datasets-0.2.2/LICENSE +21 -0
- ml_datasets-0.2.2/PKG-INFO +175 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/README.md +1 -1
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/__init__.py +2 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/__init__.py +1 -0
- ml_datasets-0.2.2/ml_datasets/loaders/cifar.py +81 -0
- ml_datasets-0.2.2/ml_datasets/loaders/mnist.py +175 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/reuters.py +5 -5
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/spacy_readers.py +4 -4
- ml_datasets-0.2.2/ml_datasets/test/__init__.py +0 -0
- ml_datasets-0.2.2/ml_datasets/test/test_datasets.py +114 -0
- ml_datasets-0.2.2/ml_datasets/test/test_spacy_readers.py +18 -0
- ml_datasets-0.2.2/ml_datasets/test/test_util.py +74 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/util.py +49 -16
- ml_datasets-0.2.2/ml_datasets.egg-info/PKG-INFO +175 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets.egg-info/SOURCES.txt +8 -1
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets.egg-info/entry_points.txt +0 -1
- ml_datasets-0.2.2/ml_datasets.egg-info/requires.txt +5 -0
- ml_datasets-0.2.2/pyproject.toml +9 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/setup.cfg +5 -4
- ml_datasets-0.2.0a0/PKG-INFO +0 -169
- ml_datasets-0.2.0a0/ml_datasets/loaders/mnist.py +0 -44
- ml_datasets-0.2.0a0/ml_datasets.egg-info/PKG-INFO +0 -169
- ml_datasets-0.2.0a0/ml_datasets.egg-info/requires.txt +0 -4
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/_registry.py +0 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/cmu.py +0 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/dbpedia.py +0 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/imdb.py +0 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/quora.py +0 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/snli.py +0 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/stack_exchange.py +0 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/universal_dependencies.py +0 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/wikiner.py +0 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets.egg-info/dependency_links.txt +0 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets.egg-info/top_level.txt +0 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets.egg-info/zip-safe +0 -0
- {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/setup.py +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2019 ExplosionAI GmbH
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ml_datasets
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Machine Learning dataset loaders
|
|
5
|
+
Home-page: https://github.com/explosion/ml-datasets
|
|
6
|
+
Author: Explosion
|
|
7
|
+
Author-email: contact@explosion.ai
|
|
8
|
+
License: MIT
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: cloudpickle>=2.2
|
|
13
|
+
Requires-Dist: numpy>=1.21.2
|
|
14
|
+
Requires-Dist: tqdm<5.0.0,>=4.10.0
|
|
15
|
+
Requires-Dist: srsly<4.0.0,>=1.0.1
|
|
16
|
+
Requires-Dist: catalogue<3.0.0,>=0.2.0
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
<a href="https://explosion.ai"><img src="https://explosion.ai/assets/img/logo.svg" width="125" height="125" align="right" /></a>
|
|
20
|
+
|
|
21
|
+
# Machine learning dataset loaders for testing and examples
|
|
22
|
+
|
|
23
|
+
Loaders for various machine learning datasets for testing and example scripts.
|
|
24
|
+
Previously in `thinc.extra.datasets`.
|
|
25
|
+
|
|
26
|
+
[](https://pypi.python.org/pypi/ml-datasets)
|
|
27
|
+
|
|
28
|
+
## Setup and installation
|
|
29
|
+
|
|
30
|
+
The package can be installed via pip:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install ml-datasets
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Loaders
|
|
37
|
+
|
|
38
|
+
Loaders can be imported directly or used via their string name (which is useful if they're set via command line arguments). Some loaders may take arguments – see the source for details.
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
# Import directly
|
|
42
|
+
from ml_datasets import imdb
|
|
43
|
+
train_data, dev_data = imdb()
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
# Load via registry
|
|
48
|
+
from ml_datasets import loaders
|
|
49
|
+
imdb_loader = loaders.get("imdb")
|
|
50
|
+
train_data, dev_data = imdb_loader()
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### Available loaders
|
|
54
|
+
|
|
55
|
+
#### NLP datasets
|
|
56
|
+
|
|
57
|
+
| ID / Function | Description | NLP task | From URL |
|
|
58
|
+
| -------------------- | -------------------------------------------- | ----------------------------------------- | :------: |
|
|
59
|
+
| `imdb` | IMDB sentiment dataset | Binary classification: sentiment analysis | ✓ |
|
|
60
|
+
| `dbpedia` | DBPedia ontology dataset | Multi-class single-label classification | ✓ |
|
|
61
|
+
| `cmu` | CMU movie genres dataset | Multi-class, multi-label classification | ✓ |
|
|
62
|
+
| `quora_questions` | Duplicate Quora questions dataset | Detecting duplicate questions | ✓ |
|
|
63
|
+
| `reuters` | Reuters dataset (texts not included) | Multi-class multi-label classification | ✓ |
|
|
64
|
+
| `snli` | Stanford Natural Language Inference corpus | Recognizing textual entailment | ✓ |
|
|
65
|
+
| `stack_exchange` | Stack Exchange dataset | Question Answering | |
|
|
66
|
+
| `ud_ancora_pos_tags` | Universal Dependencies Spanish AnCora corpus | POS tagging | ✓ |
|
|
67
|
+
| `ud_ewtb_pos_tags` | Universal Dependencies English EWT corpus | POS tagging | ✓ |
|
|
68
|
+
| `wikiner` | WikiNER data | Named entity recognition | |
|
|
69
|
+
|
|
70
|
+
#### Other ML datasets
|
|
71
|
+
|
|
72
|
+
| ID / Function | Description | ML task | From URL |
|
|
73
|
+
| ------------- | ----------- | ----------------- | :------: |
|
|
74
|
+
| `mnist` | MNIST data | Image recognition | ✓ |
|
|
75
|
+
|
|
76
|
+
### Dataset details
|
|
77
|
+
|
|
78
|
+
#### IMDB
|
|
79
|
+
|
|
80
|
+
Each instance contains the text of a movie review, and a sentiment expressed as `0` or `1`.
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
train_data, dev_data = ml_datasets.imdb()
|
|
84
|
+
for text, annot in train_data[0:5]:
|
|
85
|
+
print(f"Review: {text}")
|
|
86
|
+
print(f"Sentiment: {annot}")
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
- Download URL: [http://ai.stanford.edu/~amaas/data/sentiment/](http://ai.stanford.edu/~amaas/data/sentiment/)
|
|
90
|
+
- Citation: [Andrew L. Maas et al., 2011](https://www.aclweb.org/anthology/P11-1015/)
|
|
91
|
+
|
|
92
|
+
| Property | Training | Dev |
|
|
93
|
+
| ------------------- | ---------------- | ---------------- |
|
|
94
|
+
| # Instances | 25000 | 25000 |
|
|
95
|
+
| Label values | {`0`, `1`} | {`0`, `1`} |
|
|
96
|
+
| Labels per instance | Single | Single |
|
|
97
|
+
| Label distribution | Balanced (50/50) | Balanced (50/50) |
|
|
98
|
+
|
|
99
|
+
#### DBPedia
|
|
100
|
+
|
|
101
|
+
Each instance contains an ontological description, and a classification into one of the 14 distinct labels.
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
train_data, dev_data = ml_datasets.dbpedia()
|
|
105
|
+
for text, annot in train_data[0:5]:
|
|
106
|
+
print(f"Text: {text}")
|
|
107
|
+
print(f"Category: {annot}")
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
- Download URL: [Via fast.ai](https://course.fast.ai/datasets)
|
|
111
|
+
- Original citation: [Xiang Zhang et al., 2015](https://arxiv.org/abs/1509.01626)
|
|
112
|
+
|
|
113
|
+
| Property | Training | Dev |
|
|
114
|
+
| ------------------- | -------- | -------- |
|
|
115
|
+
| # Instances | 560000 | 70000 |
|
|
116
|
+
| Label values | `1`-`14` | `1`-`14` |
|
|
117
|
+
| Labels per instance | Single | Single |
|
|
118
|
+
| Label distribution | Balanced | Balanced |
|
|
119
|
+
|
|
120
|
+
#### CMU
|
|
121
|
+
|
|
122
|
+
Each instance contains a movie description, and a classification into a list of appropriate genres.
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
train_data, dev_data = ml_datasets.cmu()
|
|
126
|
+
for text, annot in train_data[0:5]:
|
|
127
|
+
print(f"Text: {text}")
|
|
128
|
+
print(f"Genres: {annot}")
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
- Download URL: [http://www.cs.cmu.edu/~ark/personas/](http://www.cs.cmu.edu/~ark/personas/)
|
|
132
|
+
- Original citation: [David Bamman et al., 2013](https://www.aclweb.org/anthology/P13-1035/)
|
|
133
|
+
|
|
134
|
+
| Property | Training | Dev |
|
|
135
|
+
| ------------------- | --------------------------------------------------------------------------------------------- | --- |
|
|
136
|
+
| # Instances | 41793 | 0 |
|
|
137
|
+
| Label values | 363 different genres | - |
|
|
138
|
+
| Labels per instance | Multiple | - |
|
|
139
|
+
| Label distribution | Imbalanced: 147 labels with less than 20 examples, while `Drama` occurs more than 19000 times | - |
|
|
140
|
+
|
|
141
|
+
#### Quora
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
train_data, dev_data = ml_datasets.quora_questions()
|
|
145
|
+
for questions, annot in train_data[0:50]:
|
|
146
|
+
q1, q2 = questions
|
|
147
|
+
print(f"Question 1: {q1}")
|
|
148
|
+
print(f"Question 2: {q2}")
|
|
149
|
+
print(f"Similarity: {annot}")
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Each instance contains two quora questions, and a label indicating whether or not they are duplicates (`0`: no, `1`: yes).
|
|
153
|
+
The ground-truth labels contain some amount of noise: they are not guaranteed to be perfect.
|
|
154
|
+
|
|
155
|
+
- Download URL: [http://qim.fs.quoracdn.net/quora_duplicate_questions.tsv](http://qim.fs.quoracdn.net/quora_duplicate_questions.tsv)
|
|
156
|
+
- Original citation: [Kornél Csernai et al., 2017](https://www.quora.com/q/quoradata/First-Quora-Dataset-Release-Question-Pairs)
|
|
157
|
+
|
|
158
|
+
| Property | Training | Dev |
|
|
159
|
+
| ------------------- | ------------------------- | ------------------------- |
|
|
160
|
+
| # Instances | 363859 | 40429 |
|
|
161
|
+
| Label values | {`0`, `1`} | {`0`, `1`} |
|
|
162
|
+
| Labels per instance | Single | Single |
|
|
163
|
+
| Label distribution | Imbalanced: 63% label `0` | Imbalanced: 63% label `0` |
|
|
164
|
+
|
|
165
|
+
### Registering loaders
|
|
166
|
+
|
|
167
|
+
Loaders can be registered externally using the `loaders` registry as a decorator. For example:
|
|
168
|
+
|
|
169
|
+
```python
|
|
170
|
+
@ml_datasets.loaders("my_custom_loader")
|
|
171
|
+
def my_custom_loader():
|
|
172
|
+
return load_some_data()
|
|
173
|
+
|
|
174
|
+
assert "my_custom_loader" in ml_datasets.loaders
|
|
175
|
+
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
<a href="https://explosion.ai"><img src="https://explosion.ai/assets/img/logo.svg" width="125" height="125" align="right" /></a>
|
|
2
2
|
|
|
3
|
-
# Machine learning dataset loaders
|
|
3
|
+
# Machine learning dataset loaders for testing and examples
|
|
4
4
|
|
|
5
5
|
Loaders for various machine learning datasets for testing and example scripts.
|
|
6
6
|
Previously in `thinc.extra.datasets`.
|
|
@@ -8,3 +8,5 @@ from .loaders.stack_exchange import stack_exchange
|
|
|
8
8
|
from .loaders.universal_dependencies import ud_ancora_pos_tags, ud_ewtb_pos_tags
|
|
9
9
|
from .loaders.dbpedia import dbpedia
|
|
10
10
|
from .loaders.cmu import cmu
|
|
11
|
+
from .loaders.cifar import cifar
|
|
12
|
+
from .loaders.wikiner import wikiner
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import pickle
|
|
2
|
+
import tarfile
|
|
3
|
+
import random
|
|
4
|
+
import numpy
|
|
5
|
+
|
|
6
|
+
from ..util import get_file, unzip, to_categorical
|
|
7
|
+
|
|
8
|
+
CIFAR10_URL = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
|
|
9
|
+
CIFAR100_URL = "https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def cifar(variant='10', channels_last=False, shuffle=True):
|
|
13
|
+
if variant == '10':
|
|
14
|
+
data = load_cifar10()
|
|
15
|
+
elif variant == '100':
|
|
16
|
+
data = load_cifar100(coarse=False)
|
|
17
|
+
elif variant == '100-coarse':
|
|
18
|
+
data = load_cifar100(coarse=True)
|
|
19
|
+
else:
|
|
20
|
+
raise ValueError("Variant must be one of: '10', '100', 100-coarse")
|
|
21
|
+
X_train, y_train, X_test, y_test = data
|
|
22
|
+
X_train = X_train.astype("float32")
|
|
23
|
+
X_test = X_test.astype("float32")
|
|
24
|
+
X_train /= 255.0
|
|
25
|
+
X_test /= 255.0
|
|
26
|
+
if shuffle:
|
|
27
|
+
train_data = list(zip(X_train, y_train))
|
|
28
|
+
random.shuffle(train_data)
|
|
29
|
+
X_train, y_train = unzip(train_data)
|
|
30
|
+
if channels_last:
|
|
31
|
+
X_train = X_train.reshape(X_train.shape[0], 32, 32, 3)
|
|
32
|
+
X_test = X_test.reshape(X_test.shape[0], 32, 32, 3)
|
|
33
|
+
else:
|
|
34
|
+
X_train = X_train.reshape(X_train.shape[0], 3, 32, 32)
|
|
35
|
+
X_test = X_test.reshape(X_test.shape[0], 3, 32, 32)
|
|
36
|
+
y_train = to_categorical(y_train)
|
|
37
|
+
y_test = to_categorical(y_test)
|
|
38
|
+
return (X_train, y_train), (X_test, y_test)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def load_cifar10(path='cifar-10-python.tar.gz'):
|
|
42
|
+
path = get_file(path, origin=CIFAR10_URL)
|
|
43
|
+
train_images = []
|
|
44
|
+
train_labels = []
|
|
45
|
+
with tarfile.open(path) as cifarf:
|
|
46
|
+
for name in cifarf.getnames():
|
|
47
|
+
# data is stored in batches
|
|
48
|
+
if 'data_batch' in name:
|
|
49
|
+
decompressed = cifarf.extractfile(name)
|
|
50
|
+
data = pickle.load(decompressed, encoding='bytes')
|
|
51
|
+
train_images.append(data[b'data'])
|
|
52
|
+
train_labels += data[b'labels']
|
|
53
|
+
elif 'test_batch' in name:
|
|
54
|
+
decompressed = cifarf.extractfile(name)
|
|
55
|
+
data = pickle.load(decompressed, encoding='bytes')
|
|
56
|
+
test_images = data[b'data']
|
|
57
|
+
test_labels = data[b'labels']
|
|
58
|
+
train_images = numpy.vstack(train_images)
|
|
59
|
+
train_labels = numpy.asarray(train_labels)
|
|
60
|
+
test_labels = numpy.asarray(test_labels)
|
|
61
|
+
return train_images, train_labels, test_images, test_labels
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def load_cifar100(path='cifar-100-python.tar.gz', coarse=False):
|
|
65
|
+
path = get_file(path, origin=CIFAR10_URL)
|
|
66
|
+
with tarfile.open(path) as cifarf:
|
|
67
|
+
train_decomp = cifarf.extractfile('cifar-100-python/train')
|
|
68
|
+
test_decomp = cifarf.extractfile('cifar-100-python/test')
|
|
69
|
+
train_data = pickle.load(train_decomp, encoding='bytes')
|
|
70
|
+
test_data = pickle.load(test_decomp, encoding='bytes')
|
|
71
|
+
train_images = train_data[b'data']
|
|
72
|
+
test_images = test_data[b'data']
|
|
73
|
+
if coarse:
|
|
74
|
+
train_labels = train_data[b'coarse_labels']
|
|
75
|
+
test_labels = test_data[b'coarse_labels']
|
|
76
|
+
else:
|
|
77
|
+
train_labels = train_data[b'fine_labels']
|
|
78
|
+
test_labels = test_data[b'fine_labels']
|
|
79
|
+
train_labels = numpy.asarray(train_labels)
|
|
80
|
+
test_labels = numpy.asarray(test_labels)
|
|
81
|
+
return train_images, train_labels, test_images, test_labels
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import random
|
|
2
|
+
import zipfile
|
|
3
|
+
import gzip
|
|
4
|
+
|
|
5
|
+
import cloudpickle as pickle
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
from ..util import unzip, to_categorical, get_file
|
|
9
|
+
from .._registry import register_loader
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
MNIST_URL = "https://s3.amazonaws.com/img-datasets/mnist.pkl.gz"
|
|
13
|
+
EMNIST_URL = "http://www.itl.nist.gov/iaui/vip/cs_links/EMNIST/gzip.zip"
|
|
14
|
+
EMNIST_FILE = "gzip.zip"
|
|
15
|
+
|
|
16
|
+
FA_TRAIN_IMG_URL = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz"
|
|
17
|
+
FA_TRAIN_LBL_URL = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz"
|
|
18
|
+
FA_TEST_IMG_URL = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz"
|
|
19
|
+
FA_TEST_LBL_URL = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz"
|
|
20
|
+
|
|
21
|
+
KU_TRAIN_IMG_URL = "http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-train-imgs.npz"
|
|
22
|
+
KU_TRAIN_LBL_URL = (
|
|
23
|
+
"http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-train-labels.npz"
|
|
24
|
+
)
|
|
25
|
+
KU_TEST_IMG_URL = "http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-test-imgs.npz"
|
|
26
|
+
KU_TEST_LBL_URL = "http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-test-labels.npz"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@register_loader("mnist")
|
|
30
|
+
def mnist(variant="mnist", shuffle=True):
|
|
31
|
+
if variant == "mnist":
|
|
32
|
+
(X_train, y_train), (X_test, y_test) = load_mnist()
|
|
33
|
+
elif variant == "fashion":
|
|
34
|
+
(X_train, y_train), (X_test, y_test) = load_fashion_mnist()
|
|
35
|
+
elif variant == "kuzushiji":
|
|
36
|
+
(X_train, y_train), (X_test, y_test) = load_kuzushiji_mnist()
|
|
37
|
+
elif variant.startswith("emnist-"):
|
|
38
|
+
if len(variant.split("-")) != 2:
|
|
39
|
+
raise ValueError(
|
|
40
|
+
"EMNIST data set should be given in format "
|
|
41
|
+
"'emnist-subset', where 'subset' can be "
|
|
42
|
+
"'digits', 'letters', 'balanced' "
|
|
43
|
+
"'byclass', 'bymerge' and 'mnist'. "
|
|
44
|
+
f"{variant} was provided."
|
|
45
|
+
)
|
|
46
|
+
subset = variant.split("-")[1]
|
|
47
|
+
if subset not in [
|
|
48
|
+
"digits",
|
|
49
|
+
"letters",
|
|
50
|
+
"balanced",
|
|
51
|
+
"byclass",
|
|
52
|
+
"bymerge",
|
|
53
|
+
"mnist",
|
|
54
|
+
]:
|
|
55
|
+
raise ValueError(
|
|
56
|
+
"To load EMNIST use the format "
|
|
57
|
+
"'emnist-subset' where 'subset' can be"
|
|
58
|
+
"'digits', 'letters', 'balanced' "
|
|
59
|
+
"'byclass', 'bymerge' and 'mnist'."
|
|
60
|
+
)
|
|
61
|
+
else:
|
|
62
|
+
(X_train, y_train), (X_test, y_test) = load_emnist(subset=subset)
|
|
63
|
+
else:
|
|
64
|
+
raise ValueError(
|
|
65
|
+
"Variant must be one of: "
|
|
66
|
+
"'mnist', 'fashion', 'kuzushiji', "
|
|
67
|
+
"'emnist-digits', 'emnist-letters', "
|
|
68
|
+
"'emnist-balanced', 'emnist-byclass', "
|
|
69
|
+
"'emnist-bymerge', 'emnist-mnist'."
|
|
70
|
+
)
|
|
71
|
+
n_train = X_train.shape[0]
|
|
72
|
+
n_test = X_test.shape[0]
|
|
73
|
+
n_classes = len(np.unique(y_train))
|
|
74
|
+
X_train = X_train.reshape(n_train, 784)
|
|
75
|
+
X_test = X_test.reshape(n_test, 784)
|
|
76
|
+
X_train = X_train.astype("float32")
|
|
77
|
+
X_test = X_test.astype("float32")
|
|
78
|
+
X_train /= 255.0
|
|
79
|
+
X_test /= 255.0
|
|
80
|
+
if shuffle:
|
|
81
|
+
train_data = list(zip(X_train, y_train))
|
|
82
|
+
random.shuffle(train_data)
|
|
83
|
+
X_train, y_train = unzip(train_data)
|
|
84
|
+
y_train = to_categorical(y_train, n_classes=n_classes)
|
|
85
|
+
y_test = to_categorical(y_test, n_classes=n_classes)
|
|
86
|
+
return (X_train, y_train), (X_test, y_test)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def load_mnist(path="mnist.pkl.gz"):
|
|
90
|
+
path = get_file(path, origin=MNIST_URL)
|
|
91
|
+
if path.endswith(".gz"):
|
|
92
|
+
f = gzip.open(path, "rb")
|
|
93
|
+
else:
|
|
94
|
+
f = open(path, "rb")
|
|
95
|
+
data = pickle.load(f, encoding="bytes")
|
|
96
|
+
f.close()
|
|
97
|
+
return data # (X_train, y_train), (X_test, y_test)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def load_fashion_mnist(
|
|
101
|
+
train_img_path="train-images-idx3-ubyte.gz",
|
|
102
|
+
train_label_path="train-labels-idx1-ubyte.gz",
|
|
103
|
+
test_img_path="t10k-images-idx3-ubyte.gz",
|
|
104
|
+
test_label_path="t10k-labels-idz1-ubyte.gz",
|
|
105
|
+
):
|
|
106
|
+
train_img_path = get_file(train_img_path, origin=FA_TRAIN_IMG_URL)
|
|
107
|
+
train_label_path = get_file(train_label_path, origin=FA_TRAIN_LBL_URL)
|
|
108
|
+
test_img_path = get_file(test_img_path, origin=FA_TEST_IMG_URL)
|
|
109
|
+
test_label_path = get_file(test_label_path, origin=FA_TEST_LBL_URL)
|
|
110
|
+
# Based on https://github.com/zalandoresearch/fashion-mnist/blob/master/utils/mnist_reader.py
|
|
111
|
+
with gzip.open(train_label_path, "rb") as trlbpath:
|
|
112
|
+
train_labels = np.frombuffer(trlbpath.read(), dtype=np.uint8, offset=8)
|
|
113
|
+
with gzip.open(train_img_path, "rb") as trimgpath:
|
|
114
|
+
train_images = np.frombuffer(
|
|
115
|
+
trimgpath.read(), dtype=np.uint8, offset=16
|
|
116
|
+
).reshape(len(train_labels), 28, 28)
|
|
117
|
+
with gzip.open(test_label_path, "rb") as telbpath:
|
|
118
|
+
test_labels = np.frombuffer(telbpath.read(), dtype=np.uint8, offset=8)
|
|
119
|
+
with gzip.open(test_img_path, "rb") as teimgpath:
|
|
120
|
+
test_images = np.frombuffer(
|
|
121
|
+
teimgpath.read(), dtype=np.uint8, offset=16
|
|
122
|
+
).reshape(len(test_labels), 28, 28)
|
|
123
|
+
return (train_images, train_labels), (test_images, test_labels)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def load_kuzushiji_mnist(
|
|
127
|
+
train_img_path="kmnist-train-imgs.npz",
|
|
128
|
+
train_label_path="kmnist-train-labels.npz",
|
|
129
|
+
test_img_path="kmnist-test-imgs.npz",
|
|
130
|
+
test_label_path="kmnist-test-labels.npz",
|
|
131
|
+
):
|
|
132
|
+
train_img_path = get_file(train_img_path, origin=KU_TRAIN_IMG_URL)
|
|
133
|
+
train_label_path = get_file(train_label_path, origin=KU_TRAIN_LBL_URL)
|
|
134
|
+
test_img_path = get_file(test_img_path, origin=KU_TEST_IMG_URL)
|
|
135
|
+
test_label_path = get_file(test_label_path, origin=KU_TEST_LBL_URL)
|
|
136
|
+
train_images = np.load(train_img_path)["arr_0"]
|
|
137
|
+
train_labels = np.load(train_label_path)["arr_0"]
|
|
138
|
+
test_images = np.load(test_img_path)["arr_0"]
|
|
139
|
+
test_labels = np.load(test_label_path)["arr_0"]
|
|
140
|
+
return (train_images, train_labels), (test_images, test_labels)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _decode_idx(archive, path):
|
|
144
|
+
comp = archive.read(path)
|
|
145
|
+
data = bytes(gzip.decompress(comp))
|
|
146
|
+
axes = data[3]
|
|
147
|
+
shape = []
|
|
148
|
+
dtype = np.dtype("ubyte").newbyteorder(">")
|
|
149
|
+
for axis in range(axes):
|
|
150
|
+
offset = 4 * (axis + 1)
|
|
151
|
+
size = int(np.frombuffer(data[offset : offset + 4], dtype=">u4"))
|
|
152
|
+
shape.append(size)
|
|
153
|
+
shape = tuple(shape)
|
|
154
|
+
offset = 4 * (axes + 1)
|
|
155
|
+
flat = np.frombuffer(data[offset:], dtype=dtype)
|
|
156
|
+
reshaped = flat.reshape(shape)
|
|
157
|
+
return reshaped
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def load_emnist(path=EMNIST_FILE, subset="digits"):
|
|
161
|
+
emnist_path = get_file(path, origin=EMNIST_URL)
|
|
162
|
+
train_X_path = f"gzip/emnist-{subset}-train-images-idx3-ubyte.gz"
|
|
163
|
+
train_y_path = f"gzip/emnist-{subset}-train-labels-idx1-ubyte.gz"
|
|
164
|
+
test_X_path = f"gzip/emnist-{subset}-test-images-idx3-ubyte.gz"
|
|
165
|
+
test_y_path = f"gzip/emnist-{subset}-test-labels-idx1-ubyte.gz"
|
|
166
|
+
with zipfile.ZipFile(emnist_path, "r") as archive:
|
|
167
|
+
train_X = _decode_idx(archive, train_X_path)
|
|
168
|
+
train_y = _decode_idx(archive, train_y_path)
|
|
169
|
+
test_X = _decode_idx(archive, test_X_path)
|
|
170
|
+
test_y = _decode_idx(archive, test_y_path)
|
|
171
|
+
# For some reason in this data set the labels start from 1
|
|
172
|
+
if subset == "letters":
|
|
173
|
+
train_y = train_y - 1
|
|
174
|
+
test_y = test_y - 1
|
|
175
|
+
return (train_X, train_y), (test_X, test_y)
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
import cloudpickle as pickle
|
|
2
2
|
import numpy
|
|
3
3
|
|
|
4
4
|
from ..util import get_file
|
|
5
5
|
from .._registry import register_loader
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
URL = "https://s3.amazonaws.com/text-datasets/reuters.pkl"
|
|
8
|
+
WORD_INDEX_URL = "https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl"
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
@register_loader("reuters")
|
|
@@ -15,7 +15,7 @@ def reuters():
|
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
def get_word_index(path="reuters_word_index.pkl"):
|
|
18
|
-
path = get_file(path, origin=
|
|
18
|
+
path = get_file(path, origin=WORD_INDEX_URL)
|
|
19
19
|
f = open(path, "rb")
|
|
20
20
|
data = pickle.load(f, encoding="latin1")
|
|
21
21
|
f.close()
|
|
@@ -60,7 +60,7 @@ def load_reuters(
|
|
|
60
60
|
# https://raw.githubusercontent.com/fchollet/keras/master/keras/datasets/mnist.py
|
|
61
61
|
# Copyright Francois Chollet, Google, others (2015)
|
|
62
62
|
# Under MIT license
|
|
63
|
-
path = get_file(path, origin=
|
|
63
|
+
path = get_file(path, origin=URL)
|
|
64
64
|
f = open(path, "rb")
|
|
65
65
|
X, labels = pickle.load(f)
|
|
66
66
|
f.close()
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
from functools import partial
|
|
2
|
-
from typing import Iterable, Callable, Dict
|
|
2
|
+
from typing import Iterable, Callable, Dict, Optional
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
|
|
5
5
|
from .loaders import cmu, dbpedia, imdb
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
def cmu_reader(
|
|
9
|
-
path: Path = None, *, freq_cutoff: int = 0, limit: int = 0, split=0.9
|
|
9
|
+
path: Optional[Path] = None, *, freq_cutoff: int = 0, limit: int = 0, split=0.9
|
|
10
10
|
) -> Dict[str, Callable[["Language"], Iterable["Example"]]]:
|
|
11
11
|
from spacy.training.example import Example
|
|
12
12
|
|
|
@@ -36,7 +36,7 @@ def cmu_reader(
|
|
|
36
36
|
|
|
37
37
|
|
|
38
38
|
def dbpedia_reader(
|
|
39
|
-
path: Path = None, *, train_limit: int = 0, dev_limit: int = 0
|
|
39
|
+
path: Optional[Path] = None, *, train_limit: int = 0, dev_limit: int = 0
|
|
40
40
|
) -> Dict[str, Callable[["Language"], Iterable["Example"]]]:
|
|
41
41
|
from spacy.training.example import Example
|
|
42
42
|
|
|
@@ -60,7 +60,7 @@ def dbpedia_reader(
|
|
|
60
60
|
|
|
61
61
|
|
|
62
62
|
def imdb_reader(
|
|
63
|
-
path: Path = None, *, train_limit: int = 0, dev_limit: int = 0
|
|
63
|
+
path: Optional[Path] = None, *, train_limit: int = 0, dev_limit: int = 0
|
|
64
64
|
) -> Dict[str, Callable[["Language"], Iterable["Example"]]]:
|
|
65
65
|
from spacy.training.example import Example
|
|
66
66
|
|
|
File without changes
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# TODO the tests below only verify that the various functions don't crash.
|
|
2
|
+
# Expand them to test the actual output contents.
|
|
3
|
+
|
|
4
|
+
import functools
|
|
5
|
+
import platform
|
|
6
|
+
from urllib.error import HTTPError, URLError
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
import ml_datasets
|
|
12
|
+
|
|
13
|
+
NP_VERSION = tuple(int(x) for x in np.__version__.split(".")[:2])
|
|
14
|
+
|
|
15
|
+
# FIXME warning on NumPy 2.4 when downloading pre-computed pickles:
|
|
16
|
+
# Python or NumPy boolean but got `align=0`.
|
|
17
|
+
# Did you mean to pass a tuple to create a subarray type? (Deprecated NumPy 2.4)
|
|
18
|
+
if NP_VERSION >= (2, 4):
|
|
19
|
+
np_24_deprecation = pytest.mark.filterwarnings(
|
|
20
|
+
"ignore::numpy.exceptions.VisibleDeprecationWarning",
|
|
21
|
+
|
|
22
|
+
)
|
|
23
|
+
else:
|
|
24
|
+
# Note: can't use `condition=NP_VERSION >= (2, 4)` on the decorator directly
|
|
25
|
+
# as numpy.exceptions did not exist in old NumPy versions.
|
|
26
|
+
np_24_deprecation = lambda x: x
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def skip_if_unreachable(test):
|
|
30
|
+
"""Skip a test when its download host can't be reached (connection
|
|
31
|
+
refused, timed out, DNS failure), so third-party outages don't fail CI.
|
|
32
|
+
HTTP errors still fail, as they usually mean a dataset has moved."""
|
|
33
|
+
|
|
34
|
+
@functools.wraps(test)
|
|
35
|
+
def wrapper(*args, **kwargs):
|
|
36
|
+
try:
|
|
37
|
+
return test(*args, **kwargs)
|
|
38
|
+
except HTTPError:
|
|
39
|
+
raise
|
|
40
|
+
except URLError as e:
|
|
41
|
+
pytest.skip(f"download host unreachable: {e.reason}")
|
|
42
|
+
except (ConnectionError, TimeoutError) as e:
|
|
43
|
+
pytest.skip(f"download host unreachable: {e!r}")
|
|
44
|
+
|
|
45
|
+
return wrapper
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@np_24_deprecation
|
|
49
|
+
@skip_if_unreachable
|
|
50
|
+
def test_cifar():
|
|
51
|
+
(X_train, y_train), (X_test, y_test) = ml_datasets.cifar()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@pytest.mark.skip(reason="very slow download")
|
|
55
|
+
@skip_if_unreachable
|
|
56
|
+
def test_cmu():
|
|
57
|
+
train, dev = ml_datasets.cmu()
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@skip_if_unreachable
|
|
61
|
+
def test_dbpedia():
|
|
62
|
+
train, dev = ml_datasets.dbpedia()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@skip_if_unreachable
|
|
66
|
+
def test_imdb():
|
|
67
|
+
train, dev = ml_datasets.imdb()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@np_24_deprecation
|
|
71
|
+
@skip_if_unreachable
|
|
72
|
+
def test_mnist():
|
|
73
|
+
(X_train, y_train), (X_test, y_test) = ml_datasets.mnist()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@pytest.mark.xfail(reason="403 Forbidden")
|
|
77
|
+
@skip_if_unreachable
|
|
78
|
+
def test_quora_questions():
|
|
79
|
+
train, dev = ml_datasets.quora_questions()
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@np_24_deprecation
|
|
83
|
+
@skip_if_unreachable
|
|
84
|
+
def test_reuters():
|
|
85
|
+
(X_train, y_train), (X_test, y_test) = ml_datasets.reuters()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@pytest.mark.xfail(platform.system() == "Windows", reason="path issues")
|
|
89
|
+
@skip_if_unreachable
|
|
90
|
+
def test_snli():
|
|
91
|
+
train, dev = ml_datasets.snli()
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@pytest.mark.xfail(reason="no default path")
|
|
95
|
+
@skip_if_unreachable
|
|
96
|
+
def test_stack_exchange():
|
|
97
|
+
train, dev = ml_datasets.stack_exchange()
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@skip_if_unreachable
|
|
101
|
+
def test_ud_ancora_pos_tags():
|
|
102
|
+
(train_X, train_y), (dev_X, dev_y) = ml_datasets.ud_ancora_pos_tags()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@pytest.mark.xfail(reason="str column where int expected")
|
|
106
|
+
@skip_if_unreachable
|
|
107
|
+
def test_ud_ewtb_pos_tags():
|
|
108
|
+
(train_X, train_y), (dev_X, dev_y) = ml_datasets.ud_ewtb_pos_tags()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@pytest.mark.xfail(reason="no default path")
|
|
112
|
+
@skip_if_unreachable
|
|
113
|
+
def test_wikiner():
|
|
114
|
+
train, dev = ml_datasets.wikiner()
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from ml_datasets.spacy_readers import cmu_reader, dbpedia_reader, imdb_reader
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@pytest.mark.parametrize("reader", [cmu_reader, dbpedia_reader, imdb_reader])
|
|
9
|
+
def test_reader_signature_accepts_default_path(reader):
|
|
10
|
+
# spaCy resolves registered readers through confection, which builds a
|
|
11
|
+
# pydantic model from the function signature and validates the arguments,
|
|
12
|
+
# including defaults. With `path: Path = None`, pydantic v2 rejects the
|
|
13
|
+
# default, so the readers fail unless an explicit path is given.
|
|
14
|
+
pydantic = pytest.importorskip("pydantic")
|
|
15
|
+
param = inspect.signature(reader).parameters["path"]
|
|
16
|
+
model = pydantic.create_model("ReaderArgs", path=(param.annotation, param.default))
|
|
17
|
+
assert model.model_validate({"path": None}).path is None
|
|
18
|
+
assert model.model_validate({}).path is None
|