ml-datasets 0.2.0a0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. ml_datasets-0.2.2/LICENSE +21 -0
  2. ml_datasets-0.2.2/PKG-INFO +175 -0
  3. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/README.md +1 -1
  4. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/__init__.py +2 -0
  5. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/__init__.py +1 -0
  6. ml_datasets-0.2.2/ml_datasets/loaders/cifar.py +81 -0
  7. ml_datasets-0.2.2/ml_datasets/loaders/mnist.py +175 -0
  8. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/reuters.py +5 -5
  9. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/spacy_readers.py +4 -4
  10. ml_datasets-0.2.2/ml_datasets/test/__init__.py +0 -0
  11. ml_datasets-0.2.2/ml_datasets/test/test_datasets.py +114 -0
  12. ml_datasets-0.2.2/ml_datasets/test/test_spacy_readers.py +18 -0
  13. ml_datasets-0.2.2/ml_datasets/test/test_util.py +74 -0
  14. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/util.py +49 -16
  15. ml_datasets-0.2.2/ml_datasets.egg-info/PKG-INFO +175 -0
  16. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets.egg-info/SOURCES.txt +8 -1
  17. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets.egg-info/entry_points.txt +0 -1
  18. ml_datasets-0.2.2/ml_datasets.egg-info/requires.txt +5 -0
  19. ml_datasets-0.2.2/pyproject.toml +9 -0
  20. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/setup.cfg +5 -4
  21. ml_datasets-0.2.0a0/PKG-INFO +0 -169
  22. ml_datasets-0.2.0a0/ml_datasets/loaders/mnist.py +0 -44
  23. ml_datasets-0.2.0a0/ml_datasets.egg-info/PKG-INFO +0 -169
  24. ml_datasets-0.2.0a0/ml_datasets.egg-info/requires.txt +0 -4
  25. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/_registry.py +0 -0
  26. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/cmu.py +0 -0
  27. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/dbpedia.py +0 -0
  28. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/imdb.py +0 -0
  29. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/quora.py +0 -0
  30. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/snli.py +0 -0
  31. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/stack_exchange.py +0 -0
  32. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/universal_dependencies.py +0 -0
  33. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets/loaders/wikiner.py +0 -0
  34. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets.egg-info/dependency_links.txt +0 -0
  35. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets.egg-info/top_level.txt +0 -0
  36. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/ml_datasets.egg-info/zip-safe +0 -0
  37. {ml_datasets-0.2.0a0 → ml_datasets-0.2.2}/setup.py +0 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2019 ExplosionAI GmbH
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,175 @@
1
+ Metadata-Version: 2.4
2
+ Name: ml_datasets
3
+ Version: 0.2.2
4
+ Summary: Machine Learning dataset loaders
5
+ Home-page: https://github.com/explosion/ml-datasets
6
+ Author: Explosion
7
+ Author-email: contact@explosion.ai
8
+ License: MIT
9
+ Requires-Python: >=3.10
10
+ Description-Content-Type: text/markdown
11
+ License-File: LICENSE
12
+ Requires-Dist: cloudpickle>=2.2
13
+ Requires-Dist: numpy>=1.21.2
14
+ Requires-Dist: tqdm<5.0.0,>=4.10.0
15
+ Requires-Dist: srsly<4.0.0,>=1.0.1
16
+ Requires-Dist: catalogue<3.0.0,>=0.2.0
17
+ Dynamic: license-file
18
+
19
+ <a href="https://explosion.ai"><img src="https://explosion.ai/assets/img/logo.svg" width="125" height="125" align="right" /></a>
20
+
21
+ # Machine learning dataset loaders for testing and examples
22
+
23
+ Loaders for various machine learning datasets for testing and example scripts.
24
+ Previously in `thinc.extra.datasets`.
25
+
26
+ [![PyPi Version](https://img.shields.io/pypi/v/ml-datasets.svg?style=flat-square&logo=pypi&logoColor=white)](https://pypi.python.org/pypi/ml-datasets)
27
+
28
+ ## Setup and installation
29
+
30
+ The package can be installed via pip:
31
+
32
+ ```bash
33
+ pip install ml-datasets
34
+ ```
35
+
36
+ ## Loaders
37
+
38
+ Loaders can be imported directly or used via their string name (which is useful if they're set via command line arguments). Some loaders may take arguments – see the source for details.
39
+
40
+ ```python
41
+ # Import directly
42
+ from ml_datasets import imdb
43
+ train_data, dev_data = imdb()
44
+ ```
45
+
46
+ ```python
47
+ # Load via registry
48
+ from ml_datasets import loaders
49
+ imdb_loader = loaders.get("imdb")
50
+ train_data, dev_data = imdb_loader()
51
+ ```
52
+
53
+ ### Available loaders
54
+
55
+ #### NLP datasets
56
+
57
+ | ID / Function | Description | NLP task | From URL |
58
+ | -------------------- | -------------------------------------------- | ----------------------------------------- | :------: |
59
+ | `imdb` | IMDB sentiment dataset | Binary classification: sentiment analysis | ✓ |
60
+ | `dbpedia` | DBPedia ontology dataset | Multi-class single-label classification | ✓ |
61
+ | `cmu` | CMU movie genres dataset | Multi-class, multi-label classification | ✓ |
62
+ | `quora_questions` | Duplicate Quora questions dataset | Detecting duplicate questions | ✓ |
63
+ | `reuters` | Reuters dataset (texts not included) | Multi-class multi-label classification | ✓ |
64
+ | `snli` | Stanford Natural Language Inference corpus | Recognizing textual entailment | ✓ |
65
+ | `stack_exchange` | Stack Exchange dataset | Question Answering | |
66
+ | `ud_ancora_pos_tags` | Universal Dependencies Spanish AnCora corpus | POS tagging | ✓ |
67
+ | `ud_ewtb_pos_tags` | Universal Dependencies English EWT corpus | POS tagging | ✓ |
68
+ | `wikiner` | WikiNER data | Named entity recognition | |
69
+
70
+ #### Other ML datasets
71
+
72
+ | ID / Function | Description | ML task | From URL |
73
+ | ------------- | ----------- | ----------------- | :------: |
74
+ | `mnist` | MNIST data | Image recognition | ✓ |
75
+
76
+ ### Dataset details
77
+
78
+ #### IMDB
79
+
80
+ Each instance contains the text of a movie review, and a sentiment expressed as `0` or `1`.
81
+
82
+ ```python
83
+ train_data, dev_data = ml_datasets.imdb()
84
+ for text, annot in train_data[0:5]:
85
+ print(f"Review: {text}")
86
+ print(f"Sentiment: {annot}")
87
+ ```
88
+
89
+ - Download URL: [http://ai.stanford.edu/~amaas/data/sentiment/](http://ai.stanford.edu/~amaas/data/sentiment/)
90
+ - Citation: [Andrew L. Maas et al., 2011](https://www.aclweb.org/anthology/P11-1015/)
91
+
92
+ | Property | Training | Dev |
93
+ | ------------------- | ---------------- | ---------------- |
94
+ | # Instances | 25000 | 25000 |
95
+ | Label values | {`0`, `1`} | {`0`, `1`} |
96
+ | Labels per instance | Single | Single |
97
+ | Label distribution | Balanced (50/50) | Balanced (50/50) |
98
+
99
+ #### DBPedia
100
+
101
+ Each instance contains an ontological description, and a classification into one of the 14 distinct labels.
102
+
103
+ ```python
104
+ train_data, dev_data = ml_datasets.dbpedia()
105
+ for text, annot in train_data[0:5]:
106
+ print(f"Text: {text}")
107
+ print(f"Category: {annot}")
108
+ ```
109
+
110
+ - Download URL: [Via fast.ai](https://course.fast.ai/datasets)
111
+ - Original citation: [Xiang Zhang et al., 2015](https://arxiv.org/abs/1509.01626)
112
+
113
+ | Property | Training | Dev |
114
+ | ------------------- | -------- | -------- |
115
+ | # Instances | 560000 | 70000 |
116
+ | Label values | `1`-`14` | `1`-`14` |
117
+ | Labels per instance | Single | Single |
118
+ | Label distribution | Balanced | Balanced |
119
+
120
+ #### CMU
121
+
122
+ Each instance contains a movie description, and a classification into a list of appropriate genres.
123
+
124
+ ```python
125
+ train_data, dev_data = ml_datasets.cmu()
126
+ for text, annot in train_data[0:5]:
127
+ print(f"Text: {text}")
128
+ print(f"Genres: {annot}")
129
+ ```
130
+
131
+ - Download URL: [http://www.cs.cmu.edu/~ark/personas/](http://www.cs.cmu.edu/~ark/personas/)
132
+ - Original citation: [David Bamman et al., 2013](https://www.aclweb.org/anthology/P13-1035/)
133
+
134
+ | Property | Training | Dev |
135
+ | ------------------- | --------------------------------------------------------------------------------------------- | --- |
136
+ | # Instances | 41793 | 0 |
137
+ | Label values | 363 different genres | - |
138
+ | Labels per instance | Multiple | - |
139
+ | Label distribution | Imbalanced: 147 labels with less than 20 examples, while `Drama` occurs more than 19000 times | - |
140
+
141
+ #### Quora
142
+
143
+ ```python
144
+ train_data, dev_data = ml_datasets.quora_questions()
145
+ for questions, annot in train_data[0:50]:
146
+ q1, q2 = questions
147
+ print(f"Question 1: {q1}")
148
+ print(f"Question 2: {q2}")
149
+ print(f"Similarity: {annot}")
150
+ ```
151
+
152
+ Each instance contains two quora questions, and a label indicating whether or not they are duplicates (`0`: no, `1`: yes).
153
+ The ground-truth labels contain some amount of noise: they are not guaranteed to be perfect.
154
+
155
+ - Download URL: [http://qim.fs.quoracdn.net/quora_duplicate_questions.tsv](http://qim.fs.quoracdn.net/quora_duplicate_questions.tsv)
156
+ - Original citation: [Kornél Csernai et al., 2017](https://www.quora.com/q/quoradata/First-Quora-Dataset-Release-Question-Pairs)
157
+
158
+ | Property | Training | Dev |
159
+ | ------------------- | ------------------------- | ------------------------- |
160
+ | # Instances | 363859 | 40429 |
161
+ | Label values | {`0`, `1`} | {`0`, `1`} |
162
+ | Labels per instance | Single | Single |
163
+ | Label distribution | Imbalanced: 63% label `0` | Imbalanced: 63% label `0` |
164
+
165
+ ### Registering loaders
166
+
167
+ Loaders can be registered externally using the `loaders` registry as a decorator. For example:
168
+
169
+ ```python
170
+ @ml_datasets.loaders("my_custom_loader")
171
+ def my_custom_loader():
172
+ return load_some_data()
173
+
174
+ assert "my_custom_loader" in ml_datasets.loaders
175
+ ```
@@ -1,6 +1,6 @@
1
1
  <a href="https://explosion.ai"><img src="https://explosion.ai/assets/img/logo.svg" width="125" height="125" align="right" /></a>
2
2
 
3
- # Machine learning dataset loaders
3
+ # Machine learning dataset loaders for testing and examples
4
4
 
5
5
  Loaders for various machine learning datasets for testing and example scripts.
6
6
  Previously in `thinc.extra.datasets`.
@@ -8,3 +8,5 @@ from .loaders.stack_exchange import stack_exchange
8
8
  from .loaders.universal_dependencies import ud_ancora_pos_tags, ud_ewtb_pos_tags
9
9
  from .loaders.dbpedia import dbpedia
10
10
  from .loaders.cmu import cmu
11
+ from .loaders.cifar import cifar
12
+ from .loaders.wikiner import wikiner
@@ -1,3 +1,4 @@
1
+ from .cifar import cifar
1
2
  from .cmu import cmu
2
3
  from .dbpedia import dbpedia
3
4
  from .imdb import imdb
@@ -0,0 +1,81 @@
1
+ import pickle
2
+ import tarfile
3
+ import random
4
+ import numpy
5
+
6
+ from ..util import get_file, unzip, to_categorical
7
+
8
+ CIFAR10_URL = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
9
+ CIFAR100_URL = "https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz"
10
+
11
+
12
+ def cifar(variant='10', channels_last=False, shuffle=True):
13
+ if variant == '10':
14
+ data = load_cifar10()
15
+ elif variant == '100':
16
+ data = load_cifar100(coarse=False)
17
+ elif variant == '100-coarse':
18
+ data = load_cifar100(coarse=True)
19
+ else:
20
+ raise ValueError("Variant must be one of: '10', '100', 100-coarse")
21
+ X_train, y_train, X_test, y_test = data
22
+ X_train = X_train.astype("float32")
23
+ X_test = X_test.astype("float32")
24
+ X_train /= 255.0
25
+ X_test /= 255.0
26
+ if shuffle:
27
+ train_data = list(zip(X_train, y_train))
28
+ random.shuffle(train_data)
29
+ X_train, y_train = unzip(train_data)
30
+ if channels_last:
31
+ X_train = X_train.reshape(X_train.shape[0], 32, 32, 3)
32
+ X_test = X_test.reshape(X_test.shape[0], 32, 32, 3)
33
+ else:
34
+ X_train = X_train.reshape(X_train.shape[0], 3, 32, 32)
35
+ X_test = X_test.reshape(X_test.shape[0], 3, 32, 32)
36
+ y_train = to_categorical(y_train)
37
+ y_test = to_categorical(y_test)
38
+ return (X_train, y_train), (X_test, y_test)
39
+
40
+
41
+ def load_cifar10(path='cifar-10-python.tar.gz'):
42
+ path = get_file(path, origin=CIFAR10_URL)
43
+ train_images = []
44
+ train_labels = []
45
+ with tarfile.open(path) as cifarf:
46
+ for name in cifarf.getnames():
47
+ # data is stored in batches
48
+ if 'data_batch' in name:
49
+ decompressed = cifarf.extractfile(name)
50
+ data = pickle.load(decompressed, encoding='bytes')
51
+ train_images.append(data[b'data'])
52
+ train_labels += data[b'labels']
53
+ elif 'test_batch' in name:
54
+ decompressed = cifarf.extractfile(name)
55
+ data = pickle.load(decompressed, encoding='bytes')
56
+ test_images = data[b'data']
57
+ test_labels = data[b'labels']
58
+ train_images = numpy.vstack(train_images)
59
+ train_labels = numpy.asarray(train_labels)
60
+ test_labels = numpy.asarray(test_labels)
61
+ return train_images, train_labels, test_images, test_labels
62
+
63
+
64
+ def load_cifar100(path='cifar-100-python.tar.gz', coarse=False):
65
+ path = get_file(path, origin=CIFAR10_URL)
66
+ with tarfile.open(path) as cifarf:
67
+ train_decomp = cifarf.extractfile('cifar-100-python/train')
68
+ test_decomp = cifarf.extractfile('cifar-100-python/test')
69
+ train_data = pickle.load(train_decomp, encoding='bytes')
70
+ test_data = pickle.load(test_decomp, encoding='bytes')
71
+ train_images = train_data[b'data']
72
+ test_images = test_data[b'data']
73
+ if coarse:
74
+ train_labels = train_data[b'coarse_labels']
75
+ test_labels = test_data[b'coarse_labels']
76
+ else:
77
+ train_labels = train_data[b'fine_labels']
78
+ test_labels = test_data[b'fine_labels']
79
+ train_labels = numpy.asarray(train_labels)
80
+ test_labels = numpy.asarray(test_labels)
81
+ return train_images, train_labels, test_images, test_labels
@@ -0,0 +1,175 @@
1
+ import random
2
+ import zipfile
3
+ import gzip
4
+
5
+ import cloudpickle as pickle
6
+ import numpy as np
7
+
8
+ from ..util import unzip, to_categorical, get_file
9
+ from .._registry import register_loader
10
+
11
+
12
+ MNIST_URL = "https://s3.amazonaws.com/img-datasets/mnist.pkl.gz"
13
+ EMNIST_URL = "http://www.itl.nist.gov/iaui/vip/cs_links/EMNIST/gzip.zip"
14
+ EMNIST_FILE = "gzip.zip"
15
+
16
+ FA_TRAIN_IMG_URL = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz"
17
+ FA_TRAIN_LBL_URL = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz"
18
+ FA_TEST_IMG_URL = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz"
19
+ FA_TEST_LBL_URL = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz"
20
+
21
+ KU_TRAIN_IMG_URL = "http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-train-imgs.npz"
22
+ KU_TRAIN_LBL_URL = (
23
+ "http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-train-labels.npz"
24
+ )
25
+ KU_TEST_IMG_URL = "http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-test-imgs.npz"
26
+ KU_TEST_LBL_URL = "http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-test-labels.npz"
27
+
28
+
29
+ @register_loader("mnist")
30
+ def mnist(variant="mnist", shuffle=True):
31
+ if variant == "mnist":
32
+ (X_train, y_train), (X_test, y_test) = load_mnist()
33
+ elif variant == "fashion":
34
+ (X_train, y_train), (X_test, y_test) = load_fashion_mnist()
35
+ elif variant == "kuzushiji":
36
+ (X_train, y_train), (X_test, y_test) = load_kuzushiji_mnist()
37
+ elif variant.startswith("emnist-"):
38
+ if len(variant.split("-")) != 2:
39
+ raise ValueError(
40
+ "EMNIST data set should be given in format "
41
+ "'emnist-subset', where 'subset' can be "
42
+ "'digits', 'letters', 'balanced' "
43
+ "'byclass', 'bymerge' and 'mnist'. "
44
+ f"{variant} was provided."
45
+ )
46
+ subset = variant.split("-")[1]
47
+ if subset not in [
48
+ "digits",
49
+ "letters",
50
+ "balanced",
51
+ "byclass",
52
+ "bymerge",
53
+ "mnist",
54
+ ]:
55
+ raise ValueError(
56
+ "To load EMNIST use the format "
57
+ "'emnist-subset' where 'subset' can be"
58
+ "'digits', 'letters', 'balanced' "
59
+ "'byclass', 'bymerge' and 'mnist'."
60
+ )
61
+ else:
62
+ (X_train, y_train), (X_test, y_test) = load_emnist(subset=subset)
63
+ else:
64
+ raise ValueError(
65
+ "Variant must be one of: "
66
+ "'mnist', 'fashion', 'kuzushiji', "
67
+ "'emnist-digits', 'emnist-letters', "
68
+ "'emnist-balanced', 'emnist-byclass', "
69
+ "'emnist-bymerge', 'emnist-mnist'."
70
+ )
71
+ n_train = X_train.shape[0]
72
+ n_test = X_test.shape[0]
73
+ n_classes = len(np.unique(y_train))
74
+ X_train = X_train.reshape(n_train, 784)
75
+ X_test = X_test.reshape(n_test, 784)
76
+ X_train = X_train.astype("float32")
77
+ X_test = X_test.astype("float32")
78
+ X_train /= 255.0
79
+ X_test /= 255.0
80
+ if shuffle:
81
+ train_data = list(zip(X_train, y_train))
82
+ random.shuffle(train_data)
83
+ X_train, y_train = unzip(train_data)
84
+ y_train = to_categorical(y_train, n_classes=n_classes)
85
+ y_test = to_categorical(y_test, n_classes=n_classes)
86
+ return (X_train, y_train), (X_test, y_test)
87
+
88
+
89
+ def load_mnist(path="mnist.pkl.gz"):
90
+ path = get_file(path, origin=MNIST_URL)
91
+ if path.endswith(".gz"):
92
+ f = gzip.open(path, "rb")
93
+ else:
94
+ f = open(path, "rb")
95
+ data = pickle.load(f, encoding="bytes")
96
+ f.close()
97
+ return data # (X_train, y_train), (X_test, y_test)
98
+
99
+
100
+ def load_fashion_mnist(
101
+ train_img_path="train-images-idx3-ubyte.gz",
102
+ train_label_path="train-labels-idx1-ubyte.gz",
103
+ test_img_path="t10k-images-idx3-ubyte.gz",
104
+ test_label_path="t10k-labels-idz1-ubyte.gz",
105
+ ):
106
+ train_img_path = get_file(train_img_path, origin=FA_TRAIN_IMG_URL)
107
+ train_label_path = get_file(train_label_path, origin=FA_TRAIN_LBL_URL)
108
+ test_img_path = get_file(test_img_path, origin=FA_TEST_IMG_URL)
109
+ test_label_path = get_file(test_label_path, origin=FA_TEST_LBL_URL)
110
+ # Based on https://github.com/zalandoresearch/fashion-mnist/blob/master/utils/mnist_reader.py
111
+ with gzip.open(train_label_path, "rb") as trlbpath:
112
+ train_labels = np.frombuffer(trlbpath.read(), dtype=np.uint8, offset=8)
113
+ with gzip.open(train_img_path, "rb") as trimgpath:
114
+ train_images = np.frombuffer(
115
+ trimgpath.read(), dtype=np.uint8, offset=16
116
+ ).reshape(len(train_labels), 28, 28)
117
+ with gzip.open(test_label_path, "rb") as telbpath:
118
+ test_labels = np.frombuffer(telbpath.read(), dtype=np.uint8, offset=8)
119
+ with gzip.open(test_img_path, "rb") as teimgpath:
120
+ test_images = np.frombuffer(
121
+ teimgpath.read(), dtype=np.uint8, offset=16
122
+ ).reshape(len(test_labels), 28, 28)
123
+ return (train_images, train_labels), (test_images, test_labels)
124
+
125
+
126
+ def load_kuzushiji_mnist(
127
+ train_img_path="kmnist-train-imgs.npz",
128
+ train_label_path="kmnist-train-labels.npz",
129
+ test_img_path="kmnist-test-imgs.npz",
130
+ test_label_path="kmnist-test-labels.npz",
131
+ ):
132
+ train_img_path = get_file(train_img_path, origin=KU_TRAIN_IMG_URL)
133
+ train_label_path = get_file(train_label_path, origin=KU_TRAIN_LBL_URL)
134
+ test_img_path = get_file(test_img_path, origin=KU_TEST_IMG_URL)
135
+ test_label_path = get_file(test_label_path, origin=KU_TEST_LBL_URL)
136
+ train_images = np.load(train_img_path)["arr_0"]
137
+ train_labels = np.load(train_label_path)["arr_0"]
138
+ test_images = np.load(test_img_path)["arr_0"]
139
+ test_labels = np.load(test_label_path)["arr_0"]
140
+ return (train_images, train_labels), (test_images, test_labels)
141
+
142
+
143
+ def _decode_idx(archive, path):
144
+ comp = archive.read(path)
145
+ data = bytes(gzip.decompress(comp))
146
+ axes = data[3]
147
+ shape = []
148
+ dtype = np.dtype("ubyte").newbyteorder(">")
149
+ for axis in range(axes):
150
+ offset = 4 * (axis + 1)
151
+ size = int(np.frombuffer(data[offset : offset + 4], dtype=">u4"))
152
+ shape.append(size)
153
+ shape = tuple(shape)
154
+ offset = 4 * (axes + 1)
155
+ flat = np.frombuffer(data[offset:], dtype=dtype)
156
+ reshaped = flat.reshape(shape)
157
+ return reshaped
158
+
159
+
160
+ def load_emnist(path=EMNIST_FILE, subset="digits"):
161
+ emnist_path = get_file(path, origin=EMNIST_URL)
162
+ train_X_path = f"gzip/emnist-{subset}-train-images-idx3-ubyte.gz"
163
+ train_y_path = f"gzip/emnist-{subset}-train-labels-idx1-ubyte.gz"
164
+ test_X_path = f"gzip/emnist-{subset}-test-images-idx3-ubyte.gz"
165
+ test_y_path = f"gzip/emnist-{subset}-test-labels-idx1-ubyte.gz"
166
+ with zipfile.ZipFile(emnist_path, "r") as archive:
167
+ train_X = _decode_idx(archive, train_X_path)
168
+ train_y = _decode_idx(archive, train_y_path)
169
+ test_X = _decode_idx(archive, test_X_path)
170
+ test_y = _decode_idx(archive, test_y_path)
171
+ # For some reason in this data set the labels start from 1
172
+ if subset == "letters":
173
+ train_y = train_y - 1
174
+ test_y = test_y - 1
175
+ return (train_X, train_y), (test_X, test_y)
@@ -1,11 +1,11 @@
1
- from srsly import cloudpickle as pickle
1
+ import cloudpickle as pickle
2
2
  import numpy
3
3
 
4
4
  from ..util import get_file
5
5
  from .._registry import register_loader
6
6
 
7
-
8
- URL = "https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl"
7
+ URL = "https://s3.amazonaws.com/text-datasets/reuters.pkl"
8
+ WORD_INDEX_URL = "https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl"
9
9
 
10
10
 
11
11
  @register_loader("reuters")
@@ -15,7 +15,7 @@ def reuters():
15
15
 
16
16
 
17
17
  def get_word_index(path="reuters_word_index.pkl"):
18
- path = get_file(path, origin=URL)
18
+ path = get_file(path, origin=WORD_INDEX_URL)
19
19
  f = open(path, "rb")
20
20
  data = pickle.load(f, encoding="latin1")
21
21
  f.close()
@@ -60,7 +60,7 @@ def load_reuters(
60
60
  # https://raw.githubusercontent.com/fchollet/keras/master/keras/datasets/mnist.py
61
61
  # Copyright Francois Chollet, Google, others (2015)
62
62
  # Under MIT license
63
- path = get_file(path, origin="https://s3.amazonaws.com/text-datasets/reuters.pkl")
63
+ path = get_file(path, origin=URL)
64
64
  f = open(path, "rb")
65
65
  X, labels = pickle.load(f)
66
66
  f.close()
@@ -1,12 +1,12 @@
1
1
  from functools import partial
2
- from typing import Iterable, Callable, Dict
2
+ from typing import Iterable, Callable, Dict, Optional
3
3
  from pathlib import Path
4
4
 
5
5
  from .loaders import cmu, dbpedia, imdb
6
6
 
7
7
 
8
8
  def cmu_reader(
9
- path: Path = None, *, freq_cutoff: int = 0, limit: int = 0, split=0.9
9
+ path: Optional[Path] = None, *, freq_cutoff: int = 0, limit: int = 0, split=0.9
10
10
  ) -> Dict[str, Callable[["Language"], Iterable["Example"]]]:
11
11
  from spacy.training.example import Example
12
12
 
@@ -36,7 +36,7 @@ def cmu_reader(
36
36
 
37
37
 
38
38
  def dbpedia_reader(
39
- path: Path = None, *, train_limit: int = 0, dev_limit: int = 0
39
+ path: Optional[Path] = None, *, train_limit: int = 0, dev_limit: int = 0
40
40
  ) -> Dict[str, Callable[["Language"], Iterable["Example"]]]:
41
41
  from spacy.training.example import Example
42
42
 
@@ -60,7 +60,7 @@ def dbpedia_reader(
60
60
 
61
61
 
62
62
  def imdb_reader(
63
- path: Path = None, *, train_limit: int = 0, dev_limit: int = 0
63
+ path: Optional[Path] = None, *, train_limit: int = 0, dev_limit: int = 0
64
64
  ) -> Dict[str, Callable[["Language"], Iterable["Example"]]]:
65
65
  from spacy.training.example import Example
66
66
 
File without changes
@@ -0,0 +1,114 @@
1
+ # TODO the tests below only verify that the various functions don't crash.
2
+ # Expand them to test the actual output contents.
3
+
4
+ import functools
5
+ import platform
6
+ from urllib.error import HTTPError, URLError
7
+
8
+ import pytest
9
+ import numpy as np
10
+
11
+ import ml_datasets
12
+
13
+ NP_VERSION = tuple(int(x) for x in np.__version__.split(".")[:2])
14
+
15
+ # FIXME warning on NumPy 2.4 when downloading pre-computed pickles:
16
+ # Python or NumPy boolean but got `align=0`.
17
+ # Did you mean to pass a tuple to create a subarray type? (Deprecated NumPy 2.4)
18
+ if NP_VERSION >= (2, 4):
19
+ np_24_deprecation = pytest.mark.filterwarnings(
20
+ "ignore::numpy.exceptions.VisibleDeprecationWarning",
21
+
22
+ )
23
+ else:
24
+ # Note: can't use `condition=NP_VERSION >= (2, 4)` on the decorator directly
25
+ # as numpy.exceptions did not exist in old NumPy versions.
26
+ np_24_deprecation = lambda x: x
27
+
28
+
29
+ def skip_if_unreachable(test):
30
+ """Skip a test when its download host can't be reached (connection
31
+ refused, timed out, DNS failure), so third-party outages don't fail CI.
32
+ HTTP errors still fail, as they usually mean a dataset has moved."""
33
+
34
+ @functools.wraps(test)
35
+ def wrapper(*args, **kwargs):
36
+ try:
37
+ return test(*args, **kwargs)
38
+ except HTTPError:
39
+ raise
40
+ except URLError as e:
41
+ pytest.skip(f"download host unreachable: {e.reason}")
42
+ except (ConnectionError, TimeoutError) as e:
43
+ pytest.skip(f"download host unreachable: {e!r}")
44
+
45
+ return wrapper
46
+
47
+
48
+ @np_24_deprecation
49
+ @skip_if_unreachable
50
+ def test_cifar():
51
+ (X_train, y_train), (X_test, y_test) = ml_datasets.cifar()
52
+
53
+
54
+ @pytest.mark.skip(reason="very slow download")
55
+ @skip_if_unreachable
56
+ def test_cmu():
57
+ train, dev = ml_datasets.cmu()
58
+
59
+
60
+ @skip_if_unreachable
61
+ def test_dbpedia():
62
+ train, dev = ml_datasets.dbpedia()
63
+
64
+
65
+ @skip_if_unreachable
66
+ def test_imdb():
67
+ train, dev = ml_datasets.imdb()
68
+
69
+
70
+ @np_24_deprecation
71
+ @skip_if_unreachable
72
+ def test_mnist():
73
+ (X_train, y_train), (X_test, y_test) = ml_datasets.mnist()
74
+
75
+
76
+ @pytest.mark.xfail(reason="403 Forbidden")
77
+ @skip_if_unreachable
78
+ def test_quora_questions():
79
+ train, dev = ml_datasets.quora_questions()
80
+
81
+
82
+ @np_24_deprecation
83
+ @skip_if_unreachable
84
+ def test_reuters():
85
+ (X_train, y_train), (X_test, y_test) = ml_datasets.reuters()
86
+
87
+
88
+ @pytest.mark.xfail(platform.system() == "Windows", reason="path issues")
89
+ @skip_if_unreachable
90
+ def test_snli():
91
+ train, dev = ml_datasets.snli()
92
+
93
+
94
+ @pytest.mark.xfail(reason="no default path")
95
+ @skip_if_unreachable
96
+ def test_stack_exchange():
97
+ train, dev = ml_datasets.stack_exchange()
98
+
99
+
100
+ @skip_if_unreachable
101
+ def test_ud_ancora_pos_tags():
102
+ (train_X, train_y), (dev_X, dev_y) = ml_datasets.ud_ancora_pos_tags()
103
+
104
+
105
+ @pytest.mark.xfail(reason="str column where int expected")
106
+ @skip_if_unreachable
107
+ def test_ud_ewtb_pos_tags():
108
+ (train_X, train_y), (dev_X, dev_y) = ml_datasets.ud_ewtb_pos_tags()
109
+
110
+
111
+ @pytest.mark.xfail(reason="no default path")
112
+ @skip_if_unreachable
113
+ def test_wikiner():
114
+ train, dev = ml_datasets.wikiner()
@@ -0,0 +1,18 @@
1
+ import inspect
2
+
3
+ import pytest
4
+
5
+ from ml_datasets.spacy_readers import cmu_reader, dbpedia_reader, imdb_reader
6
+
7
+
8
+ @pytest.mark.parametrize("reader", [cmu_reader, dbpedia_reader, imdb_reader])
9
+ def test_reader_signature_accepts_default_path(reader):
10
+ # spaCy resolves registered readers through confection, which builds a
11
+ # pydantic model from the function signature and validates the arguments,
12
+ # including defaults. With `path: Path = None`, pydantic v2 rejects the
13
+ # default, so the readers fail unless an explicit path is given.
14
+ pydantic = pytest.importorskip("pydantic")
15
+ param = inspect.signature(reader).parameters["path"]
16
+ model = pydantic.create_model("ReaderArgs", path=(param.annotation, param.default))
17
+ assert model.model_validate({"path": None}).path is None
18
+ assert model.model_validate({}).path is None