ml-datasets 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/PKG-INFO +3 -3
  2. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/spacy_readers.py +4 -4
  3. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/test/test_datasets.py +33 -0
  4. ml_datasets-0.2.2/ml_datasets/test/test_spacy_readers.py +18 -0
  5. ml_datasets-0.2.2/ml_datasets/test/test_util.py +74 -0
  6. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/util.py +40 -12
  7. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets.egg-info/PKG-INFO +3 -3
  8. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets.egg-info/SOURCES.txt +1 -0
  9. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets.egg-info/requires.txt +1 -1
  10. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/setup.cfg +3 -3
  11. ml_datasets-0.2.1/ml_datasets/test/test_util.py +0 -27
  12. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/LICENSE +0 -0
  13. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/README.md +0 -0
  14. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/__init__.py +0 -0
  15. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/_registry.py +0 -0
  16. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/loaders/__init__.py +0 -0
  17. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/loaders/cifar.py +0 -0
  18. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/loaders/cmu.py +0 -0
  19. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/loaders/dbpedia.py +0 -0
  20. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/loaders/imdb.py +0 -0
  21. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/loaders/mnist.py +0 -0
  22. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/loaders/quora.py +0 -0
  23. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/loaders/reuters.py +0 -0
  24. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/loaders/snli.py +0 -0
  25. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/loaders/stack_exchange.py +0 -0
  26. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/loaders/universal_dependencies.py +0 -0
  27. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/loaders/wikiner.py +0 -0
  28. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets/test/__init__.py +0 -0
  29. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets.egg-info/dependency_links.txt +0 -0
  30. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets.egg-info/entry_points.txt +0 -0
  31. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets.egg-info/top_level.txt +0 -0
  32. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/ml_datasets.egg-info/zip-safe +0 -0
  33. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/pyproject.toml +0 -0
  34. {ml_datasets-0.2.1 → ml_datasets-0.2.2}/setup.py +0 -0
@@ -1,16 +1,16 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ml_datasets
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Machine Learning dataset loaders
5
5
  Home-page: https://github.com/explosion/ml-datasets
6
6
  Author: Explosion
7
7
  Author-email: contact@explosion.ai
8
8
  License: MIT
9
- Requires-Python: >=3.8
9
+ Requires-Python: >=3.10
10
10
  Description-Content-Type: text/markdown
11
11
  License-File: LICENSE
12
12
  Requires-Dist: cloudpickle>=2.2
13
- Requires-Dist: numpy>=1.18
13
+ Requires-Dist: numpy>=1.21.2
14
14
  Requires-Dist: tqdm<5.0.0,>=4.10.0
15
15
  Requires-Dist: srsly<4.0.0,>=1.0.1
16
16
  Requires-Dist: catalogue<3.0.0,>=0.2.0
@@ -1,12 +1,12 @@
1
1
  from functools import partial
2
- from typing import Iterable, Callable, Dict
2
+ from typing import Iterable, Callable, Dict, Optional
3
3
  from pathlib import Path
4
4
 
5
5
  from .loaders import cmu, dbpedia, imdb
6
6
 
7
7
 
8
8
  def cmu_reader(
9
- path: Path = None, *, freq_cutoff: int = 0, limit: int = 0, split=0.9
9
+ path: Optional[Path] = None, *, freq_cutoff: int = 0, limit: int = 0, split=0.9
10
10
  ) -> Dict[str, Callable[["Language"], Iterable["Example"]]]:
11
11
  from spacy.training.example import Example
12
12
 
@@ -36,7 +36,7 @@ def cmu_reader(
36
36
 
37
37
 
38
38
  def dbpedia_reader(
39
- path: Path = None, *, train_limit: int = 0, dev_limit: int = 0
39
+ path: Optional[Path] = None, *, train_limit: int = 0, dev_limit: int = 0
40
40
  ) -> Dict[str, Callable[["Language"], Iterable["Example"]]]:
41
41
  from spacy.training.example import Example
42
42
 
@@ -60,7 +60,7 @@ def dbpedia_reader(
60
60
 
61
61
 
62
62
  def imdb_reader(
63
- path: Path = None, *, train_limit: int = 0, dev_limit: int = 0
63
+ path: Optional[Path] = None, *, train_limit: int = 0, dev_limit: int = 0
64
64
  ) -> Dict[str, Callable[["Language"], Iterable["Example"]]]:
65
65
  from spacy.training.example import Example
66
66
 
@@ -1,7 +1,9 @@
1
1
  # TODO the tests below only verify that the various functions don't crash.
2
2
  # Expand them to test the actual output contents.
3
3
 
4
+ import functools
4
5
  import platform
6
+ from urllib.error import HTTPError, URLError
5
7
 
6
8
  import pytest
7
9
  import numpy as np
@@ -24,58 +26,89 @@ else:
24
26
  np_24_deprecation = lambda x: x
25
27
 
26
28
 
29
+ def skip_if_unreachable(test):
30
+ """Skip a test when its download host can't be reached (connection
31
+ refused, timed out, DNS failure), so third-party outages don't fail CI.
32
+ HTTP errors still fail, as they usually mean a dataset has moved."""
33
+
34
+ @functools.wraps(test)
35
+ def wrapper(*args, **kwargs):
36
+ try:
37
+ return test(*args, **kwargs)
38
+ except HTTPError:
39
+ raise
40
+ except URLError as e:
41
+ pytest.skip(f"download host unreachable: {e.reason}")
42
+ except (ConnectionError, TimeoutError) as e:
43
+ pytest.skip(f"download host unreachable: {e!r}")
44
+
45
+ return wrapper
46
+
47
+
27
48
  @np_24_deprecation
49
+ @skip_if_unreachable
28
50
  def test_cifar():
29
51
  (X_train, y_train), (X_test, y_test) = ml_datasets.cifar()
30
52
 
31
53
 
32
54
  @pytest.mark.skip(reason="very slow download")
55
+ @skip_if_unreachable
33
56
  def test_cmu():
34
57
  train, dev = ml_datasets.cmu()
35
58
 
36
59
 
60
+ @skip_if_unreachable
37
61
  def test_dbpedia():
38
62
  train, dev = ml_datasets.dbpedia()
39
63
 
40
64
 
65
+ @skip_if_unreachable
41
66
  def test_imdb():
42
67
  train, dev = ml_datasets.imdb()
43
68
 
44
69
 
45
70
  @np_24_deprecation
71
+ @skip_if_unreachable
46
72
  def test_mnist():
47
73
  (X_train, y_train), (X_test, y_test) = ml_datasets.mnist()
48
74
 
49
75
 
50
76
  @pytest.mark.xfail(reason="403 Forbidden")
77
+ @skip_if_unreachable
51
78
  def test_quora_questions():
52
79
  train, dev = ml_datasets.quora_questions()
53
80
 
54
81
 
55
82
  @np_24_deprecation
83
+ @skip_if_unreachable
56
84
  def test_reuters():
57
85
  (X_train, y_train), (X_test, y_test) = ml_datasets.reuters()
58
86
 
59
87
 
60
88
  @pytest.mark.xfail(platform.system() == "Windows", reason="path issues")
89
+ @skip_if_unreachable
61
90
  def test_snli():
62
91
  train, dev = ml_datasets.snli()
63
92
 
64
93
 
65
94
  @pytest.mark.xfail(reason="no default path")
95
+ @skip_if_unreachable
66
96
  def test_stack_exchange():
67
97
  train, dev = ml_datasets.stack_exchange()
68
98
 
69
99
 
100
+ @skip_if_unreachable
70
101
  def test_ud_ancora_pos_tags():
71
102
  (train_X, train_y), (dev_X, dev_y) = ml_datasets.ud_ancora_pos_tags()
72
103
 
73
104
 
74
105
  @pytest.mark.xfail(reason="str column where int expected")
106
+ @skip_if_unreachable
75
107
  def test_ud_ewtb_pos_tags():
76
108
  (train_X, train_y), (dev_X, dev_y) = ml_datasets.ud_ewtb_pos_tags()
77
109
 
78
110
 
79
111
  @pytest.mark.xfail(reason="no default path")
112
+ @skip_if_unreachable
80
113
  def test_wikiner():
81
114
  train, dev = ml_datasets.wikiner()
@@ -0,0 +1,18 @@
1
+ import inspect
2
+
3
+ import pytest
4
+
5
+ from ml_datasets.spacy_readers import cmu_reader, dbpedia_reader, imdb_reader
6
+
7
+
8
+ @pytest.mark.parametrize("reader", [cmu_reader, dbpedia_reader, imdb_reader])
9
+ def test_reader_signature_accepts_default_path(reader):
10
+ # spaCy resolves registered readers through confection, which builds a
11
+ # pydantic model from the function signature and validates the arguments,
12
+ # including defaults. With `path: Path = None`, pydantic v2 rejects the
13
+ # default, so the readers fail unless an explicit path is given.
14
+ pydantic = pytest.importorskip("pydantic")
15
+ param = inspect.signature(reader).parameters["path"]
16
+ model = pydantic.create_model("ReaderArgs", path=(param.annotation, param.default))
17
+ assert model.model_validate({"path": None}).path is None
18
+ assert model.model_validate({}).path is None
@@ -0,0 +1,74 @@
1
+ import pytest
2
+ from urllib.error import HTTPError, URLError
3
+ from ml_datasets.util import get_file
4
+
5
+
6
+ def test_get_file_domain_resolution_fails():
7
+ with pytest.raises(
8
+ URLError, match=r"test_non_existent_file.*(not known|getaddrinfo failed)"
9
+ ):
10
+ get_file(
11
+ "non_existent_file.txt",
12
+ "http://test_notexist.wth/test_non_existent_file.txt"
13
+ )
14
+
15
+
16
+ def test_get_file_404_file_not_found():
17
+ with pytest.raises(HTTPError, match="test_non_existent_file.*404.*Not Found") as e:
18
+ get_file(
19
+ "non_existent_file.txt",
20
+ "http://google.com/test_non_existent_file.txt"
21
+ )
22
+ assert e.value.code == 404
23
+ # Suppress pytest.PytestUnraisableExceptionWarning:
24
+ # Exception ignored while calling deallocator
25
+ # This questionable design quirk comes from urllib.request.urlretrieve,
26
+ # so we shouldn't shim around it.
27
+ e.value.close()
28
+
29
+
30
+ def _make_tar(path, members):
31
+ import io
32
+ import tarfile
33
+
34
+ with tarfile.open(path, "w:gz") as tfile:
35
+ for name, kind in members:
36
+ info = tarfile.TarInfo(name)
37
+ if kind == "file":
38
+ data = b"hello"
39
+ info.size = len(data)
40
+ tfile.addfile(info, io.BytesIO(data))
41
+ elif kind == "symlink":
42
+ info.type = tarfile.SYMTYPE
43
+ info.linkname = "/etc/passwd"
44
+ tfile.addfile(info)
45
+
46
+
47
+ def test_safe_extract_tar_extracts_without_warning(tmp_path, recwarn):
48
+ import tarfile
49
+ from ml_datasets.util import safe_extract_tar
50
+
51
+ archive = tmp_path / "data.tar.gz"
52
+ _make_tar(archive, [("data/a.txt", "file")])
53
+ out = tmp_path / "out"
54
+ with tarfile.open(archive, "r:gz") as tfile:
55
+ safe_extract_tar(tfile, str(out))
56
+ assert (out / "data" / "a.txt").read_bytes() == b"hello"
57
+ assert not [w for w in recwarn if issubclass(w.category, DeprecationWarning)]
58
+
59
+
60
+ @pytest.mark.parametrize(
61
+ "members", [[("../evil.txt", "file")], [("data/link", "symlink")]]
62
+ )
63
+ def test_safe_extract_tar_refuses_escaping_members(tmp_path, members):
64
+ import tarfile
65
+ from ml_datasets.util import safe_extract_tar
66
+
67
+ archive = tmp_path / "data.tar.gz"
68
+ _make_tar(archive, members)
69
+ out = tmp_path / "out"
70
+ with tarfile.open(archive, "r:gz") as tfile:
71
+ with pytest.raises(tarfile.TarError):
72
+ safe_extract_tar(tfile, str(out))
73
+ assert not (tmp_path / "evil.txt").exists()
74
+ assert not (out / "data" / "link").exists()
@@ -8,6 +8,35 @@ from urllib.request import urlretrieve
8
8
  import tqdm
9
9
 
10
10
 
11
+ def safe_extract_tar(tfile, path):
12
+ """Extract a tar archive into `path`, refusing members that would be
13
+ written outside of it.
14
+
15
+ Uses the "data" extraction filter where Python supports it (3.12+ and
16
+ the security backports in 3.10.12+ and 3.11.4+), which
17
+ also avoids the DeprecationWarning that `extractall()` without a filter
18
+ raises on Python 3.12+. On Pythons without extraction filters, links and
19
+ special files are refused, and so are paths that leave `path`.
20
+ """
21
+ if hasattr(tarfile, "data_filter"):
22
+ tfile.extractall(path=path, filter="data")
23
+ return
24
+ root = os.path.realpath(path)
25
+ for member in tfile.getmembers():
26
+ if not (member.isfile() or member.isdir()):
27
+ raise tarfile.TarError(
28
+ "Refusing to extract {!r}: links and special files are not "
29
+ "supported on this Python version".format(member.name)
30
+ )
31
+ target = os.path.realpath(os.path.join(root, member.name))
32
+ if os.path.commonpath([root, target]) != root:
33
+ raise tarfile.TarError(
34
+ "Refusing to extract {!r}: path is outside of the target "
35
+ "directory".format(member.name)
36
+ )
37
+ tfile.extractall(path=path)
38
+
39
+
11
40
  def get_file(fname, origin, untar=False, unzip=False, cache_subdir="datasets"):
12
41
  """Downloads a file from a URL if it not already in the cache."""
13
42
  # https://raw.githubusercontent.com/fchollet/keras/master/keras/utils/data_utils.py
@@ -19,8 +48,8 @@ def get_file(fname, origin, untar=False, unzip=False, cache_subdir="datasets"):
19
48
  datadir = os.path.join(datadir_base, cache_subdir)
20
49
  if not os.path.exists(datadir):
21
50
  os.makedirs(datadir)
51
+ untar_fpath = os.path.join(datadir, fname)
22
52
  if untar or unzip:
23
- untar_fpath = os.path.join(datadir, fname)
24
53
  if unzip:
25
54
  fpath = untar_fpath + ".zip"
26
55
  else:
@@ -60,17 +89,16 @@ def get_file(fname, origin, untar=False, unzip=False, cache_subdir="datasets"):
60
89
  if untar:
61
90
  if not os.path.exists(untar_fpath):
62
91
  print("Untaring file...")
63
- tfile = tarfile.open(fpath, "r:gz")
64
- try:
65
- tfile.extractall(path=datadir)
66
- except (Exception, KeyboardInterrupt):
67
- if os.path.exists(untar_fpath):
68
- if os.path.isfile(untar_fpath):
69
- os.remove(untar_fpath)
70
- else:
71
- shutil.rmtree(untar_fpath)
72
- raise
73
- tfile.close()
92
+ with tarfile.open(fpath, "r:gz") as tfile:
93
+ try:
94
+ safe_extract_tar(tfile, datadir)
95
+ except (Exception, KeyboardInterrupt):
96
+ if os.path.exists(untar_fpath):
97
+ if os.path.isfile(untar_fpath):
98
+ os.remove(untar_fpath)
99
+ else:
100
+ shutil.rmtree(untar_fpath)
101
+ raise
74
102
  return untar_fpath
75
103
  elif unzip:
76
104
  if not os.path.exists(untar_fpath):
@@ -1,16 +1,16 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ml_datasets
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Machine Learning dataset loaders
5
5
  Home-page: https://github.com/explosion/ml-datasets
6
6
  Author: Explosion
7
7
  Author-email: contact@explosion.ai
8
8
  License: MIT
9
- Requires-Python: >=3.8
9
+ Requires-Python: >=3.10
10
10
  Description-Content-Type: text/markdown
11
11
  License-File: LICENSE
12
12
  Requires-Dist: cloudpickle>=2.2
13
- Requires-Dist: numpy>=1.18
13
+ Requires-Dist: numpy>=1.21.2
14
14
  Requires-Dist: tqdm<5.0.0,>=4.10.0
15
15
  Requires-Dist: srsly<4.0.0,>=1.0.1
16
16
  Requires-Dist: catalogue<3.0.0,>=0.2.0
@@ -28,4 +28,5 @@ ml_datasets/loaders/universal_dependencies.py
28
28
  ml_datasets/loaders/wikiner.py
29
29
  ml_datasets/test/__init__.py
30
30
  ml_datasets/test/test_datasets.py
31
+ ml_datasets/test/test_spacy_readers.py
31
32
  ml_datasets/test/test_util.py
@@ -1,5 +1,5 @@
1
1
  cloudpickle>=2.2
2
- numpy>=1.18
2
+ numpy>=1.21.2
3
3
  tqdm<5.0.0,>=4.10.0
4
4
  srsly<4.0.0,>=1.0.1
5
5
  catalogue<3.0.0,>=0.2.0
@@ -1,5 +1,5 @@
1
1
  [metadata]
2
- version = 0.2.1
2
+ version = 0.2.2
3
3
  description = Machine Learning dataset loaders
4
4
  url = https://github.com/explosion/ml-datasets
5
5
  author = Explosion
@@ -11,10 +11,10 @@ long_description_content_type = text/markdown
11
11
  [options]
12
12
  zip_safe = true
13
13
  include_package_data = true
14
- python_requires = >=3.8
14
+ python_requires = >=3.10
15
15
  install_requires =
16
16
  cloudpickle>=2.2
17
- numpy>=1.18
17
+ numpy>=1.21.2
18
18
  tqdm>=4.10.0,<5.0.0
19
19
  srsly>=1.0.1,<4.0.0
20
20
  catalogue>=0.2.0,<3.0.0
@@ -1,27 +0,0 @@
1
- import pytest
2
- from urllib.error import HTTPError, URLError
3
- from ml_datasets.util import get_file
4
-
5
-
6
- def test_get_file_domain_resolution_fails():
7
- with pytest.raises(
8
- URLError, match=r"test_non_existent_file.*(not known|getaddrinfo failed)"
9
- ):
10
- get_file(
11
- "non_existent_file.txt",
12
- "http://test_notexist.wth/test_non_existent_file.txt"
13
- )
14
-
15
-
16
- def test_get_file_404_file_not_found():
17
- with pytest.raises(HTTPError, match="test_non_existent_file.*404.*Not Found") as e:
18
- get_file(
19
- "non_existent_file.txt",
20
- "http://google.com/test_non_existent_file.txt"
21
- )
22
- assert e.value.code == 404
23
- # Suppress pytest.PytestUnraisableExceptionWarning:
24
- # Exception ignored while calling deallocator
25
- # This questionable design quirk comes from urllib.request.urlretrieve,
26
- # so we shouldn't shim around it.
27
- e.value.close()
File without changes
File without changes
File without changes
File without changes