acadfetch 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.2.0/acadfetch.egg-info → acadfetch-0.2.1}/PKG-INFO +1 -1
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/consts.py +1 -1
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/naming.py +11 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1/acadfetch.egg-info}/PKG-INFO +1 -1
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch.egg-info/SOURCES.txt +1 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/pyproject.toml +1 -1
- acadfetch-0.2.1/tests/test_naming_race.py +35 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/LICENSE +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/MANIFEST.in +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/README.md +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/__init__.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/__main__.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/atomic.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/auth.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/cli.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/client.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/downloader.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/errors.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/manifest.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/progress.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/py.typed +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/units.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/urls.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch/verify.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/setup.cfg +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/conftest.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/fixtures/README.md +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_atomic.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_auth.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_checksum.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_cli.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_client.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_control_names.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_disk.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_downloader.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_identity.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_isolation.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_json.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_manifest.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_naming.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_packaging.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_progress.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_redirect.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_removed.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_sources.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_urls.py +0 -0
- {acadfetch-0.2.0 → acadfetch-0.2.1}/tests/test_verify.py +0 -0
|
@@ -9,6 +9,7 @@ from __future__ import annotations
|
|
|
9
9
|
|
|
10
10
|
import os
|
|
11
11
|
import re
|
|
12
|
+
import threading
|
|
12
13
|
import unicodedata
|
|
13
14
|
from urllib.parse import unquote, urlsplit
|
|
14
15
|
|
|
@@ -74,8 +75,18 @@ def sanitize(name: str, fallback: str = "file") -> str:
|
|
|
74
75
|
return out or fallback
|
|
75
76
|
|
|
76
77
|
|
|
78
|
+
#: The check-then-add below is one step only if callers cannot interleave.
|
|
79
|
+
#: Workers share one ``taken`` set, so the window is real and small.
|
|
80
|
+
_LOCK = threading.Lock()
|
|
81
|
+
|
|
82
|
+
|
|
77
83
|
def unique(name: str, taken: set[str]) -> str:
|
|
78
84
|
"""Return a name not already in ``taken`` (case-insensitively)."""
|
|
85
|
+
with _LOCK:
|
|
86
|
+
return _unique_locked(name, taken)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _unique_locked(name: str, taken: set[str]) -> str:
|
|
79
90
|
if name.lower() not in taken:
|
|
80
91
|
taken.add(name.lower())
|
|
81
92
|
return name
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "acadfetch"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.1"
|
|
8
8
|
description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
import threading
|
|
5
|
+
|
|
6
|
+
from acadfetch import naming
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_unique_hands_out_distinct_names_under_contention():
|
|
10
|
+
"""The check-then-add needs to be one step when threads share the set.
|
|
11
|
+
|
|
12
|
+
Probabilistic by nature, so the switch interval is turned down and the
|
|
13
|
+
barrier repeated: without a lock this loses a name almost every round.
|
|
14
|
+
"""
|
|
15
|
+
old = sys.getswitchinterval()
|
|
16
|
+
sys.setswitchinterval(1e-6)
|
|
17
|
+
try:
|
|
18
|
+
for _ in range(200):
|
|
19
|
+
taken: set[str] = set()
|
|
20
|
+
barrier = threading.Barrier(8)
|
|
21
|
+
results: list[str] = []
|
|
22
|
+
|
|
23
|
+
def worker(barrier=barrier, results=results, taken=taken) -> None: # noqa: B008
|
|
24
|
+
barrier.wait()
|
|
25
|
+
results.append(naming.unique("a.bin", taken))
|
|
26
|
+
|
|
27
|
+
threads = [threading.Thread(target=worker) for _ in range(8)]
|
|
28
|
+
for thread in threads:
|
|
29
|
+
thread.start()
|
|
30
|
+
for thread in threads:
|
|
31
|
+
thread.join()
|
|
32
|
+
assert len(set(results)) == 8, results
|
|
33
|
+
assert len(taken) == 8
|
|
34
|
+
finally:
|
|
35
|
+
sys.setswitchinterval(old)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|