acadfetch 0.6.2__tar.gz → 0.6.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {acadfetch-0.6.2/acadfetch.egg-info → acadfetch-0.6.3}/PKG-INFO +1 -1
  2. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/consts.py +1 -1
  3. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/downloader.py +91 -4
  4. {acadfetch-0.6.2 → acadfetch-0.6.3/acadfetch.egg-info}/PKG-INFO +1 -1
  5. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch.egg-info/SOURCES.txt +1 -0
  6. {acadfetch-0.6.2 → acadfetch-0.6.3}/pyproject.toml +1 -1
  7. acadfetch-0.6.3/tests/test_lock.py +146 -0
  8. {acadfetch-0.6.2 → acadfetch-0.6.3}/LICENSE +0 -0
  9. {acadfetch-0.6.2 → acadfetch-0.6.3}/MANIFEST.in +0 -0
  10. {acadfetch-0.6.2 → acadfetch-0.6.3}/README.md +0 -0
  11. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/__init__.py +0 -0
  12. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/__main__.py +0 -0
  13. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/atomic.py +0 -0
  14. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/auth.py +0 -0
  15. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/cli.py +0 -0
  16. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/client.py +0 -0
  17. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/debug.py +0 -0
  18. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/errors.py +0 -0
  19. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/manifest.py +0 -0
  20. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/naming.py +0 -0
  21. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/progress.py +0 -0
  22. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/py.typed +0 -0
  23. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/ratelimit.py +0 -0
  24. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/remote.py +0 -0
  25. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/sources/__init__.py +0 -0
  26. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/sources/arxiv.py +0 -0
  27. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/sources/base.py +0 -0
  28. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/sources/direct.py +0 -0
  29. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/sources/doi.py +0 -0
  30. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/sources/registry.py +0 -0
  31. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/sources/removed.py +0 -0
  32. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/sources/zenodo.py +0 -0
  33. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/units.py +0 -0
  34. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/urls.py +0 -0
  35. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch/verify.py +0 -0
  36. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch.egg-info/dependency_links.txt +0 -0
  37. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch.egg-info/entry_points.txt +0 -0
  38. {acadfetch-0.6.2 → acadfetch-0.6.3}/acadfetch.egg-info/top_level.txt +0 -0
  39. {acadfetch-0.6.2 → acadfetch-0.6.3}/setup.cfg +0 -0
  40. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/conftest.py +0 -0
  41. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/fixtures/README.md +0 -0
  42. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/fixtures/ca.pem +0 -0
  43. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/fixtures/figshare_403_block.html +0 -0
  44. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/fixtures/figshare_403_forbidden.json +0 -0
  45. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/fixtures/figshare_article.json +0 -0
  46. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/fixtures/figshare_file.pdf +0 -0
  47. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_atomic.py +0 -0
  48. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_auth.py +0 -0
  49. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_checksum.py +0 -0
  50. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_citation.py +0 -0
  51. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_cli.py +0 -0
  52. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_client.py +0 -0
  53. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_control_names.py +0 -0
  54. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_debug.py +0 -0
  55. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_disk.py +0 -0
  56. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_downloader.py +0 -0
  57. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_hash_once.py +0 -0
  58. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_identity.py +0 -0
  59. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_isolation.py +0 -0
  60. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_json.py +0 -0
  61. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_manifest.py +0 -0
  62. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_naming.py +0 -0
  63. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_naming_race.py +0 -0
  64. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_network.py +0 -0
  65. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_packaging.py +0 -0
  66. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_progress.py +0 -0
  67. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_ratelimit.py +0 -0
  68. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_redirect.py +0 -0
  69. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_remote.py +0 -0
  70. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_removed.py +0 -0
  71. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_sources.py +0 -0
  72. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_urls.py +0 -0
  73. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_urls_glob.py +0 -0
  74. {acadfetch-0.6.2 → acadfetch-0.6.3}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.6.2
3
+ Version: 0.6.3
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -1,6 +1,6 @@
1
1
  """Package-wide constants and version."""
2
2
 
3
- __version__ = "0.6.2"
3
+ __version__ = "0.6.3"
4
4
 
5
5
  USER_AGENT = f"acadfetch/{__version__} (+https://github.com/young-ski/acadfetch)"
6
6
 
@@ -27,7 +27,7 @@ from dataclasses import dataclass, field
27
27
  from datetime import datetime, timezone
28
28
  from http.client import HTTPException
29
29
  from pathlib import Path
30
- from typing import cast
30
+ from typing import Any, cast
31
31
 
32
32
  from . import atomic, debug, naming, verify
33
33
  from .client import Client
@@ -73,6 +73,7 @@ _BINARY_EXT = frozenset(
73
73
  }
74
74
  )
75
75
  META_SUFFIX = ".part.meta.json"
76
+ LOCK_NAME = ".acadfetch.lock"
76
77
 
77
78
 
78
79
  @dataclass(frozen=True)
@@ -471,6 +472,82 @@ def check_disk(
471
472
  return (free, needed)
472
473
 
473
474
 
475
+ def _pid_alive(pid: int) -> bool:
476
+ """Is ``pid`` still running, without disturbing it?
477
+
478
+ ``os.kill(pid, 0)`` is the POSIX idiom, but on Windows it sends a console
479
+ CTRL_C: probing our own pid interrupts the process asking the question.
480
+ """
481
+ if os.name == "nt":
482
+ import ctypes
483
+
484
+ # The attribute does not exist off Windows, and both ruff and mypy
485
+ # check this file on Linux; a typed lookup keeps both quiet.
486
+ windll: Any = getattr(ctypes, "windll") # noqa: B009
487
+ kernel32 = windll.kernel32
488
+ handle = kernel32.OpenProcess(0x1000, False, pid) # query limited info
489
+ if not handle:
490
+ return False
491
+ try:
492
+ code = ctypes.c_ulong()
493
+ if not kernel32.GetExitCodeProcess(handle, ctypes.byref(code)):
494
+ return False
495
+ return code.value == 259 # STILL_ACTIVE
496
+ finally:
497
+ kernel32.CloseHandle(handle)
498
+ try:
499
+ os.kill(pid, 0)
500
+ except ProcessLookupError:
501
+ return False
502
+ except PermissionError: # exists, owned by someone else
503
+ return True
504
+ except OSError:
505
+ return False
506
+ return True
507
+
508
+
509
+ def _take_lock(out: Path) -> Path:
510
+ """Refuse to share the directory with a live concurrent run.
511
+
512
+ Two runs into one ``-o`` interleave writes into the same ``.part`` and the
513
+ survivor is a file neither of them wrote. The lock records the owner; a dead
514
+ owner, or one we cannot read, is taken over.
515
+ """
516
+ lock = out / LOCK_NAME
517
+ for _ in range(2):
518
+ try:
519
+ fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
520
+ except FileExistsError:
521
+ owner = 0
522
+ try:
523
+ owner = int(json.loads(lock.read_text("utf-8")).get("pid") or 0)
524
+ except (OSError, ValueError):
525
+ owner = 0
526
+ # A live owner is another run, even when it is this pid: two
527
+ # fetches into one directory in one process clash just as badly.
528
+ # Only a dead or unreadable owner is stale.
529
+ if owner and _pid_alive(owner):
530
+ raise TargetError(
531
+ f"{out} is being written by pid {owner}; wait for it, or "
532
+ f"delete {lock} if that process is gone"
533
+ ) from None
534
+ atomic.unlink(lock)
535
+ continue
536
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
537
+ json.dump({"pid": os.getpid(), "started": _now()}, fh)
538
+ return lock
539
+ raise TargetError(f"could not take the lock in {out}; is another run active?")
540
+
541
+
542
+ def _release_lock(lock: Path) -> None:
543
+ try:
544
+ if int(json.loads(lock.read_text("utf-8")).get("pid") or 0) != os.getpid():
545
+ return
546
+ except (OSError, ValueError):
547
+ pass
548
+ atomic.unlink(lock)
549
+
550
+
474
551
  def _reserved(files) -> set[str]: # noqa: ANN001
475
552
  """Names the tool itself will write, so a repository file cannot take them.
476
553
 
@@ -479,7 +556,7 @@ def _reserved(files) -> set[str]: # noqa: ANN001
479
556
  Reserving the derived names is cheaper than teaching every later step to
480
557
  dodge them.
481
558
  """
482
- reserved = {"manifest.json", "manifest.json.tmp"}
559
+ reserved = {"manifest.json", "manifest.json.tmp", LOCK_NAME}
483
560
  for file in files:
484
561
  name = naming.sanitize(file.name).lower()
485
562
  reserved.add(name + ".part")
@@ -736,15 +813,24 @@ def _fetch_record(
736
813
 
737
814
  if opts.max_bytes:
738
815
  check_total(record, opts.max_bytes)
816
+ lock: Path | None = None
739
817
  if not opts.dry_run:
740
818
  check_disk(files, opts.out)
819
+ lock = _take_lock(opts.out)
741
820
  opts.trusted = _trusted_urls(opts.out / "manifest.json")
742
821
 
743
822
  taken = _reserved(files)
744
- lock = threading.Lock()
745
823
  results: list[Result] = []
746
824
  opts.interrupted = False
747
825
 
826
+ try:
827
+ return _drain(files, record, opts, client, taken, results)
828
+ finally:
829
+ if lock is not None:
830
+ _release_lock(lock)
831
+
832
+
833
+ def _drain(files, record, opts, client, taken, results): # noqa: ANN001, ANN202
748
834
  if opts.concurrency <= 1:
749
835
  for file in files:
750
836
  try:
@@ -767,9 +853,10 @@ def _fetch_record(
767
853
  window = max(opts.concurrency * 2, 1)
768
854
  queue = iter(files)
769
855
  futures = {}
856
+ name_lock = threading.Lock()
770
857
 
771
858
  def submit(file: File) -> None: # noqa: ANN001
772
- with lock:
859
+ with name_lock:
773
860
  futures[pool.submit(fetch_file, client, file, opts, taken)] = file
774
861
 
775
862
  try:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.6.2
3
+ Version: 0.6.3
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -49,6 +49,7 @@ tests/test_hash_once.py
49
49
  tests/test_identity.py
50
50
  tests/test_isolation.py
51
51
  tests/test_json.py
52
+ tests/test_lock.py
52
53
  tests/test_manifest.py
53
54
  tests/test_naming.py
54
55
  tests/test_naming_race.py
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "acadfetch"
7
- version = "0.6.2"
7
+ version = "0.6.3"
8
8
  description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -0,0 +1,146 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import os
5
+ import threading
6
+ import time
7
+
8
+ import pytest
9
+
10
+ from acadfetch import cli, downloader
11
+ from acadfetch.client import Client
12
+ from acadfetch.errors import TargetError
13
+ from acadfetch.sources.base import File, Record
14
+
15
+ LOCK = downloader.LOCK_NAME
16
+
17
+
18
+ def _record(server, name="a.bin", payload=b"payload"):
19
+ server.cfg["files"]["/" + name] = payload
20
+ return Record(
21
+ source="direct",
22
+ target="t",
23
+ files=[File(server.base + "/" + name, name, len(payload))],
24
+ )
25
+
26
+
27
+ def test_a_live_owner_refuses_the_directory(server, tmp_path):
28
+ """The parent process is alive and is not us, like another run would be."""
29
+ owner = os.getpid() # alive, and not in _OWN_LOCKS, so it is another run
30
+ (tmp_path / LOCK).write_text(json.dumps({"pid": owner}), "utf-8")
31
+ record = _record(server)
32
+
33
+ with pytest.raises(TargetError) as caught:
34
+ downloader.fetch_record(record, downloader.Options(out=tmp_path), Client(rate=1000))
35
+
36
+ assert f"pid {owner}" in str(caught.value)
37
+ assert "delete" in str(caught.value)
38
+
39
+
40
+ def test_a_dead_owner_is_taken_over(server, tmp_path):
41
+ (tmp_path / LOCK).write_text(json.dumps({"pid": 999_999}), "utf-8")
42
+ record = _record(server)
43
+
44
+ results = downloader.fetch_record(record, downloader.Options(out=tmp_path), Client(rate=1000))
45
+
46
+ assert results[0].state == "downloaded"
47
+ assert not (tmp_path / LOCK).exists(), "the lock is released"
48
+
49
+
50
+ def test_an_unreadable_lock_is_taken_over(server, tmp_path):
51
+ (tmp_path / LOCK).write_text("not json", "utf-8")
52
+ record = _record(server)
53
+
54
+ results = downloader.fetch_record(record, downloader.Options(out=tmp_path), Client(rate=1000))
55
+
56
+ assert results[0].state == "downloaded"
57
+ assert not (tmp_path / LOCK).exists()
58
+
59
+
60
+ def test_the_lock_is_released_after_a_run(server, tmp_path):
61
+ record = _record(server)
62
+
63
+ downloader.fetch_record(record, downloader.Options(out=tmp_path), Client(rate=1000))
64
+
65
+ assert not (tmp_path / LOCK).exists()
66
+
67
+
68
+ def test_a_dry_run_does_not_take_the_lock(server, tmp_path):
69
+ record = _record(server)
70
+
71
+ downloader.fetch_record(
72
+ record, downloader.Options(out=tmp_path, dry_run=True), Client(rate=1000)
73
+ )
74
+
75
+ assert not (tmp_path / LOCK).exists()
76
+
77
+
78
+ def test_a_concurrent_cli_run_is_refused(server, monkeypatch, capsys, tmp_path):
79
+ record = _record(server)
80
+ monkeypatch.setattr(cli.registry, "resolve", lambda _c, _t: record)
81
+ started = threading.Event()
82
+ release = threading.Event()
83
+
84
+ def slow(*args, **kwargs):
85
+ started.set()
86
+ assert release.wait(10)
87
+ return downloader.fetch_file(*args, **kwargs)
88
+
89
+ monkeypatch.setattr(downloader, "fetch_file", slow)
90
+ first = threading.Thread(target=lambda: cli.main(["fetch", "t", "-q", "-o", str(tmp_path)]))
91
+ first.start()
92
+ assert started.wait(10)
93
+ try:
94
+ code = cli.main(["fetch", "t", "-q", "-o", str(tmp_path)])
95
+ err = capsys.readouterr().err
96
+ finally:
97
+ release.set()
98
+ first.join(10)
99
+
100
+ assert code == cli.EXIT_FAIL
101
+ assert "being written by pid" in err
102
+ assert not (tmp_path / LOCK).exists()
103
+
104
+
105
+ def test_a_repository_lock_name_does_not_collide(server, monkeypatch, tmp_path):
106
+ payload = b"theirs"
107
+ server.cfg["files"]["/" + LOCK] = payload
108
+ record = Record(
109
+ source="direct",
110
+ target="t",
111
+ files=[File(server.base + "/" + LOCK, LOCK, len(payload))],
112
+ )
113
+ monkeypatch.setattr(cli.registry, "resolve", lambda _c, _t: record)
114
+
115
+ code = cli.main(["fetch", "t", "-q", "-o", str(tmp_path)])
116
+
117
+ assert code == cli.EXIT_OK
118
+ assert (tmp_path / "acadfetch.lock").read_bytes() == payload
119
+ assert not (tmp_path / LOCK).exists(), "the run released its own lock"
120
+ assert (tmp_path / "manifest.json").exists()
121
+
122
+
123
+ def test_the_lock_file_is_not_left_by_a_failure(server, tmp_path):
124
+ record = _record(server)
125
+ record.files[0].url = server.base + "/gone.bin"
126
+ record.files[0].size = None
127
+
128
+ results = downloader.fetch_record(
129
+ record, downloader.Options(out=tmp_path, retries=0), Client(rate=1000, retries=0)
130
+ )
131
+
132
+ assert results[0].state == "failed"
133
+ assert not (tmp_path / LOCK).exists()
134
+
135
+
136
+ def test_a_stale_lock_is_replaced_not_appended(server, tmp_path):
137
+ (tmp_path / LOCK).write_text(json.dumps({"pid": 999_999}), "utf-8")
138
+ record = _record(server)
139
+
140
+ downloader.fetch_record(record, downloader.Options(out=tmp_path), Client(rate=1000))
141
+
142
+ assert os.listdir(tmp_path).count(LOCK) <= 1
143
+
144
+
145
+ def _unused() -> None: # keeps the time import honest for future timings
146
+ time.sleep(0)
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes