github-backup 0.59.0__tar.gz → 0.60.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {github_backup-0.59.0 → github_backup-0.60.0}/CHANGES.rst +10 -1
- {github_backup-0.59.0/github_backup.egg-info → github_backup-0.60.0}/PKG-INFO +3 -2
- {github_backup-0.59.0 → github_backup-0.60.0}/README +2 -1
- {github_backup-0.59.0 → github_backup-0.60.0}/README.rst +2 -1
- github_backup-0.60.0/github_backup/__init__.py +1 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/github_backup/github_backup.py +48 -24
- {github_backup-0.59.0 → github_backup-0.60.0/github_backup.egg-info}/PKG-INFO +3 -2
- {github_backup-0.59.0 → github_backup-0.60.0}/tests/test_http_451.py +31 -10
- {github_backup-0.59.0 → github_backup-0.60.0}/tests/test_pagination.py +1 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/tests/test_retrieve_data.py +195 -40
- github_backup-0.59.0/github_backup/__init__.py +0 -1
- {github_backup-0.59.0 → github_backup-0.60.0}/LICENSE.txt +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/MANIFEST.in +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/github_backup/__main__.py +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/github_backup/cli.py +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/github_backup.egg-info/SOURCES.txt +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/github_backup.egg-info/dependency_links.txt +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/github_backup.egg-info/entry_points.txt +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/github_backup.egg-info/top_level.txt +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/github_backup.egg-info/zip-safe +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/release-requirements.txt +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/requirements.txt +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/setup.cfg +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/setup.py +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/tests/test_all_starred.py +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/tests/test_attachments.py +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/tests/test_case_sensitivity.py +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/tests/test_json_dump_if_changed.py +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/tests/test_skip_assets_on.py +0 -0
- {github_backup-0.59.0 → github_backup-0.60.0}/tests/test_starred_skip_size_over.py +0 -0
|
@@ -1,9 +1,18 @@
|
|
|
1
1
|
Changelog
|
|
2
2
|
=========
|
|
3
3
|
|
|
4
|
-
0.
|
|
4
|
+
0.60.0 (2025-12-24)
|
|
5
5
|
-------------------
|
|
6
6
|
------------------------
|
|
7
|
+
- Rm max_retries.py. [michaelmartinez]
|
|
8
|
+
- Readme. [michaelmartinez]
|
|
9
|
+
- Don't use a global variable, pass the args instead. [michaelmartinez]
|
|
10
|
+
- Readme, simplify the logic a bit. [michaelmartinez]
|
|
11
|
+
- Max_retries 5. [michaelmartinez]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
0.59.0 (2025-12-21)
|
|
15
|
+
-------------------
|
|
7
16
|
- Add --starred-skip-size-over flag to limit starred repo size (#108)
|
|
8
17
|
[Rodos]
|
|
9
18
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: github-backup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.60.0
|
|
4
4
|
Summary: backup a github user or organization
|
|
5
5
|
Home-page: http://github.com/josegonzalez/python-github-backup
|
|
6
6
|
Author: Jose Diaz-Gonzalez
|
|
@@ -182,7 +182,8 @@ CLI Help output::
|
|
|
182
182
|
--throttle-limit to be set)
|
|
183
183
|
--exclude [EXCLUDE ...]
|
|
184
184
|
names of repositories to exclude
|
|
185
|
-
|
|
185
|
+
--retries MAX_RETRIES
|
|
186
|
+
maximum number of retries for API calls (default: 5)
|
|
186
187
|
|
|
187
188
|
Usage Details
|
|
188
189
|
=============
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.60.0"
|
|
@@ -74,9 +74,6 @@ else:
|
|
|
74
74
|
" 3. Debian/Ubuntu: apt-get install ca-certificates\n\n"
|
|
75
75
|
)
|
|
76
76
|
|
|
77
|
-
# Retry configuration
|
|
78
|
-
MAX_RETRIES = 5
|
|
79
|
-
|
|
80
77
|
|
|
81
78
|
def logging_subprocess(
|
|
82
79
|
popenargs, stdout_log_level=logging.DEBUG, stderr_log_level=logging.ERROR, **kwargs
|
|
@@ -144,6 +141,17 @@ def mask_password(url, secret="*****"):
|
|
|
144
141
|
return url.replace(parsed.password, secret)
|
|
145
142
|
|
|
146
143
|
|
|
144
|
+
def non_negative_int(value):
|
|
145
|
+
"""Argparse type validator for non-negative integers."""
|
|
146
|
+
try:
|
|
147
|
+
ivalue = int(value)
|
|
148
|
+
except ValueError:
|
|
149
|
+
raise argparse.ArgumentTypeError(f"'{value}' is not a valid integer")
|
|
150
|
+
if ivalue < 0:
|
|
151
|
+
raise argparse.ArgumentTypeError(f"{value} must be 0 or greater")
|
|
152
|
+
return ivalue
|
|
153
|
+
|
|
154
|
+
|
|
147
155
|
def parse_args(args=None):
|
|
148
156
|
parser = argparse.ArgumentParser(description="Backup a github account")
|
|
149
157
|
parser.add_argument("user", metavar="USER", type=str, help="github username")
|
|
@@ -468,6 +476,13 @@ def parse_args(args=None):
|
|
|
468
476
|
parser.add_argument(
|
|
469
477
|
"--exclude", dest="exclude", help="names of repositories to exclude", nargs="*"
|
|
470
478
|
)
|
|
479
|
+
parser.add_argument(
|
|
480
|
+
"--retries",
|
|
481
|
+
dest="max_retries",
|
|
482
|
+
type=non_negative_int,
|
|
483
|
+
default=5,
|
|
484
|
+
help="maximum number of retries for API calls (default: 5)",
|
|
485
|
+
)
|
|
471
486
|
return parser.parse_args(args)
|
|
472
487
|
|
|
473
488
|
|
|
@@ -622,7 +637,7 @@ def retrieve_data(args, template, query_args=None, paginated=True):
|
|
|
622
637
|
def _extract_next_page_url(link_header):
|
|
623
638
|
for link in link_header.split(","):
|
|
624
639
|
if 'rel="next"' in link:
|
|
625
|
-
return link[link.find("<") + 1:link.find(">")]
|
|
640
|
+
return link[link.find("<") + 1 : link.find(">")]
|
|
626
641
|
return None
|
|
627
642
|
|
|
628
643
|
def fetch_all() -> Generator[dict, None, None]:
|
|
@@ -631,7 +646,7 @@ def retrieve_data(args, template, query_args=None, paginated=True):
|
|
|
631
646
|
while True:
|
|
632
647
|
# FIRST: Fetch response
|
|
633
648
|
|
|
634
|
-
for attempt in range(
|
|
649
|
+
for attempt in range(args.max_retries + 1):
|
|
635
650
|
request = _construct_request(
|
|
636
651
|
per_page=per_page if paginated else None,
|
|
637
652
|
query_args=query_args,
|
|
@@ -640,7 +655,7 @@ def retrieve_data(args, template, query_args=None, paginated=True):
|
|
|
640
655
|
as_app=args.as_app,
|
|
641
656
|
fine=args.token_fine is not None,
|
|
642
657
|
)
|
|
643
|
-
http_response = make_request_with_retry(request, auth)
|
|
658
|
+
http_response = make_request_with_retry(request, auth, args.max_retries)
|
|
644
659
|
|
|
645
660
|
match http_response.getcode():
|
|
646
661
|
case 200:
|
|
@@ -654,10 +669,10 @@ def retrieve_data(args, template, query_args=None, paginated=True):
|
|
|
654
669
|
TimeoutError,
|
|
655
670
|
) as e:
|
|
656
671
|
logger.warning(f"{type(e).__name__} reading response")
|
|
657
|
-
if attempt <
|
|
672
|
+
if attempt < args.max_retries:
|
|
658
673
|
delay = calculate_retry_delay(attempt, {})
|
|
659
674
|
logger.warning(
|
|
660
|
-
f"Retrying in {delay:.1f}s (attempt {attempt + 1}/{
|
|
675
|
+
f"Retrying read in {delay:.1f}s (attempt {attempt + 1}/{args.max_retries + 1})"
|
|
661
676
|
)
|
|
662
677
|
time.sleep(delay)
|
|
663
678
|
continue # Next retry attempt
|
|
@@ -683,10 +698,10 @@ def retrieve_data(args, template, query_args=None, paginated=True):
|
|
|
683
698
|
)
|
|
684
699
|
else:
|
|
685
700
|
logger.error(
|
|
686
|
-
f"Failed to read response after {
|
|
701
|
+
f"Failed to read response after {args.max_retries + 1} attempts for {next_url or template}"
|
|
687
702
|
)
|
|
688
703
|
raise Exception(
|
|
689
|
-
f"Failed to read response after {
|
|
704
|
+
f"Failed to read response after {args.max_retries + 1} attempts for {next_url or template}"
|
|
690
705
|
)
|
|
691
706
|
|
|
692
707
|
# SECOND: Process and paginate
|
|
@@ -718,7 +733,7 @@ def retrieve_data(args, template, query_args=None, paginated=True):
|
|
|
718
733
|
return list(fetch_all())
|
|
719
734
|
|
|
720
735
|
|
|
721
|
-
def make_request_with_retry(request, auth):
|
|
736
|
+
def make_request_with_retry(request, auth, max_retries=5):
|
|
722
737
|
"""Make HTTP request with automatic retry for transient errors."""
|
|
723
738
|
|
|
724
739
|
def is_retryable_status(status_code, headers):
|
|
@@ -730,40 +745,49 @@ def make_request_with_retry(request, auth):
|
|
|
730
745
|
return int(headers.get("x-ratelimit-remaining", 1)) < 1
|
|
731
746
|
return False
|
|
732
747
|
|
|
733
|
-
for attempt in range(
|
|
748
|
+
for attempt in range(max_retries + 1):
|
|
734
749
|
try:
|
|
735
750
|
return urlopen(request, context=https_ctx)
|
|
736
751
|
|
|
737
752
|
except HTTPError as exc:
|
|
738
753
|
# HTTPError can be used as a response-like object
|
|
739
754
|
if not is_retryable_status(exc.code, exc.headers):
|
|
755
|
+
logger.error(
|
|
756
|
+
f"API Error: {exc.code} {exc.reason} for {request.full_url}"
|
|
757
|
+
)
|
|
740
758
|
raise # Non-retryable error
|
|
741
759
|
|
|
742
|
-
if attempt >=
|
|
743
|
-
logger.error(
|
|
760
|
+
if attempt >= max_retries:
|
|
761
|
+
logger.error(
|
|
762
|
+
f"HTTP {exc.code} failed after {max_retries + 1} attempts for {request.full_url}"
|
|
763
|
+
)
|
|
744
764
|
raise
|
|
745
765
|
|
|
746
766
|
delay = calculate_retry_delay(attempt, exc.headers)
|
|
747
767
|
logger.warning(
|
|
748
|
-
f"HTTP {exc.code}, retrying in {delay:.1f}s "
|
|
749
|
-
f"(attempt {attempt + 1}/{
|
|
768
|
+
f"HTTP {exc.code} ({exc.reason}), retrying in {delay:.1f}s "
|
|
769
|
+
f"(attempt {attempt + 1}/{max_retries + 1}) for {request.full_url}"
|
|
750
770
|
)
|
|
751
771
|
if auth is None and exc.code in (403, 429):
|
|
752
772
|
logger.info("Hint: Authenticate to raise your GitHub rate limit")
|
|
753
773
|
time.sleep(delay)
|
|
754
774
|
|
|
755
775
|
except (URLError, socket.error) as e:
|
|
756
|
-
if attempt >=
|
|
757
|
-
logger.error(
|
|
776
|
+
if attempt >= max_retries:
|
|
777
|
+
logger.error(
|
|
778
|
+
f"Connection error failed after {max_retries + 1} attempts: {e} for {request.full_url}"
|
|
779
|
+
)
|
|
758
780
|
raise
|
|
759
781
|
delay = calculate_retry_delay(attempt, {})
|
|
760
782
|
logger.warning(
|
|
761
783
|
f"Connection error: {e}, retrying in {delay:.1f}s "
|
|
762
|
-
f"(attempt {attempt + 1}/{
|
|
784
|
+
f"(attempt {attempt + 1}/{max_retries + 1}) for {request.full_url}"
|
|
763
785
|
)
|
|
764
786
|
time.sleep(delay)
|
|
765
787
|
|
|
766
|
-
raise Exception(
|
|
788
|
+
raise Exception(
|
|
789
|
+
f"Request failed after {max_retries + 1} attempts"
|
|
790
|
+
) # pragma: no cover
|
|
767
791
|
|
|
768
792
|
|
|
769
793
|
def _construct_request(per_page, query_args, template, auth, as_app=None, fine=False):
|
|
@@ -1579,9 +1603,7 @@ def filter_repositories(args, unfiltered_repositories):
|
|
|
1579
1603
|
repositories = [r for r in repositories if not r.get("archived")]
|
|
1580
1604
|
if args.starred_skip_size_over is not None:
|
|
1581
1605
|
if args.starred_skip_size_over <= 0:
|
|
1582
|
-
logger.warning(
|
|
1583
|
-
"--starred-skip-size-over must be greater than 0, ignoring"
|
|
1584
|
-
)
|
|
1606
|
+
logger.warning("--starred-skip-size-over must be greater than 0, ignoring")
|
|
1585
1607
|
else:
|
|
1586
1608
|
size_limit_kb = args.starred_skip_size_over * 1024
|
|
1587
1609
|
filtered = []
|
|
@@ -1590,7 +1612,9 @@ def filter_repositories(args, unfiltered_repositories):
|
|
|
1590
1612
|
size_mb = r.get("size", 0) / 1024
|
|
1591
1613
|
logger.info(
|
|
1592
1614
|
"Skipping starred repo {0} ({1:.0f} MB) due to --starred-skip-size-over {2}".format(
|
|
1593
|
-
r.get("full_name", r.get("name")),
|
|
1615
|
+
r.get("full_name", r.get("name")),
|
|
1616
|
+
size_mb,
|
|
1617
|
+
args.starred_skip_size_over,
|
|
1594
1618
|
)
|
|
1595
1619
|
)
|
|
1596
1620
|
else:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: github-backup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.60.0
|
|
4
4
|
Summary: backup a github user or organization
|
|
5
5
|
Home-page: http://github.com/josegonzalez/python-github-backup
|
|
6
6
|
Author: Jose Diaz-Gonzalez
|
|
@@ -182,7 +182,8 @@ CLI Help output::
|
|
|
182
182
|
--throttle-limit to be set)
|
|
183
183
|
--exclude [EXCLUDE ...]
|
|
184
184
|
names of repositories to exclude
|
|
185
|
-
|
|
185
|
+
--retries MAX_RETRIES
|
|
186
|
+
maximum number of retries for API calls (default: 5)
|
|
186
187
|
|
|
187
188
|
Usage Details
|
|
188
189
|
=============
|
|
@@ -21,6 +21,7 @@ class TestHTTP451Exception:
|
|
|
21
21
|
args.osx_keychain_item_account = None
|
|
22
22
|
args.throttle_limit = None
|
|
23
23
|
args.throttle_pause = 0
|
|
24
|
+
args.max_retries = 5
|
|
24
25
|
|
|
25
26
|
mock_response = Mock()
|
|
26
27
|
mock_response.getcode.return_value = 451
|
|
@@ -30,18 +31,26 @@ class TestHTTP451Exception:
|
|
|
30
31
|
"block": {
|
|
31
32
|
"reason": "dmca",
|
|
32
33
|
"created_at": "2024-11-12T14:38:04Z",
|
|
33
|
-
"html_url": "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md"
|
|
34
|
-
}
|
|
34
|
+
"html_url": "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md",
|
|
35
|
+
},
|
|
35
36
|
}
|
|
36
37
|
mock_response.read.return_value = json.dumps(dmca_data).encode("utf-8")
|
|
37
38
|
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
|
38
39
|
mock_response.reason = "Unavailable For Legal Reasons"
|
|
39
40
|
|
|
40
|
-
with patch(
|
|
41
|
+
with patch(
|
|
42
|
+
"github_backup.github_backup.make_request_with_retry",
|
|
43
|
+
return_value=mock_response,
|
|
44
|
+
):
|
|
41
45
|
with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
|
|
42
|
-
github_backup.retrieve_data(
|
|
43
|
-
|
|
44
|
-
|
|
46
|
+
github_backup.retrieve_data(
|
|
47
|
+
args, "https://api.github.com/repos/test/dmca/issues"
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
assert (
|
|
51
|
+
exc_info.value.dmca_url
|
|
52
|
+
== "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md"
|
|
53
|
+
)
|
|
45
54
|
assert "451" in str(exc_info.value)
|
|
46
55
|
|
|
47
56
|
def test_repository_unavailable_error_without_dmca_url(self):
|
|
@@ -54,6 +63,7 @@ class TestHTTP451Exception:
|
|
|
54
63
|
args.osx_keychain_item_account = None
|
|
55
64
|
args.throttle_limit = None
|
|
56
65
|
args.throttle_pause = 0
|
|
66
|
+
args.max_retries = 5
|
|
57
67
|
|
|
58
68
|
mock_response = Mock()
|
|
59
69
|
mock_response.getcode.return_value = 451
|
|
@@ -61,9 +71,14 @@ class TestHTTP451Exception:
|
|
|
61
71
|
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
|
62
72
|
mock_response.reason = "Unavailable For Legal Reasons"
|
|
63
73
|
|
|
64
|
-
with patch(
|
|
74
|
+
with patch(
|
|
75
|
+
"github_backup.github_backup.make_request_with_retry",
|
|
76
|
+
return_value=mock_response,
|
|
77
|
+
):
|
|
65
78
|
with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
|
|
66
|
-
github_backup.retrieve_data(
|
|
79
|
+
github_backup.retrieve_data(
|
|
80
|
+
args, "https://api.github.com/repos/test/dmca/issues"
|
|
81
|
+
)
|
|
67
82
|
|
|
68
83
|
assert exc_info.value.dmca_url is None
|
|
69
84
|
assert "451" in str(exc_info.value)
|
|
@@ -78,6 +93,7 @@ class TestHTTP451Exception:
|
|
|
78
93
|
args.osx_keychain_item_account = None
|
|
79
94
|
args.throttle_limit = None
|
|
80
95
|
args.throttle_pause = 0
|
|
96
|
+
args.max_retries = 5
|
|
81
97
|
|
|
82
98
|
mock_response = Mock()
|
|
83
99
|
mock_response.getcode.return_value = 451
|
|
@@ -85,9 +101,14 @@ class TestHTTP451Exception:
|
|
|
85
101
|
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
|
86
102
|
mock_response.reason = "Unavailable For Legal Reasons"
|
|
87
103
|
|
|
88
|
-
with patch(
|
|
104
|
+
with patch(
|
|
105
|
+
"github_backup.github_backup.make_request_with_retry",
|
|
106
|
+
return_value=mock_response,
|
|
107
|
+
):
|
|
89
108
|
with pytest.raises(github_backup.RepositoryUnavailableError):
|
|
90
|
-
github_backup.retrieve_data(
|
|
109
|
+
github_backup.retrieve_data(
|
|
110
|
+
args, "https://api.github.com/repos/test/dmca/issues"
|
|
111
|
+
)
|
|
91
112
|
|
|
92
113
|
|
|
93
114
|
if __name__ == "__main__":
|
|
@@ -9,26 +9,27 @@ import pytest
|
|
|
9
9
|
|
|
10
10
|
from github_backup import github_backup
|
|
11
11
|
from github_backup.github_backup import (
|
|
12
|
-
MAX_RETRIES,
|
|
13
12
|
calculate_retry_delay,
|
|
14
13
|
make_request_with_retry,
|
|
15
14
|
)
|
|
16
15
|
|
|
16
|
+
# Default retry count used in tests (matches argparse default)
|
|
17
|
+
# With max_retries=5, total attempts = 6 (1 initial + 5 retries)
|
|
18
|
+
DEFAULT_MAX_RETRIES = 5
|
|
19
|
+
|
|
17
20
|
|
|
18
21
|
class TestCalculateRetryDelay:
|
|
19
22
|
def test_respects_retry_after_header(self):
|
|
20
|
-
headers = {
|
|
23
|
+
headers = {"retry-after": "30"}
|
|
21
24
|
assert calculate_retry_delay(0, headers) == 30
|
|
22
25
|
|
|
23
26
|
def test_respects_rate_limit_reset(self):
|
|
24
27
|
import time
|
|
25
28
|
import calendar
|
|
29
|
+
|
|
26
30
|
# Set reset time 60 seconds in the future
|
|
27
31
|
future_reset = calendar.timegm(time.gmtime()) + 60
|
|
28
|
-
headers = {
|
|
29
|
-
'x-ratelimit-remaining': '0',
|
|
30
|
-
'x-ratelimit-reset': str(future_reset)
|
|
31
|
-
}
|
|
32
|
+
headers = {"x-ratelimit-remaining": "0", "x-ratelimit-reset": str(future_reset)}
|
|
32
33
|
delay = calculate_retry_delay(0, headers)
|
|
33
34
|
# Should be approximately 60 seconds (with some tolerance for execution time)
|
|
34
35
|
assert 55 <= delay <= 65
|
|
@@ -50,12 +51,10 @@ class TestCalculateRetryDelay:
|
|
|
50
51
|
def test_minimum_rate_limit_delay(self):
|
|
51
52
|
import time
|
|
52
53
|
import calendar
|
|
54
|
+
|
|
53
55
|
# Set reset time in the past (already reset)
|
|
54
56
|
past_reset = calendar.timegm(time.gmtime()) - 100
|
|
55
|
-
headers = {
|
|
56
|
-
'x-ratelimit-remaining': '0',
|
|
57
|
-
'x-ratelimit-reset': str(past_reset)
|
|
58
|
-
}
|
|
57
|
+
headers = {"x-ratelimit-remaining": "0", "x-ratelimit-reset": str(past_reset)}
|
|
59
58
|
delay = calculate_retry_delay(0, headers)
|
|
60
59
|
# Should be minimum 10 seconds even if reset time is in past
|
|
61
60
|
assert delay >= 10
|
|
@@ -74,6 +73,7 @@ class TestRetrieveDataRetry:
|
|
|
74
73
|
args.osx_keychain_item_account = None
|
|
75
74
|
args.throttle_limit = None
|
|
76
75
|
args.throttle_pause = 0
|
|
76
|
+
args.max_retries = DEFAULT_MAX_RETRIES
|
|
77
77
|
return args
|
|
78
78
|
|
|
79
79
|
def test_json_parse_error_retries_and_fails(self, mock_args):
|
|
@@ -90,13 +90,22 @@ class TestRetrieveDataRetry:
|
|
|
90
90
|
call_count += 1
|
|
91
91
|
return mock_response
|
|
92
92
|
|
|
93
|
-
with patch(
|
|
94
|
-
|
|
93
|
+
with patch(
|
|
94
|
+
"github_backup.github_backup.make_request_with_retry",
|
|
95
|
+
side_effect=mock_make_request,
|
|
96
|
+
):
|
|
97
|
+
with patch(
|
|
98
|
+
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
|
99
|
+
): # No delay in tests
|
|
95
100
|
with pytest.raises(Exception) as exc_info:
|
|
96
|
-
github_backup.retrieve_data(
|
|
101
|
+
github_backup.retrieve_data(
|
|
102
|
+
mock_args, "https://api.github.com/repos/test/repo/issues"
|
|
103
|
+
)
|
|
97
104
|
|
|
98
105
|
assert "Failed to read response after" in str(exc_info.value)
|
|
99
|
-
assert
|
|
106
|
+
assert (
|
|
107
|
+
call_count == DEFAULT_MAX_RETRIES + 1
|
|
108
|
+
) # 1 initial + 5 retries = 6 attempts
|
|
100
109
|
|
|
101
110
|
def test_json_parse_error_recovers_on_retry(self, mock_args):
|
|
102
111
|
"""HTTP 200 with invalid JSON should succeed if retry returns valid JSON."""
|
|
@@ -119,9 +128,16 @@ class TestRetrieveDataRetry:
|
|
|
119
128
|
call_count += 1
|
|
120
129
|
return result
|
|
121
130
|
|
|
122
|
-
with patch(
|
|
123
|
-
|
|
124
|
-
|
|
131
|
+
with patch(
|
|
132
|
+
"github_backup.github_backup.make_request_with_retry",
|
|
133
|
+
side_effect=mock_make_request,
|
|
134
|
+
):
|
|
135
|
+
with patch(
|
|
136
|
+
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
|
137
|
+
):
|
|
138
|
+
result = github_backup.retrieve_data(
|
|
139
|
+
mock_args, "https://api.github.com/repos/test/repo/issues"
|
|
140
|
+
)
|
|
125
141
|
|
|
126
142
|
assert result == [{"id": 1}]
|
|
127
143
|
assert call_count == 3 # Failed twice, succeeded on third
|
|
@@ -134,11 +150,18 @@ class TestRetrieveDataRetry:
|
|
|
134
150
|
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
|
135
151
|
mock_response.reason = "Not Found"
|
|
136
152
|
|
|
137
|
-
with patch(
|
|
153
|
+
with patch(
|
|
154
|
+
"github_backup.github_backup.make_request_with_retry",
|
|
155
|
+
return_value=mock_response,
|
|
156
|
+
):
|
|
138
157
|
with pytest.raises(Exception) as exc_info:
|
|
139
|
-
github_backup.retrieve_data(
|
|
158
|
+
github_backup.retrieve_data(
|
|
159
|
+
mock_args, "https://api.github.com/repos/test/notfound/issues"
|
|
160
|
+
)
|
|
140
161
|
|
|
141
|
-
assert not isinstance(
|
|
162
|
+
assert not isinstance(
|
|
163
|
+
exc_info.value, github_backup.RepositoryUnavailableError
|
|
164
|
+
)
|
|
142
165
|
assert "404" in str(exc_info.value)
|
|
143
166
|
|
|
144
167
|
|
|
@@ -151,7 +174,7 @@ class TestMakeRequestWithRetry:
|
|
|
151
174
|
good_response.read.return_value = b'{"ok": true}'
|
|
152
175
|
|
|
153
176
|
call_count = 0
|
|
154
|
-
fail_count =
|
|
177
|
+
fail_count = DEFAULT_MAX_RETRIES # Fail all retries, succeed on last attempt
|
|
155
178
|
|
|
156
179
|
def mock_urlopen(*args, **kwargs):
|
|
157
180
|
nonlocal call_count
|
|
@@ -167,14 +190,18 @@ class TestMakeRequestWithRetry:
|
|
|
167
190
|
return good_response
|
|
168
191
|
|
|
169
192
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
|
170
|
-
with patch(
|
|
193
|
+
with patch(
|
|
194
|
+
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
|
195
|
+
):
|
|
171
196
|
result = make_request_with_retry(Mock(), None)
|
|
172
197
|
|
|
173
198
|
assert result == good_response
|
|
174
|
-
assert
|
|
199
|
+
assert (
|
|
200
|
+
call_count == DEFAULT_MAX_RETRIES + 1
|
|
201
|
+
) # 1 initial + 5 retries = 6 attempts
|
|
175
202
|
|
|
176
203
|
def test_503_error_retries_until_exhausted(self):
|
|
177
|
-
"""HTTP 503 should retry
|
|
204
|
+
"""HTTP 503 should make 1 initial + DEFAULT_MAX_RETRIES retry attempts then raise."""
|
|
178
205
|
call_count = 0
|
|
179
206
|
|
|
180
207
|
def mock_urlopen(*args, **kwargs):
|
|
@@ -189,12 +216,16 @@ class TestMakeRequestWithRetry:
|
|
|
189
216
|
)
|
|
190
217
|
|
|
191
218
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
|
192
|
-
with patch(
|
|
219
|
+
with patch(
|
|
220
|
+
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
|
221
|
+
):
|
|
193
222
|
with pytest.raises(HTTPError) as exc_info:
|
|
194
223
|
make_request_with_retry(Mock(), None)
|
|
195
224
|
|
|
196
225
|
assert exc_info.value.code == 503
|
|
197
|
-
assert
|
|
226
|
+
assert (
|
|
227
|
+
call_count == DEFAULT_MAX_RETRIES + 1
|
|
228
|
+
) # 1 initial + 5 retries = 6 attempts
|
|
198
229
|
|
|
199
230
|
def test_404_error_not_retried(self):
|
|
200
231
|
"""HTTP 404 should not be retried - raise immediately."""
|
|
@@ -237,7 +268,9 @@ class TestMakeRequestWithRetry:
|
|
|
237
268
|
return good_response
|
|
238
269
|
|
|
239
270
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
|
240
|
-
with patch(
|
|
271
|
+
with patch(
|
|
272
|
+
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
|
273
|
+
):
|
|
241
274
|
result = make_request_with_retry(Mock(), None)
|
|
242
275
|
|
|
243
276
|
assert result == good_response
|
|
@@ -269,7 +302,7 @@ class TestMakeRequestWithRetry:
|
|
|
269
302
|
"""URLError (connection error) should retry and succeed if subsequent request works."""
|
|
270
303
|
good_response = Mock()
|
|
271
304
|
call_count = 0
|
|
272
|
-
fail_count =
|
|
305
|
+
fail_count = DEFAULT_MAX_RETRIES # Fail all retries, succeed on last attempt
|
|
273
306
|
|
|
274
307
|
def mock_urlopen(*args, **kwargs):
|
|
275
308
|
nonlocal call_count
|
|
@@ -279,14 +312,18 @@ class TestMakeRequestWithRetry:
|
|
|
279
312
|
return good_response
|
|
280
313
|
|
|
281
314
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
|
282
|
-
with patch(
|
|
315
|
+
with patch(
|
|
316
|
+
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
|
317
|
+
):
|
|
283
318
|
result = make_request_with_retry(Mock(), None)
|
|
284
319
|
|
|
285
320
|
assert result == good_response
|
|
286
|
-
assert
|
|
321
|
+
assert (
|
|
322
|
+
call_count == DEFAULT_MAX_RETRIES + 1
|
|
323
|
+
) # 1 initial + 5 retries = 6 attempts
|
|
287
324
|
|
|
288
325
|
def test_socket_error_retries_until_exhausted(self):
|
|
289
|
-
"""socket.error should retry
|
|
326
|
+
"""socket.error should make 1 initial + DEFAULT_MAX_RETRIES retry attempts then raise."""
|
|
290
327
|
call_count = 0
|
|
291
328
|
|
|
292
329
|
def mock_urlopen(*args, **kwargs):
|
|
@@ -295,11 +332,15 @@ class TestMakeRequestWithRetry:
|
|
|
295
332
|
raise socket.error("Connection reset by peer")
|
|
296
333
|
|
|
297
334
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
|
298
|
-
with patch(
|
|
335
|
+
with patch(
|
|
336
|
+
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
|
337
|
+
):
|
|
299
338
|
with pytest.raises(socket.error):
|
|
300
339
|
make_request_with_retry(Mock(), None)
|
|
301
340
|
|
|
302
|
-
assert
|
|
341
|
+
assert (
|
|
342
|
+
call_count == DEFAULT_MAX_RETRIES + 1
|
|
343
|
+
) # 1 initial + 5 retries = 6 attempts
|
|
303
344
|
|
|
304
345
|
|
|
305
346
|
class TestRetrieveDataThrottling:
|
|
@@ -315,6 +356,7 @@ class TestRetrieveDataThrottling:
|
|
|
315
356
|
args.osx_keychain_item_account = None
|
|
316
357
|
args.throttle_limit = 10 # Throttle when remaining <= 10
|
|
317
358
|
args.throttle_pause = 5 # Pause 5 seconds
|
|
359
|
+
args.max_retries = DEFAULT_MAX_RETRIES
|
|
318
360
|
return args
|
|
319
361
|
|
|
320
362
|
def test_throttling_pauses_when_rate_limit_low(self, mock_args):
|
|
@@ -322,11 +364,19 @@ class TestRetrieveDataThrottling:
|
|
|
322
364
|
mock_response = Mock()
|
|
323
365
|
mock_response.getcode.return_value = 200
|
|
324
366
|
mock_response.read.return_value = json.dumps([{"id": 1}]).encode("utf-8")
|
|
325
|
-
mock_response.headers = {
|
|
326
|
-
|
|
327
|
-
|
|
367
|
+
mock_response.headers = {
|
|
368
|
+
"x-ratelimit-remaining": "5",
|
|
369
|
+
"Link": "",
|
|
370
|
+
} # Below throttle_limit
|
|
371
|
+
|
|
372
|
+
with patch(
|
|
373
|
+
"github_backup.github_backup.make_request_with_retry",
|
|
374
|
+
return_value=mock_response,
|
|
375
|
+
):
|
|
328
376
|
with patch("github_backup.github_backup.time.sleep") as mock_sleep:
|
|
329
|
-
github_backup.retrieve_data(
|
|
377
|
+
github_backup.retrieve_data(
|
|
378
|
+
mock_args, "https://api.github.com/repos/test/repo/issues"
|
|
379
|
+
)
|
|
330
380
|
|
|
331
381
|
mock_sleep.assert_called_once_with(5) # throttle_pause value
|
|
332
382
|
|
|
@@ -344,16 +394,121 @@ class TestRetrieveDataSingleItem:
|
|
|
344
394
|
args.osx_keychain_item_account = None
|
|
345
395
|
args.throttle_limit = None
|
|
346
396
|
args.throttle_pause = 0
|
|
397
|
+
args.max_retries = DEFAULT_MAX_RETRIES
|
|
347
398
|
return args
|
|
348
399
|
|
|
349
400
|
def test_dict_response_returned_as_list(self, mock_args):
|
|
350
401
|
"""Single dict response should be returned as a list with one item."""
|
|
351
402
|
mock_response = Mock()
|
|
352
403
|
mock_response.getcode.return_value = 200
|
|
353
|
-
mock_response.read.return_value = json.dumps(
|
|
404
|
+
mock_response.read.return_value = json.dumps(
|
|
405
|
+
{"login": "testuser", "id": 123}
|
|
406
|
+
).encode("utf-8")
|
|
354
407
|
mock_response.headers = {"x-ratelimit-remaining": "5000", "Link": ""}
|
|
355
408
|
|
|
356
|
-
with patch(
|
|
357
|
-
|
|
409
|
+
with patch(
|
|
410
|
+
"github_backup.github_backup.make_request_with_retry",
|
|
411
|
+
return_value=mock_response,
|
|
412
|
+
):
|
|
413
|
+
result = github_backup.retrieve_data(
|
|
414
|
+
mock_args, "https://api.github.com/user"
|
|
415
|
+
)
|
|
358
416
|
|
|
359
417
|
assert result == [{"login": "testuser", "id": 123}]
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
class TestRetriesCliArgument:
|
|
421
|
+
"""Tests for --retries CLI argument validation and behavior."""
|
|
422
|
+
|
|
423
|
+
def test_retries_argument_accepted(self):
|
|
424
|
+
"""--retries flag should be accepted and parsed correctly."""
|
|
425
|
+
args = github_backup.parse_args(["--retries", "3", "testuser"])
|
|
426
|
+
assert args.max_retries == 3
|
|
427
|
+
|
|
428
|
+
def test_retries_default_value(self):
|
|
429
|
+
"""--retries should default to 5 if not specified."""
|
|
430
|
+
args = github_backup.parse_args(["testuser"])
|
|
431
|
+
assert args.max_retries == 5
|
|
432
|
+
|
|
433
|
+
def test_retries_zero_is_valid(self):
|
|
434
|
+
"""--retries 0 should be valid and mean 1 attempt (no retries)."""
|
|
435
|
+
args = github_backup.parse_args(["--retries", "0", "testuser"])
|
|
436
|
+
assert args.max_retries == 0
|
|
437
|
+
|
|
438
|
+
def test_retries_negative_rejected(self):
|
|
439
|
+
"""--retries with negative value should be rejected by argparse."""
|
|
440
|
+
with pytest.raises(SystemExit):
|
|
441
|
+
github_backup.parse_args(["--retries", "-1", "testuser"])
|
|
442
|
+
|
|
443
|
+
def test_retries_non_integer_rejected(self):
|
|
444
|
+
"""--retries with non-integer value should be rejected by argparse."""
|
|
445
|
+
with pytest.raises(SystemExit):
|
|
446
|
+
github_backup.parse_args(["--retries", "abc", "testuser"])
|
|
447
|
+
|
|
448
|
+
def test_retries_one_with_transient_error_succeeds(self):
|
|
449
|
+
"""--retries 1 should allow one retry after initial failure."""
|
|
450
|
+
good_response = Mock()
|
|
451
|
+
good_response.read.return_value = b'{"ok": true}'
|
|
452
|
+
|
|
453
|
+
call_count = 0
|
|
454
|
+
|
|
455
|
+
def mock_urlopen(*args, **kwargs):
|
|
456
|
+
nonlocal call_count
|
|
457
|
+
call_count += 1
|
|
458
|
+
if call_count == 1:
|
|
459
|
+
raise HTTPError(
|
|
460
|
+
url="https://api.github.com/test",
|
|
461
|
+
code=502,
|
|
462
|
+
msg="Bad Gateway",
|
|
463
|
+
hdrs={"x-ratelimit-remaining": "5000"},
|
|
464
|
+
fp=None,
|
|
465
|
+
)
|
|
466
|
+
return good_response
|
|
467
|
+
|
|
468
|
+
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
|
469
|
+
with patch(
|
|
470
|
+
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
|
471
|
+
):
|
|
472
|
+
result = make_request_with_retry(Mock(), None, max_retries=1)
|
|
473
|
+
|
|
474
|
+
assert result == good_response
|
|
475
|
+
assert call_count == 2 # 1 initial + 1 retry = 2 attempts
|
|
476
|
+
|
|
477
|
+
def test_custom_retry_count_limits_attempts(self):
|
|
478
|
+
"""Custom --retries value should limit actual retry attempts."""
|
|
479
|
+
args = Mock()
|
|
480
|
+
args.as_app = False
|
|
481
|
+
args.token_fine = None
|
|
482
|
+
args.token_classic = "fake_token"
|
|
483
|
+
args.osx_keychain_item_name = None
|
|
484
|
+
args.osx_keychain_item_account = None
|
|
485
|
+
args.throttle_limit = None
|
|
486
|
+
args.throttle_pause = 0
|
|
487
|
+
args.max_retries = 2 # 2 retries = 3 total attempts (1 initial + 2 retries)
|
|
488
|
+
|
|
489
|
+
mock_response = Mock()
|
|
490
|
+
mock_response.getcode.return_value = 200
|
|
491
|
+
mock_response.read.return_value = b"not valid json {"
|
|
492
|
+
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
|
493
|
+
|
|
494
|
+
call_count = 0
|
|
495
|
+
|
|
496
|
+
def mock_make_request(*args, **kwargs):
|
|
497
|
+
nonlocal call_count
|
|
498
|
+
call_count += 1
|
|
499
|
+
return mock_response
|
|
500
|
+
|
|
501
|
+
with patch(
|
|
502
|
+
"github_backup.github_backup.make_request_with_retry",
|
|
503
|
+
side_effect=mock_make_request,
|
|
504
|
+
):
|
|
505
|
+
with patch(
|
|
506
|
+
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
|
507
|
+
):
|
|
508
|
+
with pytest.raises(Exception) as exc_info:
|
|
509
|
+
github_backup.retrieve_data(
|
|
510
|
+
args, "https://api.github.com/repos/test/repo/issues"
|
|
511
|
+
)
|
|
512
|
+
|
|
513
|
+
assert "Failed to read response after 3 attempts" in str(exc_info.value)
|
|
514
|
+
assert call_count == 3 # 1 initial + 2 retries = 3 attempts
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.59.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|