github-backup 0.51.2__tar.gz → 0.52.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {github_backup-0.51.2 → github_backup-0.52.0}/CHANGES.rst +89 -1
- {github_backup-0.51.2/github_backup.egg-info → github_backup-0.52.0}/PKG-INFO +1 -1
- github_backup-0.52.0/github_backup/__init__.py +1 -0
- {github_backup-0.51.2 → github_backup-0.52.0}/github_backup/github_backup.py +98 -58
- {github_backup-0.51.2 → github_backup-0.52.0/github_backup.egg-info}/PKG-INFO +1 -1
- {github_backup-0.51.2 → github_backup-0.52.0}/github_backup.egg-info/SOURCES.txt +3 -1
- {github_backup-0.51.2 → github_backup-0.52.0}/release-requirements.txt +4 -4
- github_backup-0.52.0/tests/test_http_451.py +143 -0
- github_backup-0.52.0/tests/test_pagination.py +153 -0
- github_backup-0.51.2/github_backup/__init__.py +0 -1
- {github_backup-0.51.2 → github_backup-0.52.0}/LICENSE.txt +0 -0
- {github_backup-0.51.2 → github_backup-0.52.0}/MANIFEST.in +0 -0
- {github_backup-0.51.2 → github_backup-0.52.0}/README +0 -0
- {github_backup-0.51.2 → github_backup-0.52.0}/README.rst +0 -0
- {github_backup-0.51.2 → github_backup-0.52.0}/bin/github-backup +0 -0
- {github_backup-0.51.2 → github_backup-0.52.0}/github_backup.egg-info/dependency_links.txt +0 -0
- {github_backup-0.51.2 → github_backup-0.52.0}/github_backup.egg-info/top_level.txt +0 -0
- {github_backup-0.51.2 → github_backup-0.52.0}/github_backup.egg-info/zip-safe +0 -0
- {github_backup-0.51.2 → github_backup-0.52.0}/requirements.txt +0 -0
- {github_backup-0.51.2 → github_backup-0.52.0}/setup.cfg +0 -0
- {github_backup-0.51.2 → github_backup-0.52.0}/setup.py +0 -0
- {github_backup-0.51.2 → github_backup-0.52.0}/tests/test_attachments.py +0 -0
|
@@ -1,9 +1,97 @@
|
|
|
1
1
|
Changelog
|
|
2
2
|
=========
|
|
3
3
|
|
|
4
|
-
0.
|
|
4
|
+
0.52.0 (2025-11-28)
|
|
5
5
|
-------------------
|
|
6
6
|
------------------------
|
|
7
|
+
- Skip DMCA'd repos which return a 451 response. [Rodos]
|
|
8
|
+
|
|
9
|
+
Log a warning and the link to the DMCA notice. Continue backing up
|
|
10
|
+
other repositories instead of crashing.
|
|
11
|
+
|
|
12
|
+
Closes #163
|
|
13
|
+
- Chore(deps): bump restructuredtext-lint in the python-packages group.
|
|
14
|
+
[dependabot[bot]]
|
|
15
|
+
|
|
16
|
+
Bumps the python-packages group with 1 update: [restructuredtext-lint](https://github.com/twolfson/restructuredtext-lint).
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
Updates `restructuredtext-lint` from 1.4.0 to 2.0.2
|
|
20
|
+
- [Changelog](https://github.com/twolfson/restructuredtext-lint/blob/master/CHANGELOG.rst)
|
|
21
|
+
- [Commits](https://github.com/twolfson/restructuredtext-lint/compare/1.4.0...2.0.2)
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
updated-dependencies:
|
|
25
|
+
- dependency-name: restructuredtext-lint
|
|
26
|
+
dependency-version: 2.0.2
|
|
27
|
+
dependency-type: direct:production
|
|
28
|
+
update-type: version-update:semver-major
|
|
29
|
+
dependency-group: python-packages
|
|
30
|
+
...
|
|
31
|
+
- Chore(deps): bump actions/checkout from 5 to 6. [dependabot[bot]]
|
|
32
|
+
|
|
33
|
+
Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6.
|
|
34
|
+
- [Release notes](https://github.com/actions/checkout/releases)
|
|
35
|
+
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
|
|
36
|
+
- [Commits](https://github.com/actions/checkout/compare/v5...v6)
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
updated-dependencies:
|
|
40
|
+
- dependency-name: actions/checkout
|
|
41
|
+
dependency-version: '6'
|
|
42
|
+
dependency-type: direct:production
|
|
43
|
+
update-type: version-update:semver-major
|
|
44
|
+
...
|
|
45
|
+
- Chore(deps): bump the python-packages group with 3 updates.
|
|
46
|
+
[dependabot[bot]]
|
|
47
|
+
|
|
48
|
+
Bumps the python-packages group with 3 updates: [click](https://github.com/pallets/click), [pytest](https://github.com/pytest-dev/pytest) and [keyring](https://github.com/jaraco/keyring).
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
Updates `click` from 8.3.0 to 8.3.1
|
|
52
|
+
- [Release notes](https://github.com/pallets/click/releases)
|
|
53
|
+
- [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst)
|
|
54
|
+
- [Commits](https://github.com/pallets/click/compare/8.3.0...8.3.1)
|
|
55
|
+
|
|
56
|
+
Updates `pytest` from 8.3.3 to 9.0.1
|
|
57
|
+
- [Release notes](https://github.com/pytest-dev/pytest/releases)
|
|
58
|
+
- [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
|
|
59
|
+
- [Commits](https://github.com/pytest-dev/pytest/compare/8.3.3...9.0.1)
|
|
60
|
+
|
|
61
|
+
Updates `keyring` from 25.6.0 to 25.7.0
|
|
62
|
+
- [Release notes](https://github.com/jaraco/keyring/releases)
|
|
63
|
+
- [Changelog](https://github.com/jaraco/keyring/blob/main/NEWS.rst)
|
|
64
|
+
- [Commits](https://github.com/jaraco/keyring/compare/v25.6.0...v25.7.0)
|
|
65
|
+
|
|
66
|
+
---
|
|
67
|
+
updated-dependencies:
|
|
68
|
+
- dependency-name: click
|
|
69
|
+
dependency-version: 8.3.1
|
|
70
|
+
dependency-type: direct:production
|
|
71
|
+
update-type: version-update:semver-patch
|
|
72
|
+
dependency-group: python-packages
|
|
73
|
+
- dependency-name: pytest
|
|
74
|
+
dependency-version: 9.0.1
|
|
75
|
+
dependency-type: direct:production
|
|
76
|
+
update-type: version-update:semver-major
|
|
77
|
+
dependency-group: python-packages
|
|
78
|
+
- dependency-name: keyring
|
|
79
|
+
dependency-version: 25.7.0
|
|
80
|
+
dependency-type: direct:production
|
|
81
|
+
update-type: version-update:semver-minor
|
|
82
|
+
dependency-group: python-packages
|
|
83
|
+
...
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
0.51.3 (2025-11-18)
|
|
87
|
+
-------------------
|
|
88
|
+
- Test: Add pagination tests for cursor and page-based Link headers.
|
|
89
|
+
[Rodos]
|
|
90
|
+
- Use cursor based pagination. [Helio Machado]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
0.51.2 (2025-11-16)
|
|
94
|
+
-------------------
|
|
7
95
|
|
|
8
96
|
Fix
|
|
9
97
|
~~~
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.52.0"
|
|
@@ -37,6 +37,15 @@ FNULL = open(os.devnull, "w")
|
|
|
37
37
|
FILE_URI_PREFIX = "file://"
|
|
38
38
|
logger = logging.getLogger(__name__)
|
|
39
39
|
|
|
40
|
+
|
|
41
|
+
class RepositoryUnavailableError(Exception):
|
|
42
|
+
"""Raised when a repository is unavailable due to legal reasons (e.g., DMCA takedown)."""
|
|
43
|
+
|
|
44
|
+
def __init__(self, message, dmca_url=None):
|
|
45
|
+
super().__init__(message)
|
|
46
|
+
self.dmca_url = dmca_url
|
|
47
|
+
|
|
48
|
+
|
|
40
49
|
# Setup SSL context with fallback chain
|
|
41
50
|
https_ctx = ssl.create_default_context()
|
|
42
51
|
if https_ctx.get_ca_certs():
|
|
@@ -592,27 +601,39 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
|
|
|
592
601
|
auth = get_auth(args, encode=not args.as_app)
|
|
593
602
|
query_args = get_query_args(query_args)
|
|
594
603
|
per_page = 100
|
|
595
|
-
|
|
604
|
+
next_url = None
|
|
596
605
|
|
|
597
606
|
while True:
|
|
598
607
|
if single_request:
|
|
599
|
-
|
|
608
|
+
request_per_page = None
|
|
600
609
|
else:
|
|
601
|
-
|
|
602
|
-
request_page, request_per_page = page, per_page
|
|
610
|
+
request_per_page = per_page
|
|
603
611
|
|
|
604
612
|
request = _construct_request(
|
|
605
613
|
request_per_page,
|
|
606
|
-
request_page,
|
|
607
614
|
query_args,
|
|
608
|
-
template,
|
|
615
|
+
next_url or template,
|
|
609
616
|
auth,
|
|
610
617
|
as_app=args.as_app,
|
|
611
618
|
fine=True if args.token_fine is not None else False,
|
|
612
619
|
) # noqa
|
|
613
|
-
r, errors = _get_response(request, auth, template)
|
|
620
|
+
r, errors = _get_response(request, auth, next_url or template)
|
|
614
621
|
|
|
615
622
|
status_code = int(r.getcode())
|
|
623
|
+
|
|
624
|
+
# Handle DMCA takedown (HTTP 451) - raise exception to skip entire repository
|
|
625
|
+
if status_code == 451:
|
|
626
|
+
dmca_url = None
|
|
627
|
+
try:
|
|
628
|
+
response_data = json.loads(r.read().decode("utf-8"))
|
|
629
|
+
dmca_url = response_data.get("block", {}).get("html_url")
|
|
630
|
+
except Exception:
|
|
631
|
+
pass
|
|
632
|
+
raise RepositoryUnavailableError(
|
|
633
|
+
"Repository unavailable due to legal reasons (HTTP 451)",
|
|
634
|
+
dmca_url=dmca_url
|
|
635
|
+
)
|
|
636
|
+
|
|
616
637
|
# Check if we got correct data
|
|
617
638
|
try:
|
|
618
639
|
response = json.loads(r.read().decode("utf-8"))
|
|
@@ -644,15 +665,14 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
|
|
|
644
665
|
retries += 1
|
|
645
666
|
time.sleep(5)
|
|
646
667
|
request = _construct_request(
|
|
647
|
-
|
|
648
|
-
page,
|
|
668
|
+
request_per_page,
|
|
649
669
|
query_args,
|
|
650
|
-
template,
|
|
670
|
+
next_url or template,
|
|
651
671
|
auth,
|
|
652
672
|
as_app=args.as_app,
|
|
653
673
|
fine=True if args.token_fine is not None else False,
|
|
654
674
|
) # noqa
|
|
655
|
-
r, errors = _get_response(request, auth, template)
|
|
675
|
+
r, errors = _get_response(request, auth, next_url or template)
|
|
656
676
|
|
|
657
677
|
status_code = int(r.getcode())
|
|
658
678
|
try:
|
|
@@ -682,7 +702,16 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
|
|
|
682
702
|
if type(response) is list:
|
|
683
703
|
for resp in response:
|
|
684
704
|
yield resp
|
|
685
|
-
|
|
705
|
+
# Parse Link header for next page URL (cursor-based pagination)
|
|
706
|
+
link_header = r.headers.get("Link", "")
|
|
707
|
+
next_url = None
|
|
708
|
+
if link_header:
|
|
709
|
+
# Parse Link header: <https://api.github.com/...?per_page=100&after=cursor>; rel="next"
|
|
710
|
+
for link in link_header.split(","):
|
|
711
|
+
if 'rel="next"' in link:
|
|
712
|
+
next_url = link[link.find("<") + 1:link.find(">")]
|
|
713
|
+
break
|
|
714
|
+
if not next_url:
|
|
686
715
|
break
|
|
687
716
|
elif type(response) is dict and single_request:
|
|
688
717
|
yield response
|
|
@@ -735,22 +764,27 @@ def _get_response(request, auth, template):
|
|
|
735
764
|
|
|
736
765
|
|
|
737
766
|
def _construct_request(
|
|
738
|
-
per_page,
|
|
767
|
+
per_page, query_args, template, auth, as_app=None, fine=False
|
|
739
768
|
):
|
|
740
|
-
|
|
741
|
-
if
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
if query_args:
|
|
746
|
-
all_query_args.update(query_args)
|
|
747
|
-
|
|
748
|
-
request_url = template
|
|
749
|
-
if all_query_args:
|
|
750
|
-
querystring = urlencode(all_query_args)
|
|
751
|
-
request_url = template + "?" + querystring
|
|
769
|
+
# If template is already a full URL with query params (from Link header), use it directly
|
|
770
|
+
if "?" in template and template.startswith("http"):
|
|
771
|
+
request_url = template
|
|
772
|
+
# Extract query string for logging
|
|
773
|
+
querystring = template.split("?", 1)[1]
|
|
752
774
|
else:
|
|
753
|
-
|
|
775
|
+
# Build URL with query parameters
|
|
776
|
+
all_query_args = {}
|
|
777
|
+
if per_page:
|
|
778
|
+
all_query_args["per_page"] = per_page
|
|
779
|
+
if query_args:
|
|
780
|
+
all_query_args.update(query_args)
|
|
781
|
+
|
|
782
|
+
request_url = template
|
|
783
|
+
if all_query_args:
|
|
784
|
+
querystring = urlencode(all_query_args)
|
|
785
|
+
request_url = template + "?" + querystring
|
|
786
|
+
else:
|
|
787
|
+
querystring = ""
|
|
754
788
|
|
|
755
789
|
request = Request(request_url)
|
|
756
790
|
if auth is not None:
|
|
@@ -766,7 +800,7 @@ def _construct_request(
|
|
|
766
800
|
"Accept", "application/vnd.github.machine-man-preview+json"
|
|
767
801
|
)
|
|
768
802
|
|
|
769
|
-
log_url = template
|
|
803
|
+
log_url = template if "?" not in template else template.split("?")[0]
|
|
770
804
|
if querystring:
|
|
771
805
|
log_url += "?" + querystring
|
|
772
806
|
logger.info("Requesting {}".format(log_url))
|
|
@@ -843,8 +877,7 @@ def download_file(url, path, auth, as_app=False, fine=False):
|
|
|
843
877
|
return
|
|
844
878
|
|
|
845
879
|
request = _construct_request(
|
|
846
|
-
per_page=
|
|
847
|
-
page=1,
|
|
880
|
+
per_page=None,
|
|
848
881
|
query_args={},
|
|
849
882
|
template=url,
|
|
850
883
|
auth=auth,
|
|
@@ -1657,40 +1690,47 @@ def backup_repositories(args, output_directory, repositories):
|
|
|
1657
1690
|
|
|
1658
1691
|
continue # don't try to back anything else for a gist; it doesn't exist
|
|
1659
1692
|
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1693
|
+
try:
|
|
1694
|
+
download_wiki = args.include_wiki or args.include_everything
|
|
1695
|
+
if repository["has_wiki"] and download_wiki:
|
|
1696
|
+
fetch_repository(
|
|
1697
|
+
repository["name"],
|
|
1698
|
+
repo_url.replace(".git", ".wiki.git"),
|
|
1699
|
+
os.path.join(repo_cwd, "wiki"),
|
|
1700
|
+
skip_existing=args.skip_existing,
|
|
1701
|
+
bare_clone=args.bare_clone,
|
|
1702
|
+
lfs_clone=args.lfs_clone,
|
|
1703
|
+
no_prune=args.no_prune,
|
|
1704
|
+
)
|
|
1705
|
+
if args.include_issues or args.include_everything:
|
|
1706
|
+
backup_issues(args, repo_cwd, repository, repos_template)
|
|
1673
1707
|
|
|
1674
|
-
|
|
1675
|
-
|
|
1708
|
+
if args.include_pulls or args.include_everything:
|
|
1709
|
+
backup_pulls(args, repo_cwd, repository, repos_template)
|
|
1676
1710
|
|
|
1677
|
-
|
|
1678
|
-
|
|
1711
|
+
if args.include_milestones or args.include_everything:
|
|
1712
|
+
backup_milestones(args, repo_cwd, repository, repos_template)
|
|
1679
1713
|
|
|
1680
|
-
|
|
1681
|
-
|
|
1714
|
+
if args.include_labels or args.include_everything:
|
|
1715
|
+
backup_labels(args, repo_cwd, repository, repos_template)
|
|
1682
1716
|
|
|
1683
|
-
|
|
1684
|
-
|
|
1717
|
+
if args.include_hooks or args.include_everything:
|
|
1718
|
+
backup_hooks(args, repo_cwd, repository, repos_template)
|
|
1685
1719
|
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1720
|
+
if args.include_releases or args.include_everything:
|
|
1721
|
+
backup_releases(
|
|
1722
|
+
args,
|
|
1723
|
+
repo_cwd,
|
|
1724
|
+
repository,
|
|
1725
|
+
repos_template,
|
|
1726
|
+
include_assets=args.include_assets or args.include_everything,
|
|
1727
|
+
)
|
|
1728
|
+
except RepositoryUnavailableError as e:
|
|
1729
|
+
logger.warning(f"Repository {repository['full_name']} is unavailable (HTTP 451)")
|
|
1730
|
+
if e.dmca_url:
|
|
1731
|
+
logger.warning(f"DMCA notice: {e.dmca_url}")
|
|
1732
|
+
logger.info(f"Skipping remaining resources for {repository['full_name']}")
|
|
1733
|
+
continue
|
|
1694
1734
|
|
|
1695
1735
|
if args.incremental:
|
|
1696
1736
|
if last_update == "0000-00-00T00:00:00Z":
|
|
@@ -14,4 +14,6 @@ github_backup.egg-info/SOURCES.txt
|
|
|
14
14
|
github_backup.egg-info/dependency_links.txt
|
|
15
15
|
github_backup.egg-info/top_level.txt
|
|
16
16
|
github_backup.egg-info/zip-safe
|
|
17
|
-
tests/test_attachments.py
|
|
17
|
+
tests/test_attachments.py
|
|
18
|
+
tests/test_http_451.py
|
|
19
|
+
tests/test_pagination.py
|
|
@@ -3,16 +3,16 @@ black==25.11.0
|
|
|
3
3
|
bleach==6.3.0
|
|
4
4
|
certifi==2025.11.12
|
|
5
5
|
charset-normalizer==3.4.4
|
|
6
|
-
click==8.3.
|
|
6
|
+
click==8.3.1
|
|
7
7
|
colorama==0.4.6
|
|
8
8
|
docutils==0.22.3
|
|
9
9
|
flake8==7.3.0
|
|
10
10
|
gitchangelog==3.0.4
|
|
11
|
-
pytest==
|
|
11
|
+
pytest==9.0.1
|
|
12
12
|
idna==3.11
|
|
13
13
|
importlib-metadata==8.7.0
|
|
14
14
|
jaraco.classes==3.4.0
|
|
15
|
-
keyring==25.
|
|
15
|
+
keyring==25.7.0
|
|
16
16
|
markdown-it-py==4.0.0
|
|
17
17
|
mccabe==0.7.0
|
|
18
18
|
mdurl==0.1.2
|
|
@@ -28,7 +28,7 @@ Pygments==2.19.2
|
|
|
28
28
|
readme-renderer==44.0
|
|
29
29
|
requests==2.32.5
|
|
30
30
|
requests-toolbelt==1.0.0
|
|
31
|
-
restructuredtext-lint==
|
|
31
|
+
restructuredtext-lint==2.0.2
|
|
32
32
|
rfc3986==2.0.0
|
|
33
33
|
rich==14.2.0
|
|
34
34
|
setuptools==80.9.0
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""Tests for HTTP 451 (DMCA takedown) handling."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from unittest.mock import Mock, patch
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from github_backup import github_backup
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TestHTTP451Exception:
|
|
12
|
+
"""Test suite for HTTP 451 DMCA takedown exception handling."""
|
|
13
|
+
|
|
14
|
+
def test_repository_unavailable_error_raised(self):
|
|
15
|
+
"""HTTP 451 should raise RepositoryUnavailableError with DMCA URL."""
|
|
16
|
+
# Create mock args
|
|
17
|
+
args = Mock()
|
|
18
|
+
args.as_app = False
|
|
19
|
+
args.token_fine = None
|
|
20
|
+
args.token_classic = None
|
|
21
|
+
args.username = None
|
|
22
|
+
args.password = None
|
|
23
|
+
args.osx_keychain_item_name = None
|
|
24
|
+
args.osx_keychain_item_account = None
|
|
25
|
+
args.throttle_limit = None
|
|
26
|
+
args.throttle_pause = 0
|
|
27
|
+
|
|
28
|
+
# Mock HTTPError 451 response
|
|
29
|
+
mock_response = Mock()
|
|
30
|
+
mock_response.getcode.return_value = 451
|
|
31
|
+
|
|
32
|
+
dmca_data = {
|
|
33
|
+
"message": "Repository access blocked",
|
|
34
|
+
"block": {
|
|
35
|
+
"reason": "dmca",
|
|
36
|
+
"created_at": "2024-11-12T14:38:04Z",
|
|
37
|
+
"html_url": "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md"
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
mock_response.read.return_value = json.dumps(dmca_data).encode("utf-8")
|
|
41
|
+
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
|
42
|
+
mock_response.reason = "Unavailable For Legal Reasons"
|
|
43
|
+
|
|
44
|
+
def mock_get_response(request, auth, template):
|
|
45
|
+
return mock_response, []
|
|
46
|
+
|
|
47
|
+
with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
|
|
48
|
+
with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
|
|
49
|
+
list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/dmca/issues"))
|
|
50
|
+
|
|
51
|
+
# Check exception has DMCA URL
|
|
52
|
+
assert exc_info.value.dmca_url == "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md"
|
|
53
|
+
assert "451" in str(exc_info.value)
|
|
54
|
+
|
|
55
|
+
def test_repository_unavailable_error_without_dmca_url(self):
|
|
56
|
+
"""HTTP 451 without DMCA details should still raise exception."""
|
|
57
|
+
args = Mock()
|
|
58
|
+
args.as_app = False
|
|
59
|
+
args.token_fine = None
|
|
60
|
+
args.token_classic = None
|
|
61
|
+
args.username = None
|
|
62
|
+
args.password = None
|
|
63
|
+
args.osx_keychain_item_name = None
|
|
64
|
+
args.osx_keychain_item_account = None
|
|
65
|
+
args.throttle_limit = None
|
|
66
|
+
args.throttle_pause = 0
|
|
67
|
+
|
|
68
|
+
mock_response = Mock()
|
|
69
|
+
mock_response.getcode.return_value = 451
|
|
70
|
+
mock_response.read.return_value = b'{"message": "Blocked"}'
|
|
71
|
+
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
|
72
|
+
mock_response.reason = "Unavailable For Legal Reasons"
|
|
73
|
+
|
|
74
|
+
def mock_get_response(request, auth, template):
|
|
75
|
+
return mock_response, []
|
|
76
|
+
|
|
77
|
+
with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
|
|
78
|
+
with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
|
|
79
|
+
list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/dmca/issues"))
|
|
80
|
+
|
|
81
|
+
# Exception raised even without DMCA URL
|
|
82
|
+
assert exc_info.value.dmca_url is None
|
|
83
|
+
assert "451" in str(exc_info.value)
|
|
84
|
+
|
|
85
|
+
def test_repository_unavailable_error_with_malformed_json(self):
|
|
86
|
+
"""HTTP 451 with malformed JSON should still raise exception."""
|
|
87
|
+
args = Mock()
|
|
88
|
+
args.as_app = False
|
|
89
|
+
args.token_fine = None
|
|
90
|
+
args.token_classic = None
|
|
91
|
+
args.username = None
|
|
92
|
+
args.password = None
|
|
93
|
+
args.osx_keychain_item_name = None
|
|
94
|
+
args.osx_keychain_item_account = None
|
|
95
|
+
args.throttle_limit = None
|
|
96
|
+
args.throttle_pause = 0
|
|
97
|
+
|
|
98
|
+
mock_response = Mock()
|
|
99
|
+
mock_response.getcode.return_value = 451
|
|
100
|
+
mock_response.read.return_value = b"invalid json {"
|
|
101
|
+
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
|
102
|
+
mock_response.reason = "Unavailable For Legal Reasons"
|
|
103
|
+
|
|
104
|
+
def mock_get_response(request, auth, template):
|
|
105
|
+
return mock_response, []
|
|
106
|
+
|
|
107
|
+
with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
|
|
108
|
+
with pytest.raises(github_backup.RepositoryUnavailableError):
|
|
109
|
+
list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/dmca/issues"))
|
|
110
|
+
|
|
111
|
+
def test_other_http_errors_unchanged(self):
|
|
112
|
+
"""Other HTTP errors should still raise generic Exception."""
|
|
113
|
+
args = Mock()
|
|
114
|
+
args.as_app = False
|
|
115
|
+
args.token_fine = None
|
|
116
|
+
args.token_classic = None
|
|
117
|
+
args.username = None
|
|
118
|
+
args.password = None
|
|
119
|
+
args.osx_keychain_item_name = None
|
|
120
|
+
args.osx_keychain_item_account = None
|
|
121
|
+
args.throttle_limit = None
|
|
122
|
+
args.throttle_pause = 0
|
|
123
|
+
|
|
124
|
+
mock_response = Mock()
|
|
125
|
+
mock_response.getcode.return_value = 404
|
|
126
|
+
mock_response.read.return_value = b'{"message": "Not Found"}'
|
|
127
|
+
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
|
128
|
+
mock_response.reason = "Not Found"
|
|
129
|
+
|
|
130
|
+
def mock_get_response(request, auth, template):
|
|
131
|
+
return mock_response, []
|
|
132
|
+
|
|
133
|
+
with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
|
|
134
|
+
# Should raise generic Exception, not RepositoryUnavailableError
|
|
135
|
+
with pytest.raises(Exception) as exc_info:
|
|
136
|
+
list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/notfound/issues"))
|
|
137
|
+
|
|
138
|
+
assert not isinstance(exc_info.value, github_backup.RepositoryUnavailableError)
|
|
139
|
+
assert "404" in str(exc_info.value)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
if __name__ == "__main__":
|
|
143
|
+
pytest.main([__file__, "-v"])
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""Tests for Link header pagination handling."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from unittest.mock import Mock, patch
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from github_backup import github_backup
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class MockHTTPResponse:
|
|
12
|
+
"""Mock HTTP response for paginated API calls."""
|
|
13
|
+
|
|
14
|
+
def __init__(self, data, link_header=None):
|
|
15
|
+
self._content = json.dumps(data).encode("utf-8")
|
|
16
|
+
self._link_header = link_header
|
|
17
|
+
self._read = False
|
|
18
|
+
self.reason = "OK"
|
|
19
|
+
|
|
20
|
+
def getcode(self):
|
|
21
|
+
return 200
|
|
22
|
+
|
|
23
|
+
def read(self):
|
|
24
|
+
if self._read:
|
|
25
|
+
return b""
|
|
26
|
+
self._read = True
|
|
27
|
+
return self._content
|
|
28
|
+
|
|
29
|
+
def get_header(self, name, default=None):
|
|
30
|
+
"""Mock method for headers.get()."""
|
|
31
|
+
return self.headers.get(name, default)
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def headers(self):
|
|
35
|
+
headers = {"x-ratelimit-remaining": "5000"}
|
|
36
|
+
if self._link_header:
|
|
37
|
+
headers["Link"] = self._link_header
|
|
38
|
+
return headers
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@pytest.fixture
|
|
42
|
+
def mock_args():
|
|
43
|
+
"""Mock args for retrieve_data_gen."""
|
|
44
|
+
args = Mock()
|
|
45
|
+
args.as_app = False
|
|
46
|
+
args.token_fine = None
|
|
47
|
+
args.token_classic = "fake_token"
|
|
48
|
+
args.username = None
|
|
49
|
+
args.password = None
|
|
50
|
+
args.osx_keychain_item_name = None
|
|
51
|
+
args.osx_keychain_item_account = None
|
|
52
|
+
args.throttle_limit = None
|
|
53
|
+
args.throttle_pause = 0
|
|
54
|
+
return args
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_cursor_based_pagination(mock_args):
|
|
58
|
+
"""Link header with 'after' cursor parameter works correctly."""
|
|
59
|
+
|
|
60
|
+
# Simulate issues endpoint behavior: returns cursor in Link header
|
|
61
|
+
responses = [
|
|
62
|
+
# Issues endpoint returns 'after' cursor parameter (not 'page')
|
|
63
|
+
MockHTTPResponse(
|
|
64
|
+
data=[{"issue": i} for i in range(1, 101)], # Page 1 contents
|
|
65
|
+
link_header='<https://api.github.com/repos/owner/repo/issues?per_page=100&after=ABC123&page=2>; rel="next"',
|
|
66
|
+
),
|
|
67
|
+
MockHTTPResponse(
|
|
68
|
+
data=[{"issue": i} for i in range(101, 151)], # Page 2 contents
|
|
69
|
+
link_header=None, # No Link header - signals end of pagination
|
|
70
|
+
),
|
|
71
|
+
]
|
|
72
|
+
requests_made = []
|
|
73
|
+
|
|
74
|
+
def mock_urlopen(request, *args, **kwargs):
|
|
75
|
+
url = request.get_full_url()
|
|
76
|
+
requests_made.append(url)
|
|
77
|
+
return responses[len(requests_made) - 1]
|
|
78
|
+
|
|
79
|
+
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
|
80
|
+
results = list(
|
|
81
|
+
github_backup.retrieve_data_gen(
|
|
82
|
+
mock_args, "https://api.github.com/repos/owner/repo/issues"
|
|
83
|
+
)
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
# Verify all items retrieved and cursor was used in second request
|
|
87
|
+
assert len(results) == 150
|
|
88
|
+
assert len(requests_made) == 2
|
|
89
|
+
assert "after=ABC123" in requests_made[1]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_page_based_pagination(mock_args):
|
|
93
|
+
"""Link header with 'page' parameter works correctly."""
|
|
94
|
+
|
|
95
|
+
# Simulate pulls/repos endpoint behavior: returns page numbers in Link header
|
|
96
|
+
responses = [
|
|
97
|
+
# Pulls endpoint uses traditional 'page' parameter (not cursor)
|
|
98
|
+
MockHTTPResponse(
|
|
99
|
+
data=[{"pull": i} for i in range(1, 101)], # Page 1 contents
|
|
100
|
+
link_header='<https://api.github.com/repos/owner/repo/pulls?per_page=100&page=2>; rel="next"',
|
|
101
|
+
),
|
|
102
|
+
MockHTTPResponse(
|
|
103
|
+
data=[{"pull": i} for i in range(101, 181)], # Page 2 contents
|
|
104
|
+
link_header=None, # No Link header - signals end of pagination
|
|
105
|
+
),
|
|
106
|
+
]
|
|
107
|
+
requests_made = []
|
|
108
|
+
|
|
109
|
+
def mock_urlopen(request, *args, **kwargs):
|
|
110
|
+
url = request.get_full_url()
|
|
111
|
+
requests_made.append(url)
|
|
112
|
+
return responses[len(requests_made) - 1]
|
|
113
|
+
|
|
114
|
+
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
|
115
|
+
results = list(
|
|
116
|
+
github_backup.retrieve_data_gen(
|
|
117
|
+
mock_args, "https://api.github.com/repos/owner/repo/pulls"
|
|
118
|
+
)
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
# Verify all items retrieved and page parameter was used (not cursor)
|
|
122
|
+
assert len(results) == 180
|
|
123
|
+
assert len(requests_made) == 2
|
|
124
|
+
assert "page=2" in requests_made[1]
|
|
125
|
+
assert "after" not in requests_made[1]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def test_no_link_header_stops_pagination(mock_args):
|
|
129
|
+
"""Pagination stops when Link header is absent."""
|
|
130
|
+
|
|
131
|
+
# Simulate endpoint with results that fit in a single page
|
|
132
|
+
responses = [
|
|
133
|
+
MockHTTPResponse(
|
|
134
|
+
data=[{"label": i} for i in range(1, 51)], # Page contents
|
|
135
|
+
link_header=None, # No Link header - signals end of pagination
|
|
136
|
+
)
|
|
137
|
+
]
|
|
138
|
+
requests_made = []
|
|
139
|
+
|
|
140
|
+
def mock_urlopen(request, *args, **kwargs):
|
|
141
|
+
requests_made.append(request.get_full_url())
|
|
142
|
+
return responses[len(requests_made) - 1]
|
|
143
|
+
|
|
144
|
+
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
|
145
|
+
results = list(
|
|
146
|
+
github_backup.retrieve_data_gen(
|
|
147
|
+
mock_args, "https://api.github.com/repos/owner/repo/labels"
|
|
148
|
+
)
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
# Verify pagination stopped after first request
|
|
152
|
+
assert len(results) == 50
|
|
153
|
+
assert len(requests_made) == 1
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.51.2"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|