github-backup 0.51.2__tar.gz → 0.52.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. {github_backup-0.51.2 → github_backup-0.52.0}/CHANGES.rst +89 -1
  2. {github_backup-0.51.2/github_backup.egg-info → github_backup-0.52.0}/PKG-INFO +1 -1
  3. github_backup-0.52.0/github_backup/__init__.py +1 -0
  4. {github_backup-0.51.2 → github_backup-0.52.0}/github_backup/github_backup.py +98 -58
  5. {github_backup-0.51.2 → github_backup-0.52.0/github_backup.egg-info}/PKG-INFO +1 -1
  6. {github_backup-0.51.2 → github_backup-0.52.0}/github_backup.egg-info/SOURCES.txt +3 -1
  7. {github_backup-0.51.2 → github_backup-0.52.0}/release-requirements.txt +4 -4
  8. github_backup-0.52.0/tests/test_http_451.py +143 -0
  9. github_backup-0.52.0/tests/test_pagination.py +153 -0
  10. github_backup-0.51.2/github_backup/__init__.py +0 -1
  11. {github_backup-0.51.2 → github_backup-0.52.0}/LICENSE.txt +0 -0
  12. {github_backup-0.51.2 → github_backup-0.52.0}/MANIFEST.in +0 -0
  13. {github_backup-0.51.2 → github_backup-0.52.0}/README +0 -0
  14. {github_backup-0.51.2 → github_backup-0.52.0}/README.rst +0 -0
  15. {github_backup-0.51.2 → github_backup-0.52.0}/bin/github-backup +0 -0
  16. {github_backup-0.51.2 → github_backup-0.52.0}/github_backup.egg-info/dependency_links.txt +0 -0
  17. {github_backup-0.51.2 → github_backup-0.52.0}/github_backup.egg-info/top_level.txt +0 -0
  18. {github_backup-0.51.2 → github_backup-0.52.0}/github_backup.egg-info/zip-safe +0 -0
  19. {github_backup-0.51.2 → github_backup-0.52.0}/requirements.txt +0 -0
  20. {github_backup-0.51.2 → github_backup-0.52.0}/setup.cfg +0 -0
  21. {github_backup-0.51.2 → github_backup-0.52.0}/setup.py +0 -0
  22. {github_backup-0.51.2 → github_backup-0.52.0}/tests/test_attachments.py +0 -0
@@ -1,9 +1,97 @@
1
1
  Changelog
2
2
  =========
3
3
 
4
- 0.51.2 (2025-11-16)
4
+ 0.52.0 (2025-11-28)
5
5
  -------------------
6
6
  ------------------------
7
+ - Skip DMCA'd repos which return a 451 response. [Rodos]
8
+
9
+ Log a warning and the link to the DMCA notice. Continue backing up
10
+ other repositories instead of crashing.
11
+
12
+ Closes #163
13
+ - Chore(deps): bump restructuredtext-lint in the python-packages group.
14
+ [dependabot[bot]]
15
+
16
+ Bumps the python-packages group with 1 update: [restructuredtext-lint](https://github.com/twolfson/restructuredtext-lint).
17
+
18
+
19
+ Updates `restructuredtext-lint` from 1.4.0 to 2.0.2
20
+ - [Changelog](https://github.com/twolfson/restructuredtext-lint/blob/master/CHANGELOG.rst)
21
+ - [Commits](https://github.com/twolfson/restructuredtext-lint/compare/1.4.0...2.0.2)
22
+
23
+ ---
24
+ updated-dependencies:
25
+ - dependency-name: restructuredtext-lint
26
+ dependency-version: 2.0.2
27
+ dependency-type: direct:production
28
+ update-type: version-update:semver-major
29
+ dependency-group: python-packages
30
+ ...
31
+ - Chore(deps): bump actions/checkout from 5 to 6. [dependabot[bot]]
32
+
33
+ Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6.
34
+ - [Release notes](https://github.com/actions/checkout/releases)
35
+ - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
36
+ - [Commits](https://github.com/actions/checkout/compare/v5...v6)
37
+
38
+ ---
39
+ updated-dependencies:
40
+ - dependency-name: actions/checkout
41
+ dependency-version: '6'
42
+ dependency-type: direct:production
43
+ update-type: version-update:semver-major
44
+ ...
45
+ - Chore(deps): bump the python-packages group with 3 updates.
46
+ [dependabot[bot]]
47
+
48
+ Bumps the python-packages group with 3 updates: [click](https://github.com/pallets/click), [pytest](https://github.com/pytest-dev/pytest) and [keyring](https://github.com/jaraco/keyring).
49
+
50
+
51
+ Updates `click` from 8.3.0 to 8.3.1
52
+ - [Release notes](https://github.com/pallets/click/releases)
53
+ - [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst)
54
+ - [Commits](https://github.com/pallets/click/compare/8.3.0...8.3.1)
55
+
56
+ Updates `pytest` from 8.3.3 to 9.0.1
57
+ - [Release notes](https://github.com/pytest-dev/pytest/releases)
58
+ - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
59
+ - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.3...9.0.1)
60
+
61
+ Updates `keyring` from 25.6.0 to 25.7.0
62
+ - [Release notes](https://github.com/jaraco/keyring/releases)
63
+ - [Changelog](https://github.com/jaraco/keyring/blob/main/NEWS.rst)
64
+ - [Commits](https://github.com/jaraco/keyring/compare/v25.6.0...v25.7.0)
65
+
66
+ ---
67
+ updated-dependencies:
68
+ - dependency-name: click
69
+ dependency-version: 8.3.1
70
+ dependency-type: direct:production
71
+ update-type: version-update:semver-patch
72
+ dependency-group: python-packages
73
+ - dependency-name: pytest
74
+ dependency-version: 9.0.1
75
+ dependency-type: direct:production
76
+ update-type: version-update:semver-major
77
+ dependency-group: python-packages
78
+ - dependency-name: keyring
79
+ dependency-version: 25.7.0
80
+ dependency-type: direct:production
81
+ update-type: version-update:semver-minor
82
+ dependency-group: python-packages
83
+ ...
84
+
85
+
86
+ 0.51.3 (2025-11-18)
87
+ -------------------
88
+ - Test: Add pagination tests for cursor and page-based Link headers.
89
+ [Rodos]
90
+ - Use cursor based pagination. [Helio Machado]
91
+
92
+
93
+ 0.51.2 (2025-11-16)
94
+ -------------------
7
95
 
8
96
  Fix
9
97
  ~~~
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: github-backup
3
- Version: 0.51.2
3
+ Version: 0.52.0
4
4
  Summary: backup a github user or organization
5
5
  Home-page: http://github.com/josegonzalez/python-github-backup
6
6
  Author: Jose Diaz-Gonzalez
@@ -0,0 +1 @@
1
+ __version__ = "0.52.0"
@@ -37,6 +37,15 @@ FNULL = open(os.devnull, "w")
37
37
  FILE_URI_PREFIX = "file://"
38
38
  logger = logging.getLogger(__name__)
39
39
 
40
+
41
+ class RepositoryUnavailableError(Exception):
42
+ """Raised when a repository is unavailable due to legal reasons (e.g., DMCA takedown)."""
43
+
44
+ def __init__(self, message, dmca_url=None):
45
+ super().__init__(message)
46
+ self.dmca_url = dmca_url
47
+
48
+
40
49
  # Setup SSL context with fallback chain
41
50
  https_ctx = ssl.create_default_context()
42
51
  if https_ctx.get_ca_certs():
@@ -592,27 +601,39 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
592
601
  auth = get_auth(args, encode=not args.as_app)
593
602
  query_args = get_query_args(query_args)
594
603
  per_page = 100
595
- page = 0
604
+ next_url = None
596
605
 
597
606
  while True:
598
607
  if single_request:
599
- request_page, request_per_page = None, None
608
+ request_per_page = None
600
609
  else:
601
- page = page + 1
602
- request_page, request_per_page = page, per_page
610
+ request_per_page = per_page
603
611
 
604
612
  request = _construct_request(
605
613
  request_per_page,
606
- request_page,
607
614
  query_args,
608
- template,
615
+ next_url or template,
609
616
  auth,
610
617
  as_app=args.as_app,
611
618
  fine=True if args.token_fine is not None else False,
612
619
  ) # noqa
613
- r, errors = _get_response(request, auth, template)
620
+ r, errors = _get_response(request, auth, next_url or template)
614
621
 
615
622
  status_code = int(r.getcode())
623
+
624
+ # Handle DMCA takedown (HTTP 451) - raise exception to skip entire repository
625
+ if status_code == 451:
626
+ dmca_url = None
627
+ try:
628
+ response_data = json.loads(r.read().decode("utf-8"))
629
+ dmca_url = response_data.get("block", {}).get("html_url")
630
+ except Exception:
631
+ pass
632
+ raise RepositoryUnavailableError(
633
+ "Repository unavailable due to legal reasons (HTTP 451)",
634
+ dmca_url=dmca_url
635
+ )
636
+
616
637
  # Check if we got correct data
617
638
  try:
618
639
  response = json.loads(r.read().decode("utf-8"))
@@ -644,15 +665,14 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
644
665
  retries += 1
645
666
  time.sleep(5)
646
667
  request = _construct_request(
647
- per_page,
648
- page,
668
+ request_per_page,
649
669
  query_args,
650
- template,
670
+ next_url or template,
651
671
  auth,
652
672
  as_app=args.as_app,
653
673
  fine=True if args.token_fine is not None else False,
654
674
  ) # noqa
655
- r, errors = _get_response(request, auth, template)
675
+ r, errors = _get_response(request, auth, next_url or template)
656
676
 
657
677
  status_code = int(r.getcode())
658
678
  try:
@@ -682,7 +702,16 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
682
702
  if type(response) is list:
683
703
  for resp in response:
684
704
  yield resp
685
- if len(response) < per_page:
705
+ # Parse Link header for next page URL (cursor-based pagination)
706
+ link_header = r.headers.get("Link", "")
707
+ next_url = None
708
+ if link_header:
709
+ # Parse Link header: <https://api.github.com/...?per_page=100&after=cursor>; rel="next"
710
+ for link in link_header.split(","):
711
+ if 'rel="next"' in link:
712
+ next_url = link[link.find("<") + 1:link.find(">")]
713
+ break
714
+ if not next_url:
686
715
  break
687
716
  elif type(response) is dict and single_request:
688
717
  yield response
@@ -735,22 +764,27 @@ def _get_response(request, auth, template):
735
764
 
736
765
 
737
766
  def _construct_request(
738
- per_page, page, query_args, template, auth, as_app=None, fine=False
767
+ per_page, query_args, template, auth, as_app=None, fine=False
739
768
  ):
740
- all_query_args = {}
741
- if per_page:
742
- all_query_args["per_page"] = per_page
743
- if page:
744
- all_query_args["page"] = page
745
- if query_args:
746
- all_query_args.update(query_args)
747
-
748
- request_url = template
749
- if all_query_args:
750
- querystring = urlencode(all_query_args)
751
- request_url = template + "?" + querystring
769
+ # If template is already a full URL with query params (from Link header), use it directly
770
+ if "?" in template and template.startswith("http"):
771
+ request_url = template
772
+ # Extract query string for logging
773
+ querystring = template.split("?", 1)[1]
752
774
  else:
753
- querystring = ""
775
+ # Build URL with query parameters
776
+ all_query_args = {}
777
+ if per_page:
778
+ all_query_args["per_page"] = per_page
779
+ if query_args:
780
+ all_query_args.update(query_args)
781
+
782
+ request_url = template
783
+ if all_query_args:
784
+ querystring = urlencode(all_query_args)
785
+ request_url = template + "?" + querystring
786
+ else:
787
+ querystring = ""
754
788
 
755
789
  request = Request(request_url)
756
790
  if auth is not None:
@@ -766,7 +800,7 @@ def _construct_request(
766
800
  "Accept", "application/vnd.github.machine-man-preview+json"
767
801
  )
768
802
 
769
- log_url = template
803
+ log_url = template if "?" not in template else template.split("?")[0]
770
804
  if querystring:
771
805
  log_url += "?" + querystring
772
806
  logger.info("Requesting {}".format(log_url))
@@ -843,8 +877,7 @@ def download_file(url, path, auth, as_app=False, fine=False):
843
877
  return
844
878
 
845
879
  request = _construct_request(
846
- per_page=100,
847
- page=1,
880
+ per_page=None,
848
881
  query_args={},
849
882
  template=url,
850
883
  auth=auth,
@@ -1657,40 +1690,47 @@ def backup_repositories(args, output_directory, repositories):
1657
1690
 
1658
1691
  continue # don't try to back anything else for a gist; it doesn't exist
1659
1692
 
1660
- download_wiki = args.include_wiki or args.include_everything
1661
- if repository["has_wiki"] and download_wiki:
1662
- fetch_repository(
1663
- repository["name"],
1664
- repo_url.replace(".git", ".wiki.git"),
1665
- os.path.join(repo_cwd, "wiki"),
1666
- skip_existing=args.skip_existing,
1667
- bare_clone=args.bare_clone,
1668
- lfs_clone=args.lfs_clone,
1669
- no_prune=args.no_prune,
1670
- )
1671
- if args.include_issues or args.include_everything:
1672
- backup_issues(args, repo_cwd, repository, repos_template)
1693
+ try:
1694
+ download_wiki = args.include_wiki or args.include_everything
1695
+ if repository["has_wiki"] and download_wiki:
1696
+ fetch_repository(
1697
+ repository["name"],
1698
+ repo_url.replace(".git", ".wiki.git"),
1699
+ os.path.join(repo_cwd, "wiki"),
1700
+ skip_existing=args.skip_existing,
1701
+ bare_clone=args.bare_clone,
1702
+ lfs_clone=args.lfs_clone,
1703
+ no_prune=args.no_prune,
1704
+ )
1705
+ if args.include_issues or args.include_everything:
1706
+ backup_issues(args, repo_cwd, repository, repos_template)
1673
1707
 
1674
- if args.include_pulls or args.include_everything:
1675
- backup_pulls(args, repo_cwd, repository, repos_template)
1708
+ if args.include_pulls or args.include_everything:
1709
+ backup_pulls(args, repo_cwd, repository, repos_template)
1676
1710
 
1677
- if args.include_milestones or args.include_everything:
1678
- backup_milestones(args, repo_cwd, repository, repos_template)
1711
+ if args.include_milestones or args.include_everything:
1712
+ backup_milestones(args, repo_cwd, repository, repos_template)
1679
1713
 
1680
- if args.include_labels or args.include_everything:
1681
- backup_labels(args, repo_cwd, repository, repos_template)
1714
+ if args.include_labels or args.include_everything:
1715
+ backup_labels(args, repo_cwd, repository, repos_template)
1682
1716
 
1683
- if args.include_hooks or args.include_everything:
1684
- backup_hooks(args, repo_cwd, repository, repos_template)
1717
+ if args.include_hooks or args.include_everything:
1718
+ backup_hooks(args, repo_cwd, repository, repos_template)
1685
1719
 
1686
- if args.include_releases or args.include_everything:
1687
- backup_releases(
1688
- args,
1689
- repo_cwd,
1690
- repository,
1691
- repos_template,
1692
- include_assets=args.include_assets or args.include_everything,
1693
- )
1720
+ if args.include_releases or args.include_everything:
1721
+ backup_releases(
1722
+ args,
1723
+ repo_cwd,
1724
+ repository,
1725
+ repos_template,
1726
+ include_assets=args.include_assets or args.include_everything,
1727
+ )
1728
+ except RepositoryUnavailableError as e:
1729
+ logger.warning(f"Repository {repository['full_name']} is unavailable (HTTP 451)")
1730
+ if e.dmca_url:
1731
+ logger.warning(f"DMCA notice: {e.dmca_url}")
1732
+ logger.info(f"Skipping remaining resources for {repository['full_name']}")
1733
+ continue
1694
1734
 
1695
1735
  if args.incremental:
1696
1736
  if last_update == "0000-00-00T00:00:00Z":
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: github-backup
3
- Version: 0.51.2
3
+ Version: 0.52.0
4
4
  Summary: backup a github user or organization
5
5
  Home-page: http://github.com/josegonzalez/python-github-backup
6
6
  Author: Jose Diaz-Gonzalez
@@ -14,4 +14,6 @@ github_backup.egg-info/SOURCES.txt
14
14
  github_backup.egg-info/dependency_links.txt
15
15
  github_backup.egg-info/top_level.txt
16
16
  github_backup.egg-info/zip-safe
17
- tests/test_attachments.py
17
+ tests/test_attachments.py
18
+ tests/test_http_451.py
19
+ tests/test_pagination.py
@@ -3,16 +3,16 @@ black==25.11.0
3
3
  bleach==6.3.0
4
4
  certifi==2025.11.12
5
5
  charset-normalizer==3.4.4
6
- click==8.3.0
6
+ click==8.3.1
7
7
  colorama==0.4.6
8
8
  docutils==0.22.3
9
9
  flake8==7.3.0
10
10
  gitchangelog==3.0.4
11
- pytest==8.3.3
11
+ pytest==9.0.1
12
12
  idna==3.11
13
13
  importlib-metadata==8.7.0
14
14
  jaraco.classes==3.4.0
15
- keyring==25.6.0
15
+ keyring==25.7.0
16
16
  markdown-it-py==4.0.0
17
17
  mccabe==0.7.0
18
18
  mdurl==0.1.2
@@ -28,7 +28,7 @@ Pygments==2.19.2
28
28
  readme-renderer==44.0
29
29
  requests==2.32.5
30
30
  requests-toolbelt==1.0.0
31
- restructuredtext-lint==1.4.0
31
+ restructuredtext-lint==2.0.2
32
32
  rfc3986==2.0.0
33
33
  rich==14.2.0
34
34
  setuptools==80.9.0
@@ -0,0 +1,143 @@
1
+ """Tests for HTTP 451 (DMCA takedown) handling."""
2
+
3
+ import json
4
+ from unittest.mock import Mock, patch
5
+
6
+ import pytest
7
+
8
+ from github_backup import github_backup
9
+
10
+
11
+ class TestHTTP451Exception:
12
+ """Test suite for HTTP 451 DMCA takedown exception handling."""
13
+
14
+ def test_repository_unavailable_error_raised(self):
15
+ """HTTP 451 should raise RepositoryUnavailableError with DMCA URL."""
16
+ # Create mock args
17
+ args = Mock()
18
+ args.as_app = False
19
+ args.token_fine = None
20
+ args.token_classic = None
21
+ args.username = None
22
+ args.password = None
23
+ args.osx_keychain_item_name = None
24
+ args.osx_keychain_item_account = None
25
+ args.throttle_limit = None
26
+ args.throttle_pause = 0
27
+
28
+ # Mock HTTPError 451 response
29
+ mock_response = Mock()
30
+ mock_response.getcode.return_value = 451
31
+
32
+ dmca_data = {
33
+ "message": "Repository access blocked",
34
+ "block": {
35
+ "reason": "dmca",
36
+ "created_at": "2024-11-12T14:38:04Z",
37
+ "html_url": "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md"
38
+ }
39
+ }
40
+ mock_response.read.return_value = json.dumps(dmca_data).encode("utf-8")
41
+ mock_response.headers = {"x-ratelimit-remaining": "5000"}
42
+ mock_response.reason = "Unavailable For Legal Reasons"
43
+
44
+ def mock_get_response(request, auth, template):
45
+ return mock_response, []
46
+
47
+ with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
48
+ with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
49
+ list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/dmca/issues"))
50
+
51
+ # Check exception has DMCA URL
52
+ assert exc_info.value.dmca_url == "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md"
53
+ assert "451" in str(exc_info.value)
54
+
55
+ def test_repository_unavailable_error_without_dmca_url(self):
56
+ """HTTP 451 without DMCA details should still raise exception."""
57
+ args = Mock()
58
+ args.as_app = False
59
+ args.token_fine = None
60
+ args.token_classic = None
61
+ args.username = None
62
+ args.password = None
63
+ args.osx_keychain_item_name = None
64
+ args.osx_keychain_item_account = None
65
+ args.throttle_limit = None
66
+ args.throttle_pause = 0
67
+
68
+ mock_response = Mock()
69
+ mock_response.getcode.return_value = 451
70
+ mock_response.read.return_value = b'{"message": "Blocked"}'
71
+ mock_response.headers = {"x-ratelimit-remaining": "5000"}
72
+ mock_response.reason = "Unavailable For Legal Reasons"
73
+
74
+ def mock_get_response(request, auth, template):
75
+ return mock_response, []
76
+
77
+ with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
78
+ with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
79
+ list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/dmca/issues"))
80
+
81
+ # Exception raised even without DMCA URL
82
+ assert exc_info.value.dmca_url is None
83
+ assert "451" in str(exc_info.value)
84
+
85
+ def test_repository_unavailable_error_with_malformed_json(self):
86
+ """HTTP 451 with malformed JSON should still raise exception."""
87
+ args = Mock()
88
+ args.as_app = False
89
+ args.token_fine = None
90
+ args.token_classic = None
91
+ args.username = None
92
+ args.password = None
93
+ args.osx_keychain_item_name = None
94
+ args.osx_keychain_item_account = None
95
+ args.throttle_limit = None
96
+ args.throttle_pause = 0
97
+
98
+ mock_response = Mock()
99
+ mock_response.getcode.return_value = 451
100
+ mock_response.read.return_value = b"invalid json {"
101
+ mock_response.headers = {"x-ratelimit-remaining": "5000"}
102
+ mock_response.reason = "Unavailable For Legal Reasons"
103
+
104
+ def mock_get_response(request, auth, template):
105
+ return mock_response, []
106
+
107
+ with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
108
+ with pytest.raises(github_backup.RepositoryUnavailableError):
109
+ list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/dmca/issues"))
110
+
111
+ def test_other_http_errors_unchanged(self):
112
+ """Other HTTP errors should still raise generic Exception."""
113
+ args = Mock()
114
+ args.as_app = False
115
+ args.token_fine = None
116
+ args.token_classic = None
117
+ args.username = None
118
+ args.password = None
119
+ args.osx_keychain_item_name = None
120
+ args.osx_keychain_item_account = None
121
+ args.throttle_limit = None
122
+ args.throttle_pause = 0
123
+
124
+ mock_response = Mock()
125
+ mock_response.getcode.return_value = 404
126
+ mock_response.read.return_value = b'{"message": "Not Found"}'
127
+ mock_response.headers = {"x-ratelimit-remaining": "5000"}
128
+ mock_response.reason = "Not Found"
129
+
130
+ def mock_get_response(request, auth, template):
131
+ return mock_response, []
132
+
133
+ with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
134
+ # Should raise generic Exception, not RepositoryUnavailableError
135
+ with pytest.raises(Exception) as exc_info:
136
+ list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/notfound/issues"))
137
+
138
+ assert not isinstance(exc_info.value, github_backup.RepositoryUnavailableError)
139
+ assert "404" in str(exc_info.value)
140
+
141
+
142
+ if __name__ == "__main__":
143
+ pytest.main([__file__, "-v"])
@@ -0,0 +1,153 @@
1
+ """Tests for Link header pagination handling."""
2
+
3
+ import json
4
+ from unittest.mock import Mock, patch
5
+
6
+ import pytest
7
+
8
+ from github_backup import github_backup
9
+
10
+
11
+ class MockHTTPResponse:
12
+ """Mock HTTP response for paginated API calls."""
13
+
14
+ def __init__(self, data, link_header=None):
15
+ self._content = json.dumps(data).encode("utf-8")
16
+ self._link_header = link_header
17
+ self._read = False
18
+ self.reason = "OK"
19
+
20
+ def getcode(self):
21
+ return 200
22
+
23
+ def read(self):
24
+ if self._read:
25
+ return b""
26
+ self._read = True
27
+ return self._content
28
+
29
+ def get_header(self, name, default=None):
30
+ """Mock method for headers.get()."""
31
+ return self.headers.get(name, default)
32
+
33
+ @property
34
+ def headers(self):
35
+ headers = {"x-ratelimit-remaining": "5000"}
36
+ if self._link_header:
37
+ headers["Link"] = self._link_header
38
+ return headers
39
+
40
+
41
+ @pytest.fixture
42
+ def mock_args():
43
+ """Mock args for retrieve_data_gen."""
44
+ args = Mock()
45
+ args.as_app = False
46
+ args.token_fine = None
47
+ args.token_classic = "fake_token"
48
+ args.username = None
49
+ args.password = None
50
+ args.osx_keychain_item_name = None
51
+ args.osx_keychain_item_account = None
52
+ args.throttle_limit = None
53
+ args.throttle_pause = 0
54
+ return args
55
+
56
+
57
+ def test_cursor_based_pagination(mock_args):
58
+ """Link header with 'after' cursor parameter works correctly."""
59
+
60
+ # Simulate issues endpoint behavior: returns cursor in Link header
61
+ responses = [
62
+ # Issues endpoint returns 'after' cursor parameter (not 'page')
63
+ MockHTTPResponse(
64
+ data=[{"issue": i} for i in range(1, 101)], # Page 1 contents
65
+ link_header='<https://api.github.com/repos/owner/repo/issues?per_page=100&after=ABC123&page=2>; rel="next"',
66
+ ),
67
+ MockHTTPResponse(
68
+ data=[{"issue": i} for i in range(101, 151)], # Page 2 contents
69
+ link_header=None, # No Link header - signals end of pagination
70
+ ),
71
+ ]
72
+ requests_made = []
73
+
74
+ def mock_urlopen(request, *args, **kwargs):
75
+ url = request.get_full_url()
76
+ requests_made.append(url)
77
+ return responses[len(requests_made) - 1]
78
+
79
+ with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
80
+ results = list(
81
+ github_backup.retrieve_data_gen(
82
+ mock_args, "https://api.github.com/repos/owner/repo/issues"
83
+ )
84
+ )
85
+
86
+ # Verify all items retrieved and cursor was used in second request
87
+ assert len(results) == 150
88
+ assert len(requests_made) == 2
89
+ assert "after=ABC123" in requests_made[1]
90
+
91
+
92
+ def test_page_based_pagination(mock_args):
93
+ """Link header with 'page' parameter works correctly."""
94
+
95
+ # Simulate pulls/repos endpoint behavior: returns page numbers in Link header
96
+ responses = [
97
+ # Pulls endpoint uses traditional 'page' parameter (not cursor)
98
+ MockHTTPResponse(
99
+ data=[{"pull": i} for i in range(1, 101)], # Page 1 contents
100
+ link_header='<https://api.github.com/repos/owner/repo/pulls?per_page=100&page=2>; rel="next"',
101
+ ),
102
+ MockHTTPResponse(
103
+ data=[{"pull": i} for i in range(101, 181)], # Page 2 contents
104
+ link_header=None, # No Link header - signals end of pagination
105
+ ),
106
+ ]
107
+ requests_made = []
108
+
109
+ def mock_urlopen(request, *args, **kwargs):
110
+ url = request.get_full_url()
111
+ requests_made.append(url)
112
+ return responses[len(requests_made) - 1]
113
+
114
+ with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
115
+ results = list(
116
+ github_backup.retrieve_data_gen(
117
+ mock_args, "https://api.github.com/repos/owner/repo/pulls"
118
+ )
119
+ )
120
+
121
+ # Verify all items retrieved and page parameter was used (not cursor)
122
+ assert len(results) == 180
123
+ assert len(requests_made) == 2
124
+ assert "page=2" in requests_made[1]
125
+ assert "after" not in requests_made[1]
126
+
127
+
128
+ def test_no_link_header_stops_pagination(mock_args):
129
+ """Pagination stops when Link header is absent."""
130
+
131
+ # Simulate endpoint with results that fit in a single page
132
+ responses = [
133
+ MockHTTPResponse(
134
+ data=[{"label": i} for i in range(1, 51)], # Page contents
135
+ link_header=None, # No Link header - signals end of pagination
136
+ )
137
+ ]
138
+ requests_made = []
139
+
140
+ def mock_urlopen(request, *args, **kwargs):
141
+ requests_made.append(request.get_full_url())
142
+ return responses[len(requests_made) - 1]
143
+
144
+ with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
145
+ results = list(
146
+ github_backup.retrieve_data_gen(
147
+ mock_args, "https://api.github.com/repos/owner/repo/labels"
148
+ )
149
+ )
150
+
151
+ # Verify pagination stopped after first request
152
+ assert len(results) == 50
153
+ assert len(requests_made) == 1
@@ -1 +0,0 @@
1
- __version__ = "0.51.2"
File without changes
File without changes
File without changes