github-backup 0.61.0__tar.gz → 0.61.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {github_backup-0.61.0 → github_backup-0.61.1}/CHANGES.rst +36 -1
- {github_backup-0.61.0/github_backup.egg-info → github_backup-0.61.1}/PKG-INFO +3 -1
- {github_backup-0.61.0 → github_backup-0.61.1}/README +2 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/README.rst +2 -0
- github_backup-0.61.1/github_backup/__init__.py +1 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/github_backup/cli.py +10 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/github_backup/github_backup.py +100 -8
- {github_backup-0.61.0 → github_backup-0.61.1/github_backup.egg-info}/PKG-INFO +3 -1
- {github_backup-0.61.0 → github_backup-0.61.1}/tests/test_all_starred.py +9 -53
- {github_backup-0.61.0 → github_backup-0.61.1}/tests/test_attachments.py +147 -13
- {github_backup-0.61.0 → github_backup-0.61.1}/tests/test_case_sensitivity.py +6 -40
- {github_backup-0.61.0 → github_backup-0.61.1}/tests/test_http_451.py +6 -30
- {github_backup-0.61.0 → github_backup-0.61.1}/tests/test_pagination.py +10 -24
- {github_backup-0.61.0 → github_backup-0.61.1}/tests/test_retrieve_data.py +26 -61
- {github_backup-0.61.0 → github_backup-0.61.1}/tests/test_skip_assets_on.py +15 -61
- {github_backup-0.61.0 → github_backup-0.61.1}/tests/test_starred_skip_size_over.py +26 -49
- github_backup-0.61.0/github_backup/__init__.py +0 -1
- {github_backup-0.61.0 → github_backup-0.61.1}/LICENSE.txt +0 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/MANIFEST.in +0 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/github_backup/__main__.py +0 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/github_backup.egg-info/SOURCES.txt +0 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/github_backup.egg-info/dependency_links.txt +0 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/github_backup.egg-info/entry_points.txt +0 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/github_backup.egg-info/top_level.txt +0 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/github_backup.egg-info/zip-safe +0 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/release-requirements.txt +0 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/requirements.txt +0 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/setup.cfg +0 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/setup.py +0 -0
- {github_backup-0.61.0 → github_backup-0.61.1}/tests/test_json_dump_if_changed.py +0 -0
|
@@ -1,9 +1,44 @@
|
|
|
1
1
|
Changelog
|
|
2
2
|
=========
|
|
3
3
|
|
|
4
|
-
0.61.
|
|
4
|
+
0.61.1 (2026-01-13)
|
|
5
5
|
-------------------
|
|
6
6
|
------------------------
|
|
7
|
+
- Refactor test fixtures to use shared create_args helper. [Rodos]
|
|
8
|
+
|
|
9
|
+
Uses the real parse_args() function to get CLI defaults, so when
|
|
10
|
+
new arguments are added they're automatically available to all tests.
|
|
11
|
+
|
|
12
|
+
Changes:
|
|
13
|
+
- Add tests/conftest.py with create_args fixture
|
|
14
|
+
- Update 8 test files to use shared fixture
|
|
15
|
+
- Remove duplicate _create_mock_args methods
|
|
16
|
+
- Remove redundant @pytest.fixture mock_args definitions
|
|
17
|
+
|
|
18
|
+
This eliminates the need to update multiple test files when
|
|
19
|
+
adding new CLI arguments.
|
|
20
|
+
- Fix fine-grained PAT attachment downloads for private repos (#477)
|
|
21
|
+
[Rodos]
|
|
22
|
+
|
|
23
|
+
Fine-grained personal access tokens cannot download attachments from
|
|
24
|
+
private repositories directly due to a GitHub platform limitation.
|
|
25
|
+
|
|
26
|
+
This adds a workaround for image attachments (/assets/ URLs) using
|
|
27
|
+
GitHub's Markdown API to convert URLs to JWT-signed URLs that can be
|
|
28
|
+
downloaded without authentication.
|
|
29
|
+
|
|
30
|
+
Changes:
|
|
31
|
+
- Add get_jwt_signed_url_via_markdown_api() function
|
|
32
|
+
- Detect fine-grained token + private repo + /assets/ URL upfront
|
|
33
|
+
- Use JWT workaround for those cases, mark success with jwt_workaround flag
|
|
34
|
+
- Skip download with skipped_at when workaround fails
|
|
35
|
+
- Add startup warning when using --attachments with fine-grained tokens
|
|
36
|
+
- Document limitation in README (file attachments still fail)
|
|
37
|
+
- Add 6 unit tests for JWT workaround logic
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
0.61.0 (2026-01-12)
|
|
41
|
+
-------------------
|
|
7
42
|
- Docs: Add missing `--retries` argument to README. [Lukas Bestle]
|
|
8
43
|
- Test: Adapt tests to new argument. [Lukas Bestle]
|
|
9
44
|
- Feat: Backup of repository security advisories. [Lukas Bestle]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: github-backup
|
|
3
|
-
Version: 0.61.
|
|
3
|
+
Version: 0.61.1
|
|
4
4
|
Summary: backup a github user or organization
|
|
5
5
|
Home-page: http://github.com/josegonzalez/python-github-backup
|
|
6
6
|
Author: Jose Diaz-Gonzalez
|
|
@@ -311,6 +311,8 @@ The tool automatically extracts file extensions from HTTP headers to ensure file
|
|
|
311
311
|
|
|
312
312
|
**Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer).
|
|
313
313
|
|
|
314
|
+
**Fine-grained token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) cannot download attachments from private repositories directly. This affects both ``/assets/`` (images) and ``/files/`` (documents) URLs. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using fine-grained tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead of a fine-grained token (``-f``). See `#477 <https://github.com/josegonzalez/python-github-backup/issues/477>`_ for details.
|
|
315
|
+
|
|
314
316
|
|
|
315
317
|
Run in Docker container
|
|
316
318
|
-----------------------
|
|
@@ -281,6 +281,8 @@ The tool automatically extracts file extensions from HTTP headers to ensure file
|
|
|
281
281
|
|
|
282
282
|
**Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer).
|
|
283
283
|
|
|
284
|
+
**Fine-grained token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) cannot download attachments from private repositories directly. This affects both ``/assets/`` (images) and ``/files/`` (documents) URLs. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using fine-grained tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead of a fine-grained token (``-f``). See `#477 <https://github.com/josegonzalez/python-github-backup/issues/477>`_ for details.
|
|
285
|
+
|
|
284
286
|
|
|
285
287
|
Run in Docker container
|
|
286
288
|
-----------------------
|
|
@@ -281,6 +281,8 @@ The tool automatically extracts file extensions from HTTP headers to ensure file
|
|
|
281
281
|
|
|
282
282
|
**Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer).
|
|
283
283
|
|
|
284
|
+
**Fine-grained token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) cannot download attachments from private repositories directly. This affects both ``/assets/`` (images) and ``/files/`` (documents) URLs. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using fine-grained tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead of a fine-grained token (``-f``). See `#477 <https://github.com/josegonzalez/python-github-backup/issues/477>`_ for details.
|
|
285
|
+
|
|
284
286
|
|
|
285
287
|
Run in Docker container
|
|
286
288
|
-----------------------
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.61.1"
|
|
@@ -46,6 +46,16 @@ def main():
|
|
|
46
46
|
"Use -t/--token or -f/--token-fine to authenticate."
|
|
47
47
|
)
|
|
48
48
|
|
|
49
|
+
# Issue #477: Fine-grained PATs cannot download all attachment types from
|
|
50
|
+
# private repos. Image attachments will be retried via Markdown API workaround.
|
|
51
|
+
if args.include_attachments and args.token_fine:
|
|
52
|
+
logger.warning(
|
|
53
|
+
"Using --attachments with fine-grained token. Due to GitHub platform "
|
|
54
|
+
"limitations, file attachments (PDFs, etc.) from private repos may fail. "
|
|
55
|
+
"Image attachments will be retried via workaround. For full attachment "
|
|
56
|
+
"support, use --token-classic instead."
|
|
57
|
+
)
|
|
58
|
+
|
|
49
59
|
if args.quiet:
|
|
50
60
|
logger.setLevel(logging.WARNING)
|
|
51
61
|
|
|
@@ -1062,6 +1062,65 @@ def download_attachment_file(url, path, auth, as_app=False, fine=False):
|
|
|
1062
1062
|
return metadata
|
|
1063
1063
|
|
|
1064
1064
|
|
|
1065
|
+
def get_jwt_signed_url_via_markdown_api(url, token, repo_context):
|
|
1066
|
+
"""Convert a user-attachments/assets URL to a JWT-signed URL via Markdown API.
|
|
1067
|
+
|
|
1068
|
+
GitHub's Markdown API renders image URLs and returns HTML containing
|
|
1069
|
+
JWT-signed private-user-images.githubusercontent.com URLs that work
|
|
1070
|
+
without token authentication.
|
|
1071
|
+
|
|
1072
|
+
This is a workaround for issue #477 where fine-grained PATs cannot
|
|
1073
|
+
download user-attachments URLs from private repos directly.
|
|
1074
|
+
|
|
1075
|
+
Limitations:
|
|
1076
|
+
- Only works for /assets/ URLs (images)
|
|
1077
|
+
- Does NOT work for /files/ URLs (PDFs, text files, etc.)
|
|
1078
|
+
- JWT URLs expire after ~5 minutes
|
|
1079
|
+
|
|
1080
|
+
Args:
|
|
1081
|
+
url: The github.com/user-attachments/assets/UUID URL
|
|
1082
|
+
token: Raw fine-grained PAT (github_pat_...)
|
|
1083
|
+
repo_context: Repository context as "owner/repo"
|
|
1084
|
+
|
|
1085
|
+
Returns:
|
|
1086
|
+
str: JWT-signed URL from private-user-images.githubusercontent.com
|
|
1087
|
+
None: If conversion fails
|
|
1088
|
+
"""
|
|
1089
|
+
|
|
1090
|
+
try:
|
|
1091
|
+
payload = json.dumps(
|
|
1092
|
+
{"text": f"", "mode": "gfm", "context": repo_context}
|
|
1093
|
+
).encode("utf-8")
|
|
1094
|
+
|
|
1095
|
+
request = Request("https://api.github.com/markdown", data=payload, method="POST")
|
|
1096
|
+
request.add_header("Authorization", f"token {token}")
|
|
1097
|
+
request.add_header("Content-Type", "application/json")
|
|
1098
|
+
request.add_header("Accept", "application/vnd.github+json")
|
|
1099
|
+
|
|
1100
|
+
html = urlopen(request, timeout=30).read().decode("utf-8")
|
|
1101
|
+
|
|
1102
|
+
# Parse JWT-signed URL from HTML response
|
|
1103
|
+
# Format: <img src="https://private-user-images.githubusercontent.com/...?jwt=..." ...>
|
|
1104
|
+
if match := re.search(
|
|
1105
|
+
r'src="(https://private-user-images\.githubusercontent\.com/[^"]+)"', html
|
|
1106
|
+
):
|
|
1107
|
+
jwt_url = match.group(1)
|
|
1108
|
+
logger.debug("Converted attachment URL to JWT-signed URL via Markdown API")
|
|
1109
|
+
return jwt_url
|
|
1110
|
+
|
|
1111
|
+
logger.debug("Markdown API response did not contain JWT-signed URL")
|
|
1112
|
+
return None
|
|
1113
|
+
|
|
1114
|
+
except HTTPError as e:
|
|
1115
|
+
logger.debug(
|
|
1116
|
+
"Markdown API request failed with HTTP {0}: {1}".format(e.code, e.reason)
|
|
1117
|
+
)
|
|
1118
|
+
return None
|
|
1119
|
+
except Exception as e:
|
|
1120
|
+
logger.debug("Markdown API request failed: {0}".format(str(e)))
|
|
1121
|
+
return None
|
|
1122
|
+
|
|
1123
|
+
|
|
1065
1124
|
def extract_attachment_urls(item_data, issue_number=None, repository_full_name=None):
|
|
1066
1125
|
"""Extract GitHub-hosted attachment URLs from issue/PR body and comments.
|
|
1067
1126
|
|
|
@@ -1415,15 +1474,46 @@ def download_attachments(
|
|
|
1415
1474
|
filename = get_attachment_filename(url)
|
|
1416
1475
|
filepath = os.path.join(attachments_dir, filename)
|
|
1417
1476
|
|
|
1418
|
-
#
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1477
|
+
# Issue #477: Fine-grained PATs cannot download user-attachments/assets
|
|
1478
|
+
# from private repos directly (404). Use Markdown API workaround to get
|
|
1479
|
+
# a JWT-signed URL. Only works for /assets/ (images), not /files/.
|
|
1480
|
+
needs_jwt = (
|
|
1481
|
+
args.token_fine is not None
|
|
1482
|
+
and repository.get("private", False)
|
|
1483
|
+
and "github.com/user-attachments/assets/" in url
|
|
1425
1484
|
)
|
|
1426
1485
|
|
|
1486
|
+
if not needs_jwt:
|
|
1487
|
+
# NORMAL download path
|
|
1488
|
+
metadata = download_attachment_file(
|
|
1489
|
+
url,
|
|
1490
|
+
filepath,
|
|
1491
|
+
get_auth(args, encode=not args.as_app),
|
|
1492
|
+
as_app=args.as_app,
|
|
1493
|
+
fine=args.token_fine is not None,
|
|
1494
|
+
)
|
|
1495
|
+
elif jwt_url := get_jwt_signed_url_via_markdown_api(
|
|
1496
|
+
url, args.token_fine, repository["full_name"]
|
|
1497
|
+
):
|
|
1498
|
+
# JWT needed and extracted, download via JWT
|
|
1499
|
+
metadata = download_attachment_file(
|
|
1500
|
+
jwt_url, filepath, auth=None, as_app=False, fine=False
|
|
1501
|
+
)
|
|
1502
|
+
metadata["url"] = url # Apply back the original URL
|
|
1503
|
+
metadata["jwt_workaround"] = True
|
|
1504
|
+
else:
|
|
1505
|
+
# Markdown API workaround failed - skip download we know will fail
|
|
1506
|
+
metadata = {
|
|
1507
|
+
"url": url,
|
|
1508
|
+
"success": False,
|
|
1509
|
+
"skipped_at": datetime.now(timezone.utc).isoformat(),
|
|
1510
|
+
"error": "Fine-grained token cannot download private repo attachments. "
|
|
1511
|
+
"Markdown API workaround failed. Use --token-classic instead.",
|
|
1512
|
+
}
|
|
1513
|
+
logger.warning(
|
|
1514
|
+
"Skipping attachment {0}: {1}".format(url, metadata["error"])
|
|
1515
|
+
)
|
|
1516
|
+
|
|
1427
1517
|
# If download succeeded but we got an extension from Content-Disposition,
|
|
1428
1518
|
# we may need to rename the file to add the extension
|
|
1429
1519
|
if metadata["success"] and metadata.get("original_filename"):
|
|
@@ -1951,7 +2041,9 @@ def backup_security_advisories(args, repo_cwd, repository, repos_template):
|
|
|
1951
2041
|
logger.info("Retrieving {0} security advisories".format(repository["full_name"]))
|
|
1952
2042
|
mkdir_p(repo_cwd, advisory_cwd)
|
|
1953
2043
|
|
|
1954
|
-
template = "{0}/{1}/security-advisories".format(
|
|
2044
|
+
template = "{0}/{1}/security-advisories".format(
|
|
2045
|
+
repos_template, repository["full_name"]
|
|
2046
|
+
)
|
|
1955
2047
|
|
|
1956
2048
|
_advisories = retrieve_data(args, template)
|
|
1957
2049
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: github-backup
|
|
3
|
-
Version: 0.61.
|
|
3
|
+
Version: 0.61.1
|
|
4
4
|
Summary: backup a github user or organization
|
|
5
5
|
Home-page: http://github.com/josegonzalez/python-github-backup
|
|
6
6
|
Author: Jose Diaz-Gonzalez
|
|
@@ -311,6 +311,8 @@ The tool automatically extracts file extensions from HTTP headers to ensure file
|
|
|
311
311
|
|
|
312
312
|
**Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer).
|
|
313
313
|
|
|
314
|
+
**Fine-grained token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) cannot download attachments from private repositories directly. This affects both ``/assets/`` (images) and ``/files/`` (documents) URLs. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using fine-grained tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead of a fine-grained token (``-f``). See `#477 <https://github.com/josegonzalez/python-github-backup/issues/477>`_ for details.
|
|
315
|
+
|
|
314
316
|
|
|
315
317
|
Run in Docker container
|
|
316
318
|
-----------------------
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Tests for --all-starred flag behavior (issue #225)."""
|
|
2
2
|
|
|
3
3
|
import pytest
|
|
4
|
-
from unittest.mock import
|
|
4
|
+
from unittest.mock import patch
|
|
5
5
|
|
|
6
6
|
from github_backup import github_backup
|
|
7
7
|
|
|
@@ -12,58 +12,14 @@ class TestAllStarredCloning:
|
|
|
12
12
|
Issue #225: --all-starred should clone starred repos without requiring --repositories.
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
|
-
def _create_mock_args(self, **overrides):
|
|
16
|
-
"""Create a mock args object with sensible defaults."""
|
|
17
|
-
args = Mock()
|
|
18
|
-
args.user = "testuser"
|
|
19
|
-
args.output_directory = "/tmp/backup"
|
|
20
|
-
args.include_repository = False
|
|
21
|
-
args.include_everything = False
|
|
22
|
-
args.include_gists = False
|
|
23
|
-
args.include_starred_gists = False
|
|
24
|
-
args.all_starred = False
|
|
25
|
-
args.skip_existing = False
|
|
26
|
-
args.bare_clone = False
|
|
27
|
-
args.lfs_clone = False
|
|
28
|
-
args.no_prune = False
|
|
29
|
-
args.include_wiki = False
|
|
30
|
-
args.include_issues = False
|
|
31
|
-
args.include_issue_comments = False
|
|
32
|
-
args.include_issue_events = False
|
|
33
|
-
args.include_pulls = False
|
|
34
|
-
args.include_pull_comments = False
|
|
35
|
-
args.include_pull_commits = False
|
|
36
|
-
args.include_pull_details = False
|
|
37
|
-
args.include_labels = False
|
|
38
|
-
args.include_hooks = False
|
|
39
|
-
args.include_milestones = False
|
|
40
|
-
args.include_security_advisories = False
|
|
41
|
-
args.include_releases = False
|
|
42
|
-
args.include_assets = False
|
|
43
|
-
args.include_attachments = False
|
|
44
|
-
args.incremental = False
|
|
45
|
-
args.incremental_by_files = False
|
|
46
|
-
args.github_host = None
|
|
47
|
-
args.prefer_ssh = False
|
|
48
|
-
args.token_classic = None
|
|
49
|
-
args.token_fine = None
|
|
50
|
-
args.as_app = False
|
|
51
|
-
args.osx_keychain_item_name = None
|
|
52
|
-
args.osx_keychain_item_account = None
|
|
53
|
-
|
|
54
|
-
for key, value in overrides.items():
|
|
55
|
-
setattr(args, key, value)
|
|
56
|
-
|
|
57
|
-
return args
|
|
58
|
-
|
|
59
15
|
@patch('github_backup.github_backup.fetch_repository')
|
|
60
16
|
@patch('github_backup.github_backup.get_github_repo_url')
|
|
61
|
-
def test_all_starred_clones_without_repositories_flag(self, mock_get_url, mock_fetch):
|
|
17
|
+
def test_all_starred_clones_without_repositories_flag(self, mock_get_url, mock_fetch, create_args):
|
|
62
18
|
"""--all-starred should clone starred repos without --repositories flag.
|
|
63
19
|
|
|
64
20
|
This is the core fix for issue #225.
|
|
65
21
|
"""
|
|
66
|
-
args =
|
|
22
|
+
args = create_args(all_starred=True)
|
|
67
23
|
mock_get_url.return_value = "https://github.com/otheruser/awesome-project.git"
|
|
68
24
|
|
|
69
25
|
# A starred repository (is_starred flag set by retrieve_repositories)
|
|
@@ -88,9 +44,9 @@ class TestAllStarredCloning:
|
|
|
88
44
|
|
|
89
45
|
@patch('github_backup.github_backup.fetch_repository')
|
|
90
46
|
@patch('github_backup.github_backup.get_github_repo_url')
|
|
91
|
-
def test_starred_repo_not_cloned_without_all_starred_flag(self, mock_get_url, mock_fetch):
|
|
47
|
+
def test_starred_repo_not_cloned_without_all_starred_flag(self, mock_get_url, mock_fetch, create_args):
|
|
92
48
|
"""Starred repos should NOT be cloned if --all-starred is not set."""
|
|
93
|
-
args =
|
|
49
|
+
args = create_args(all_starred=False)
|
|
94
50
|
mock_get_url.return_value = "https://github.com/otheruser/awesome-project.git"
|
|
95
51
|
|
|
96
52
|
starred_repo = {
|
|
@@ -111,9 +67,9 @@ class TestAllStarredCloning:
|
|
|
111
67
|
|
|
112
68
|
@patch('github_backup.github_backup.fetch_repository')
|
|
113
69
|
@patch('github_backup.github_backup.get_github_repo_url')
|
|
114
|
-
def test_non_starred_repo_not_cloned_with_only_all_starred(self, mock_get_url, mock_fetch):
|
|
70
|
+
def test_non_starred_repo_not_cloned_with_only_all_starred(self, mock_get_url, mock_fetch, create_args):
|
|
115
71
|
"""Non-starred repos should NOT be cloned when only --all-starred is set."""
|
|
116
|
-
args =
|
|
72
|
+
args = create_args(all_starred=True)
|
|
117
73
|
mock_get_url.return_value = "https://github.com/testuser/my-project.git"
|
|
118
74
|
|
|
119
75
|
# A regular (non-starred) repository
|
|
@@ -135,9 +91,9 @@ class TestAllStarredCloning:
|
|
|
135
91
|
|
|
136
92
|
@patch('github_backup.github_backup.fetch_repository')
|
|
137
93
|
@patch('github_backup.github_backup.get_github_repo_url')
|
|
138
|
-
def test_repositories_flag_still_works(self, mock_get_url, mock_fetch):
|
|
94
|
+
def test_repositories_flag_still_works(self, mock_get_url, mock_fetch, create_args):
|
|
139
95
|
"""--repositories flag should still clone repos as before."""
|
|
140
|
-
args =
|
|
96
|
+
args = create_args(include_repository=True)
|
|
141
97
|
mock_get_url.return_value = "https://github.com/testuser/my-project.git"
|
|
142
98
|
|
|
143
99
|
regular_repo = {
|
|
@@ -4,7 +4,7 @@ import json
|
|
|
4
4
|
import os
|
|
5
5
|
import tempfile
|
|
6
6
|
from pathlib import Path
|
|
7
|
-
from unittest.mock import Mock
|
|
7
|
+
from unittest.mock import Mock, patch
|
|
8
8
|
|
|
9
9
|
import pytest
|
|
10
10
|
|
|
@@ -12,22 +12,13 @@ from github_backup import github_backup
|
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
@pytest.fixture
|
|
15
|
-
def attachment_test_setup(tmp_path):
|
|
15
|
+
def attachment_test_setup(tmp_path, create_args):
|
|
16
16
|
"""Fixture providing setup and helper for attachment download tests."""
|
|
17
|
-
from unittest.mock import patch
|
|
18
|
-
|
|
19
17
|
issue_cwd = tmp_path / "issues"
|
|
20
18
|
issue_cwd.mkdir()
|
|
21
19
|
|
|
22
|
-
#
|
|
23
|
-
args =
|
|
24
|
-
args.as_app = False
|
|
25
|
-
args.token_fine = None
|
|
26
|
-
args.token_classic = None
|
|
27
|
-
args.osx_keychain_item_name = None
|
|
28
|
-
args.osx_keychain_item_account = None
|
|
29
|
-
args.user = "testuser"
|
|
30
|
-
args.repository = "testrepo"
|
|
20
|
+
# Create args using shared fixture
|
|
21
|
+
args = create_args(user="testuser", repository="testrepo")
|
|
31
22
|
|
|
32
23
|
repository = {"full_name": "testuser/testrepo"}
|
|
33
24
|
|
|
@@ -349,3 +340,146 @@ class TestManifestDuplicatePrevention:
|
|
|
349
340
|
downloaded_urls[0]
|
|
350
341
|
== "https://github.com/user-attachments/assets/unavailable"
|
|
351
342
|
)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
class TestJWTWorkaround:
|
|
346
|
+
"""Test JWT workaround for fine-grained tokens on private repos (issue #477)."""
|
|
347
|
+
|
|
348
|
+
def test_markdown_api_extracts_jwt_url(self):
|
|
349
|
+
"""Markdown API response with JWT URL is extracted correctly."""
|
|
350
|
+
html_response = (
|
|
351
|
+
'<p><a href="https://private-user-images.githubusercontent.com'
|
|
352
|
+
'/123/abc.png?jwt=eyJhbGciOiJ"><img src="https://private-user-'
|
|
353
|
+
'images.githubusercontent.com/123/abc.png?jwt=eyJhbGciOiJ" '
|
|
354
|
+
'alt="img"></a></p>'
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
mock_response = Mock()
|
|
358
|
+
mock_response.read.return_value = html_response.encode("utf-8")
|
|
359
|
+
|
|
360
|
+
with patch("github_backup.github_backup.urlopen", return_value=mock_response):
|
|
361
|
+
result = github_backup.get_jwt_signed_url_via_markdown_api(
|
|
362
|
+
"https://github.com/user-attachments/assets/abc123",
|
|
363
|
+
"github_pat_token",
|
|
364
|
+
"owner/repo"
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
expected = (
|
|
368
|
+
"https://private-user-images.githubusercontent.com"
|
|
369
|
+
"/123/abc.png?jwt=eyJhbGciOiJ"
|
|
370
|
+
)
|
|
371
|
+
assert result == expected
|
|
372
|
+
|
|
373
|
+
def test_markdown_api_returns_none_on_http_error(self):
|
|
374
|
+
"""HTTP errors return None."""
|
|
375
|
+
from urllib.error import HTTPError
|
|
376
|
+
|
|
377
|
+
error = HTTPError("http://test", 403, "Forbidden", {}, None)
|
|
378
|
+
with patch("github_backup.github_backup.urlopen", side_effect=error):
|
|
379
|
+
result = github_backup.get_jwt_signed_url_via_markdown_api(
|
|
380
|
+
"https://github.com/user-attachments/assets/abc123",
|
|
381
|
+
"github_pat_token",
|
|
382
|
+
"owner/repo"
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
assert result is None
|
|
386
|
+
|
|
387
|
+
def test_markdown_api_returns_none_when_no_jwt_url(self):
|
|
388
|
+
"""Response without JWT URL returns None."""
|
|
389
|
+
mock_response = Mock()
|
|
390
|
+
mock_response.read.return_value = b"<p>No image here</p>"
|
|
391
|
+
|
|
392
|
+
with patch("github_backup.github_backup.urlopen", return_value=mock_response):
|
|
393
|
+
result = github_backup.get_jwt_signed_url_via_markdown_api(
|
|
394
|
+
"https://github.com/user-attachments/assets/abc123",
|
|
395
|
+
"github_pat_token",
|
|
396
|
+
"owner/repo"
|
|
397
|
+
)
|
|
398
|
+
|
|
399
|
+
assert result is None
|
|
400
|
+
|
|
401
|
+
def test_needs_jwt_only_for_fine_grained_private_assets(self):
|
|
402
|
+
"""needs_jwt is True only for fine-grained + private + /assets/ URL."""
|
|
403
|
+
assets_url = "https://github.com/user-attachments/assets/abc123"
|
|
404
|
+
files_url = "https://github.com/user-attachments/files/123/doc.pdf"
|
|
405
|
+
token_fine = "github_pat_test"
|
|
406
|
+
private = True
|
|
407
|
+
public = False
|
|
408
|
+
|
|
409
|
+
# Fine-grained + private + assets = True
|
|
410
|
+
needs_jwt = (
|
|
411
|
+
token_fine is not None
|
|
412
|
+
and private
|
|
413
|
+
and "github.com/user-attachments/assets/" in assets_url
|
|
414
|
+
)
|
|
415
|
+
assert needs_jwt is True
|
|
416
|
+
|
|
417
|
+
# Fine-grained + private + files = False
|
|
418
|
+
needs_jwt = (
|
|
419
|
+
token_fine is not None
|
|
420
|
+
and private
|
|
421
|
+
and "github.com/user-attachments/assets/" in files_url
|
|
422
|
+
)
|
|
423
|
+
assert needs_jwt is False
|
|
424
|
+
|
|
425
|
+
# Fine-grained + public + assets = False
|
|
426
|
+
needs_jwt = (
|
|
427
|
+
token_fine is not None
|
|
428
|
+
and public
|
|
429
|
+
and "github.com/user-attachments/assets/" in assets_url
|
|
430
|
+
)
|
|
431
|
+
assert needs_jwt is False
|
|
432
|
+
|
|
433
|
+
def test_jwt_workaround_sets_manifest_flag(self, attachment_test_setup):
|
|
434
|
+
"""Successful JWT workaround sets jwt_workaround flag in manifest."""
|
|
435
|
+
setup = attachment_test_setup
|
|
436
|
+
setup["args"].token_fine = "github_pat_test"
|
|
437
|
+
setup["repository"]["private"] = True
|
|
438
|
+
|
|
439
|
+
issue_data = {"body": "https://github.com/user-attachments/assets/abc123"}
|
|
440
|
+
|
|
441
|
+
jwt_url = "https://private-user-images.githubusercontent.com/123/abc.png?jwt=token"
|
|
442
|
+
|
|
443
|
+
with patch(
|
|
444
|
+
"github_backup.github_backup.get_jwt_signed_url_via_markdown_api",
|
|
445
|
+
return_value=jwt_url
|
|
446
|
+
), patch(
|
|
447
|
+
"github_backup.github_backup.download_attachment_file",
|
|
448
|
+
return_value={"success": True, "http_status": 200, "url": jwt_url}
|
|
449
|
+
):
|
|
450
|
+
github_backup.download_attachments(
|
|
451
|
+
setup["args"], setup["issue_cwd"], issue_data, 123, setup["repository"]
|
|
452
|
+
)
|
|
453
|
+
|
|
454
|
+
manifest_path = os.path.join(setup["issue_cwd"], "attachments", "123", "manifest.json")
|
|
455
|
+
with open(manifest_path) as f:
|
|
456
|
+
manifest = json.load(f)
|
|
457
|
+
|
|
458
|
+
assert manifest["attachments"][0]["jwt_workaround"] is True
|
|
459
|
+
assert manifest["attachments"][0]["url"] == "https://github.com/user-attachments/assets/abc123"
|
|
460
|
+
|
|
461
|
+
def test_jwt_workaround_failure_uses_skipped_at(self, attachment_test_setup):
|
|
462
|
+
"""Failed JWT workaround uses skipped_at instead of downloaded_at."""
|
|
463
|
+
setup = attachment_test_setup
|
|
464
|
+
setup["args"].token_fine = "github_pat_test"
|
|
465
|
+
setup["repository"]["private"] = True
|
|
466
|
+
|
|
467
|
+
issue_data = {"body": "https://github.com/user-attachments/assets/abc123"}
|
|
468
|
+
|
|
469
|
+
with patch(
|
|
470
|
+
"github_backup.github_backup.get_jwt_signed_url_via_markdown_api",
|
|
471
|
+
return_value=None # Markdown API failed
|
|
472
|
+
):
|
|
473
|
+
github_backup.download_attachments(
|
|
474
|
+
setup["args"], setup["issue_cwd"], issue_data, 123, setup["repository"]
|
|
475
|
+
)
|
|
476
|
+
|
|
477
|
+
manifest_path = os.path.join(setup["issue_cwd"], "attachments", "123", "manifest.json")
|
|
478
|
+
with open(manifest_path) as f:
|
|
479
|
+
manifest = json.load(f)
|
|
480
|
+
|
|
481
|
+
attachment = manifest["attachments"][0]
|
|
482
|
+
assert attachment["success"] is False
|
|
483
|
+
assert "skipped_at" in attachment
|
|
484
|
+
assert "downloaded_at" not in attachment
|
|
485
|
+
assert "Use --token-classic" in attachment["error"]
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"""Tests for case-insensitive username/organization filtering."""
|
|
2
2
|
|
|
3
3
|
import pytest
|
|
4
|
-
from unittest.mock import Mock
|
|
5
4
|
|
|
6
5
|
from github_backup import github_backup
|
|
7
6
|
|
|
@@ -9,25 +8,14 @@ from github_backup import github_backup
|
|
|
9
8
|
class TestCaseSensitivity:
|
|
10
9
|
"""Test suite for case-insensitive username matching in filter_repositories."""
|
|
11
10
|
|
|
12
|
-
def test_filter_repositories_case_insensitive_user(self):
|
|
11
|
+
def test_filter_repositories_case_insensitive_user(self, create_args):
|
|
13
12
|
"""Should filter repositories case-insensitively for usernames.
|
|
14
13
|
|
|
15
14
|
Reproduces issue #198 where typing 'iamrodos' fails to match
|
|
16
15
|
repositories with owner.login='Iamrodos' (the canonical case from GitHub API).
|
|
17
16
|
"""
|
|
18
17
|
# Simulate user typing lowercase username
|
|
19
|
-
args =
|
|
20
|
-
args.user = "iamrodos" # lowercase (what user typed)
|
|
21
|
-
args.repository = None
|
|
22
|
-
args.name_regex = None
|
|
23
|
-
args.languages = None
|
|
24
|
-
args.exclude = None
|
|
25
|
-
args.fork = False
|
|
26
|
-
args.private = False
|
|
27
|
-
args.public = False
|
|
28
|
-
args.all = True
|
|
29
|
-
args.skip_archived = False
|
|
30
|
-
args.starred_skip_size_over = None
|
|
18
|
+
args = create_args(user="iamrodos")
|
|
31
19
|
|
|
32
20
|
# Simulate GitHub API returning canonical case
|
|
33
21
|
repos = [
|
|
@@ -52,23 +40,12 @@ class TestCaseSensitivity:
|
|
|
52
40
|
assert filtered[0]["name"] == "repo1"
|
|
53
41
|
assert filtered[1]["name"] == "repo2"
|
|
54
42
|
|
|
55
|
-
def test_filter_repositories_case_insensitive_org(self):
|
|
43
|
+
def test_filter_repositories_case_insensitive_org(self, create_args):
|
|
56
44
|
"""Should filter repositories case-insensitively for organizations.
|
|
57
45
|
|
|
58
46
|
Tests the example from issue #198 where 'prai-org' doesn't match 'PRAI-Org'.
|
|
59
47
|
"""
|
|
60
|
-
args =
|
|
61
|
-
args.user = "prai-org" # lowercase (what user typed)
|
|
62
|
-
args.repository = None
|
|
63
|
-
args.name_regex = None
|
|
64
|
-
args.languages = None
|
|
65
|
-
args.exclude = None
|
|
66
|
-
args.fork = False
|
|
67
|
-
args.private = False
|
|
68
|
-
args.public = False
|
|
69
|
-
args.all = True
|
|
70
|
-
args.skip_archived = False
|
|
71
|
-
args.starred_skip_size_over = None
|
|
48
|
+
args = create_args(user="prai-org")
|
|
72
49
|
|
|
73
50
|
repos = [
|
|
74
51
|
{
|
|
@@ -85,20 +62,9 @@ class TestCaseSensitivity:
|
|
|
85
62
|
assert len(filtered) == 1
|
|
86
63
|
assert filtered[0]["name"] == "repo1"
|
|
87
64
|
|
|
88
|
-
def test_filter_repositories_case_variations(self):
|
|
65
|
+
def test_filter_repositories_case_variations(self, create_args):
|
|
89
66
|
"""Should handle various case combinations correctly."""
|
|
90
|
-
args =
|
|
91
|
-
args.user = "TeSt-UsEr" # Mixed case
|
|
92
|
-
args.repository = None
|
|
93
|
-
args.name_regex = None
|
|
94
|
-
args.languages = None
|
|
95
|
-
args.exclude = None
|
|
96
|
-
args.fork = False
|
|
97
|
-
args.private = False
|
|
98
|
-
args.public = False
|
|
99
|
-
args.all = True
|
|
100
|
-
args.skip_archived = False
|
|
101
|
-
args.starred_skip_size_over = None
|
|
67
|
+
args = create_args(user="TeSt-UsEr")
|
|
102
68
|
|
|
103
69
|
repos = [
|
|
104
70
|
{"name": "repo1", "owner": {"login": "test-user"}, "private": False, "fork": False},
|