github-backup 0.60.0__tar.gz → 0.61.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {github_backup-0.60.0 → github_backup-0.61.1}/CHANGES.rst +43 -1
- {github_backup-0.60.0/github_backup.egg-info → github_backup-0.61.1}/PKG-INFO +10 -6
- {github_backup-0.60.0 → github_backup-0.61.1}/README +9 -5
- {github_backup-0.60.0 → github_backup-0.61.1}/README.rst +9 -5
- github_backup-0.61.1/github_backup/__init__.py +1 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/github_backup/cli.py +10 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/github_backup/github_backup.py +143 -7
- {github_backup-0.60.0 → github_backup-0.61.1/github_backup.egg-info}/PKG-INFO +10 -6
- {github_backup-0.60.0 → github_backup-0.61.1}/tests/test_all_starred.py +9 -52
- {github_backup-0.60.0 → github_backup-0.61.1}/tests/test_attachments.py +147 -13
- {github_backup-0.60.0 → github_backup-0.61.1}/tests/test_case_sensitivity.py +6 -40
- {github_backup-0.60.0 → github_backup-0.61.1}/tests/test_http_451.py +6 -30
- {github_backup-0.60.0 → github_backup-0.61.1}/tests/test_pagination.py +10 -24
- {github_backup-0.60.0 → github_backup-0.61.1}/tests/test_retrieve_data.py +26 -61
- {github_backup-0.60.0 → github_backup-0.61.1}/tests/test_skip_assets_on.py +15 -61
- {github_backup-0.60.0 → github_backup-0.61.1}/tests/test_starred_skip_size_over.py +26 -49
- github_backup-0.60.0/github_backup/__init__.py +0 -1
- {github_backup-0.60.0 → github_backup-0.61.1}/LICENSE.txt +0 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/MANIFEST.in +0 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/github_backup/__main__.py +0 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/github_backup.egg-info/SOURCES.txt +0 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/github_backup.egg-info/dependency_links.txt +0 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/github_backup.egg-info/entry_points.txt +0 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/github_backup.egg-info/top_level.txt +0 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/github_backup.egg-info/zip-safe +0 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/release-requirements.txt +0 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/requirements.txt +0 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/setup.cfg +0 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/setup.py +0 -0
- {github_backup-0.60.0 → github_backup-0.61.1}/tests/test_json_dump_if_changed.py +0 -0
|
@@ -1,9 +1,51 @@
|
|
|
1
1
|
Changelog
|
|
2
2
|
=========
|
|
3
3
|
|
|
4
|
-
0.
|
|
4
|
+
0.61.1 (2026-01-13)
|
|
5
5
|
-------------------
|
|
6
6
|
------------------------
|
|
7
|
+
- Refactor test fixtures to use shared create_args helper. [Rodos]
|
|
8
|
+
|
|
9
|
+
Uses the real parse_args() function to get CLI defaults, so when
|
|
10
|
+
new arguments are added they're automatically available to all tests.
|
|
11
|
+
|
|
12
|
+
Changes:
|
|
13
|
+
- Add tests/conftest.py with create_args fixture
|
|
14
|
+
- Update 8 test files to use shared fixture
|
|
15
|
+
- Remove duplicate _create_mock_args methods
|
|
16
|
+
- Remove redundant @pytest.fixture mock_args definitions
|
|
17
|
+
|
|
18
|
+
This eliminates the need to update multiple test files when
|
|
19
|
+
adding new CLI arguments.
|
|
20
|
+
- Fix fine-grained PAT attachment downloads for private repos (#477)
|
|
21
|
+
[Rodos]
|
|
22
|
+
|
|
23
|
+
Fine-grained personal access tokens cannot download attachments from
|
|
24
|
+
private repositories directly due to a GitHub platform limitation.
|
|
25
|
+
|
|
26
|
+
This adds a workaround for image attachments (/assets/ URLs) using
|
|
27
|
+
GitHub's Markdown API to convert URLs to JWT-signed URLs that can be
|
|
28
|
+
downloaded without authentication.
|
|
29
|
+
|
|
30
|
+
Changes:
|
|
31
|
+
- Add get_jwt_signed_url_via_markdown_api() function
|
|
32
|
+
- Detect fine-grained token + private repo + /assets/ URL upfront
|
|
33
|
+
- Use JWT workaround for those cases, mark success with jwt_workaround flag
|
|
34
|
+
- Skip download with skipped_at when workaround fails
|
|
35
|
+
- Add startup warning when using --attachments with fine-grained tokens
|
|
36
|
+
- Document limitation in README (file attachments still fail)
|
|
37
|
+
- Add 6 unit tests for JWT workaround logic
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
0.61.0 (2026-01-12)
|
|
41
|
+
-------------------
|
|
42
|
+
- Docs: Add missing `--retries` argument to README. [Lukas Bestle]
|
|
43
|
+
- Test: Adapt tests to new argument. [Lukas Bestle]
|
|
44
|
+
- Feat: Backup of repository security advisories. [Lukas Bestle]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
0.60.0 (2025-12-24)
|
|
48
|
+
-------------------
|
|
7
49
|
- Rm max_retries.py. [michaelmartinez]
|
|
8
50
|
- Readme. [michaelmartinez]
|
|
9
51
|
- Don't use a global variable, pass the args instead. [michaelmartinez]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: github-backup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.61.1
|
|
4
4
|
Summary: backup a github user or organization
|
|
5
5
|
Home-page: http://github.com/josegonzalez/python-github-backup
|
|
6
6
|
Author: Jose Diaz-Gonzalez
|
|
@@ -73,9 +73,9 @@ CLI Help output::
|
|
|
73
73
|
[--watched] [--followers] [--following] [--all]
|
|
74
74
|
[--issues] [--issue-comments] [--issue-events] [--pulls]
|
|
75
75
|
[--pull-comments] [--pull-commits] [--pull-details]
|
|
76
|
-
[--labels] [--hooks] [--milestones] [--
|
|
77
|
-
[--bare] [--no-prune] [--lfs] [--wikis]
|
|
78
|
-
[--starred-gists] [--skip-archived] [--skip-existing]
|
|
76
|
+
[--labels] [--hooks] [--milestones] [--security-advisories]
|
|
77
|
+
[--repositories] [--bare] [--no-prune] [--lfs] [--wikis]
|
|
78
|
+
[--gists] [--starred-gists] [--skip-archived] [--skip-existing]
|
|
79
79
|
[-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST]
|
|
80
80
|
[-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v]
|
|
81
81
|
[--keychain-name OSX_KEYCHAIN_ITEM_NAME]
|
|
@@ -85,7 +85,7 @@ CLI Help output::
|
|
|
85
85
|
[--skip-assets-on [SKIP_ASSETS_ON ...]] [--attachments]
|
|
86
86
|
[--throttle-limit THROTTLE_LIMIT]
|
|
87
87
|
[--throttle-pause THROTTLE_PAUSE]
|
|
88
|
-
[--exclude [EXCLUDE ...]]
|
|
88
|
+
[--exclude [EXCLUDE ...]] [--retries MAX_RETRIES]
|
|
89
89
|
USER
|
|
90
90
|
|
|
91
91
|
Backup a github account
|
|
@@ -131,6 +131,8 @@ CLI Help output::
|
|
|
131
131
|
--hooks include hooks in backup (works only when
|
|
132
132
|
authenticated)
|
|
133
133
|
--milestones include milestones in backup
|
|
134
|
+
--security-advisories
|
|
135
|
+
include security advisories in backup
|
|
134
136
|
--repositories include repository clone in backup
|
|
135
137
|
--bare clone bare repositories
|
|
136
138
|
--no-prune disable prune option for git fetch
|
|
@@ -309,6 +311,8 @@ The tool automatically extracts file extensions from HTTP headers to ensure file
|
|
|
309
311
|
|
|
310
312
|
**Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer).
|
|
311
313
|
|
|
314
|
+
**Fine-grained token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) cannot download attachments from private repositories directly. This affects both ``/assets/`` (images) and ``/files/`` (documents) URLs. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using fine-grained tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead of a fine-grained token (``-f``). See `#477 <https://github.com/josegonzalez/python-github-backup/issues/477>`_ for details.
|
|
315
|
+
|
|
312
316
|
|
|
313
317
|
Run in Docker container
|
|
314
318
|
-----------------------
|
|
@@ -431,7 +435,7 @@ Quietly and incrementally backup useful Github user data (public and private rep
|
|
|
431
435
|
export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
432
436
|
GH_USER=YOUR-GITHUB-USER
|
|
433
437
|
|
|
434
|
-
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
438
|
+
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --security-advisories --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
435
439
|
|
|
436
440
|
Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
|
|
437
441
|
|
|
@@ -43,9 +43,9 @@ CLI Help output::
|
|
|
43
43
|
[--watched] [--followers] [--following] [--all]
|
|
44
44
|
[--issues] [--issue-comments] [--issue-events] [--pulls]
|
|
45
45
|
[--pull-comments] [--pull-commits] [--pull-details]
|
|
46
|
-
[--labels] [--hooks] [--milestones] [--
|
|
47
|
-
[--bare] [--no-prune] [--lfs] [--wikis]
|
|
48
|
-
[--starred-gists] [--skip-archived] [--skip-existing]
|
|
46
|
+
[--labels] [--hooks] [--milestones] [--security-advisories]
|
|
47
|
+
[--repositories] [--bare] [--no-prune] [--lfs] [--wikis]
|
|
48
|
+
[--gists] [--starred-gists] [--skip-archived] [--skip-existing]
|
|
49
49
|
[-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST]
|
|
50
50
|
[-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v]
|
|
51
51
|
[--keychain-name OSX_KEYCHAIN_ITEM_NAME]
|
|
@@ -55,7 +55,7 @@ CLI Help output::
|
|
|
55
55
|
[--skip-assets-on [SKIP_ASSETS_ON ...]] [--attachments]
|
|
56
56
|
[--throttle-limit THROTTLE_LIMIT]
|
|
57
57
|
[--throttle-pause THROTTLE_PAUSE]
|
|
58
|
-
[--exclude [EXCLUDE ...]]
|
|
58
|
+
[--exclude [EXCLUDE ...]] [--retries MAX_RETRIES]
|
|
59
59
|
USER
|
|
60
60
|
|
|
61
61
|
Backup a github account
|
|
@@ -101,6 +101,8 @@ CLI Help output::
|
|
|
101
101
|
--hooks include hooks in backup (works only when
|
|
102
102
|
authenticated)
|
|
103
103
|
--milestones include milestones in backup
|
|
104
|
+
--security-advisories
|
|
105
|
+
include security advisories in backup
|
|
104
106
|
--repositories include repository clone in backup
|
|
105
107
|
--bare clone bare repositories
|
|
106
108
|
--no-prune disable prune option for git fetch
|
|
@@ -279,6 +281,8 @@ The tool automatically extracts file extensions from HTTP headers to ensure file
|
|
|
279
281
|
|
|
280
282
|
**Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer).
|
|
281
283
|
|
|
284
|
+
**Fine-grained token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) cannot download attachments from private repositories directly. This affects both ``/assets/`` (images) and ``/files/`` (documents) URLs. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using fine-grained tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead of a fine-grained token (``-f``). See `#477 <https://github.com/josegonzalez/python-github-backup/issues/477>`_ for details.
|
|
285
|
+
|
|
282
286
|
|
|
283
287
|
Run in Docker container
|
|
284
288
|
-----------------------
|
|
@@ -401,7 +405,7 @@ Quietly and incrementally backup useful Github user data (public and private rep
|
|
|
401
405
|
export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
402
406
|
GH_USER=YOUR-GITHUB-USER
|
|
403
407
|
|
|
404
|
-
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
408
|
+
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --security-advisories --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
405
409
|
|
|
406
410
|
Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
|
|
407
411
|
|
|
@@ -43,9 +43,9 @@ CLI Help output::
|
|
|
43
43
|
[--watched] [--followers] [--following] [--all]
|
|
44
44
|
[--issues] [--issue-comments] [--issue-events] [--pulls]
|
|
45
45
|
[--pull-comments] [--pull-commits] [--pull-details]
|
|
46
|
-
[--labels] [--hooks] [--milestones] [--
|
|
47
|
-
[--bare] [--no-prune] [--lfs] [--wikis]
|
|
48
|
-
[--starred-gists] [--skip-archived] [--skip-existing]
|
|
46
|
+
[--labels] [--hooks] [--milestones] [--security-advisories]
|
|
47
|
+
[--repositories] [--bare] [--no-prune] [--lfs] [--wikis]
|
|
48
|
+
[--gists] [--starred-gists] [--skip-archived] [--skip-existing]
|
|
49
49
|
[-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST]
|
|
50
50
|
[-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v]
|
|
51
51
|
[--keychain-name OSX_KEYCHAIN_ITEM_NAME]
|
|
@@ -55,7 +55,7 @@ CLI Help output::
|
|
|
55
55
|
[--skip-assets-on [SKIP_ASSETS_ON ...]] [--attachments]
|
|
56
56
|
[--throttle-limit THROTTLE_LIMIT]
|
|
57
57
|
[--throttle-pause THROTTLE_PAUSE]
|
|
58
|
-
[--exclude [EXCLUDE ...]]
|
|
58
|
+
[--exclude [EXCLUDE ...]] [--retries MAX_RETRIES]
|
|
59
59
|
USER
|
|
60
60
|
|
|
61
61
|
Backup a github account
|
|
@@ -101,6 +101,8 @@ CLI Help output::
|
|
|
101
101
|
--hooks include hooks in backup (works only when
|
|
102
102
|
authenticated)
|
|
103
103
|
--milestones include milestones in backup
|
|
104
|
+
--security-advisories
|
|
105
|
+
include security advisories in backup
|
|
104
106
|
--repositories include repository clone in backup
|
|
105
107
|
--bare clone bare repositories
|
|
106
108
|
--no-prune disable prune option for git fetch
|
|
@@ -279,6 +281,8 @@ The tool automatically extracts file extensions from HTTP headers to ensure file
|
|
|
279
281
|
|
|
280
282
|
**Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer).
|
|
281
283
|
|
|
284
|
+
**Fine-grained token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) cannot download attachments from private repositories directly. This affects both ``/assets/`` (images) and ``/files/`` (documents) URLs. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using fine-grained tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead of a fine-grained token (``-f``). See `#477 <https://github.com/josegonzalez/python-github-backup/issues/477>`_ for details.
|
|
285
|
+
|
|
282
286
|
|
|
283
287
|
Run in Docker container
|
|
284
288
|
-----------------------
|
|
@@ -401,7 +405,7 @@ Quietly and incrementally backup useful Github user data (public and private rep
|
|
|
401
405
|
export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
402
406
|
GH_USER=YOUR-GITHUB-USER
|
|
403
407
|
|
|
404
|
-
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
408
|
+
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --security-advisories --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
405
409
|
|
|
406
410
|
Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
|
|
407
411
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.61.1"
|
|
@@ -46,6 +46,16 @@ def main():
|
|
|
46
46
|
"Use -t/--token or -f/--token-fine to authenticate."
|
|
47
47
|
)
|
|
48
48
|
|
|
49
|
+
# Issue #477: Fine-grained PATs cannot download all attachment types from
|
|
50
|
+
# private repos. Image attachments will be retried via Markdown API workaround.
|
|
51
|
+
if args.include_attachments and args.token_fine:
|
|
52
|
+
logger.warning(
|
|
53
|
+
"Using --attachments with fine-grained token. Due to GitHub platform "
|
|
54
|
+
"limitations, file attachments (PDFs, etc.) from private repos may fail. "
|
|
55
|
+
"Image attachments will be retried via workaround. For full attachment "
|
|
56
|
+
"support, use --token-classic instead."
|
|
57
|
+
)
|
|
58
|
+
|
|
49
59
|
if args.quiet:
|
|
50
60
|
logger.setLevel(logging.WARNING)
|
|
51
61
|
|
|
@@ -310,6 +310,12 @@ def parse_args(args=None):
|
|
|
310
310
|
dest="include_milestones",
|
|
311
311
|
help="include milestones in backup",
|
|
312
312
|
)
|
|
313
|
+
parser.add_argument(
|
|
314
|
+
"--security-advisories",
|
|
315
|
+
action="store_true",
|
|
316
|
+
dest="include_security_advisories",
|
|
317
|
+
help="include security advisories in backup",
|
|
318
|
+
)
|
|
313
319
|
parser.add_argument(
|
|
314
320
|
"--repositories",
|
|
315
321
|
action="store_true",
|
|
@@ -1056,6 +1062,65 @@ def download_attachment_file(url, path, auth, as_app=False, fine=False):
|
|
|
1056
1062
|
return metadata
|
|
1057
1063
|
|
|
1058
1064
|
|
|
1065
|
+
def get_jwt_signed_url_via_markdown_api(url, token, repo_context):
|
|
1066
|
+
"""Convert a user-attachments/assets URL to a JWT-signed URL via Markdown API.
|
|
1067
|
+
|
|
1068
|
+
GitHub's Markdown API renders image URLs and returns HTML containing
|
|
1069
|
+
JWT-signed private-user-images.githubusercontent.com URLs that work
|
|
1070
|
+
without token authentication.
|
|
1071
|
+
|
|
1072
|
+
This is a workaround for issue #477 where fine-grained PATs cannot
|
|
1073
|
+
download user-attachments URLs from private repos directly.
|
|
1074
|
+
|
|
1075
|
+
Limitations:
|
|
1076
|
+
- Only works for /assets/ URLs (images)
|
|
1077
|
+
- Does NOT work for /files/ URLs (PDFs, text files, etc.)
|
|
1078
|
+
- JWT URLs expire after ~5 minutes
|
|
1079
|
+
|
|
1080
|
+
Args:
|
|
1081
|
+
url: The github.com/user-attachments/assets/UUID URL
|
|
1082
|
+
token: Raw fine-grained PAT (github_pat_...)
|
|
1083
|
+
repo_context: Repository context as "owner/repo"
|
|
1084
|
+
|
|
1085
|
+
Returns:
|
|
1086
|
+
str: JWT-signed URL from private-user-images.githubusercontent.com
|
|
1087
|
+
None: If conversion fails
|
|
1088
|
+
"""
|
|
1089
|
+
|
|
1090
|
+
try:
|
|
1091
|
+
payload = json.dumps(
|
|
1092
|
+
{"text": f"", "mode": "gfm", "context": repo_context}
|
|
1093
|
+
).encode("utf-8")
|
|
1094
|
+
|
|
1095
|
+
request = Request("https://api.github.com/markdown", data=payload, method="POST")
|
|
1096
|
+
request.add_header("Authorization", f"token {token}")
|
|
1097
|
+
request.add_header("Content-Type", "application/json")
|
|
1098
|
+
request.add_header("Accept", "application/vnd.github+json")
|
|
1099
|
+
|
|
1100
|
+
html = urlopen(request, timeout=30).read().decode("utf-8")
|
|
1101
|
+
|
|
1102
|
+
# Parse JWT-signed URL from HTML response
|
|
1103
|
+
# Format: <img src="https://private-user-images.githubusercontent.com/...?jwt=..." ...>
|
|
1104
|
+
if match := re.search(
|
|
1105
|
+
r'src="(https://private-user-images\.githubusercontent\.com/[^"]+)"', html
|
|
1106
|
+
):
|
|
1107
|
+
jwt_url = match.group(1)
|
|
1108
|
+
logger.debug("Converted attachment URL to JWT-signed URL via Markdown API")
|
|
1109
|
+
return jwt_url
|
|
1110
|
+
|
|
1111
|
+
logger.debug("Markdown API response did not contain JWT-signed URL")
|
|
1112
|
+
return None
|
|
1113
|
+
|
|
1114
|
+
except HTTPError as e:
|
|
1115
|
+
logger.debug(
|
|
1116
|
+
"Markdown API request failed with HTTP {0}: {1}".format(e.code, e.reason)
|
|
1117
|
+
)
|
|
1118
|
+
return None
|
|
1119
|
+
except Exception as e:
|
|
1120
|
+
logger.debug("Markdown API request failed: {0}".format(str(e)))
|
|
1121
|
+
return None
|
|
1122
|
+
|
|
1123
|
+
|
|
1059
1124
|
def extract_attachment_urls(item_data, issue_number=None, repository_full_name=None):
|
|
1060
1125
|
"""Extract GitHub-hosted attachment URLs from issue/PR body and comments.
|
|
1061
1126
|
|
|
@@ -1409,15 +1474,46 @@ def download_attachments(
|
|
|
1409
1474
|
filename = get_attachment_filename(url)
|
|
1410
1475
|
filepath = os.path.join(attachments_dir, filename)
|
|
1411
1476
|
|
|
1412
|
-
#
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1477
|
+
# Issue #477: Fine-grained PATs cannot download user-attachments/assets
|
|
1478
|
+
# from private repos directly (404). Use Markdown API workaround to get
|
|
1479
|
+
# a JWT-signed URL. Only works for /assets/ (images), not /files/.
|
|
1480
|
+
needs_jwt = (
|
|
1481
|
+
args.token_fine is not None
|
|
1482
|
+
and repository.get("private", False)
|
|
1483
|
+
and "github.com/user-attachments/assets/" in url
|
|
1419
1484
|
)
|
|
1420
1485
|
|
|
1486
|
+
if not needs_jwt:
|
|
1487
|
+
# NORMAL download path
|
|
1488
|
+
metadata = download_attachment_file(
|
|
1489
|
+
url,
|
|
1490
|
+
filepath,
|
|
1491
|
+
get_auth(args, encode=not args.as_app),
|
|
1492
|
+
as_app=args.as_app,
|
|
1493
|
+
fine=args.token_fine is not None,
|
|
1494
|
+
)
|
|
1495
|
+
elif jwt_url := get_jwt_signed_url_via_markdown_api(
|
|
1496
|
+
url, args.token_fine, repository["full_name"]
|
|
1497
|
+
):
|
|
1498
|
+
# JWT needed and extracted, download via JWT
|
|
1499
|
+
metadata = download_attachment_file(
|
|
1500
|
+
jwt_url, filepath, auth=None, as_app=False, fine=False
|
|
1501
|
+
)
|
|
1502
|
+
metadata["url"] = url # Apply back the original URL
|
|
1503
|
+
metadata["jwt_workaround"] = True
|
|
1504
|
+
else:
|
|
1505
|
+
# Markdown API workaround failed - skip download we know will fail
|
|
1506
|
+
metadata = {
|
|
1507
|
+
"url": url,
|
|
1508
|
+
"success": False,
|
|
1509
|
+
"skipped_at": datetime.now(timezone.utc).isoformat(),
|
|
1510
|
+
"error": "Fine-grained token cannot download private repo attachments. "
|
|
1511
|
+
"Markdown API workaround failed. Use --token-classic instead.",
|
|
1512
|
+
}
|
|
1513
|
+
logger.warning(
|
|
1514
|
+
"Skipping attachment {0}: {1}".format(url, metadata["error"])
|
|
1515
|
+
)
|
|
1516
|
+
|
|
1421
1517
|
# If download succeeded but we got an extension from Content-Disposition,
|
|
1422
1518
|
# we may need to rename the file to add the extension
|
|
1423
1519
|
if metadata["success"] and metadata.get("original_filename"):
|
|
@@ -1718,6 +1814,9 @@ def backup_repositories(args, output_directory, repositories):
|
|
|
1718
1814
|
if args.include_milestones or args.include_everything:
|
|
1719
1815
|
backup_milestones(args, repo_cwd, repository, repos_template)
|
|
1720
1816
|
|
|
1817
|
+
if args.include_security_advisories or args.include_everything:
|
|
1818
|
+
backup_security_advisories(args, repo_cwd, repository, repos_template)
|
|
1819
|
+
|
|
1721
1820
|
if args.include_labels or args.include_everything:
|
|
1722
1821
|
backup_labels(args, repo_cwd, repository, repos_template)
|
|
1723
1822
|
|
|
@@ -1934,6 +2033,43 @@ def backup_milestones(args, repo_cwd, repository, repos_template):
|
|
|
1934
2033
|
)
|
|
1935
2034
|
|
|
1936
2035
|
|
|
2036
|
+
def backup_security_advisories(args, repo_cwd, repository, repos_template):
|
|
2037
|
+
advisory_cwd = os.path.join(repo_cwd, "security-advisories")
|
|
2038
|
+
if args.skip_existing and os.path.isdir(advisory_cwd):
|
|
2039
|
+
return
|
|
2040
|
+
|
|
2041
|
+
logger.info("Retrieving {0} security advisories".format(repository["full_name"]))
|
|
2042
|
+
mkdir_p(repo_cwd, advisory_cwd)
|
|
2043
|
+
|
|
2044
|
+
template = "{0}/{1}/security-advisories".format(
|
|
2045
|
+
repos_template, repository["full_name"]
|
|
2046
|
+
)
|
|
2047
|
+
|
|
2048
|
+
_advisories = retrieve_data(args, template)
|
|
2049
|
+
|
|
2050
|
+
advisories = {}
|
|
2051
|
+
for advisory in _advisories:
|
|
2052
|
+
advisories[advisory["ghsa_id"]] = advisory
|
|
2053
|
+
|
|
2054
|
+
written_count = 0
|
|
2055
|
+
for ghsa_id, advisory in list(advisories.items()):
|
|
2056
|
+
advisory_file = "{0}/{1}.json".format(advisory_cwd, ghsa_id)
|
|
2057
|
+
if json_dump_if_changed(advisory, advisory_file):
|
|
2058
|
+
written_count += 1
|
|
2059
|
+
|
|
2060
|
+
total = len(advisories)
|
|
2061
|
+
if written_count == total:
|
|
2062
|
+
logger.info("Saved {0} security advisories to disk".format(total))
|
|
2063
|
+
elif written_count == 0:
|
|
2064
|
+
logger.info("{0} security advisories unchanged, skipped write".format(total))
|
|
2065
|
+
else:
|
|
2066
|
+
logger.info(
|
|
2067
|
+
"Saved {0} of {1} security advisories to disk ({2} unchanged)".format(
|
|
2068
|
+
written_count, total, total - written_count
|
|
2069
|
+
)
|
|
2070
|
+
)
|
|
2071
|
+
|
|
2072
|
+
|
|
1937
2073
|
def backup_labels(args, repo_cwd, repository, repos_template):
|
|
1938
2074
|
label_cwd = os.path.join(repo_cwd, "labels")
|
|
1939
2075
|
output_file = "{0}/labels.json".format(label_cwd)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: github-backup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.61.1
|
|
4
4
|
Summary: backup a github user or organization
|
|
5
5
|
Home-page: http://github.com/josegonzalez/python-github-backup
|
|
6
6
|
Author: Jose Diaz-Gonzalez
|
|
@@ -73,9 +73,9 @@ CLI Help output::
|
|
|
73
73
|
[--watched] [--followers] [--following] [--all]
|
|
74
74
|
[--issues] [--issue-comments] [--issue-events] [--pulls]
|
|
75
75
|
[--pull-comments] [--pull-commits] [--pull-details]
|
|
76
|
-
[--labels] [--hooks] [--milestones] [--
|
|
77
|
-
[--bare] [--no-prune] [--lfs] [--wikis]
|
|
78
|
-
[--starred-gists] [--skip-archived] [--skip-existing]
|
|
76
|
+
[--labels] [--hooks] [--milestones] [--security-advisories]
|
|
77
|
+
[--repositories] [--bare] [--no-prune] [--lfs] [--wikis]
|
|
78
|
+
[--gists] [--starred-gists] [--skip-archived] [--skip-existing]
|
|
79
79
|
[-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST]
|
|
80
80
|
[-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v]
|
|
81
81
|
[--keychain-name OSX_KEYCHAIN_ITEM_NAME]
|
|
@@ -85,7 +85,7 @@ CLI Help output::
|
|
|
85
85
|
[--skip-assets-on [SKIP_ASSETS_ON ...]] [--attachments]
|
|
86
86
|
[--throttle-limit THROTTLE_LIMIT]
|
|
87
87
|
[--throttle-pause THROTTLE_PAUSE]
|
|
88
|
-
[--exclude [EXCLUDE ...]]
|
|
88
|
+
[--exclude [EXCLUDE ...]] [--retries MAX_RETRIES]
|
|
89
89
|
USER
|
|
90
90
|
|
|
91
91
|
Backup a github account
|
|
@@ -131,6 +131,8 @@ CLI Help output::
|
|
|
131
131
|
--hooks include hooks in backup (works only when
|
|
132
132
|
authenticated)
|
|
133
133
|
--milestones include milestones in backup
|
|
134
|
+
--security-advisories
|
|
135
|
+
include security advisories in backup
|
|
134
136
|
--repositories include repository clone in backup
|
|
135
137
|
--bare clone bare repositories
|
|
136
138
|
--no-prune disable prune option for git fetch
|
|
@@ -309,6 +311,8 @@ The tool automatically extracts file extensions from HTTP headers to ensure file
|
|
|
309
311
|
|
|
310
312
|
**Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer).
|
|
311
313
|
|
|
314
|
+
**Fine-grained token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) cannot download attachments from private repositories directly. This affects both ``/assets/`` (images) and ``/files/`` (documents) URLs. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using fine-grained tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead of a fine-grained token (``-f``). See `#477 <https://github.com/josegonzalez/python-github-backup/issues/477>`_ for details.
|
|
315
|
+
|
|
312
316
|
|
|
313
317
|
Run in Docker container
|
|
314
318
|
-----------------------
|
|
@@ -431,7 +435,7 @@ Quietly and incrementally backup useful Github user data (public and private rep
|
|
|
431
435
|
export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
432
436
|
GH_USER=YOUR-GITHUB-USER
|
|
433
437
|
|
|
434
|
-
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
438
|
+
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --security-advisories --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
435
439
|
|
|
436
440
|
Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
|
|
437
441
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Tests for --all-starred flag behavior (issue #225)."""
|
|
2
2
|
|
|
3
3
|
import pytest
|
|
4
|
-
from unittest.mock import
|
|
4
|
+
from unittest.mock import patch
|
|
5
5
|
|
|
6
6
|
from github_backup import github_backup
|
|
7
7
|
|
|
@@ -12,57 +12,14 @@ class TestAllStarredCloning:
|
|
|
12
12
|
Issue #225: --all-starred should clone starred repos without requiring --repositories.
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
|
-
def _create_mock_args(self, **overrides):
|
|
16
|
-
"""Create a mock args object with sensible defaults."""
|
|
17
|
-
args = Mock()
|
|
18
|
-
args.user = "testuser"
|
|
19
|
-
args.output_directory = "/tmp/backup"
|
|
20
|
-
args.include_repository = False
|
|
21
|
-
args.include_everything = False
|
|
22
|
-
args.include_gists = False
|
|
23
|
-
args.include_starred_gists = False
|
|
24
|
-
args.all_starred = False
|
|
25
|
-
args.skip_existing = False
|
|
26
|
-
args.bare_clone = False
|
|
27
|
-
args.lfs_clone = False
|
|
28
|
-
args.no_prune = False
|
|
29
|
-
args.include_wiki = False
|
|
30
|
-
args.include_issues = False
|
|
31
|
-
args.include_issue_comments = False
|
|
32
|
-
args.include_issue_events = False
|
|
33
|
-
args.include_pulls = False
|
|
34
|
-
args.include_pull_comments = False
|
|
35
|
-
args.include_pull_commits = False
|
|
36
|
-
args.include_pull_details = False
|
|
37
|
-
args.include_labels = False
|
|
38
|
-
args.include_hooks = False
|
|
39
|
-
args.include_milestones = False
|
|
40
|
-
args.include_releases = False
|
|
41
|
-
args.include_assets = False
|
|
42
|
-
args.include_attachments = False
|
|
43
|
-
args.incremental = False
|
|
44
|
-
args.incremental_by_files = False
|
|
45
|
-
args.github_host = None
|
|
46
|
-
args.prefer_ssh = False
|
|
47
|
-
args.token_classic = None
|
|
48
|
-
args.token_fine = None
|
|
49
|
-
args.as_app = False
|
|
50
|
-
args.osx_keychain_item_name = None
|
|
51
|
-
args.osx_keychain_item_account = None
|
|
52
|
-
|
|
53
|
-
for key, value in overrides.items():
|
|
54
|
-
setattr(args, key, value)
|
|
55
|
-
|
|
56
|
-
return args
|
|
57
|
-
|
|
58
15
|
@patch('github_backup.github_backup.fetch_repository')
|
|
59
16
|
@patch('github_backup.github_backup.get_github_repo_url')
|
|
60
|
-
def test_all_starred_clones_without_repositories_flag(self, mock_get_url, mock_fetch):
|
|
17
|
+
def test_all_starred_clones_without_repositories_flag(self, mock_get_url, mock_fetch, create_args):
|
|
61
18
|
"""--all-starred should clone starred repos without --repositories flag.
|
|
62
19
|
|
|
63
20
|
This is the core fix for issue #225.
|
|
64
21
|
"""
|
|
65
|
-
args =
|
|
22
|
+
args = create_args(all_starred=True)
|
|
66
23
|
mock_get_url.return_value = "https://github.com/otheruser/awesome-project.git"
|
|
67
24
|
|
|
68
25
|
# A starred repository (is_starred flag set by retrieve_repositories)
|
|
@@ -87,9 +44,9 @@ class TestAllStarredCloning:
|
|
|
87
44
|
|
|
88
45
|
@patch('github_backup.github_backup.fetch_repository')
|
|
89
46
|
@patch('github_backup.github_backup.get_github_repo_url')
|
|
90
|
-
def test_starred_repo_not_cloned_without_all_starred_flag(self, mock_get_url, mock_fetch):
|
|
47
|
+
def test_starred_repo_not_cloned_without_all_starred_flag(self, mock_get_url, mock_fetch, create_args):
|
|
91
48
|
"""Starred repos should NOT be cloned if --all-starred is not set."""
|
|
92
|
-
args =
|
|
49
|
+
args = create_args(all_starred=False)
|
|
93
50
|
mock_get_url.return_value = "https://github.com/otheruser/awesome-project.git"
|
|
94
51
|
|
|
95
52
|
starred_repo = {
|
|
@@ -110,9 +67,9 @@ class TestAllStarredCloning:
|
|
|
110
67
|
|
|
111
68
|
@patch('github_backup.github_backup.fetch_repository')
|
|
112
69
|
@patch('github_backup.github_backup.get_github_repo_url')
|
|
113
|
-
def test_non_starred_repo_not_cloned_with_only_all_starred(self, mock_get_url, mock_fetch):
|
|
70
|
+
def test_non_starred_repo_not_cloned_with_only_all_starred(self, mock_get_url, mock_fetch, create_args):
|
|
114
71
|
"""Non-starred repos should NOT be cloned when only --all-starred is set."""
|
|
115
|
-
args =
|
|
72
|
+
args = create_args(all_starred=True)
|
|
116
73
|
mock_get_url.return_value = "https://github.com/testuser/my-project.git"
|
|
117
74
|
|
|
118
75
|
# A regular (non-starred) repository
|
|
@@ -134,9 +91,9 @@ class TestAllStarredCloning:
|
|
|
134
91
|
|
|
135
92
|
@patch('github_backup.github_backup.fetch_repository')
|
|
136
93
|
@patch('github_backup.github_backup.get_github_repo_url')
|
|
137
|
-
def test_repositories_flag_still_works(self, mock_get_url, mock_fetch):
|
|
94
|
+
def test_repositories_flag_still_works(self, mock_get_url, mock_fetch, create_args):
|
|
138
95
|
"""--repositories flag should still clone repos as before."""
|
|
139
|
-
args =
|
|
96
|
+
args = create_args(include_repository=True)
|
|
140
97
|
mock_get_url.return_value = "https://github.com/testuser/my-project.git"
|
|
141
98
|
|
|
142
99
|
regular_repo = {
|