github-backup 0.61.4__tar.gz → 0.62.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {github_backup-0.61.4 → github_backup-0.62.0}/CHANGES.rst +165 -1
- {github_backup-0.61.4/github_backup.egg-info → github_backup-0.62.0}/PKG-INFO +47 -18
- {github_backup-0.61.4 → github_backup-0.62.0}/README +46 -17
- {github_backup-0.61.4 → github_backup-0.62.0}/README.rst +46 -17
- github_backup-0.62.0/github_backup/__init__.py +1 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/github_backup/github_backup.py +810 -71
- github_backup-0.62.0/github_backup/graphql_queries.py +292 -0
- {github_backup-0.61.4 → github_backup-0.62.0/github_backup.egg-info}/PKG-INFO +47 -18
- {github_backup-0.61.4 → github_backup-0.62.0}/github_backup.egg-info/SOURCES.txt +7 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/release-requirements.txt +3 -3
- github_backup-0.62.0/tests/test_auth.py +75 -0
- github_backup-0.62.0/tests/test_discussions.py +257 -0
- github_backup-0.62.0/tests/test_incremental_per_repository.py +189 -0
- github_backup-0.62.0/tests/test_pull_incremental_pagination.py +131 -0
- github_backup-0.62.0/tests/test_pull_reviews.py +237 -0
- github_backup-0.62.0/tests/test_releases.py +95 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/tests/test_retrieve_data.py +28 -0
- github_backup-0.61.4/github_backup/__init__.py +0 -1
- {github_backup-0.61.4 → github_backup-0.62.0}/LICENSE.txt +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/MANIFEST.in +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/github_backup/__main__.py +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/github_backup/cli.py +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/github_backup.egg-info/dependency_links.txt +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/github_backup.egg-info/entry_points.txt +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/github_backup.egg-info/top_level.txt +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/github_backup.egg-info/zip-safe +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/requirements.txt +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/setup.cfg +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/setup.py +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/tests/test_all_starred.py +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/tests/test_attachments.py +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/tests/test_case_sensitivity.py +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/tests/test_http_451.py +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/tests/test_json_dump_if_changed.py +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/tests/test_pagination.py +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/tests/test_skip_assets_on.py +0 -0
- {github_backup-0.61.4 → github_backup-0.62.0}/tests/test_starred_skip_size_over.py +0 -0
|
@@ -1,9 +1,173 @@
|
|
|
1
1
|
Changelog
|
|
2
2
|
=========
|
|
3
3
|
|
|
4
|
-
0.
|
|
4
|
+
0.62.0 (2026-04-29)
|
|
5
5
|
-------------------
|
|
6
6
|
------------------------
|
|
7
|
+
- Skip checkpoint-equal incremental items. [Duncan Ogilvie]
|
|
8
|
+
- Avoid redundant release asset list requests. [Duncan Ogilvie]
|
|
9
|
+
- Reduce unnecessary pull requests with incremental fetching. [Duncan
|
|
10
|
+
Ogilvie]
|
|
11
|
+
- Implement per-resource last_update timestamps. [Duncan Ogilvie]
|
|
12
|
+
|
|
13
|
+
Closes #62
|
|
14
|
+
- Add support for pull request reviews. [Duncan Ogilvie]
|
|
15
|
+
|
|
16
|
+
Closes #124
|
|
17
|
+
- Add support for discussions. [Duncan Ogilvie]
|
|
18
|
+
|
|
19
|
+
Closes #290
|
|
20
|
+
- Add --token-from-gh authentication option. [Duncan Ogilvie]
|
|
21
|
+
- Chore(deps): bump pytest in the python-packages group.
|
|
22
|
+
[dependabot[bot]]
|
|
23
|
+
|
|
24
|
+
Bumps the python-packages group with 1 update: [pytest](https://github.com/pytest-dev/pytest).
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
Updates `pytest` from 9.0.2 to 9.0.3
|
|
28
|
+
- [Release notes](https://github.com/pytest-dev/pytest/releases)
|
|
29
|
+
- [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
|
|
30
|
+
- [Commits](https://github.com/pytest-dev/pytest/compare/9.0.2...9.0.3)
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
updated-dependencies:
|
|
34
|
+
- dependency-name: pytest
|
|
35
|
+
dependency-version: 9.0.3
|
|
36
|
+
dependency-type: direct:production
|
|
37
|
+
update-type: version-update:semver-patch
|
|
38
|
+
dependency-group: python-packages
|
|
39
|
+
...
|
|
40
|
+
- Chore(deps): bump black in the python-packages group.
|
|
41
|
+
[dependabot[bot]]
|
|
42
|
+
|
|
43
|
+
Bumps the python-packages group with 1 update: [black](https://github.com/psf/black).
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
Updates `black` from 26.3.0 to 26.3.1
|
|
47
|
+
- [Release notes](https://github.com/psf/black/releases)
|
|
48
|
+
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
|
|
49
|
+
- [Commits](https://github.com/psf/black/compare/26.3.0...26.3.1)
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
updated-dependencies:
|
|
53
|
+
- dependency-name: black
|
|
54
|
+
dependency-version: 26.3.1
|
|
55
|
+
dependency-type: direct:production
|
|
56
|
+
update-type: version-update:semver-patch
|
|
57
|
+
dependency-group: python-packages
|
|
58
|
+
...
|
|
59
|
+
- Chore(deps): bump docker/login-action from 3 to 4. [dependabot[bot]]
|
|
60
|
+
|
|
61
|
+
Bumps [docker/login-action](https://github.com/docker/login-action) from 3 to 4.
|
|
62
|
+
- [Release notes](https://github.com/docker/login-action/releases)
|
|
63
|
+
- [Commits](https://github.com/docker/login-action/compare/v3...v4)
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
updated-dependencies:
|
|
67
|
+
- dependency-name: docker/login-action
|
|
68
|
+
dependency-version: '4'
|
|
69
|
+
dependency-type: direct:production
|
|
70
|
+
update-type: version-update:semver-major
|
|
71
|
+
...
|
|
72
|
+
- Chore(deps): bump docker/setup-qemu-action from 3 to 4.
|
|
73
|
+
[dependabot[bot]]
|
|
74
|
+
|
|
75
|
+
Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 3 to 4.
|
|
76
|
+
- [Release notes](https://github.com/docker/setup-qemu-action/releases)
|
|
77
|
+
- [Commits](https://github.com/docker/setup-qemu-action/compare/v3...v4)
|
|
78
|
+
|
|
79
|
+
---
|
|
80
|
+
updated-dependencies:
|
|
81
|
+
- dependency-name: docker/setup-qemu-action
|
|
82
|
+
dependency-version: '4'
|
|
83
|
+
dependency-type: direct:production
|
|
84
|
+
update-type: version-update:semver-major
|
|
85
|
+
...
|
|
86
|
+
- Chore(deps): bump docker/build-push-action from 6 to 7.
|
|
87
|
+
[dependabot[bot]]
|
|
88
|
+
|
|
89
|
+
Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6 to 7.
|
|
90
|
+
- [Release notes](https://github.com/docker/build-push-action/releases)
|
|
91
|
+
- [Commits](https://github.com/docker/build-push-action/compare/v6...v7)
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
updated-dependencies:
|
|
95
|
+
- dependency-name: docker/build-push-action
|
|
96
|
+
dependency-version: '7'
|
|
97
|
+
dependency-type: direct:production
|
|
98
|
+
update-type: version-update:semver-major
|
|
99
|
+
...
|
|
100
|
+
- Chore(deps): bump docker/setup-buildx-action from 3 to 4.
|
|
101
|
+
[dependabot[bot]]
|
|
102
|
+
|
|
103
|
+
Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3 to 4.
|
|
104
|
+
- [Release notes](https://github.com/docker/setup-buildx-action/releases)
|
|
105
|
+
- [Commits](https://github.com/docker/setup-buildx-action/compare/v3...v4)
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
updated-dependencies:
|
|
109
|
+
- dependency-name: docker/setup-buildx-action
|
|
110
|
+
dependency-version: '4'
|
|
111
|
+
dependency-type: direct:production
|
|
112
|
+
update-type: version-update:semver-major
|
|
113
|
+
...
|
|
114
|
+
- Chore(deps): bump docker/metadata-action from 5 to 6.
|
|
115
|
+
[dependabot[bot]]
|
|
116
|
+
|
|
117
|
+
Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 5 to 6.
|
|
118
|
+
- [Release notes](https://github.com/docker/metadata-action/releases)
|
|
119
|
+
- [Commits](https://github.com/docker/metadata-action/compare/v5...v6)
|
|
120
|
+
|
|
121
|
+
---
|
|
122
|
+
updated-dependencies:
|
|
123
|
+
- dependency-name: docker/metadata-action
|
|
124
|
+
dependency-version: '6'
|
|
125
|
+
dependency-type: direct:production
|
|
126
|
+
update-type: version-update:semver-major
|
|
127
|
+
...
|
|
128
|
+
- Chore(deps): bump the python-packages group with 2 updates.
|
|
129
|
+
[dependabot[bot]]
|
|
130
|
+
|
|
131
|
+
Bumps the python-packages group with 2 updates: [black](https://github.com/psf/black) and [setuptools](https://github.com/pypa/setuptools).
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
Updates `black` from 26.1.0 to 26.3.0
|
|
135
|
+
- [Release notes](https://github.com/psf/black/releases)
|
|
136
|
+
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
|
|
137
|
+
- [Commits](https://github.com/psf/black/compare/26.1.0...26.3.0)
|
|
138
|
+
|
|
139
|
+
Updates `setuptools` from 82.0.0 to 82.0.1
|
|
140
|
+
- [Release notes](https://github.com/pypa/setuptools/releases)
|
|
141
|
+
- [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst)
|
|
142
|
+
- [Commits](https://github.com/pypa/setuptools/compare/v82.0.0...v82.0.1)
|
|
143
|
+
|
|
144
|
+
---
|
|
145
|
+
updated-dependencies:
|
|
146
|
+
- dependency-name: black
|
|
147
|
+
dependency-version: 26.3.0
|
|
148
|
+
dependency-type: direct:production
|
|
149
|
+
update-type: version-update:semver-minor
|
|
150
|
+
dependency-group: python-packages
|
|
151
|
+
- dependency-name: setuptools
|
|
152
|
+
dependency-version: 82.0.1
|
|
153
|
+
dependency-type: direct:production
|
|
154
|
+
update-type: version-update:semver-patch
|
|
155
|
+
dependency-group: python-packages
|
|
156
|
+
...
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
0.61.5 (2026-02-18)
|
|
160
|
+
-------------------
|
|
161
|
+
- Fix empty repository crash due to None timestamp comparison (#489)
|
|
162
|
+
[Rodos]
|
|
163
|
+
|
|
164
|
+
Empty repositories have None for pushed_at/updated_at, causing a
|
|
165
|
+
TypeError when compared to the last_update string. Use .get() with
|
|
166
|
+
truthiness check to skip None timestamps in incremental tracking.
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
0.61.4 (2026-02-16)
|
|
170
|
+
-------------------
|
|
7
171
|
- Fix HTTP 451 DMCA and 403 TOS handling regression (#487) [Rodos]
|
|
8
172
|
|
|
9
173
|
The DMCA handling added in PR #454 had a bug: make_request_with_retry()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: github-backup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.62.0
|
|
4
4
|
Summary: backup a github user or organization
|
|
5
5
|
Home-page: http://github.com/josegonzalez/python-github-backup
|
|
6
6
|
Author: Jose Diaz-Gonzalez
|
|
@@ -34,7 +34,7 @@ github-backup
|
|
|
34
34
|
|
|
35
35
|
|PyPI| |Python Versions|
|
|
36
36
|
|
|
37
|
-
The package can be used to backup an *entire* `Github <https://github.com/>`_ organization, repository or user account, including starred repos, issues and wikis in the most appropriate format (clones for wikis, json files for issues).
|
|
37
|
+
The package can be used to backup an *entire* `Github <https://github.com/>`_ organization, repository or user account, including starred repos, issues, discussions and wikis in the most appropriate format (clones for wikis, json files for issues and discussions).
|
|
38
38
|
|
|
39
39
|
Requirements
|
|
40
40
|
============
|
|
@@ -66,16 +66,18 @@ Show the CLI help output::
|
|
|
66
66
|
|
|
67
67
|
CLI Help output::
|
|
68
68
|
|
|
69
|
-
github-backup [-h] [-t TOKEN_CLASSIC] [-f TOKEN_FINE] [
|
|
70
|
-
[-o OUTPUT_DIRECTORY] [-l LOG_LEVEL] [-i]
|
|
69
|
+
github-backup [-h] [-t TOKEN_CLASSIC] [-f TOKEN_FINE] [--token-from-gh]
|
|
70
|
+
[-q] [--as-app] [-o OUTPUT_DIRECTORY] [-l LOG_LEVEL] [-i]
|
|
71
71
|
[--incremental-by-files]
|
|
72
72
|
[--starred] [--all-starred] [--starred-skip-size-over MB]
|
|
73
73
|
[--watched] [--followers] [--following] [--all]
|
|
74
74
|
[--issues] [--issue-comments] [--issue-events] [--pulls]
|
|
75
|
-
[--pull-comments] [--pull-
|
|
75
|
+
[--pull-comments] [--pull-reviews] [--pull-commits]
|
|
76
|
+
[--pull-details]
|
|
76
77
|
[--labels] [--hooks] [--milestones] [--security-advisories]
|
|
77
|
-
[--repositories] [--bare] [--no-prune]
|
|
78
|
-
[--
|
|
78
|
+
[--discussions] [--repositories] [--bare] [--no-prune]
|
|
79
|
+
[--lfs] [--wikis] [--gists] [--starred-gists]
|
|
80
|
+
[--skip-archived] [--skip-existing]
|
|
79
81
|
[-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST]
|
|
80
82
|
[-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v]
|
|
81
83
|
[--keychain-name OSX_KEYCHAIN_ITEM_NAME]
|
|
@@ -101,6 +103,7 @@ CLI Help output::
|
|
|
101
103
|
-f, --token-fine TOKEN_FINE
|
|
102
104
|
fine-grained personal access token (github_pat_....),
|
|
103
105
|
or path to token (file://...)
|
|
106
|
+
--token-from-gh read token from GitHub CLI (gh auth token)
|
|
104
107
|
-q, --quiet supress log messages less severe than warning, e.g.
|
|
105
108
|
info
|
|
106
109
|
--as-app authenticate as github app instead of as a user.
|
|
@@ -125,6 +128,7 @@ CLI Help output::
|
|
|
125
128
|
--issue-events include issue events in backup
|
|
126
129
|
--pulls include pull requests in backup
|
|
127
130
|
--pull-comments include pull request review comments in backup
|
|
131
|
+
--pull-reviews include pull request reviews in backup
|
|
128
132
|
--pull-commits include pull request commits in backup
|
|
129
133
|
--pull-details include more pull request details in backup [*]
|
|
130
134
|
--labels include labels in backup
|
|
@@ -133,6 +137,7 @@ CLI Help output::
|
|
|
133
137
|
--milestones include milestones in backup
|
|
134
138
|
--security-advisories
|
|
135
139
|
include security advisories in backup
|
|
140
|
+
--discussions include discussions in backup
|
|
136
141
|
--repositories include repository clone in backup
|
|
137
142
|
--bare clone bare repositories
|
|
138
143
|
--no-prune disable prune option for git fetch
|
|
@@ -173,8 +178,8 @@ CLI Help output::
|
|
|
173
178
|
applies if including releases
|
|
174
179
|
--skip-assets-on [SKIP_ASSETS_ON ...]
|
|
175
180
|
skip asset downloads for these repositories
|
|
176
|
-
--attachments download user-attachments from issues
|
|
177
|
-
|
|
181
|
+
--attachments download user-attachments from issues, pull requests,
|
|
182
|
+
and discussions
|
|
178
183
|
--throttle-limit THROTTLE_LIMIT
|
|
179
184
|
start throttling of GitHub API requests after this
|
|
180
185
|
amount of API requests remain
|
|
@@ -201,6 +206,8 @@ The positional argument ``USER`` specifies the user or organization account you
|
|
|
201
206
|
|
|
202
207
|
**Classic tokens** (``-t TOKEN``) are `slightly less secure <https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#personal-access-tokens-classic>`_ as they provide very coarse-grained permissions.
|
|
203
208
|
|
|
209
|
+
If you already authenticate with the `GitHub CLI <https://cli.github.com/>`_, you can use ``--token-from-gh`` to read the token with ``gh auth token`` instead of passing a token directly. This avoids placing the token in shell history or process arguments. When ``--github-host`` is set, the token is read with ``gh auth token --hostname HOST``.
|
|
210
|
+
|
|
204
211
|
|
|
205
212
|
Fine Tokens
|
|
206
213
|
~~~~~~~~~~~
|
|
@@ -211,7 +218,7 @@ Customise the permissions for your use case, but for a personal account full bac
|
|
|
211
218
|
|
|
212
219
|
**User permissions**: Read access to followers, starring, and watching.
|
|
213
220
|
|
|
214
|
-
**Repository permissions**: Read access to contents, issues, metadata, pull requests, and webhooks.
|
|
221
|
+
**Repository permissions**: Read access to contents, discussions, issues, metadata, pull requests, and webhooks.
|
|
215
222
|
|
|
216
223
|
|
|
217
224
|
GitHub Apps
|
|
@@ -292,9 +299,9 @@ LFS objects are fetched for all refs, not just the current checkout, ensuring a
|
|
|
292
299
|
About Attachments
|
|
293
300
|
-----------------
|
|
294
301
|
|
|
295
|
-
When you use the ``--attachments`` option with ``--issues`` or ``--
|
|
302
|
+
When you use the ``--attachments`` option with ``--issues``, ``--pulls`` or ``--discussions``, the tool will download user-uploaded attachments (images, videos, documents, etc.) from issue, pull request and discussion descriptions and comments. In some circumstances attachments contain valuable data related to the topic, and without their backup important information or context might be lost inadvertently.
|
|
296
303
|
|
|
297
|
-
Attachments are saved to ``issues/attachments/{issue_number}/`` and ``
|
|
304
|
+
Attachments are saved to ``issues/attachments/{issue_number}/``, ``pulls/attachments/{pull_number}/`` and ``discussions/attachments/{discussion_number}/`` directories, where ``{issue_number}`` is the GitHub issue number (e.g., issue #123 saves to ``issues/attachments/123/``). Each attachment directory contains:
|
|
298
305
|
|
|
299
306
|
- The downloaded attachment files (named by their GitHub identifier with appropriate file extensions)
|
|
300
307
|
- If multiple attachments have the same filename, conflicts are resolved with numeric suffixes (e.g., ``report.pdf``, ``report_1.pdf``, ``report_2.pdf``)
|
|
@@ -314,6 +321,16 @@ The tool automatically extracts file extensions from HTTP headers to ensure file
|
|
|
314
321
|
**Fine-grained token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) cannot download attachments from private repositories directly. This affects both ``/assets/`` (images) and ``/files/`` (documents) URLs. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using fine-grained tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead of a fine-grained token (``-f``). See `#477 <https://github.com/josegonzalez/python-github-backup/issues/477>`_ for details.
|
|
315
322
|
|
|
316
323
|
|
|
324
|
+
About Discussions
|
|
325
|
+
-----------------
|
|
326
|
+
|
|
327
|
+
GitHub Discussions are backed up with GitHub's GraphQL API because the REST API does not expose discussions. Use ``--discussions`` to save each discussion as JSON under ``repositories/{repo}/discussions/{number}.json``. Discussion backups include the discussion body and metadata, category information, comments, and comment replies.
|
|
328
|
+
|
|
329
|
+
``--discussions`` is included in ``--all``. Unlike most REST API-backed resources, discussions require authentication because GitHub's GraphQL API requires a token. Fine-grained personal access tokens and GitHub Apps need read access to the repository's Discussions permission.
|
|
330
|
+
|
|
331
|
+
Incremental backups use a per-repository checkpoint at ``repositories/{repo}/discussions/last_update`` based on discussion ``updatedAt`` timestamps. This is separate from the repository-level ``last_update`` file so discussion activity is not missed if the repository's own update timestamp does not change. If you enable ``--discussions`` on an existing incremental backup, the first run performs a full discussions backup for each repository and creates the discussions checkpoint for future runs.
|
|
332
|
+
|
|
333
|
+
|
|
317
334
|
About security advisories
|
|
318
335
|
-------------------------
|
|
319
336
|
|
|
@@ -355,12 +372,24 @@ For finer control, avoid using ``--assets`` with starred repos, or use ``--skip-
|
|
|
355
372
|
|
|
356
373
|
Alternatively, consider just storing links to starred repos in JSON format with ``--starred``.
|
|
357
374
|
|
|
375
|
+
About pull request reviews
|
|
376
|
+
--------------------------
|
|
377
|
+
|
|
378
|
+
Use ``--pull-reviews`` with ``--pulls`` to include GitHub pull request review metadata under each pull request's ``review_data`` key. Reviews are separate from review comments: ``--pull-comments`` backs up inline review comments via ``comment_data`` and regular PR conversation comments via ``comment_regular_data``, while ``--pull-reviews`` backs up review state, submitted time, commit ID, and the top-level review body.
|
|
379
|
+
|
|
380
|
+
``--pull-reviews`` is included in ``--all``. Incremental backups use a per-repository checkpoint at ``repositories/{repo}/pulls/reviews_last_update``. If ``--pull-reviews`` is enabled on an existing incremental backup, the first run performs a one-time backfill for pull request reviews so older PRs are not skipped by the existing pull request checkpoint. Existing ``comment_data``, ``comment_regular_data`` and ``commit_data`` fields are preserved when only review data is being added.
|
|
381
|
+
|
|
382
|
+
|
|
358
383
|
Incremental Backup
|
|
359
384
|
------------------
|
|
360
385
|
|
|
361
|
-
Using (``-i, --incremental``) will only request new data from the API **since the last
|
|
386
|
+
Using (``-i, --incremental``) will only request new data from the API **since the last successful resource backup**. e.g. only request issues from the API since the last issue backup for that repository.
|
|
387
|
+
|
|
388
|
+
Incremental checkpoints for issue and pull request API backups are stored per resource in that repository's backup directory (for example ``repositories/{repo}/issues/last_update``, ``repositories/{repo}/pulls/last_update`` or ``starred/{owner}/{repo}/pulls/last_update``). Older versions stored a single global ``last_update`` file in the output directory root. During migration, the legacy global checkpoint is used as a fallback only for resource directories that already contain backup data but do not yet have their own checkpoint. New repositories or newly enabled resources with no existing data get a full backup instead of inheriting an unrelated global checkpoint.
|
|
389
|
+
|
|
390
|
+
After all existing issue and pull request resource directories have per-resource checkpoints, the legacy global ``last_update`` file is removed automatically.
|
|
362
391
|
|
|
363
|
-
This means any blocking errors on previous runs can cause
|
|
392
|
+
This means any blocking errors on previous runs can cause missing data in backups for the affected repository resource.
|
|
364
393
|
|
|
365
394
|
Using (``--incremental-by-files``) will request new data from the API **based on when the file was modified on filesystem**. e.g. if you modify the file yourself you may miss something.
|
|
366
395
|
|
|
@@ -373,7 +402,7 @@ Known blocking errors
|
|
|
373
402
|
|
|
374
403
|
Some errors will block the backup run by exiting the script. e.g. receiving a 403 Forbidden error from the Github API.
|
|
375
404
|
|
|
376
|
-
If the incremental argument is used,
|
|
405
|
+
If the incremental argument is used, per-resource checkpoints are only advanced after that resource's backup work completes. A blocking error can still abort the overall run, but repositories and resources that were not processed will keep their previous checkpoints.
|
|
377
406
|
|
|
378
407
|
It's therefore recommended to only use the incremental argument if the output/result is being actively monitored, or complimented with periodic full non-incremental runs, to avoid unexpected missing data in a regular backup runs.
|
|
379
408
|
|
|
@@ -446,14 +475,14 @@ Quietly and incrementally backup useful Github user data (public and private rep
|
|
|
446
475
|
export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
447
476
|
GH_USER=YOUR-GITHUB-USER
|
|
448
477
|
|
|
449
|
-
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --security-advisories --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
478
|
+
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --security-advisories --discussions --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
450
479
|
|
|
451
480
|
Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
|
|
452
481
|
|
|
453
482
|
export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
454
483
|
GH_USER=YOUR-GITHUB-USER
|
|
455
484
|
|
|
456
|
-
github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
|
|
485
|
+
github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --discussions --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
|
|
457
486
|
|
|
458
487
|
Pipe a token from stdin to avoid storing it in environment variables or command history (Unix-like systems only)::
|
|
459
488
|
|
|
@@ -469,7 +498,7 @@ This tool creates backups only, there is no inbuilt restore command.
|
|
|
469
498
|
cd /tmp/white-house/repositories/petitions/repository
|
|
470
499
|
git push --mirror git@github.com:WhiteHouse/petitions.git
|
|
471
500
|
|
|
472
|
-
**Issues, pull requests, comments, and other metadata** are saved as JSON files for archival purposes. The GitHub API does not support recreating this data faithfully, creating issues via the API has limitations:
|
|
501
|
+
**Issues, pull requests, discussions, comments, and other metadata** are saved as JSON files for archival purposes. The GitHub API does not support recreating this data faithfully, creating issues via the API has limitations:
|
|
473
502
|
|
|
474
503
|
- New issue/PR numbers are assigned (original numbers cannot be set)
|
|
475
504
|
- Timestamps reflect creation time (original dates cannot be set)
|
|
@@ -4,7 +4,7 @@ github-backup
|
|
|
4
4
|
|
|
5
5
|
|PyPI| |Python Versions|
|
|
6
6
|
|
|
7
|
-
The package can be used to backup an *entire* `Github <https://github.com/>`_ organization, repository or user account, including starred repos, issues and wikis in the most appropriate format (clones for wikis, json files for issues).
|
|
7
|
+
The package can be used to backup an *entire* `Github <https://github.com/>`_ organization, repository or user account, including starred repos, issues, discussions and wikis in the most appropriate format (clones for wikis, json files for issues and discussions).
|
|
8
8
|
|
|
9
9
|
Requirements
|
|
10
10
|
============
|
|
@@ -36,16 +36,18 @@ Show the CLI help output::
|
|
|
36
36
|
|
|
37
37
|
CLI Help output::
|
|
38
38
|
|
|
39
|
-
github-backup [-h] [-t TOKEN_CLASSIC] [-f TOKEN_FINE] [
|
|
40
|
-
[-o OUTPUT_DIRECTORY] [-l LOG_LEVEL] [-i]
|
|
39
|
+
github-backup [-h] [-t TOKEN_CLASSIC] [-f TOKEN_FINE] [--token-from-gh]
|
|
40
|
+
[-q] [--as-app] [-o OUTPUT_DIRECTORY] [-l LOG_LEVEL] [-i]
|
|
41
41
|
[--incremental-by-files]
|
|
42
42
|
[--starred] [--all-starred] [--starred-skip-size-over MB]
|
|
43
43
|
[--watched] [--followers] [--following] [--all]
|
|
44
44
|
[--issues] [--issue-comments] [--issue-events] [--pulls]
|
|
45
|
-
[--pull-comments] [--pull-
|
|
45
|
+
[--pull-comments] [--pull-reviews] [--pull-commits]
|
|
46
|
+
[--pull-details]
|
|
46
47
|
[--labels] [--hooks] [--milestones] [--security-advisories]
|
|
47
|
-
[--repositories] [--bare] [--no-prune]
|
|
48
|
-
[--
|
|
48
|
+
[--discussions] [--repositories] [--bare] [--no-prune]
|
|
49
|
+
[--lfs] [--wikis] [--gists] [--starred-gists]
|
|
50
|
+
[--skip-archived] [--skip-existing]
|
|
49
51
|
[-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST]
|
|
50
52
|
[-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v]
|
|
51
53
|
[--keychain-name OSX_KEYCHAIN_ITEM_NAME]
|
|
@@ -71,6 +73,7 @@ CLI Help output::
|
|
|
71
73
|
-f, --token-fine TOKEN_FINE
|
|
72
74
|
fine-grained personal access token (github_pat_....),
|
|
73
75
|
or path to token (file://...)
|
|
76
|
+
--token-from-gh read token from GitHub CLI (gh auth token)
|
|
74
77
|
-q, --quiet supress log messages less severe than warning, e.g.
|
|
75
78
|
info
|
|
76
79
|
--as-app authenticate as github app instead of as a user.
|
|
@@ -95,6 +98,7 @@ CLI Help output::
|
|
|
95
98
|
--issue-events include issue events in backup
|
|
96
99
|
--pulls include pull requests in backup
|
|
97
100
|
--pull-comments include pull request review comments in backup
|
|
101
|
+
--pull-reviews include pull request reviews in backup
|
|
98
102
|
--pull-commits include pull request commits in backup
|
|
99
103
|
--pull-details include more pull request details in backup [*]
|
|
100
104
|
--labels include labels in backup
|
|
@@ -103,6 +107,7 @@ CLI Help output::
|
|
|
103
107
|
--milestones include milestones in backup
|
|
104
108
|
--security-advisories
|
|
105
109
|
include security advisories in backup
|
|
110
|
+
--discussions include discussions in backup
|
|
106
111
|
--repositories include repository clone in backup
|
|
107
112
|
--bare clone bare repositories
|
|
108
113
|
--no-prune disable prune option for git fetch
|
|
@@ -143,8 +148,8 @@ CLI Help output::
|
|
|
143
148
|
applies if including releases
|
|
144
149
|
--skip-assets-on [SKIP_ASSETS_ON ...]
|
|
145
150
|
skip asset downloads for these repositories
|
|
146
|
-
--attachments download user-attachments from issues
|
|
147
|
-
|
|
151
|
+
--attachments download user-attachments from issues, pull requests,
|
|
152
|
+
and discussions
|
|
148
153
|
--throttle-limit THROTTLE_LIMIT
|
|
149
154
|
start throttling of GitHub API requests after this
|
|
150
155
|
amount of API requests remain
|
|
@@ -171,6 +176,8 @@ The positional argument ``USER`` specifies the user or organization account you
|
|
|
171
176
|
|
|
172
177
|
**Classic tokens** (``-t TOKEN``) are `slightly less secure <https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#personal-access-tokens-classic>`_ as they provide very coarse-grained permissions.
|
|
173
178
|
|
|
179
|
+
If you already authenticate with the `GitHub CLI <https://cli.github.com/>`_, you can use ``--token-from-gh`` to read the token with ``gh auth token`` instead of passing a token directly. This avoids placing the token in shell history or process arguments. When ``--github-host`` is set, the token is read with ``gh auth token --hostname HOST``.
|
|
180
|
+
|
|
174
181
|
|
|
175
182
|
Fine Tokens
|
|
176
183
|
~~~~~~~~~~~
|
|
@@ -181,7 +188,7 @@ Customise the permissions for your use case, but for a personal account full bac
|
|
|
181
188
|
|
|
182
189
|
**User permissions**: Read access to followers, starring, and watching.
|
|
183
190
|
|
|
184
|
-
**Repository permissions**: Read access to contents, issues, metadata, pull requests, and webhooks.
|
|
191
|
+
**Repository permissions**: Read access to contents, discussions, issues, metadata, pull requests, and webhooks.
|
|
185
192
|
|
|
186
193
|
|
|
187
194
|
GitHub Apps
|
|
@@ -262,9 +269,9 @@ LFS objects are fetched for all refs, not just the current checkout, ensuring a
|
|
|
262
269
|
About Attachments
|
|
263
270
|
-----------------
|
|
264
271
|
|
|
265
|
-
When you use the ``--attachments`` option with ``--issues`` or ``--
|
|
272
|
+
When you use the ``--attachments`` option with ``--issues``, ``--pulls`` or ``--discussions``, the tool will download user-uploaded attachments (images, videos, documents, etc.) from issue, pull request and discussion descriptions and comments. In some circumstances attachments contain valuable data related to the topic, and without their backup important information or context might be lost inadvertently.
|
|
266
273
|
|
|
267
|
-
Attachments are saved to ``issues/attachments/{issue_number}/`` and ``
|
|
274
|
+
Attachments are saved to ``issues/attachments/{issue_number}/``, ``pulls/attachments/{pull_number}/`` and ``discussions/attachments/{discussion_number}/`` directories, where ``{issue_number}`` is the GitHub issue number (e.g., issue #123 saves to ``issues/attachments/123/``). Each attachment directory contains:
|
|
268
275
|
|
|
269
276
|
- The downloaded attachment files (named by their GitHub identifier with appropriate file extensions)
|
|
270
277
|
- If multiple attachments have the same filename, conflicts are resolved with numeric suffixes (e.g., ``report.pdf``, ``report_1.pdf``, ``report_2.pdf``)
|
|
@@ -284,6 +291,16 @@ The tool automatically extracts file extensions from HTTP headers to ensure file
|
|
|
284
291
|
**Fine-grained token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) cannot download attachments from private repositories directly. This affects both ``/assets/`` (images) and ``/files/`` (documents) URLs. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using fine-grained tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead of a fine-grained token (``-f``). See `#477 <https://github.com/josegonzalez/python-github-backup/issues/477>`_ for details.
|
|
285
292
|
|
|
286
293
|
|
|
294
|
+
About Discussions
|
|
295
|
+
-----------------
|
|
296
|
+
|
|
297
|
+
GitHub Discussions are backed up with GitHub's GraphQL API because the REST API does not expose discussions. Use ``--discussions`` to save each discussion as JSON under ``repositories/{repo}/discussions/{number}.json``. Discussion backups include the discussion body and metadata, category information, comments, and comment replies.
|
|
298
|
+
|
|
299
|
+
``--discussions`` is included in ``--all``. Unlike most REST API-backed resources, discussions require authentication because GitHub's GraphQL API requires a token. Fine-grained personal access tokens and GitHub Apps need read access to the repository's Discussions permission.
|
|
300
|
+
|
|
301
|
+
Incremental backups use a per-repository checkpoint at ``repositories/{repo}/discussions/last_update`` based on discussion ``updatedAt`` timestamps. This is separate from the repository-level ``last_update`` file so discussion activity is not missed if the repository's own update timestamp does not change. If you enable ``--discussions`` on an existing incremental backup, the first run performs a full discussions backup for each repository and creates the discussions checkpoint for future runs.
|
|
302
|
+
|
|
303
|
+
|
|
287
304
|
About security advisories
|
|
288
305
|
-------------------------
|
|
289
306
|
|
|
@@ -325,12 +342,24 @@ For finer control, avoid using ``--assets`` with starred repos, or use ``--skip-
|
|
|
325
342
|
|
|
326
343
|
Alternatively, consider just storing links to starred repos in JSON format with ``--starred``.
|
|
327
344
|
|
|
345
|
+
About pull request reviews
|
|
346
|
+
--------------------------
|
|
347
|
+
|
|
348
|
+
Use ``--pull-reviews`` with ``--pulls`` to include GitHub pull request review metadata under each pull request's ``review_data`` key. Reviews are separate from review comments: ``--pull-comments`` backs up inline review comments via ``comment_data`` and regular PR conversation comments via ``comment_regular_data``, while ``--pull-reviews`` backs up review state, submitted time, commit ID, and the top-level review body.
|
|
349
|
+
|
|
350
|
+
``--pull-reviews`` is included in ``--all``. Incremental backups use a per-repository checkpoint at ``repositories/{repo}/pulls/reviews_last_update``. If ``--pull-reviews`` is enabled on an existing incremental backup, the first run performs a one-time backfill for pull request reviews so older PRs are not skipped by the existing pull request checkpoint. Existing ``comment_data``, ``comment_regular_data`` and ``commit_data`` fields are preserved when only review data is being added.
|
|
351
|
+
|
|
352
|
+
|
|
328
353
|
Incremental Backup
|
|
329
354
|
------------------
|
|
330
355
|
|
|
331
|
-
Using (``-i, --incremental``) will only request new data from the API **since the last
|
|
356
|
+
Using (``-i, --incremental``) will only request new data from the API **since the last successful resource backup**. e.g. only request issues from the API since the last issue backup for that repository.
|
|
357
|
+
|
|
358
|
+
Incremental checkpoints for issue and pull request API backups are stored per resource in that repository's backup directory (for example ``repositories/{repo}/issues/last_update``, ``repositories/{repo}/pulls/last_update`` or ``starred/{owner}/{repo}/pulls/last_update``). Older versions stored a single global ``last_update`` file in the output directory root. During migration, the legacy global checkpoint is used as a fallback only for resource directories that already contain backup data but do not yet have their own checkpoint. New repositories or newly enabled resources with no existing data get a full backup instead of inheriting an unrelated global checkpoint.
|
|
359
|
+
|
|
360
|
+
After all existing issue and pull request resource directories have per-resource checkpoints, the legacy global ``last_update`` file is removed automatically.
|
|
332
361
|
|
|
333
|
-
This means any blocking errors on previous runs can cause
|
|
362
|
+
This means any blocking errors on previous runs can cause missing data in backups for the affected repository resource.
|
|
334
363
|
|
|
335
364
|
Using (``--incremental-by-files``) will request new data from the API **based on when the file was modified on filesystem**. e.g. if you modify the file yourself you may miss something.
|
|
336
365
|
|
|
@@ -343,7 +372,7 @@ Known blocking errors
|
|
|
343
372
|
|
|
344
373
|
Some errors will block the backup run by exiting the script. e.g. receiving a 403 Forbidden error from the Github API.
|
|
345
374
|
|
|
346
|
-
If the incremental argument is used,
|
|
375
|
+
If the incremental argument is used, per-resource checkpoints are only advanced after that resource's backup work completes. A blocking error can still abort the overall run, but repositories and resources that were not processed will keep their previous checkpoints.
|
|
347
376
|
|
|
348
377
|
It's therefore recommended to only use the incremental argument if the output/result is being actively monitored, or complimented with periodic full non-incremental runs, to avoid unexpected missing data in a regular backup runs.
|
|
349
378
|
|
|
@@ -416,14 +445,14 @@ Quietly and incrementally backup useful Github user data (public and private rep
|
|
|
416
445
|
export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
417
446
|
GH_USER=YOUR-GITHUB-USER
|
|
418
447
|
|
|
419
|
-
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --security-advisories --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
448
|
+
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --security-advisories --discussions --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
420
449
|
|
|
421
450
|
Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
|
|
422
451
|
|
|
423
452
|
export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
424
453
|
GH_USER=YOUR-GITHUB-USER
|
|
425
454
|
|
|
426
|
-
github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
|
|
455
|
+
github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --discussions --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
|
|
427
456
|
|
|
428
457
|
Pipe a token from stdin to avoid storing it in environment variables or command history (Unix-like systems only)::
|
|
429
458
|
|
|
@@ -439,7 +468,7 @@ This tool creates backups only, there is no inbuilt restore command.
|
|
|
439
468
|
cd /tmp/white-house/repositories/petitions/repository
|
|
440
469
|
git push --mirror git@github.com:WhiteHouse/petitions.git
|
|
441
470
|
|
|
442
|
-
**Issues, pull requests, comments, and other metadata** are saved as JSON files for archival purposes. The GitHub API does not support recreating this data faithfully, creating issues via the API has limitations:
|
|
471
|
+
**Issues, pull requests, discussions, comments, and other metadata** are saved as JSON files for archival purposes. The GitHub API does not support recreating this data faithfully, creating issues via the API has limitations:
|
|
443
472
|
|
|
444
473
|
- New issue/PR numbers are assigned (original numbers cannot be set)
|
|
445
474
|
- Timestamps reflect creation time (original dates cannot be set)
|