github-backup 0.62.0__tar.gz → 0.62.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {github_backup-0.62.0 → github_backup-0.62.1}/CHANGES.rst +57 -1
- {github_backup-0.62.0/github_backup.egg-info → github_backup-0.62.1}/PKG-INFO +16 -11
- {github_backup-0.62.0 → github_backup-0.62.1}/README +15 -10
- {github_backup-0.62.0 → github_backup-0.62.1}/README.rst +15 -10
- github_backup-0.62.1/github_backup/__init__.py +1 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/github_backup/github_backup.py +49 -46
- {github_backup-0.62.0 → github_backup-0.62.1/github_backup.egg-info}/PKG-INFO +16 -11
- github_backup-0.62.0/github_backup/__init__.py +0 -1
- {github_backup-0.62.0 → github_backup-0.62.1}/LICENSE.txt +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/MANIFEST.in +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/github_backup/__main__.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/github_backup/cli.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/github_backup/graphql_queries.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/github_backup.egg-info/SOURCES.txt +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/github_backup.egg-info/dependency_links.txt +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/github_backup.egg-info/entry_points.txt +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/github_backup.egg-info/top_level.txt +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/github_backup.egg-info/zip-safe +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/release-requirements.txt +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/requirements.txt +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/setup.cfg +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/setup.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_all_starred.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_attachments.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_auth.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_case_sensitivity.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_discussions.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_http_451.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_incremental_per_repository.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_json_dump_if_changed.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_pagination.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_pull_incremental_pagination.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_pull_reviews.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_releases.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_retrieve_data.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_skip_assets_on.py +0 -0
- {github_backup-0.62.0 → github_backup-0.62.1}/tests/test_starred_skip_size_over.py +0 -0
|
@@ -1,9 +1,65 @@
|
|
|
1
1
|
Changelog
|
|
2
2
|
=========
|
|
3
3
|
|
|
4
|
-
0.62.
|
|
4
|
+
0.62.1 (2026-04-30)
|
|
5
5
|
-------------------
|
|
6
6
|
------------------------
|
|
7
|
+
- Document that nothing is saved by default. [Changaco]
|
|
8
|
+
- Eliminate trailing spaces. [Changaco]
|
|
9
|
+
- Remove pointless and unsafe `export`s in examples. [Changaco]
|
|
10
|
+
- Try to clarify what `--incremental` actually does. [Changaco]
|
|
11
|
+
- Fix a typo in the README. [Changaco]
|
|
12
|
+
- Document that `--all` doesn't imply `--attachments` [Changaco]
|
|
13
|
+
- Rename a function to match what it actually does. [Changaco]
|
|
14
|
+
- Don't leave files open. [Changaco]
|
|
15
|
+
- Remove legacy code in `mkdir_p` function. [Changaco]
|
|
16
|
+
- Don't pass stdin when doing so can't do any good. [Changaco]
|
|
17
|
+
|
|
18
|
+
When the child process doesn't inherit stderr, it can't ask the user for input, so it shouldn't inherit stdin either.
|
|
19
|
+
- Use `subprocess.DEVNULL` instead of emulating it. [Changaco]
|
|
20
|
+
- Remove bad invocation of the system shell. [Changaco]
|
|
21
|
+
- Add missing `context` argument to `urlopen` call. [Changaco]
|
|
22
|
+
- Suppress output of call to `git lfs version` [Changaco]
|
|
23
|
+
- Handle more network errors. [Changaco]
|
|
24
|
+
|
|
25
|
+
```python-traceback
|
|
26
|
+
Traceback (most recent call last):
|
|
27
|
+
File ".local/bin/github-backup", line 6, in <module>
|
|
28
|
+
sys.exit(main())
|
|
29
|
+
~~~~^^
|
|
30
|
+
File ".local/share/pipx/venvs/github-backup/lib/python3.14/site-packages/github_backup/cli.py", line 83, in main
|
|
31
|
+
backup_repositories(args, output_directory, repositories)
|
|
32
|
+
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
33
|
+
File ".local/share/pipx/venvs/github-backup/lib/python3.14/site-packages/github_backup/github_backup.py", line 1845, in backup_repositories
|
|
34
|
+
backup_pulls(args, repo_cwd, repository, repos_template)
|
|
35
|
+
~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
36
|
+
File ".local/share/pipx/venvs/github-backup/lib/python3.14/site-packages/github_backup/github_backup.py", line 2019, in backup_pulls
|
|
37
|
+
pulls[number]["commit_data"] = retrieve_data(args, template)
|
|
38
|
+
~~~~~~~~~~~~~^^^^^^^^^^^^^^^^
|
|
39
|
+
File ".local/share/pipx/venvs/github-backup/lib/python3.14/site-packages/github_backup/github_backup.py", line 766, in retrieve_data
|
|
40
|
+
return list(fetch_all())
|
|
41
|
+
File ".local/share/pipx/venvs/github-backup/lib/python3.14/site-packages/github_backup/github_backup.py", line 717, in fetch_all
|
|
42
|
+
response = json.loads(http_response.read().decode("utf-8"))
|
|
43
|
+
~~~~~~~~~~~~~~~~~~^^
|
|
44
|
+
File "/usr/lib/python3.14/http/client.py", line 500, in read
|
|
45
|
+
s = self._safe_read(self.length)
|
|
46
|
+
File "/usr/lib/python3.14/http/client.py", line 648, in _safe_read
|
|
47
|
+
data = self.fp.read(cursize)
|
|
48
|
+
File "/usr/lib/python3.14/socket.py", line 725, in readinto
|
|
49
|
+
return self._sock.recv_into(b)
|
|
50
|
+
~~~~~~~~~~~~~~~~~~~~^^^
|
|
51
|
+
File "/usr/lib/python3.14/ssl.py", line 1304, in recv_into
|
|
52
|
+
return self.read(nbytes, buffer)
|
|
53
|
+
~~~~~~~~~^^^^^^^^^^^^^^^^
|
|
54
|
+
File "/usr/lib/python3.14/ssl.py", line 1138, in read
|
|
55
|
+
return self._sslobj.read(len, buffer)
|
|
56
|
+
~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
|
|
57
|
+
ConnectionResetError: [Errno 104] Connection reset by peer
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
0.62.0 (2026-04-29)
|
|
62
|
+
-------------------
|
|
7
63
|
- Skip checkpoint-equal incremental items. [Duncan Ogilvie]
|
|
8
64
|
- Avoid redundant release asset list requests. [Duncan Ogilvie]
|
|
9
65
|
- Reduce unnecessary pull requests with incremental fetching. [Duncan
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: github-backup
|
|
3
|
-
Version: 0.62.
|
|
3
|
+
Version: 0.62.1
|
|
4
4
|
Summary: backup a github user or organization
|
|
5
5
|
Home-page: http://github.com/josegonzalez/python-github-backup
|
|
6
6
|
Author: Jose Diaz-Gonzalez
|
|
@@ -52,7 +52,7 @@ Using PIP via PyPI::
|
|
|
52
52
|
Using PIP via Github (more likely the latest version)::
|
|
53
53
|
|
|
54
54
|
pip install git+https://github.com/josegonzalez/python-github-backup.git#egg=github-backup
|
|
55
|
-
|
|
55
|
+
|
|
56
56
|
*Install note for python newcomers:*
|
|
57
57
|
|
|
58
58
|
Python scripts are unlikely to be included in your ``$PATH`` by default, this means it cannot be run directly in terminal with ``$ github-backup ...``, you can either add python's install path to your environments ``$PATH`` or call the script directly e.g. using ``$ ~/.local/bin/github-backup``.*
|
|
@@ -279,7 +279,7 @@ Note: When you run github-backup, you will be asked whether you want to allow "
|
|
|
279
279
|
Github Rate-limit and Throttling
|
|
280
280
|
--------------------------------
|
|
281
281
|
|
|
282
|
-
"github-backup" will automatically throttle itself based on feedback from the Github API.
|
|
282
|
+
"github-backup" will automatically throttle itself based on feedback from the Github API.
|
|
283
283
|
|
|
284
284
|
Their API is usually rate-limited to 5000 calls per hour. The API will ask github-backup to pause until a specific time when the limit is reset again (at the start of the next hour). This continues until the backup is complete.
|
|
285
285
|
|
|
@@ -355,7 +355,12 @@ Gotchas / Known-issues
|
|
|
355
355
|
All is not everything
|
|
356
356
|
---------------------
|
|
357
357
|
|
|
358
|
-
The ``--all`` argument does not include: cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
|
|
358
|
+
The ``--all`` argument does not include: downloading attachments from issue and pull request comments (``--attachments``), cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
|
|
359
|
+
|
|
360
|
+
Saves nothing if no arguments are passed
|
|
361
|
+
----------------------------------------
|
|
362
|
+
|
|
363
|
+
At least one argument like ``--all`` or ``--repositories`` is needed for github-backup to actually save data. Without relevant arguments, github-backup fetches some data from GitHub but doesn't put any of it into files.
|
|
359
364
|
|
|
360
365
|
Starred repository size
|
|
361
366
|
-----------------------
|
|
@@ -393,9 +398,9 @@ This means any blocking errors on previous runs can cause missing data in backup
|
|
|
393
398
|
|
|
394
399
|
Using (``--incremental-by-files``) will request new data from the API **based on when the file was modified on filesystem**. e.g. if you modify the file yourself you may miss something.
|
|
395
400
|
|
|
396
|
-
Still
|
|
401
|
+
Still safer than the previous version.
|
|
397
402
|
|
|
398
|
-
|
|
403
|
+
Incremental backup only changes how issue and pull request data is fetched.
|
|
399
404
|
|
|
400
405
|
Known blocking errors
|
|
401
406
|
---------------------
|
|
@@ -459,12 +464,12 @@ Github Backup Examples
|
|
|
459
464
|
|
|
460
465
|
Backup all repositories, including private ones using a classic token::
|
|
461
466
|
|
|
462
|
-
|
|
467
|
+
ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
463
468
|
github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories --private
|
|
464
469
|
|
|
465
470
|
Use a fine-grained access token to backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
|
|
466
471
|
|
|
467
|
-
|
|
472
|
+
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
468
473
|
ORGANIZATION=docker
|
|
469
474
|
REPO=cli
|
|
470
475
|
# e.g. git@github.com:docker/cli.git
|
|
@@ -472,14 +477,14 @@ Use a fine-grained access token to backup a single organization repository with
|
|
|
472
477
|
|
|
473
478
|
Quietly and incrementally backup useful Github user data (public and private repos with SSH) including; all issues, pulls, all public starred repos and gists (omitting "hooks", "releases" and therefore "assets" to prevent blocking). *Great for a cron job.* ::
|
|
474
479
|
|
|
475
|
-
|
|
480
|
+
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
476
481
|
GH_USER=YOUR-GITHUB-USER
|
|
477
482
|
|
|
478
483
|
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --security-advisories --discussions --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
479
|
-
|
|
484
|
+
|
|
480
485
|
Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
|
|
481
486
|
|
|
482
|
-
|
|
487
|
+
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
483
488
|
GH_USER=YOUR-GITHUB-USER
|
|
484
489
|
|
|
485
490
|
github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --discussions --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
|
|
@@ -22,7 +22,7 @@ Using PIP via PyPI::
|
|
|
22
22
|
Using PIP via Github (more likely the latest version)::
|
|
23
23
|
|
|
24
24
|
pip install git+https://github.com/josegonzalez/python-github-backup.git#egg=github-backup
|
|
25
|
-
|
|
25
|
+
|
|
26
26
|
*Install note for python newcomers:*
|
|
27
27
|
|
|
28
28
|
Python scripts are unlikely to be included in your ``$PATH`` by default, this means it cannot be run directly in terminal with ``$ github-backup ...``, you can either add python's install path to your environments ``$PATH`` or call the script directly e.g. using ``$ ~/.local/bin/github-backup``.*
|
|
@@ -249,7 +249,7 @@ Note: When you run github-backup, you will be asked whether you want to allow "
|
|
|
249
249
|
Github Rate-limit and Throttling
|
|
250
250
|
--------------------------------
|
|
251
251
|
|
|
252
|
-
"github-backup" will automatically throttle itself based on feedback from the Github API.
|
|
252
|
+
"github-backup" will automatically throttle itself based on feedback from the Github API.
|
|
253
253
|
|
|
254
254
|
Their API is usually rate-limited to 5000 calls per hour. The API will ask github-backup to pause until a specific time when the limit is reset again (at the start of the next hour). This continues until the backup is complete.
|
|
255
255
|
|
|
@@ -325,7 +325,12 @@ Gotchas / Known-issues
|
|
|
325
325
|
All is not everything
|
|
326
326
|
---------------------
|
|
327
327
|
|
|
328
|
-
The ``--all`` argument does not include: cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
|
|
328
|
+
The ``--all`` argument does not include: downloading attachments from issue and pull request comments (``--attachments``), cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
|
|
329
|
+
|
|
330
|
+
Saves nothing if no arguments are passed
|
|
331
|
+
----------------------------------------
|
|
332
|
+
|
|
333
|
+
At least one argument like ``--all`` or ``--repositories`` is needed for github-backup to actually save data. Without relevant arguments, github-backup fetches some data from GitHub but doesn't put any of it into files.
|
|
329
334
|
|
|
330
335
|
Starred repository size
|
|
331
336
|
-----------------------
|
|
@@ -363,9 +368,9 @@ This means any blocking errors on previous runs can cause missing data in backup
|
|
|
363
368
|
|
|
364
369
|
Using (``--incremental-by-files``) will request new data from the API **based on when the file was modified on filesystem**. e.g. if you modify the file yourself you may miss something.
|
|
365
370
|
|
|
366
|
-
Still
|
|
371
|
+
Still safer than the previous version.
|
|
367
372
|
|
|
368
|
-
|
|
373
|
+
Incremental backup only changes how issue and pull request data is fetched.
|
|
369
374
|
|
|
370
375
|
Known blocking errors
|
|
371
376
|
---------------------
|
|
@@ -429,12 +434,12 @@ Github Backup Examples
|
|
|
429
434
|
|
|
430
435
|
Backup all repositories, including private ones using a classic token::
|
|
431
436
|
|
|
432
|
-
|
|
437
|
+
ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
433
438
|
github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories --private
|
|
434
439
|
|
|
435
440
|
Use a fine-grained access token to backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
|
|
436
441
|
|
|
437
|
-
|
|
442
|
+
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
438
443
|
ORGANIZATION=docker
|
|
439
444
|
REPO=cli
|
|
440
445
|
# e.g. git@github.com:docker/cli.git
|
|
@@ -442,14 +447,14 @@ Use a fine-grained access token to backup a single organization repository with
|
|
|
442
447
|
|
|
443
448
|
Quietly and incrementally backup useful Github user data (public and private repos with SSH) including; all issues, pulls, all public starred repos and gists (omitting "hooks", "releases" and therefore "assets" to prevent blocking). *Great for a cron job.* ::
|
|
444
449
|
|
|
445
|
-
|
|
450
|
+
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
446
451
|
GH_USER=YOUR-GITHUB-USER
|
|
447
452
|
|
|
448
453
|
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --security-advisories --discussions --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
449
|
-
|
|
454
|
+
|
|
450
455
|
Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
|
|
451
456
|
|
|
452
|
-
|
|
457
|
+
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
453
458
|
GH_USER=YOUR-GITHUB-USER
|
|
454
459
|
|
|
455
460
|
github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --discussions --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
|
|
@@ -22,7 +22,7 @@ Using PIP via PyPI::
|
|
|
22
22
|
Using PIP via Github (more likely the latest version)::
|
|
23
23
|
|
|
24
24
|
pip install git+https://github.com/josegonzalez/python-github-backup.git#egg=github-backup
|
|
25
|
-
|
|
25
|
+
|
|
26
26
|
*Install note for python newcomers:*
|
|
27
27
|
|
|
28
28
|
Python scripts are unlikely to be included in your ``$PATH`` by default, this means it cannot be run directly in terminal with ``$ github-backup ...``, you can either add python's install path to your environments ``$PATH`` or call the script directly e.g. using ``$ ~/.local/bin/github-backup``.*
|
|
@@ -249,7 +249,7 @@ Note: When you run github-backup, you will be asked whether you want to allow "
|
|
|
249
249
|
Github Rate-limit and Throttling
|
|
250
250
|
--------------------------------
|
|
251
251
|
|
|
252
|
-
"github-backup" will automatically throttle itself based on feedback from the Github API.
|
|
252
|
+
"github-backup" will automatically throttle itself based on feedback from the Github API.
|
|
253
253
|
|
|
254
254
|
Their API is usually rate-limited to 5000 calls per hour. The API will ask github-backup to pause until a specific time when the limit is reset again (at the start of the next hour). This continues until the backup is complete.
|
|
255
255
|
|
|
@@ -325,7 +325,12 @@ Gotchas / Known-issues
|
|
|
325
325
|
All is not everything
|
|
326
326
|
---------------------
|
|
327
327
|
|
|
328
|
-
The ``--all`` argument does not include: cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
|
|
328
|
+
The ``--all`` argument does not include: downloading attachments from issue and pull request comments (``--attachments``), cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
|
|
329
|
+
|
|
330
|
+
Saves nothing if no arguments are passed
|
|
331
|
+
----------------------------------------
|
|
332
|
+
|
|
333
|
+
At least one argument like ``--all`` or ``--repositories`` is needed for github-backup to actually save data. Without relevant arguments, github-backup fetches some data from GitHub but doesn't put any of it into files.
|
|
329
334
|
|
|
330
335
|
Starred repository size
|
|
331
336
|
-----------------------
|
|
@@ -363,9 +368,9 @@ This means any blocking errors on previous runs can cause missing data in backup
|
|
|
363
368
|
|
|
364
369
|
Using (``--incremental-by-files``) will request new data from the API **based on when the file was modified on filesystem**. e.g. if you modify the file yourself you may miss something.
|
|
365
370
|
|
|
366
|
-
Still
|
|
371
|
+
Still safer than the previous version.
|
|
367
372
|
|
|
368
|
-
|
|
373
|
+
Incremental backup only changes how issue and pull request data is fetched.
|
|
369
374
|
|
|
370
375
|
Known blocking errors
|
|
371
376
|
---------------------
|
|
@@ -429,12 +434,12 @@ Github Backup Examples
|
|
|
429
434
|
|
|
430
435
|
Backup all repositories, including private ones using a classic token::
|
|
431
436
|
|
|
432
|
-
|
|
437
|
+
ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
433
438
|
github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories --private
|
|
434
439
|
|
|
435
440
|
Use a fine-grained access token to backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
|
|
436
441
|
|
|
437
|
-
|
|
442
|
+
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
438
443
|
ORGANIZATION=docker
|
|
439
444
|
REPO=cli
|
|
440
445
|
# e.g. git@github.com:docker/cli.git
|
|
@@ -442,14 +447,14 @@ Use a fine-grained access token to backup a single organization repository with
|
|
|
442
447
|
|
|
443
448
|
Quietly and incrementally backup useful Github user data (public and private repos with SSH) including; all issues, pulls, all public starred repos and gists (omitting "hooks", "releases" and therefore "assets" to prevent blocking). *Great for a cron job.* ::
|
|
444
449
|
|
|
445
|
-
|
|
450
|
+
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
446
451
|
GH_USER=YOUR-GITHUB-USER
|
|
447
452
|
|
|
448
453
|
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --security-advisories --discussions --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
449
|
-
|
|
454
|
+
|
|
450
455
|
Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
|
|
451
456
|
|
|
452
|
-
|
|
457
|
+
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
453
458
|
GH_USER=YOUR-GITHUB-USER
|
|
454
459
|
|
|
455
460
|
github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --discussions --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.62.1"
|
|
@@ -6,7 +6,6 @@ import argparse
|
|
|
6
6
|
import base64
|
|
7
7
|
import calendar
|
|
8
8
|
import codecs
|
|
9
|
-
import errno
|
|
10
9
|
import json
|
|
11
10
|
import logging
|
|
12
11
|
import os
|
|
@@ -40,7 +39,6 @@ from .graphql_queries import (
|
|
|
40
39
|
DISCUSSION_REPLIES_QUERY,
|
|
41
40
|
)
|
|
42
41
|
|
|
43
|
-
FNULL = open(os.devnull, "w")
|
|
44
42
|
FILE_URI_PREFIX = "file://"
|
|
45
43
|
logger = logging.getLogger(__name__)
|
|
46
44
|
|
|
@@ -128,13 +126,7 @@ def logging_subprocess(
|
|
|
128
126
|
|
|
129
127
|
def mkdir_p(*args):
|
|
130
128
|
for path in args:
|
|
131
|
-
|
|
132
|
-
os.makedirs(path)
|
|
133
|
-
except OSError as exc: # Python >2.5
|
|
134
|
-
if exc.errno == errno.EEXIST and os.path.isdir(path):
|
|
135
|
-
pass
|
|
136
|
-
else:
|
|
137
|
-
raise
|
|
129
|
+
os.makedirs(path, exist_ok=True)
|
|
138
130
|
|
|
139
131
|
|
|
140
132
|
def mask_password(url, secret="*****"):
|
|
@@ -488,7 +480,7 @@ def parse_args(args=None):
|
|
|
488
480
|
"--attachments",
|
|
489
481
|
action="store_true",
|
|
490
482
|
dest="include_attachments",
|
|
491
|
-
help="download user-attachments from issues, pull requests, and discussions",
|
|
483
|
+
help="download user-attachments from issues, pull requests, and discussions [*]",
|
|
492
484
|
)
|
|
493
485
|
parser.add_argument(
|
|
494
486
|
"--throttle-limit",
|
|
@@ -529,19 +521,18 @@ def get_auth(args, encode=True, for_git_cli=False):
|
|
|
529
521
|
if platform.system() != "Darwin":
|
|
530
522
|
raise Exception("Keychain arguments are only supported on Mac OSX")
|
|
531
523
|
try:
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
).strip()
|
|
524
|
+
token = subprocess.check_output(
|
|
525
|
+
[
|
|
526
|
+
"security",
|
|
527
|
+
"find-generic-password",
|
|
528
|
+
"-s",
|
|
529
|
+
args.osx_keychain_item_name,
|
|
530
|
+
"-a",
|
|
531
|
+
args.osx_keychain_item_account,
|
|
532
|
+
"-w",
|
|
533
|
+
],
|
|
534
|
+
stderr=subprocess.DEVNULL,
|
|
535
|
+
).strip()
|
|
545
536
|
token = token.decode("utf-8")
|
|
546
537
|
auth = token + ":" + "x-oauth-basic"
|
|
547
538
|
except subprocess.SubprocessError:
|
|
@@ -554,7 +545,7 @@ def get_auth(args, encode=True, for_git_cli=False):
|
|
|
554
545
|
)
|
|
555
546
|
elif args.token_fine:
|
|
556
547
|
if args.token_fine.startswith(FILE_URI_PREFIX):
|
|
557
|
-
args.token_fine =
|
|
548
|
+
args.token_fine = read_first_line(args.token_fine)
|
|
558
549
|
|
|
559
550
|
if args.token_fine.startswith("github_pat_"):
|
|
560
551
|
auth = args.token_fine
|
|
@@ -570,7 +561,7 @@ def get_auth(args, encode=True, for_git_cli=False):
|
|
|
570
561
|
)
|
|
571
562
|
args.token_classic = read_token_from_gh_cli(args)
|
|
572
563
|
elif args.token_classic.startswith(FILE_URI_PREFIX):
|
|
573
|
-
args.token_classic =
|
|
564
|
+
args.token_classic = read_first_line(args.token_classic)
|
|
574
565
|
|
|
575
566
|
if not args.as_app:
|
|
576
567
|
auth = args.token_classic + ":" + "x-oauth-basic"
|
|
@@ -632,8 +623,9 @@ def get_github_host(args):
|
|
|
632
623
|
return host
|
|
633
624
|
|
|
634
625
|
|
|
635
|
-
def
|
|
636
|
-
|
|
626
|
+
def read_first_line(file_uri):
|
|
627
|
+
with open(file_uri[len(FILE_URI_PREFIX) :], "rt") as f:
|
|
628
|
+
return f.readline().strip()
|
|
637
629
|
|
|
638
630
|
|
|
639
631
|
def read_token_from_gh_cli(args):
|
|
@@ -806,6 +798,7 @@ def retrieve_data(args, template, query_args=None, paginated=True, lazy=False):
|
|
|
806
798
|
response = json.loads(http_response.read().decode("utf-8"))
|
|
807
799
|
break # Exit retry loop and handle the data returned
|
|
808
800
|
except (
|
|
801
|
+
ConnectionError,
|
|
809
802
|
IncompleteRead,
|
|
810
803
|
json.decoder.JSONDecodeError,
|
|
811
804
|
TimeoutError,
|
|
@@ -1297,7 +1290,7 @@ def get_jwt_signed_url_via_markdown_api(url, token, repo_context):
|
|
|
1297
1290
|
request.add_header("Content-Type", "application/json")
|
|
1298
1291
|
request.add_header("Accept", "application/vnd.github+json")
|
|
1299
1292
|
|
|
1300
|
-
html = urlopen(request, timeout=30).read().decode("utf-8")
|
|
1293
|
+
html = urlopen(request, context=https_ctx, timeout=30).read().decode("utf-8")
|
|
1301
1294
|
|
|
1302
1295
|
# Parse JWT-signed URL from HTML response
|
|
1303
1296
|
# Format: <img src="https://private-user-images.githubusercontent.com/...?jwt=..." ...>
|
|
@@ -1781,7 +1774,10 @@ def get_authenticated_user(args):
|
|
|
1781
1774
|
|
|
1782
1775
|
|
|
1783
1776
|
def check_git_lfs_install():
|
|
1784
|
-
exit_code = subprocess.call(
|
|
1777
|
+
exit_code = subprocess.call(
|
|
1778
|
+
["git", "lfs", "version"], stdin=subprocess.DEVNULL,
|
|
1779
|
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
1780
|
+
)
|
|
1785
1781
|
if exit_code != 0:
|
|
1786
1782
|
raise Exception(
|
|
1787
1783
|
"The argument --lfs requires you to have Git LFS installed.\nYou can get it from https://git-lfs.github.com."
|
|
@@ -1969,10 +1965,11 @@ def read_legacy_last_update(args, output_directory):
|
|
|
1969
1965
|
return None, None
|
|
1970
1966
|
|
|
1971
1967
|
last_update_path = os.path.join(output_directory, INCREMENTAL_LAST_UPDATE_FILENAME)
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1968
|
+
try:
|
|
1969
|
+
with open(last_update_path) as f:
|
|
1970
|
+
return last_update_path, f.read().strip()
|
|
1971
|
+
except FileNotFoundError:
|
|
1972
|
+
return last_update_path, None
|
|
1976
1973
|
|
|
1977
1974
|
|
|
1978
1975
|
def read_resource_last_update(args, resource_cwd, legacy_last_update=None):
|
|
@@ -1980,13 +1977,13 @@ def read_resource_last_update(args, resource_cwd, legacy_last_update=None):
|
|
|
1980
1977
|
return None
|
|
1981
1978
|
|
|
1982
1979
|
last_update_path = os.path.join(resource_cwd, INCREMENTAL_LAST_UPDATE_FILENAME)
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1980
|
+
try:
|
|
1981
|
+
with open(last_update_path) as f:
|
|
1982
|
+
return f.read().strip()
|
|
1983
|
+
except FileNotFoundError:
|
|
1984
|
+
if legacy_last_update and resource_backup_exists(resource_cwd):
|
|
1985
|
+
return legacy_last_update
|
|
1986
|
+
return None
|
|
1990
1987
|
|
|
1991
1988
|
|
|
1992
1989
|
def write_resource_last_update(args, resource_cwd, repository):
|
|
@@ -1995,7 +1992,8 @@ def write_resource_last_update(args, resource_cwd, repository):
|
|
|
1995
1992
|
|
|
1996
1993
|
mkdir_p(resource_cwd)
|
|
1997
1994
|
last_update_path = os.path.join(resource_cwd, INCREMENTAL_LAST_UPDATE_FILENAME)
|
|
1998
|
-
open(last_update_path, "w")
|
|
1995
|
+
with open(last_update_path, "w") as f:
|
|
1996
|
+
f.write(get_repository_checkpoint_time(repository))
|
|
1999
1997
|
|
|
2000
1998
|
|
|
2001
1999
|
def iter_incremental_resource_dirs(output_directory):
|
|
@@ -2383,7 +2381,8 @@ def backup_discussions(args, repo_cwd, repository):
|
|
|
2383
2381
|
discussions_since = None
|
|
2384
2382
|
discussion_last_update_path = os.path.join(discussion_cwd, "last_update")
|
|
2385
2383
|
if args.incremental and os.path.exists(discussion_last_update_path):
|
|
2386
|
-
|
|
2384
|
+
with open(discussion_last_update_path) as f:
|
|
2385
|
+
discussions_since = f.read().strip()
|
|
2387
2386
|
|
|
2388
2387
|
logger.info("Retrieving {0} discussions".format(repository["full_name"]))
|
|
2389
2388
|
try:
|
|
@@ -2469,7 +2468,8 @@ def backup_discussions(args, repo_cwd, repository):
|
|
|
2469
2468
|
and newest_seen
|
|
2470
2469
|
and (not discussions_since or newest_seen > discussions_since)
|
|
2471
2470
|
):
|
|
2472
|
-
open(discussion_last_update_path, "w")
|
|
2471
|
+
with open(discussion_last_update_path, "w") as f:
|
|
2472
|
+
f.write(newest_seen)
|
|
2473
2473
|
|
|
2474
2474
|
attempted_count = len(summaries) - skipped_count
|
|
2475
2475
|
if not summaries:
|
|
@@ -2606,7 +2606,8 @@ def get_pull_reviews_since(args, pulls_cwd):
|
|
|
2606
2606
|
# repository-level checkpoint would otherwise skip old PRs forever.
|
|
2607
2607
|
return None, None, reviews_last_update_path
|
|
2608
2608
|
|
|
2609
|
-
|
|
2609
|
+
with open(reviews_last_update_path) as f:
|
|
2610
|
+
reviews_since = f.read().strip()
|
|
2610
2611
|
if args_since and reviews_since:
|
|
2611
2612
|
return min(args_since, reviews_since), reviews_since, reviews_last_update_path
|
|
2612
2613
|
|
|
@@ -2758,7 +2759,8 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
|
|
|
2758
2759
|
and not pull_review_errors
|
|
2759
2760
|
and (not pull_reviews_since or newest_pull_update > pull_reviews_since)
|
|
2760
2761
|
):
|
|
2761
|
-
open(pull_reviews_last_update_path, "w")
|
|
2762
|
+
with open(pull_reviews_last_update_path, "w") as f:
|
|
2763
|
+
f.write(newest_pull_update)
|
|
2762
2764
|
|
|
2763
2765
|
|
|
2764
2766
|
def backup_milestones(args, repo_cwd, repository, repos_template):
|
|
@@ -2980,7 +2982,8 @@ def fetch_repository(
|
|
|
2980
2982
|
masked_remote_url = mask_password(remote_url)
|
|
2981
2983
|
|
|
2982
2984
|
initialized = subprocess.call(
|
|
2983
|
-
"git ls-remote
|
|
2985
|
+
["git", "ls-remote", remote_url], stdin=subprocess.DEVNULL,
|
|
2986
|
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
2984
2987
|
)
|
|
2985
2988
|
if initialized == 128:
|
|
2986
2989
|
if ".wiki.git" in remote_url:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: github-backup
|
|
3
|
-
Version: 0.62.
|
|
3
|
+
Version: 0.62.1
|
|
4
4
|
Summary: backup a github user or organization
|
|
5
5
|
Home-page: http://github.com/josegonzalez/python-github-backup
|
|
6
6
|
Author: Jose Diaz-Gonzalez
|
|
@@ -52,7 +52,7 @@ Using PIP via PyPI::
|
|
|
52
52
|
Using PIP via Github (more likely the latest version)::
|
|
53
53
|
|
|
54
54
|
pip install git+https://github.com/josegonzalez/python-github-backup.git#egg=github-backup
|
|
55
|
-
|
|
55
|
+
|
|
56
56
|
*Install note for python newcomers:*
|
|
57
57
|
|
|
58
58
|
Python scripts are unlikely to be included in your ``$PATH`` by default, this means it cannot be run directly in terminal with ``$ github-backup ...``, you can either add python's install path to your environments ``$PATH`` or call the script directly e.g. using ``$ ~/.local/bin/github-backup``.*
|
|
@@ -279,7 +279,7 @@ Note: When you run github-backup, you will be asked whether you want to allow "
|
|
|
279
279
|
Github Rate-limit and Throttling
|
|
280
280
|
--------------------------------
|
|
281
281
|
|
|
282
|
-
"github-backup" will automatically throttle itself based on feedback from the Github API.
|
|
282
|
+
"github-backup" will automatically throttle itself based on feedback from the Github API.
|
|
283
283
|
|
|
284
284
|
Their API is usually rate-limited to 5000 calls per hour. The API will ask github-backup to pause until a specific time when the limit is reset again (at the start of the next hour). This continues until the backup is complete.
|
|
285
285
|
|
|
@@ -355,7 +355,12 @@ Gotchas / Known-issues
|
|
|
355
355
|
All is not everything
|
|
356
356
|
---------------------
|
|
357
357
|
|
|
358
|
-
The ``--all`` argument does not include: cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
|
|
358
|
+
The ``--all`` argument does not include: downloading attachments from issue and pull request comments (``--attachments``), cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
|
|
359
|
+
|
|
360
|
+
Saves nothing if no arguments are passed
|
|
361
|
+
----------------------------------------
|
|
362
|
+
|
|
363
|
+
At least one argument like ``--all`` or ``--repositories`` is needed for github-backup to actually save data. Without relevant arguments, github-backup fetches some data from GitHub but doesn't put any of it into files.
|
|
359
364
|
|
|
360
365
|
Starred repository size
|
|
361
366
|
-----------------------
|
|
@@ -393,9 +398,9 @@ This means any blocking errors on previous runs can cause missing data in backup
|
|
|
393
398
|
|
|
394
399
|
Using (``--incremental-by-files``) will request new data from the API **based on when the file was modified on filesystem**. e.g. if you modify the file yourself you may miss something.
|
|
395
400
|
|
|
396
|
-
Still
|
|
401
|
+
Still safer than the previous version.
|
|
397
402
|
|
|
398
|
-
|
|
403
|
+
Incremental backup only changes how issue and pull request data is fetched.
|
|
399
404
|
|
|
400
405
|
Known blocking errors
|
|
401
406
|
---------------------
|
|
@@ -459,12 +464,12 @@ Github Backup Examples
|
|
|
459
464
|
|
|
460
465
|
Backup all repositories, including private ones using a classic token::
|
|
461
466
|
|
|
462
|
-
|
|
467
|
+
ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
463
468
|
github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories --private
|
|
464
469
|
|
|
465
470
|
Use a fine-grained access token to backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
|
|
466
471
|
|
|
467
|
-
|
|
472
|
+
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
468
473
|
ORGANIZATION=docker
|
|
469
474
|
REPO=cli
|
|
470
475
|
# e.g. git@github.com:docker/cli.git
|
|
@@ -472,14 +477,14 @@ Use a fine-grained access token to backup a single organization repository with
|
|
|
472
477
|
|
|
473
478
|
Quietly and incrementally backup useful Github user data (public and private repos with SSH) including; all issues, pulls, all public starred repos and gists (omitting "hooks", "releases" and therefore "assets" to prevent blocking). *Great for a cron job.* ::
|
|
474
479
|
|
|
475
|
-
|
|
480
|
+
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
476
481
|
GH_USER=YOUR-GITHUB-USER
|
|
477
482
|
|
|
478
483
|
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --security-advisories --discussions --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
479
|
-
|
|
484
|
+
|
|
480
485
|
Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
|
|
481
486
|
|
|
482
|
-
|
|
487
|
+
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
483
488
|
GH_USER=YOUR-GITHUB-USER
|
|
484
489
|
|
|
485
490
|
github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --discussions --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.62.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|