dump-github 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dump_github-0.2.0/CHANGELOG.md +39 -0
- dump_github-0.2.0/LICENSE +21 -0
- dump_github-0.2.0/MANIFEST.in +4 -0
- dump_github-0.2.0/PKG-INFO +144 -0
- dump_github-0.2.0/README.md +112 -0
- dump_github-0.2.0/dump_github/__init__.py +4 -0
- dump_github-0.2.0/dump_github/_version.py +1 -0
- dump_github-0.2.0/dump_github/lib.py +158 -0
- dump_github-0.2.0/dump_github/main.py +131 -0
- dump_github-0.2.0/dump_github.egg-info/PKG-INFO +144 -0
- dump_github-0.2.0/dump_github.egg-info/SOURCES.txt +18 -0
- dump_github-0.2.0/dump_github.egg-info/dependency_links.txt +1 -0
- dump_github-0.2.0/dump_github.egg-info/entry_points.txt +2 -0
- dump_github-0.2.0/dump_github.egg-info/requires.txt +5 -0
- dump_github-0.2.0/dump_github.egg-info/top_level.txt +1 -0
- dump_github-0.2.0/pyproject.toml +47 -0
- dump_github-0.2.0/setup.cfg +4 -0
- dump_github-0.2.0/tests/__init__.py +0 -0
- dump_github-0.2.0/tests/test_lib.py +294 -0
- dump_github-0.2.0/tests/test_main.py +104 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/).
|
|
7
|
+
|
|
8
|
+
## [0.2.0] - 2026-11-12
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Organisation support: `dump-github <org>` lists only repositories owned by
|
|
13
|
+
that organisation, through `GET /orgs/{org}/repos`.
|
|
14
|
+
- `--version` flag.
|
|
15
|
+
- Modern packaging: PEP 621 `pyproject.toml`, sdist **and** wheel builds,
|
|
16
|
+
`requires-python >= 3.10`, project URLs and PEP 639 license metadata.
|
|
17
|
+
- CI workflow (tests on Python 3.9/3.12/3.13) and a tag-triggered PyPI release
|
|
18
|
+
workflow using Trusted Publishing.
|
|
19
|
+
|
|
20
|
+
### Fixed
|
|
21
|
+
|
|
22
|
+
- Token mode listed `GET /user/repos`, which returned the token owner's
|
|
23
|
+
personal repositories plus the repositories of every organisation the token
|
|
24
|
+
owner belongs to, and ignored the requested username. Listing is now scoped
|
|
25
|
+
to the requested account (user or organisation) and filters out repositories
|
|
26
|
+
owned by anybody else.
|
|
27
|
+
- Zip archive URLs now use the repository's real `full_name` instead of the
|
|
28
|
+
positional username, so archives are always downloaded from the correct
|
|
29
|
+
owner.
|
|
30
|
+
- Clone no longer embeds the token in the remote URL. A one-shot credential
|
|
31
|
+
helper supplies it through the child process environment, so the token is
|
|
32
|
+
not written to `.git/config` and does not show up in `ps` output.
|
|
33
|
+
- Removed the accidental `argparse` runtime dependency (it is a standard
|
|
34
|
+
library module).
|
|
35
|
+
|
|
36
|
+
### Tests
|
|
37
|
+
|
|
38
|
+
- 15 unit and integration tests covering listing scopes, organisation
|
|
39
|
+
handling, pagination, foreign-owner filtering and the git credential helper.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Core2002
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dump-github
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Back up users' GitHub repositories in case they need them later.
|
|
5
|
+
Author-email: core2002 <core2002@aliyun.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Core2002/dump_github
|
|
8
|
+
Project-URL: Repository, https://github.com/Core2002/dump_github
|
|
9
|
+
Project-URL: Issues, https://github.com/Core2002/dump_github/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/Core2002/dump_github/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: github,backup,repositories,cli
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Software Development :: Version Control :: Git
|
|
23
|
+
Classifier: Topic :: System :: Archiving :: Backup
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Requires-Dist: requests>=2.28
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
30
|
+
Requires-Dist: twine>=5; extra == "dev"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
### Github Dumper
|
|
34
|
+
|
|
35
|
+
[](https://github.com/Core2002/dump_github/actions/workflows/ci.yml)
|
|
36
|
+
[](https://pypi.org/project/dump-github/)
|
|
37
|
+
|
|
38
|
+
Github Dumper is a Python program designed to search for users' GitHub repositories and download the repositories. This tool can be used to back up users' GitHub repositories in case they need them later.
|
|
39
|
+
|
|
40
|
+
**Install (Python 3.10+):**
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
pip install -U dump-github
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
**Usage:**
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
usage: dump-github [-h] [-d] [-p] [--version] [--token TOKEN]
|
|
50
|
+
[--limit_size LIMIT_SIZE]
|
|
51
|
+
username
|
|
52
|
+
|
|
53
|
+
Backup users github repo.
|
|
54
|
+
|
|
55
|
+
positional arguments:
|
|
56
|
+
username
|
|
57
|
+
|
|
58
|
+
options:
|
|
59
|
+
-h, --help show this help message and exit
|
|
60
|
+
-d, --download_zip only download zip file
|
|
61
|
+
-p, --print only print urls
|
|
62
|
+
--version show program's version number and exit
|
|
63
|
+
--token TOKEN GitHub personal access token (or set GITHUB_TOKEN).
|
|
64
|
+
Required for private repos and higher API rate limits.
|
|
65
|
+
--limit_size LIMIT_SIZE
|
|
66
|
+
limit size(MB) of download zip file, if 0 then no
|
|
67
|
+
limit(default:100).
|
|
68
|
+
|
|
69
|
+
https://github.com/Core2002/dump_github
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
### Authentication (token)
|
|
73
|
+
|
|
74
|
+
GitHub’s API requires a proper `Authorization: Bearer <token>` header. This tool accepts a **personal access token** in two ways (first match wins):
|
|
75
|
+
|
|
76
|
+
1. **`--token`** on the command line
|
|
77
|
+
2. **`GITHUB_TOKEN`** environment variable (if `--token` is empty)
|
|
78
|
+
|
|
79
|
+
**Why use a token?**
|
|
80
|
+
|
|
81
|
+
- **Private repositories**: Without a token, only public repos are listed and can be cloned or archived. With a token that has the right scopes, you can include private repos you are allowed to access (for example when backing up your own account).
|
|
82
|
+
- **Rate limits**: Authenticated API requests get a much higher limit than anonymous requests.
|
|
83
|
+
|
|
84
|
+
**How the token is used**
|
|
85
|
+
|
|
86
|
+
- **Listing repos**: sent as `Bearer` on the REST API and scoped strictly to the requested account:
|
|
87
|
+
- **organisation** → `GET /orgs/{org}/repos?type=all` — only repositories owned by that organisation (private ones included when the token may access them);
|
|
88
|
+
- **your own user account** → `GET /user/repos?affiliation=owner` — private and public repos, without organisation/collaborator repos;
|
|
89
|
+
- **any other user** → `GET /users/{username}/repos?type=owner`.
|
|
90
|
+
|
|
91
|
+
Repositories whose owner is not the requested account are discarded as a final guard, so a user dump never includes organisation repositories and an organisation dump never includes user repositories.
|
|
92
|
+
- **Clone** (default mode): the token is handed to `git` through a one-shot credential helper (`credential.helper` reads `DUMP_GITHUB_TOKEN` from the child process environment), so the clone URL stays `https://github.com/...`. The token is never shown in `ps` output and never written to the cloned repository's `.git/config`; existing credential helpers are reset for that clone so a stale stored credential cannot shadow the token.
|
|
93
|
+
- **Zip download** (`-d`): uses the [repository archive (zipball)](https://docs.github.com/en/rest/repos/repos#download-a-repository-archive-zip) API with the same `Bearer` header (via `curl`), which works for private repos; without a token, public archive URLs are used instead.
|
|
94
|
+
|
|
95
|
+
Create a token under GitHub → **Settings** → **Developer settings** → **Personal access tokens**. For private repo backup, enable access to **repository contents** (classic PAT: `repo` scope; fine-grained: read access to the repositories you need).
|
|
96
|
+
|
|
97
|
+
**Examples**
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
# Public repos only (no token)
|
|
101
|
+
dump-github Core2002 -p
|
|
102
|
+
|
|
103
|
+
# Token on the command line (private + higher API limits)
|
|
104
|
+
dump-github username --token ghp_xxxxxxxxxxxx
|
|
105
|
+
|
|
106
|
+
# Back up an organisation (only that organisation's repositories)
|
|
107
|
+
dump-github some-org --token ghp_xxxxxxxxxxxx
|
|
108
|
+
|
|
109
|
+
# Same, via environment variable (recommended to avoid shell history)
|
|
110
|
+
export GITHUB_TOKEN=ghp_xxxxxxxxxxxx
|
|
111
|
+
dump-github Core2002 -d
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
**Security note:** Do not commit tokens or paste them into shared logs. Prefer `GITHUB_TOKEN` in a local shell profile or a secret manager, and rotate the token if it leaks. Cloned repositories keep a clean remote URL (`https://github.com/...`), so `.git/config` does not contain the token.
|
|
115
|
+
|
|
116
|
+
### Development
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
python -m venv .venv
|
|
120
|
+
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
121
|
+
python -m pip install -e ".[dev]"
|
|
122
|
+
python -m unittest discover -s tests -v
|
|
123
|
+
python -m build # dist/: sdist + wheel
|
|
124
|
+
python -m twine check dist/*
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
CI runs the test suite on Python 3.10, 3.12 and 3.13, and checks the build metadata with `twine check` (`.github/workflows/ci.yml`).
|
|
128
|
+
|
|
129
|
+
### Releasing
|
|
130
|
+
|
|
131
|
+
1. Bump `__version__` in `dump_github/_version.py` and move the changelog entries in `CHANGELOG.md` to the new version.
|
|
132
|
+
2. Commit, tag and push:
|
|
133
|
+
```bash
|
|
134
|
+
git tag v0.2.0
|
|
135
|
+
git push origin main v0.2.0
|
|
136
|
+
```
|
|
137
|
+
3. The `Release` workflow runs the tests, builds the sdist and wheel, checks them with `twine check` and publishes both to PyPI through **Trusted Publishing**.
|
|
138
|
+
|
|
139
|
+
One-time setup on PyPI: add a trusted publisher for the `Core2002/dump_github` repository and the `release.yml` workflow. No PyPI API token is stored in the repository. Manual fallback:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
python -m build
|
|
143
|
+
python -m twine upload dist/*
|
|
144
|
+
```
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
### Github Dumper
|
|
2
|
+
|
|
3
|
+
[](https://github.com/Core2002/dump_github/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/dump-github/)
|
|
5
|
+
|
|
6
|
+
Github Dumper is a Python program designed to search for users' GitHub repositories and download the repositories. This tool can be used to back up users' GitHub repositories in case they need them later.
|
|
7
|
+
|
|
8
|
+
**Install (Python 3.10+):**
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
pip install -U dump-github
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
**Usage:**
|
|
15
|
+
|
|
16
|
+
```
|
|
17
|
+
usage: dump-github [-h] [-d] [-p] [--version] [--token TOKEN]
|
|
18
|
+
[--limit_size LIMIT_SIZE]
|
|
19
|
+
username
|
|
20
|
+
|
|
21
|
+
Backup users github repo.
|
|
22
|
+
|
|
23
|
+
positional arguments:
|
|
24
|
+
username
|
|
25
|
+
|
|
26
|
+
options:
|
|
27
|
+
-h, --help show this help message and exit
|
|
28
|
+
-d, --download_zip only download zip file
|
|
29
|
+
-p, --print only print urls
|
|
30
|
+
--version show program's version number and exit
|
|
31
|
+
--token TOKEN GitHub personal access token (or set GITHUB_TOKEN).
|
|
32
|
+
Required for private repos and higher API rate limits.
|
|
33
|
+
--limit_size LIMIT_SIZE
|
|
34
|
+
limit size(MB) of download zip file, if 0 then no
|
|
35
|
+
limit(default:100).
|
|
36
|
+
|
|
37
|
+
https://github.com/Core2002/dump_github
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
### Authentication (token)
|
|
41
|
+
|
|
42
|
+
GitHub’s API requires a proper `Authorization: Bearer <token>` header. This tool accepts a **personal access token** in two ways (first match wins):
|
|
43
|
+
|
|
44
|
+
1. **`--token`** on the command line
|
|
45
|
+
2. **`GITHUB_TOKEN`** environment variable (if `--token` is empty)
|
|
46
|
+
|
|
47
|
+
**Why use a token?**
|
|
48
|
+
|
|
49
|
+
- **Private repositories**: Without a token, only public repos are listed and can be cloned or archived. With a token that has the right scopes, you can include private repos you are allowed to access (for example when backing up your own account).
|
|
50
|
+
- **Rate limits**: Authenticated API requests get a much higher limit than anonymous requests.
|
|
51
|
+
|
|
52
|
+
**How the token is used**
|
|
53
|
+
|
|
54
|
+
- **Listing repos**: sent as `Bearer` on the REST API and scoped strictly to the requested account:
|
|
55
|
+
- **organisation** → `GET /orgs/{org}/repos?type=all` — only repositories owned by that organisation (private ones included when the token may access them);
|
|
56
|
+
- **your own user account** → `GET /user/repos?affiliation=owner` — private and public repos, without organisation/collaborator repos;
|
|
57
|
+
- **any other user** → `GET /users/{username}/repos?type=owner`.
|
|
58
|
+
|
|
59
|
+
Repositories whose owner is not the requested account are discarded as a final guard, so a user dump never includes organisation repositories and an organisation dump never includes user repositories.
|
|
60
|
+
- **Clone** (default mode): the token is handed to `git` through a one-shot credential helper (`credential.helper` reads `DUMP_GITHUB_TOKEN` from the child process environment), so the clone URL stays `https://github.com/...`. The token is never shown in `ps` output and never written to the cloned repository's `.git/config`; existing credential helpers are reset for that clone so a stale stored credential cannot shadow the token.
|
|
61
|
+
- **Zip download** (`-d`): uses the [repository archive (zipball)](https://docs.github.com/en/rest/repos/repos#download-a-repository-archive-zip) API with the same `Bearer` header (via `curl`), which works for private repos; without a token, public archive URLs are used instead.
|
|
62
|
+
|
|
63
|
+
Create a token under GitHub → **Settings** → **Developer settings** → **Personal access tokens**. For private repo backup, enable access to **repository contents** (classic PAT: `repo` scope; fine-grained: read access to the repositories you need).
|
|
64
|
+
|
|
65
|
+
**Examples**
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
# Public repos only (no token)
|
|
69
|
+
dump-github Core2002 -p
|
|
70
|
+
|
|
71
|
+
# Token on the command line (private + higher API limits)
|
|
72
|
+
dump-github username --token ghp_xxxxxxxxxxxx
|
|
73
|
+
|
|
74
|
+
# Back up an organisation (only that organisation's repositories)
|
|
75
|
+
dump-github some-org --token ghp_xxxxxxxxxxxx
|
|
76
|
+
|
|
77
|
+
# Same, via environment variable (recommended to avoid shell history)
|
|
78
|
+
export GITHUB_TOKEN=ghp_xxxxxxxxxxxx
|
|
79
|
+
dump-github Core2002 -d
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
**Security note:** Do not commit tokens or paste them into shared logs. Prefer `GITHUB_TOKEN` in a local shell profile or a secret manager, and rotate the token if it leaks. Cloned repositories keep a clean remote URL (`https://github.com/...`), so `.git/config` does not contain the token.
|
|
83
|
+
|
|
84
|
+
### Development
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
python -m venv .venv
|
|
88
|
+
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
89
|
+
python -m pip install -e ".[dev]"
|
|
90
|
+
python -m unittest discover -s tests -v
|
|
91
|
+
python -m build # dist/: sdist + wheel
|
|
92
|
+
python -m twine check dist/*
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
CI runs the test suite on Python 3.10, 3.12 and 3.13, and checks the build metadata with `twine check` (`.github/workflows/ci.yml`).
|
|
96
|
+
|
|
97
|
+
### Releasing
|
|
98
|
+
|
|
99
|
+
1. Bump `__version__` in `dump_github/_version.py` and move the changelog entries in `CHANGELOG.md` to the new version.
|
|
100
|
+
2. Commit, tag and push:
|
|
101
|
+
```bash
|
|
102
|
+
git tag v0.2.0
|
|
103
|
+
git push origin main v0.2.0
|
|
104
|
+
```
|
|
105
|
+
3. The `Release` workflow runs the tests, builds the sdist and wheel, checks them with `twine check` and publishes both to PyPI through **Trusted Publishing**.
|
|
106
|
+
|
|
107
|
+
One-time setup on PyPI: add a trusted publisher for the `Core2002/dump_github` repository and the `release.yml` workflow. No PyPI API token is stored in the repository. Manual fallback:
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
python -m build
|
|
111
|
+
python -m twine upload dist/*
|
|
112
|
+
```
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.0"
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
from urllib.parse import quote
|
|
2
|
+
|
|
3
|
+
import requests
|
|
4
|
+
|
|
5
|
+
GITHUB_API = "https://api.github.com"
|
|
6
|
+
PER_PAGE = 100
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _github_auth_headers(token):
|
|
10
|
+
"""GitHub REST API expects Authorization: Bearer <token> (PAT classic / fine-grained)."""
|
|
11
|
+
if not token:
|
|
12
|
+
return {}
|
|
13
|
+
token = token.strip()
|
|
14
|
+
if not token:
|
|
15
|
+
return {}
|
|
16
|
+
if token.lower().startswith(("bearer ", "token ", "basic ")):
|
|
17
|
+
return {"Authorization": token}
|
|
18
|
+
return {"Authorization": "Bearer {}".format(token)}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _api_headers(token=""):
|
|
22
|
+
headers = {
|
|
23
|
+
"Accept": "application/vnd.github+json",
|
|
24
|
+
"X-GitHub-Api-Version": "2022-11-28",
|
|
25
|
+
}
|
|
26
|
+
headers.update(_github_auth_headers(token))
|
|
27
|
+
return headers
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _api_get(path, token=""):
|
|
31
|
+
"""GET one GitHub REST API path and return the decoded JSON, or None on failure."""
|
|
32
|
+
try:
|
|
33
|
+
response = requests.get(
|
|
34
|
+
GITHUB_API + path, headers=_api_headers(token), timeout=30
|
|
35
|
+
)
|
|
36
|
+
except requests.RequestException:
|
|
37
|
+
return None
|
|
38
|
+
if response.status_code != 200:
|
|
39
|
+
return None
|
|
40
|
+
try:
|
|
41
|
+
return response.json()
|
|
42
|
+
except ValueError:
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def get_authenticated_login(token=""):
|
|
47
|
+
"""Return the login of the token owner, or None if there is no usable token."""
|
|
48
|
+
if not _github_auth_headers(token):
|
|
49
|
+
return None
|
|
50
|
+
account = _api_get("/user", token)
|
|
51
|
+
if not isinstance(account, dict):
|
|
52
|
+
return None
|
|
53
|
+
return account.get("login") or None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def get_account_type(username, token=""):
|
|
57
|
+
"""Return ``"Organization"``/``"User"`` for ``username``, or None if unknown."""
|
|
58
|
+
quoted = quote(username, safe="")
|
|
59
|
+
account = _api_get("/users/{}".format(quoted), token)
|
|
60
|
+
if isinstance(account, dict) and account.get("type"):
|
|
61
|
+
return account["type"]
|
|
62
|
+
# ``/users/{org}`` is not guaranteed to resolve for organisations, so fall
|
|
63
|
+
# back to the org endpoint, which unambiguously identifies one.
|
|
64
|
+
org = _api_get("/orgs/{}".format(quoted), token)
|
|
65
|
+
if isinstance(org, dict) and org.get("login"):
|
|
66
|
+
return "Organization"
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _repos_path(username, page, authenticated_login, account_type):
|
|
71
|
+
"""Build a listing API path that only contains repositories owned by ``username``.
|
|
72
|
+
|
|
73
|
+
``GET /user/repos`` (used by an earlier version whenever a token was
|
|
74
|
+
supplied) lists every repository the token owner can see, which includes
|
|
75
|
+
repositories of the organisations the token owner is a member of, and it
|
|
76
|
+
ignores ``username`` entirely. Listing must therefore be scoped to the
|
|
77
|
+
requested account:
|
|
78
|
+
|
|
79
|
+
* organisation -> ``/orgs/{org}/repos?type=all``; only repositories owned
|
|
80
|
+
by that organisation, including private ones the token may access
|
|
81
|
+
* own user account -> ``/user/repos?affiliation=owner``; keeps private
|
|
82
|
+
repos, drops organisation/collaborator repos
|
|
83
|
+
* any other user account -> ``/users/{username}/repos?type=owner``
|
|
84
|
+
"""
|
|
85
|
+
quoted = quote(username, safe="")
|
|
86
|
+
if (account_type or "").lower() == "organization":
|
|
87
|
+
return "/orgs/{}/repos?type=all&per_page={}&page={}".format(
|
|
88
|
+
quoted, PER_PAGE, page
|
|
89
|
+
)
|
|
90
|
+
own_account = bool(authenticated_login) and (
|
|
91
|
+
authenticated_login.lower() == username.lower()
|
|
92
|
+
)
|
|
93
|
+
if own_account:
|
|
94
|
+
return "/user/repos?affiliation=owner&per_page={}&page={}".format(
|
|
95
|
+
PER_PAGE, page
|
|
96
|
+
)
|
|
97
|
+
return "/users/{}/repos?type=owner&per_page={}&page={}".format(
|
|
98
|
+
quoted, PER_PAGE, page
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _is_owned_by(repo, username):
|
|
103
|
+
owner = repo.get("owner") or {}
|
|
104
|
+
login = owner.get("login") or ""
|
|
105
|
+
if login:
|
|
106
|
+
return login.lower() == username.lower()
|
|
107
|
+
full_name = repo.get("full_name") or ""
|
|
108
|
+
return full_name.split("/", 1)[0].lower() == username.lower()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _fetch_repos_page(username, page, token="", authenticated_login=None,
|
|
112
|
+
account_type=None):
|
|
113
|
+
"""Fetch one raw page of repositories owned by ``username``."""
|
|
114
|
+
path = _repos_path(username, page, authenticated_login, account_type)
|
|
115
|
+
repos = _api_get(path, token)
|
|
116
|
+
if not isinstance(repos, list):
|
|
117
|
+
return []
|
|
118
|
+
return [repo for repo in repos if isinstance(repo, dict)]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def get_page_of_repos(username, page, token="", authenticated_login=None,
|
|
122
|
+
account_type=None):
|
|
123
|
+
username = username.strip()
|
|
124
|
+
if account_type is None:
|
|
125
|
+
account_type = get_account_type(username, token)
|
|
126
|
+
if authenticated_login is None:
|
|
127
|
+
authenticated_login = get_authenticated_login(token)
|
|
128
|
+
repos = _fetch_repos_page(
|
|
129
|
+
username, page, token, authenticated_login, account_type
|
|
130
|
+
)
|
|
131
|
+
return [repo for repo in repos if _is_owned_by(repo, username)]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def search_user_repos(username, token=""):
|
|
135
|
+
username = username.strip()
|
|
136
|
+
authenticated_login = get_authenticated_login(token)
|
|
137
|
+
account_type = get_account_type(username, token)
|
|
138
|
+
page = 0
|
|
139
|
+
res_repos = []
|
|
140
|
+
|
|
141
|
+
while True:
|
|
142
|
+
page += 1
|
|
143
|
+
repos = _fetch_repos_page(
|
|
144
|
+
username, page, token, authenticated_login, account_type
|
|
145
|
+
)
|
|
146
|
+
if len(repos) == 0:
|
|
147
|
+
break
|
|
148
|
+
res_repos += repos
|
|
149
|
+
if len(repos) < PER_PAGE:
|
|
150
|
+
break
|
|
151
|
+
|
|
152
|
+
# Never return repositories that belong to somebody else (e.g. a user repo
|
|
153
|
+
# returned for an organisation, or an organisation the token owner is in).
|
|
154
|
+
return [repo for repo in res_repos if _is_owned_by(repo, username)]
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
# Backward-compatible alias for the historical typo
|
|
158
|
+
serach_user_repos = search_user_repos
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import os
|
|
3
|
+
import subprocess
|
|
4
|
+
|
|
5
|
+
from ._version import __version__
|
|
6
|
+
from .lib import search_user_repos
|
|
7
|
+
|
|
8
|
+
# One-shot credential helper: git asks for credentials, the helper answers with
|
|
9
|
+
# the token from the environment. The token is therefore never part of the
|
|
10
|
+
# clone URL and never gets written to the cloned repository's .git/config.
|
|
11
|
+
GIT_CREDENTIAL_HELPER = (
|
|
12
|
+
'!f() { test "$1" = get || return 0; '
|
|
13
|
+
'echo username=x-access-token; '
|
|
14
|
+
'echo "password=$DUMP_GITHUB_TOKEN"; }; f'
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
parser = argparse.ArgumentParser(
|
|
18
|
+
prog="dump-github",
|
|
19
|
+
description="Backup users github repo.",
|
|
20
|
+
epilog="https://github.com/Core2002/dump_github",
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
parser.add_argument("username")
|
|
24
|
+
parser.add_argument(
|
|
25
|
+
"-d", "--download_zip", action="store_true", help="only download zip file"
|
|
26
|
+
)
|
|
27
|
+
parser.add_argument("-p", "--print", action="store_true", help="only print urls")
|
|
28
|
+
parser.add_argument(
|
|
29
|
+
"--version", action="version", version="%(prog)s {}".format(__version__)
|
|
30
|
+
)
|
|
31
|
+
parser.add_argument(
|
|
32
|
+
"--token",
|
|
33
|
+
action="store",
|
|
34
|
+
default="",
|
|
35
|
+
help="GitHub personal access token (or set GITHUB_TOKEN). Required for private repos and higher API rate limits.",
|
|
36
|
+
)
|
|
37
|
+
parser.add_argument(
|
|
38
|
+
"--limit_size",
|
|
39
|
+
type=int,
|
|
40
|
+
default=100,
|
|
41
|
+
help="limit size(MB) of download zip file, if 0 then no limit(default:100).",
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def clone_repo(repo, token=""):
|
|
46
|
+
name = repo["name"]
|
|
47
|
+
url = repo["clone_url"]
|
|
48
|
+
cmd = ["git", "clone", url]
|
|
49
|
+
env = None
|
|
50
|
+
if token:
|
|
51
|
+
# Use a credential helper instead of https://x-access-token:TOKEN@...
|
|
52
|
+
# so the token does not leak into `ps` output or .git/config.
|
|
53
|
+
cmd = [
|
|
54
|
+
"git",
|
|
55
|
+
"-c",
|
|
56
|
+
"credential.helper=",
|
|
57
|
+
"-c",
|
|
58
|
+
"credential.helper={}".format(GIT_CREDENTIAL_HELPER),
|
|
59
|
+
"clone",
|
|
60
|
+
url,
|
|
61
|
+
]
|
|
62
|
+
env = os.environ.copy()
|
|
63
|
+
env["DUMP_GITHUB_TOKEN"] = token
|
|
64
|
+
env["GIT_TERMINAL_PROMPT"] = "0"
|
|
65
|
+
print("Clone repo: {}".format(name))
|
|
66
|
+
subprocess.run(cmd, env=env, check=False)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def download_zip(user_name, reop, token="", limit_size=100):
|
|
70
|
+
ref = reop.get("default_branch") or "main"
|
|
71
|
+
# Always archive the repository the API actually listed. Using the
|
|
72
|
+
# positional username here would point at the wrong owner whenever the
|
|
73
|
+
# repo does not belong to that user.
|
|
74
|
+
full_name = reop.get("full_name") or "{}/{}".format(user_name, reop["name"])
|
|
75
|
+
max_size = limit_size * 1000 * 1000
|
|
76
|
+
file_name = "./{}.zip".format(full_name.replace("/", "_"))
|
|
77
|
+
curl_cmd = [
|
|
78
|
+
"curl",
|
|
79
|
+
"-sS",
|
|
80
|
+
"-L",
|
|
81
|
+
"--connect-timeout",
|
|
82
|
+
"5",
|
|
83
|
+
"--retry",
|
|
84
|
+
"3",
|
|
85
|
+
"--retry-delay",
|
|
86
|
+
"3",
|
|
87
|
+
]
|
|
88
|
+
if max_size > 0:
|
|
89
|
+
curl_cmd.extend(["--max-filesize", str(max_size)])
|
|
90
|
+
if token:
|
|
91
|
+
url = "https://api.github.com/repos/{}/zipball/{}".format(full_name, ref)
|
|
92
|
+
curl_cmd.extend(
|
|
93
|
+
[
|
|
94
|
+
"-H",
|
|
95
|
+
"Authorization: Bearer {}".format(token),
|
|
96
|
+
"-H",
|
|
97
|
+
"Accept: application/vnd.github+json",
|
|
98
|
+
"-o",
|
|
99
|
+
file_name,
|
|
100
|
+
url,
|
|
101
|
+
]
|
|
102
|
+
)
|
|
103
|
+
else:
|
|
104
|
+
url = "https://github.com/{}/archive/refs/heads/{}.zip".format(full_name, ref)
|
|
105
|
+
curl_cmd.extend(["-o", file_name, url])
|
|
106
|
+
print("Download {} : {}".format(reop["name"], url))
|
|
107
|
+
subprocess.run(curl_cmd, check=False)
|
|
108
|
+
if (
|
|
109
|
+
max_size > 0
|
|
110
|
+
and os.path.exists(file_name)
|
|
111
|
+
and os.stat(file_name).st_size > max_size
|
|
112
|
+
):
|
|
113
|
+
os.remove(file_name)
|
|
114
|
+
print("Removed {} because it exceeds the limit size.".format(file_name))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def main(argv=None):
|
|
118
|
+
args = parser.parse_args(argv)
|
|
119
|
+
token = (args.token or "").strip() or (os.environ.get("GITHUB_TOKEN") or "").strip()
|
|
120
|
+
repos = search_user_repos(args.username, token)
|
|
121
|
+
for repo in repos:
|
|
122
|
+
if args.print:
|
|
123
|
+
print(repo["clone_url"])
|
|
124
|
+
elif args.download_zip:
|
|
125
|
+
download_zip(args.username, repo, token, args.limit_size)
|
|
126
|
+
else:
|
|
127
|
+
clone_repo(repo, token)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
if __name__ == "__main__":
|
|
131
|
+
main()
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dump-github
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Back up users' GitHub repositories in case they need them later.
|
|
5
|
+
Author-email: core2002 <core2002@aliyun.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Core2002/dump_github
|
|
8
|
+
Project-URL: Repository, https://github.com/Core2002/dump_github
|
|
9
|
+
Project-URL: Issues, https://github.com/Core2002/dump_github/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/Core2002/dump_github/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: github,backup,repositories,cli
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Software Development :: Version Control :: Git
|
|
23
|
+
Classifier: Topic :: System :: Archiving :: Backup
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Requires-Dist: requests>=2.28
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
30
|
+
Requires-Dist: twine>=5; extra == "dev"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
### Github Dumper
|
|
34
|
+
|
|
35
|
+
[](https://github.com/Core2002/dump_github/actions/workflows/ci.yml)
|
|
36
|
+
[](https://pypi.org/project/dump-github/)
|
|
37
|
+
|
|
38
|
+
Github Dumper is a Python program designed to search for users' GitHub repositories and download the repositories. This tool can be used to back up users' GitHub repositories in case they need them later.
|
|
39
|
+
|
|
40
|
+
**Install (Python 3.10+):**
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
pip install -U dump-github
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
**Usage:**
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
usage: dump-github [-h] [-d] [-p] [--version] [--token TOKEN]
|
|
50
|
+
[--limit_size LIMIT_SIZE]
|
|
51
|
+
username
|
|
52
|
+
|
|
53
|
+
Backup users github repo.
|
|
54
|
+
|
|
55
|
+
positional arguments:
|
|
56
|
+
username
|
|
57
|
+
|
|
58
|
+
options:
|
|
59
|
+
-h, --help show this help message and exit
|
|
60
|
+
-d, --download_zip only download zip file
|
|
61
|
+
-p, --print only print urls
|
|
62
|
+
--version show program's version number and exit
|
|
63
|
+
--token TOKEN GitHub personal access token (or set GITHUB_TOKEN).
|
|
64
|
+
Required for private repos and higher API rate limits.
|
|
65
|
+
--limit_size LIMIT_SIZE
|
|
66
|
+
limit size(MB) of download zip file, if 0 then no
|
|
67
|
+
limit(default:100).
|
|
68
|
+
|
|
69
|
+
https://github.com/Core2002/dump_github
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
### Authentication (token)
|
|
73
|
+
|
|
74
|
+
GitHub’s API requires a proper `Authorization: Bearer <token>` header. This tool accepts a **personal access token** in two ways (first match wins):
|
|
75
|
+
|
|
76
|
+
1. **`--token`** on the command line
|
|
77
|
+
2. **`GITHUB_TOKEN`** environment variable (if `--token` is empty)
|
|
78
|
+
|
|
79
|
+
**Why use a token?**
|
|
80
|
+
|
|
81
|
+
- **Private repositories**: Without a token, only public repos are listed and can be cloned or archived. With a token that has the right scopes, you can include private repos you are allowed to access (for example when backing up your own account).
|
|
82
|
+
- **Rate limits**: Authenticated API requests get a much higher limit than anonymous requests.
|
|
83
|
+
|
|
84
|
+
**How the token is used**
|
|
85
|
+
|
|
86
|
+
- **Listing repos**: sent as `Bearer` on the REST API and scoped strictly to the requested account:
|
|
87
|
+
- **organisation** → `GET /orgs/{org}/repos?type=all` — only repositories owned by that organisation (private ones included when the token may access them);
|
|
88
|
+
- **your own user account** → `GET /user/repos?affiliation=owner` — private and public repos, without organisation/collaborator repos;
|
|
89
|
+
- **any other user** → `GET /users/{username}/repos?type=owner`.
|
|
90
|
+
|
|
91
|
+
Repositories whose owner is not the requested account are discarded as a final guard, so a user dump never includes organisation repositories and an organisation dump never includes user repositories.
|
|
92
|
+
- **Clone** (default mode): the token is handed to `git` through a one-shot credential helper (`credential.helper` reads `DUMP_GITHUB_TOKEN` from the child process environment), so the clone URL stays `https://github.com/...`. The token is never shown in `ps` output and never written to the cloned repository's `.git/config`; existing credential helpers are reset for that clone so a stale stored credential cannot shadow the token.
|
|
93
|
+
- **Zip download** (`-d`): uses the [repository archive (zipball)](https://docs.github.com/en/rest/repos/repos#download-a-repository-archive-zip) API with the same `Bearer` header (via `curl`), which works for private repos; without a token, public archive URLs are used instead.
|
|
94
|
+
|
|
95
|
+
Create a token under GitHub → **Settings** → **Developer settings** → **Personal access tokens**. For private repo backup, enable access to **repository contents** (classic PAT: `repo` scope; fine-grained: read access to the repositories you need).
|
|
96
|
+
|
|
97
|
+
**Examples**
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
# Public repos only (no token)
|
|
101
|
+
dump-github Core2002 -p
|
|
102
|
+
|
|
103
|
+
# Token on the command line (private + higher API limits)
|
|
104
|
+
dump-github username --token ghp_xxxxxxxxxxxx
|
|
105
|
+
|
|
106
|
+
# Back up an organisation (only that organisation's repositories)
|
|
107
|
+
dump-github some-org --token ghp_xxxxxxxxxxxx
|
|
108
|
+
|
|
109
|
+
# Same, via environment variable (recommended to avoid shell history)
|
|
110
|
+
export GITHUB_TOKEN=ghp_xxxxxxxxxxxx
|
|
111
|
+
dump-github Core2002 -d
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
**Security note:** Do not commit tokens or paste them into shared logs. Prefer `GITHUB_TOKEN` in a local shell profile or a secret manager, and rotate the token if it leaks. Cloned repositories keep a clean remote URL (`https://github.com/...`), so `.git/config` does not contain the token.
|
|
115
|
+
|
|
116
|
+
### Development
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
python -m venv .venv
|
|
120
|
+
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
121
|
+
python -m pip install -e ".[dev]"
|
|
122
|
+
python -m unittest discover -s tests -v
|
|
123
|
+
python -m build # dist/: sdist + wheel
|
|
124
|
+
python -m twine check dist/*
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
CI runs the test suite on Python 3.10, 3.12 and 3.13, and checks the build metadata with `twine check` (`.github/workflows/ci.yml`).
|
|
128
|
+
|
|
129
|
+
### Releasing
|
|
130
|
+
|
|
131
|
+
1. Bump `__version__` in `dump_github/_version.py` and move the changelog entries in `CHANGELOG.md` to the new version.
|
|
132
|
+
2. Commit, tag and push:
|
|
133
|
+
```bash
|
|
134
|
+
git tag v0.2.0
|
|
135
|
+
git push origin main v0.2.0
|
|
136
|
+
```
|
|
137
|
+
3. The `Release` workflow runs the tests, builds the sdist and wheel, checks them with `twine check` and publishes both to PyPI through **Trusted Publishing**.
|
|
138
|
+
|
|
139
|
+
One-time setup on PyPI: add a trusted publisher for the `Core2002/dump_github` repository and the `release.yml` workflow. No PyPI API token is stored in the repository. Manual fallback:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
python -m build
|
|
143
|
+
python -m twine upload dist/*
|
|
144
|
+
```
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
CHANGELOG.md
|
|
2
|
+
LICENSE
|
|
3
|
+
MANIFEST.in
|
|
4
|
+
README.md
|
|
5
|
+
pyproject.toml
|
|
6
|
+
dump_github/__init__.py
|
|
7
|
+
dump_github/_version.py
|
|
8
|
+
dump_github/lib.py
|
|
9
|
+
dump_github/main.py
|
|
10
|
+
dump_github.egg-info/PKG-INFO
|
|
11
|
+
dump_github.egg-info/SOURCES.txt
|
|
12
|
+
dump_github.egg-info/dependency_links.txt
|
|
13
|
+
dump_github.egg-info/entry_points.txt
|
|
14
|
+
dump_github.egg-info/requires.txt
|
|
15
|
+
dump_github.egg-info/top_level.txt
|
|
16
|
+
tests/__init__.py
|
|
17
|
+
tests/test_lib.py
|
|
18
|
+
tests/test_main.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
dump_github
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "dump-github"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Back up users' GitHub repositories in case they need them later."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "core2002", email = "core2002@aliyun.com" }]
|
|
14
|
+
keywords = ["github", "backup", "repositories", "cli"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
22
|
+
"Programming Language :: Python :: 3.10",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Topic :: Software Development :: Version Control :: Git",
|
|
27
|
+
"Topic :: System :: Archiving :: Backup",
|
|
28
|
+
]
|
|
29
|
+
dependencies = ["requests>=2.28"]
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
dev = ["build>=1.2", "twine>=5"]
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
Homepage = "https://github.com/Core2002/dump_github"
|
|
36
|
+
Repository = "https://github.com/Core2002/dump_github"
|
|
37
|
+
Issues = "https://github.com/Core2002/dump_github/issues"
|
|
38
|
+
Changelog = "https://github.com/Core2002/dump_github/blob/main/CHANGELOG.md"
|
|
39
|
+
|
|
40
|
+
[project.scripts]
|
|
41
|
+
dump-github = "dump_github:main"
|
|
42
|
+
|
|
43
|
+
[tool.setuptools.packages.find]
|
|
44
|
+
include = ["dump_github*"]
|
|
45
|
+
|
|
46
|
+
[tool.setuptools.dynamic]
|
|
47
|
+
version = { attr = "dump_github._version.__version__" }
|
|
File without changes
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
import unittest
|
|
4
|
+
from unittest import mock
|
|
5
|
+
|
|
6
|
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
7
|
+
|
|
8
|
+
from dump_github import lib
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class FakeResponse:
|
|
12
|
+
def __init__(self, status_code, payload):
|
|
13
|
+
self.status_code = status_code
|
|
14
|
+
self._payload = payload
|
|
15
|
+
|
|
16
|
+
def json(self):
|
|
17
|
+
return self._payload
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def make_repo(owner, name):
|
|
21
|
+
return {
|
|
22
|
+
"name": name,
|
|
23
|
+
"full_name": "{}/{}".format(owner, name),
|
|
24
|
+
"owner": {"login": owner},
|
|
25
|
+
"clone_url": "https://github.com/{}/{}.git".format(owner, name),
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ListingTests(unittest.TestCase):
|
|
30
|
+
def setUp(self):
|
|
31
|
+
self.calls = []
|
|
32
|
+
|
|
33
|
+
@staticmethod
|
|
34
|
+
def _matches(url, prefix):
|
|
35
|
+
return (
|
|
36
|
+
url == prefix
|
|
37
|
+
or url.startswith(prefix + "?")
|
|
38
|
+
or url.startswith(prefix + "/")
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def route(self, routes):
|
|
42
|
+
"""requests.get replacement: records calls, matches exact URL or prefix."""
|
|
43
|
+
|
|
44
|
+
def fake_get(url, headers=None, timeout=None):
|
|
45
|
+
self.calls.append((url, headers or {}))
|
|
46
|
+
for prefix, payload in routes:
|
|
47
|
+
if self._matches(url, prefix):
|
|
48
|
+
return FakeResponse(200, payload)
|
|
49
|
+
return FakeResponse(404, {"message": "Not Found"})
|
|
50
|
+
|
|
51
|
+
return fake_get
|
|
52
|
+
|
|
53
|
+
def urls(self):
|
|
54
|
+
return [url for url, _ in self.calls]
|
|
55
|
+
|
|
56
|
+
def repos_calls(self):
|
|
57
|
+
return [(url, headers) for url, headers in self.calls if "/repos?" in url]
|
|
58
|
+
|
|
59
|
+
def test_token_own_account_lists_only_owned_repos(self):
|
|
60
|
+
routes = [
|
|
61
|
+
("https://api.github.com/user/repos", [
|
|
62
|
+
make_repo("alice", "private-project"),
|
|
63
|
+
make_repo("acme-org", "org-project"), # must never leak through
|
|
64
|
+
]),
|
|
65
|
+
("https://api.github.com/users/alice",
|
|
66
|
+
{"login": "alice", "type": "User"}),
|
|
67
|
+
("https://api.github.com/user", {"login": "alice"}),
|
|
68
|
+
]
|
|
69
|
+
with mock.patch.object(lib.requests, "get", self.route(routes)):
|
|
70
|
+
repos = lib.search_user_repos("alice", token="ghp_test")
|
|
71
|
+
|
|
72
|
+
self.assertEqual([repo["name"] for repo in repos], ["private-project"])
|
|
73
|
+
self.assertFalse(any("/users/alice/repos" in url for url in self.urls()))
|
|
74
|
+
own = [url for url in self.urls() if "/user/repos" in url]
|
|
75
|
+
self.assertEqual(len(own), 1)
|
|
76
|
+
self.assertIn("affiliation=owner", own[0])
|
|
77
|
+
self.assertTrue(
|
|
78
|
+
all(
|
|
79
|
+
headers.get("Authorization") == "Bearer ghp_test"
|
|
80
|
+
for _, headers in self.calls
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
def test_token_for_another_user_uses_username_endpoint(self):
|
|
85
|
+
routes = [
|
|
86
|
+
("https://api.github.com/users/alice/repos", [
|
|
87
|
+
make_repo("alice", "alice-repo"),
|
|
88
|
+
make_repo("acme-org", "org-project"),
|
|
89
|
+
]),
|
|
90
|
+
("https://api.github.com/users/alice",
|
|
91
|
+
{"login": "alice", "type": "User"}),
|
|
92
|
+
("https://api.github.com/user", {"login": "bob"}),
|
|
93
|
+
]
|
|
94
|
+
with mock.patch.object(lib.requests, "get", self.route(routes)):
|
|
95
|
+
repos = lib.search_user_repos("alice", token="ghp_test")
|
|
96
|
+
|
|
97
|
+
self.assertEqual([repo["name"] for repo in repos], ["alice-repo"])
|
|
98
|
+
self.assertIn(
|
|
99
|
+
"https://api.github.com/users/alice/repos?type=owner&per_page=100&page=1",
|
|
100
|
+
self.urls(),
|
|
101
|
+
)
|
|
102
|
+
self.assertFalse(any("/user/repos" in url for url in self.urls()))
|
|
103
|
+
repos_calls = self.repos_calls()
|
|
104
|
+
self.assertEqual(len(repos_calls), 1)
|
|
105
|
+
self.assertEqual(
|
|
106
|
+
repos_calls[0][1]["Authorization"], "Bearer ghp_test"
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
def test_no_token_uses_public_user_endpoint_without_auth(self):
|
|
110
|
+
routes = [
|
|
111
|
+
("https://api.github.com/users/alice/repos", [
|
|
112
|
+
make_repo("alice", "alice-repo"),
|
|
113
|
+
]),
|
|
114
|
+
("https://api.github.com/users/alice",
|
|
115
|
+
{"login": "alice", "type": "User"}),
|
|
116
|
+
]
|
|
117
|
+
with mock.patch.object(lib.requests, "get", self.route(routes)):
|
|
118
|
+
repos = lib.search_user_repos("alice")
|
|
119
|
+
|
|
120
|
+
self.assertEqual([repo["name"] for repo in repos], ["alice-repo"])
|
|
121
|
+
self.assertEqual(
|
|
122
|
+
self.urls(),
|
|
123
|
+
[
|
|
124
|
+
"https://api.github.com/users/alice",
|
|
125
|
+
"https://api.github.com/users/alice/repos?type=owner&per_page=100&page=1",
|
|
126
|
+
],
|
|
127
|
+
)
|
|
128
|
+
self.assertTrue(
|
|
129
|
+
all("Authorization" not in headers for _, headers in self.calls)
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
def test_pagination_stops_on_short_page_and_resolves_once(self):
|
|
133
|
+
first_page = [make_repo("alice", "repo-{}".format(i)) for i in range(100)]
|
|
134
|
+
routes = [
|
|
135
|
+
("https://api.github.com/user/repos?affiliation=owner&per_page=100&page=1",
|
|
136
|
+
first_page),
|
|
137
|
+
("https://api.github.com/user/repos?affiliation=owner&per_page=100&page=2",
|
|
138
|
+
[make_repo("alice", "last-repo")]),
|
|
139
|
+
("https://api.github.com/users/alice",
|
|
140
|
+
{"login": "alice", "type": "User"}),
|
|
141
|
+
("https://api.github.com/user", {"login": "alice"}),
|
|
142
|
+
]
|
|
143
|
+
with mock.patch.object(lib.requests, "get", self.route(routes)):
|
|
144
|
+
repos = lib.search_user_repos("alice", token="ghp_test")
|
|
145
|
+
|
|
146
|
+
self.assertEqual(len(repos), 101)
|
|
147
|
+
self.assertEqual(sum(1 for url in self.urls() if url.endswith("/user")), 1)
|
|
148
|
+
self.assertEqual(
|
|
149
|
+
sum(1 for url in self.urls() if url.endswith("/users/alice")), 1
|
|
150
|
+
)
|
|
151
|
+
self.assertFalse(any("page=3" in url for url in self.urls()))
|
|
152
|
+
|
|
153
|
+
def test_username_is_url_quoted(self):
|
|
154
|
+
routes = [
|
|
155
|
+
("https://api.github.com/users/some%20user/repos", []),
|
|
156
|
+
("https://api.github.com/users/some%20user",
|
|
157
|
+
{"login": "some user", "type": "User"}),
|
|
158
|
+
]
|
|
159
|
+
|
|
160
|
+
with mock.patch.object(lib.requests, "get", self.route(routes)):
|
|
161
|
+
lib.get_page_of_repos("some user", 1)
|
|
162
|
+
|
|
163
|
+
self.assertEqual(
|
|
164
|
+
self.urls()[0], "https://api.github.com/users/some%20user"
|
|
165
|
+
)
|
|
166
|
+
self.assertIn(
|
|
167
|
+
"https://api.github.com/users/some%20user/repos", self.urls()[1]
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
def test_page_of_repos_filters_foreign_owners(self):
|
|
171
|
+
routes = [
|
|
172
|
+
("https://api.github.com/users/alice/repos", [
|
|
173
|
+
make_repo("alice", "mine"),
|
|
174
|
+
make_repo("acme-org", "not-mine"),
|
|
175
|
+
]),
|
|
176
|
+
("https://api.github.com/users/alice",
|
|
177
|
+
{"login": "alice", "type": "User"}),
|
|
178
|
+
("https://api.github.com/user", {"login": "bob"}),
|
|
179
|
+
]
|
|
180
|
+
with mock.patch.object(lib.requests, "get", self.route(routes)):
|
|
181
|
+
repos = lib.get_page_of_repos("alice", 1, token="ghp_test")
|
|
182
|
+
|
|
183
|
+
self.assertEqual([repo["name"] for repo in repos], ["mine"])
|
|
184
|
+
|
|
185
|
+
def test_organization_with_token_uses_org_endpoint_and_filters(self):
|
|
186
|
+
routes = [
|
|
187
|
+
("https://api.github.com/orgs/acme-org/repos", [
|
|
188
|
+
make_repo("acme-org", "org-private"),
|
|
189
|
+
make_repo("alice", "alice-personal"), # must never leak through
|
|
190
|
+
]),
|
|
191
|
+
("https://api.github.com/users/acme-org",
|
|
192
|
+
{"login": "acme-org", "type": "Organization"}),
|
|
193
|
+
("https://api.github.com/user", {"login": "alice"}),
|
|
194
|
+
]
|
|
195
|
+
with mock.patch.object(lib.requests, "get", self.route(routes)):
|
|
196
|
+
repos = lib.search_user_repos("acme-org", token="ghp_test")
|
|
197
|
+
|
|
198
|
+
self.assertEqual([repo["name"] for repo in repos], ["org-private"])
|
|
199
|
+
self.assertIn(
|
|
200
|
+
"https://api.github.com/orgs/acme-org/repos?type=all&per_page=100&page=1",
|
|
201
|
+
self.urls(),
|
|
202
|
+
)
|
|
203
|
+
self.assertFalse(any("/users/acme-org/repos" in url for url in self.urls()))
|
|
204
|
+
self.assertFalse(any("/user/repos" in url for url in self.urls()))
|
|
205
|
+
repos_calls = self.repos_calls()
|
|
206
|
+
self.assertEqual(len(repos_calls), 1)
|
|
207
|
+
self.assertEqual(
|
|
208
|
+
repos_calls[0][1]["Authorization"], "Bearer ghp_test"
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
def test_organization_without_token_uses_org_endpoint(self):
|
|
212
|
+
routes = [
|
|
213
|
+
("https://api.github.com/orgs/acme-org/repos", [
|
|
214
|
+
make_repo("acme-org", "org-public"),
|
|
215
|
+
]),
|
|
216
|
+
("https://api.github.com/users/acme-org",
|
|
217
|
+
{"login": "acme-org", "type": "Organization"}),
|
|
218
|
+
]
|
|
219
|
+
with mock.patch.object(lib.requests, "get", self.route(routes)):
|
|
220
|
+
repos = lib.search_user_repos("acme-org")
|
|
221
|
+
|
|
222
|
+
self.assertEqual([repo["name"] for repo in repos], ["org-public"])
|
|
223
|
+
self.assertEqual(
|
|
224
|
+
self.urls(),
|
|
225
|
+
[
|
|
226
|
+
"https://api.github.com/users/acme-org",
|
|
227
|
+
"https://api.github.com/orgs/acme-org/repos?type=all&per_page=100&page=1",
|
|
228
|
+
],
|
|
229
|
+
)
|
|
230
|
+
self.assertTrue(
|
|
231
|
+
all("Authorization" not in headers for _, headers in self.calls)
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
def test_organization_detected_via_org_endpoint_fallback(self):
|
|
235
|
+
def fake_get(url, headers=None, timeout=None):
|
|
236
|
+
self.calls.append((url, headers or {}))
|
|
237
|
+
if url.startswith("https://api.github.com/users/acme-org"):
|
|
238
|
+
return FakeResponse(404, {"message": "Not Found"})
|
|
239
|
+
if url == "https://api.github.com/orgs/acme-org":
|
|
240
|
+
return FakeResponse(200, {"login": "acme-org"})
|
|
241
|
+
if url.startswith("https://api.github.com/orgs/acme-org/repos"):
|
|
242
|
+
return FakeResponse(200, [
|
|
243
|
+
make_repo("acme-org", "org-repo"),
|
|
244
|
+
make_repo("bob", "bob-repo"),
|
|
245
|
+
])
|
|
246
|
+
return FakeResponse(404, {})
|
|
247
|
+
|
|
248
|
+
with mock.patch.object(lib.requests, "get", fake_get):
|
|
249
|
+
repos = lib.search_user_repos("acme-org")
|
|
250
|
+
|
|
251
|
+
self.assertEqual([repo["name"] for repo in repos], ["org-repo"])
|
|
252
|
+
self.assertIn(
|
|
253
|
+
"https://api.github.com/orgs/acme-org/repos?type=all&per_page=100&page=1",
|
|
254
|
+
self.urls(),
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
def test_unknown_account_type_falls_back_to_user_endpoint(self):
|
|
258
|
+
def fake_get(url, headers=None, timeout=None):
|
|
259
|
+
self.calls.append((url, headers or {}))
|
|
260
|
+
if url.startswith("https://api.github.com/users/foo/repos"):
|
|
261
|
+
return FakeResponse(200, [
|
|
262
|
+
make_repo("foo", "foo-repo"),
|
|
263
|
+
make_repo("bar", "bar-repo"),
|
|
264
|
+
])
|
|
265
|
+
return FakeResponse(404, {})
|
|
266
|
+
|
|
267
|
+
with mock.patch.object(lib.requests, "get", fake_get):
|
|
268
|
+
repos = lib.search_user_repos("foo")
|
|
269
|
+
|
|
270
|
+
self.assertEqual([repo["name"] for repo in repos], ["foo-repo"])
|
|
271
|
+
self.assertIn(
|
|
272
|
+
"https://api.github.com/users/foo/repos?type=owner&per_page=100&page=1",
|
|
273
|
+
self.urls(),
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
def test_login_lookup_failure_still_uses_username_endpoint(self):
|
|
277
|
+
def fake_get(url, headers=None, timeout=None):
|
|
278
|
+
self.calls.append((url, headers or {}))
|
|
279
|
+
if url == "https://api.github.com/user":
|
|
280
|
+
return FakeResponse(500, {"message": "boom"})
|
|
281
|
+
if url == "https://api.github.com/users/alice":
|
|
282
|
+
return FakeResponse(200, {"login": "alice", "type": "User"})
|
|
283
|
+
if url.startswith("https://api.github.com/users/alice/repos"):
|
|
284
|
+
return FakeResponse(200, [make_repo("alice", "public-repo")])
|
|
285
|
+
return FakeResponse(404, {})
|
|
286
|
+
|
|
287
|
+
with mock.patch.object(lib.requests, "get", fake_get):
|
|
288
|
+
repos = lib.search_user_repos("alice", token="ghp_test")
|
|
289
|
+
|
|
290
|
+
self.assertEqual([repo["name"] for repo in repos], ["public-repo"])
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
if __name__ == "__main__":
|
|
294
|
+
unittest.main()
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
import unittest
|
|
5
|
+
from unittest import mock
|
|
6
|
+
|
|
7
|
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
8
|
+
|
|
9
|
+
main = importlib.import_module("dump_github.main")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class CloneTests(unittest.TestCase):
|
|
13
|
+
def setUp(self):
|
|
14
|
+
self.calls = []
|
|
15
|
+
|
|
16
|
+
def fake_run(self, cmd, **kwargs):
|
|
17
|
+
self.calls.append((cmd, kwargs))
|
|
18
|
+
return mock.Mock(returncode=0)
|
|
19
|
+
|
|
20
|
+
def test_token_is_passed_through_env_not_url(self):
|
|
21
|
+
repo = {"name": "repo", "clone_url": "https://github.com/alice/repo.git"}
|
|
22
|
+
with mock.patch.object(main.subprocess, "run", self.fake_run):
|
|
23
|
+
main.clone_repo(repo, token="ghp_secret")
|
|
24
|
+
|
|
25
|
+
cmd, kwargs = self.calls[0]
|
|
26
|
+
self.assertEqual(cmd[0], "git")
|
|
27
|
+
self.assertIn("clone", cmd)
|
|
28
|
+
# Existing credential helpers are reset so a stale stored credential
|
|
29
|
+
# cannot shadow the token supplied here.
|
|
30
|
+
self.assertEqual(
|
|
31
|
+
cmd[cmd.index("credential.helper=") - 1], "-c"
|
|
32
|
+
)
|
|
33
|
+
self.assertIn(
|
|
34
|
+
"credential.helper={}".format(main.GIT_CREDENTIAL_HELPER), cmd
|
|
35
|
+
)
|
|
36
|
+
self.assertEqual(cmd[-1], "https://github.com/alice/repo.git")
|
|
37
|
+
self.assertNotIn("ghp_secret", " ".join(cmd))
|
|
38
|
+
self.assertNotIn("@github.com", cmd[-1])
|
|
39
|
+
env = kwargs["env"]
|
|
40
|
+
self.assertEqual(env["DUMP_GITHUB_TOKEN"], "ghp_secret")
|
|
41
|
+
self.assertEqual(env["GIT_TERMINAL_PROMPT"], "0")
|
|
42
|
+
|
|
43
|
+
def test_env_keeps_the_rest_of_the_environment(self):
|
|
44
|
+
repo = {"name": "repo", "clone_url": "https://github.com/alice/repo.git"}
|
|
45
|
+
with mock.patch.dict(os.environ, {"KEEP_ME": "yes"}):
|
|
46
|
+
with mock.patch.object(main.subprocess, "run", self.fake_run):
|
|
47
|
+
main.clone_repo(repo, token="ghp_secret")
|
|
48
|
+
|
|
49
|
+
env = self.calls[0][1]["env"]
|
|
50
|
+
self.assertEqual(env["KEEP_ME"], "yes")
|
|
51
|
+
self.assertIsNot(env, os.environ)
|
|
52
|
+
|
|
53
|
+
def test_clone_without_token_stays_anonymous(self):
|
|
54
|
+
repo = {"name": "repo", "clone_url": "https://github.com/alice/repo.git"}
|
|
55
|
+
with mock.patch.object(main.subprocess, "run", self.fake_run):
|
|
56
|
+
main.clone_repo(repo)
|
|
57
|
+
|
|
58
|
+
cmd, kwargs = self.calls[0]
|
|
59
|
+
self.assertEqual(cmd, ["git", "clone", "https://github.com/alice/repo.git"])
|
|
60
|
+
self.assertIsNone(kwargs["env"])
|
|
61
|
+
|
|
62
|
+
def test_version_flag(self):
|
|
63
|
+
import contextlib
|
|
64
|
+
import io
|
|
65
|
+
|
|
66
|
+
buf = io.StringIO()
|
|
67
|
+
with self.assertRaises(SystemExit) as ctx:
|
|
68
|
+
with contextlib.redirect_stdout(buf):
|
|
69
|
+
main.main(["--version"])
|
|
70
|
+
|
|
71
|
+
self.assertEqual(ctx.exception.code, 0)
|
|
72
|
+
self.assertEqual(buf.getvalue().strip(), "dump-github " + main.__version__)
|
|
73
|
+
|
|
74
|
+
def test_credential_helper_outputs_the_env_token(self):
|
|
75
|
+
"""Run the real helper snippet through `git credential fill` if git exists."""
|
|
76
|
+
import shutil
|
|
77
|
+
import subprocess
|
|
78
|
+
|
|
79
|
+
if not shutil.which("git"):
|
|
80
|
+
self.skipTest("git is not installed")
|
|
81
|
+
|
|
82
|
+
result = subprocess.run(
|
|
83
|
+
[
|
|
84
|
+
"git",
|
|
85
|
+
"-c",
|
|
86
|
+
"credential.helper=",
|
|
87
|
+
"-c",
|
|
88
|
+
"credential.helper={}".format(main.GIT_CREDENTIAL_HELPER),
|
|
89
|
+
"credential",
|
|
90
|
+
"fill",
|
|
91
|
+
],
|
|
92
|
+
input="protocol=https\nhost=github.com\n\n",
|
|
93
|
+
capture_output=True,
|
|
94
|
+
text=True,
|
|
95
|
+
env={**os.environ, "DUMP_GITHUB_TOKEN": "ghp_secret"},
|
|
96
|
+
)
|
|
97
|
+
self.assertEqual(result.returncode, 0, result.stderr)
|
|
98
|
+
self.assertIn("username=x-access-token", result.stdout)
|
|
99
|
+
self.assertIn("password=ghp_secret", result.stdout)
|
|
100
|
+
self.assertNotIn("DUMP_GITHUB_TOKEN", result.stdout)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
if __name__ == "__main__":
|
|
104
|
+
unittest.main()
|