githubtakeout 0.1.15__tar.gz → 0.1.17__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {githubtakeout-0.1.15/src/githubtakeout.egg-info → githubtakeout-0.1.17}/PKG-INFO +22 -3
- {githubtakeout-0.1.15 → githubtakeout-0.1.17}/README.md +20 -2
- {githubtakeout-0.1.15 → githubtakeout-0.1.17}/pyproject.toml +13 -5
- {githubtakeout-0.1.15 → githubtakeout-0.1.17/src/githubtakeout.egg-info}/PKG-INFO +22 -3
- {githubtakeout-0.1.15 → githubtakeout-0.1.17}/src/githubtakeout.egg-info/SOURCES.txt +2 -1
- {githubtakeout-0.1.15 → githubtakeout-0.1.17}/src/githubtakeout.egg-info/requires.txt +1 -0
- {githubtakeout-0.1.15 → githubtakeout-0.1.17}/src/githubtakeout.py +120 -39
- {githubtakeout-0.1.15 → githubtakeout-0.1.17}/src/progress.py +3 -1
- githubtakeout-0.1.17/tests/test_githubtakeout.py +262 -0
- {githubtakeout-0.1.15 → githubtakeout-0.1.17}/LICENSE +0 -0
- {githubtakeout-0.1.15 → githubtakeout-0.1.17}/setup.cfg +0 -0
- {githubtakeout-0.1.15 → githubtakeout-0.1.17}/src/githubtakeout.egg-info/dependency_links.txt +0 -0
- {githubtakeout-0.1.15 → githubtakeout-0.1.17}/src/githubtakeout.egg-info/entry_points.txt +0 -0
- {githubtakeout-0.1.15 → githubtakeout-0.1.17}/src/githubtakeout.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: githubtakeout
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.17
|
|
4
4
|
Summary: Backup and archive Git Repos and Gists from GitHub
|
|
5
5
|
Author: Corey Goldberg
|
|
6
6
|
Maintainer: Corey Goldberg
|
|
@@ -21,6 +21,7 @@ Description-Content-Type: text/markdown
|
|
|
21
21
|
License-File: LICENSE
|
|
22
22
|
Requires-Dist: GitPython==3.1.45
|
|
23
23
|
Requires-Dist: PyGithub==2.8.1
|
|
24
|
+
Requires-Dist: python-dotenv==1.2.1
|
|
24
25
|
Requires-Dist: rich==14.2.0
|
|
25
26
|
Dynamic: license-file
|
|
26
27
|
|
|
@@ -57,6 +58,10 @@ Archives are saved in compressed zip format (`.zip`) by default. You can also
|
|
|
57
58
|
save as tarballs (`.tar.gz`) using the `--format=tar` option, or skip archiving
|
|
58
59
|
using the `--format=none` option.
|
|
59
60
|
|
|
61
|
+
If a repo with history exists from a previous run, it will pull new changes.
|
|
62
|
+
Otherwise, it will clone the repo. Use the `--keep` if you don't want repos
|
|
63
|
+
deleted after an archive is created.
|
|
64
|
+
|
|
60
65
|
## Requirements:
|
|
61
66
|
|
|
62
67
|
- Python 3.12+
|
|
@@ -83,13 +88,24 @@ can set the `GITHUB_TOKEN` environment variable:
|
|
|
83
88
|
$ export GITHUB_TOKEN=<access token>
|
|
84
89
|
```
|
|
85
90
|
|
|
91
|
+
You can also set this in an `.env` file in the current directory. It should
|
|
92
|
+
contain:
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
GITHUB_TOKEN=<access token>
|
|
96
|
+
```
|
|
97
|
+
|
|
86
98
|
If you prefer to be prompted for your token each time you run the program, use
|
|
87
99
|
the `--token` argument.
|
|
88
100
|
|
|
89
101
|
## CLI Options:
|
|
90
102
|
|
|
91
103
|
```
|
|
92
|
-
|
|
104
|
+
$ githubtakeout --help
|
|
105
|
+
usage: githubtakeout [-h] [--dir DIR] [--pattern PATTERN] [--skip_pattern PATTERN]
|
|
106
|
+
[--format {tar,zip,none}] [--gists] [--history]
|
|
107
|
+
[--skip_forks] [--keep] [--list] [--token]
|
|
108
|
+
username
|
|
93
109
|
|
|
94
110
|
positional arguments:
|
|
95
111
|
username github username
|
|
@@ -97,10 +113,13 @@ positional arguments:
|
|
|
97
113
|
options:
|
|
98
114
|
-h, --help show this help message and exit
|
|
99
115
|
--dir DIR output directory (default: .)
|
|
100
|
-
--pattern PATTERN regex matching repo names
|
|
116
|
+
--pattern PATTERN regex matching repo names to include
|
|
117
|
+
--skip_pattern PATTERN regex matching repo names to skip
|
|
101
118
|
--format {tar,zip,none} archive format (default: zip)
|
|
102
119
|
--gists include gists
|
|
103
120
|
--history include commit history and branches (.git directory)
|
|
121
|
+
--skip_forks skip repos that are forks
|
|
122
|
+
--keep keep repos after archiving
|
|
104
123
|
--list list repos only
|
|
105
124
|
--token prompt for auth token
|
|
106
125
|
```
|
|
@@ -31,6 +31,10 @@ Archives are saved in compressed zip format (`.zip`) by default. You can also
|
|
|
31
31
|
save as tarballs (`.tar.gz`) using the `--format=tar` option, or skip archiving
|
|
32
32
|
using the `--format=none` option.
|
|
33
33
|
|
|
34
|
+
If a repo with history exists from a previous run, it will pull new changes.
|
|
35
|
+
Otherwise, it will clone the repo. Use the `--keep` if you don't want repos
|
|
36
|
+
deleted after an archive is created.
|
|
37
|
+
|
|
34
38
|
## Requirements:
|
|
35
39
|
|
|
36
40
|
- Python 3.12+
|
|
@@ -57,13 +61,24 @@ can set the `GITHUB_TOKEN` environment variable:
|
|
|
57
61
|
$ export GITHUB_TOKEN=<access token>
|
|
58
62
|
```
|
|
59
63
|
|
|
64
|
+
You can also set this in an `.env` file in the current directory. It should
|
|
65
|
+
contain:
|
|
66
|
+
|
|
67
|
+
```
|
|
68
|
+
GITHUB_TOKEN=<access token>
|
|
69
|
+
```
|
|
70
|
+
|
|
60
71
|
If you prefer to be prompted for your token each time you run the program, use
|
|
61
72
|
the `--token` argument.
|
|
62
73
|
|
|
63
74
|
## CLI Options:
|
|
64
75
|
|
|
65
76
|
```
|
|
66
|
-
|
|
77
|
+
$ githubtakeout --help
|
|
78
|
+
usage: githubtakeout [-h] [--dir DIR] [--pattern PATTERN] [--skip_pattern PATTERN]
|
|
79
|
+
[--format {tar,zip,none}] [--gists] [--history]
|
|
80
|
+
[--skip_forks] [--keep] [--list] [--token]
|
|
81
|
+
username
|
|
67
82
|
|
|
68
83
|
positional arguments:
|
|
69
84
|
username github username
|
|
@@ -71,10 +86,13 @@ positional arguments:
|
|
|
71
86
|
options:
|
|
72
87
|
-h, --help show this help message and exit
|
|
73
88
|
--dir DIR output directory (default: .)
|
|
74
|
-
--pattern PATTERN regex matching repo names
|
|
89
|
+
--pattern PATTERN regex matching repo names to include
|
|
90
|
+
--skip_pattern PATTERN regex matching repo names to skip
|
|
75
91
|
--format {tar,zip,none} archive format (default: zip)
|
|
76
92
|
--gists include gists
|
|
77
93
|
--history include commit history and branches (.git directory)
|
|
94
|
+
--skip_forks skip repos that are forks
|
|
95
|
+
--keep keep repos after archiving
|
|
78
96
|
--list list repos only
|
|
79
97
|
--token prompt for auth token
|
|
80
98
|
```
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "githubtakeout"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.17"
|
|
8
8
|
description = "Backup and archive Git Repos and Gists from GitHub"
|
|
9
9
|
license = "MIT"
|
|
10
10
|
license-files = ["LICENSE"]
|
|
@@ -25,6 +25,7 @@ classifiers = [
|
|
|
25
25
|
dependencies = [
|
|
26
26
|
"GitPython==3.1.45",
|
|
27
27
|
"PyGithub==2.8.1",
|
|
28
|
+
"python-dotenv==1.2.1",
|
|
28
29
|
"rich==14.2.0",
|
|
29
30
|
]
|
|
30
31
|
|
|
@@ -45,6 +46,9 @@ lint = [
|
|
|
45
46
|
"refurb",
|
|
46
47
|
"ruff",
|
|
47
48
|
]
|
|
49
|
+
test = [
|
|
50
|
+
"pytest",
|
|
51
|
+
]
|
|
48
52
|
validate = [
|
|
49
53
|
"packaging",
|
|
50
54
|
"validate-pyproject",
|
|
@@ -53,14 +57,18 @@ validate = [
|
|
|
53
57
|
[tool.setuptools]
|
|
54
58
|
package-dir = {"" = "src"}
|
|
55
59
|
|
|
60
|
+
[tool.pytest.ini_options]
|
|
61
|
+
addopts = "-vv" # extra verbose
|
|
62
|
+
tmp_path_retention_policy = "none"
|
|
63
|
+
|
|
56
64
|
[tool.black]
|
|
57
|
-
line-length =
|
|
65
|
+
line-length = 88
|
|
58
66
|
target-version = [
|
|
59
67
|
"py312", "py313", "py314"
|
|
60
68
|
]
|
|
61
69
|
|
|
62
70
|
[tool.ruff]
|
|
63
|
-
line-length =
|
|
71
|
+
line-length = 88
|
|
64
72
|
respect-gitignore = true
|
|
65
73
|
target-version = "py312"
|
|
66
74
|
|
|
@@ -76,10 +84,10 @@ fixable = ["ALL"]
|
|
|
76
84
|
|
|
77
85
|
[tool.ruff.format]
|
|
78
86
|
docstring-code-format = true
|
|
79
|
-
docstring-code-line-length =
|
|
87
|
+
docstring-code-line-length = 88
|
|
80
88
|
|
|
81
89
|
[tool.refurb]
|
|
82
|
-
python_version = "3.
|
|
90
|
+
python_version = "3.12"
|
|
83
91
|
enable_all = true
|
|
84
92
|
disable = [
|
|
85
93
|
"FURB104", "FURB107", "FURB147", "FURB173", "FURB183", "FURB184"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: githubtakeout
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.17
|
|
4
4
|
Summary: Backup and archive Git Repos and Gists from GitHub
|
|
5
5
|
Author: Corey Goldberg
|
|
6
6
|
Maintainer: Corey Goldberg
|
|
@@ -21,6 +21,7 @@ Description-Content-Type: text/markdown
|
|
|
21
21
|
License-File: LICENSE
|
|
22
22
|
Requires-Dist: GitPython==3.1.45
|
|
23
23
|
Requires-Dist: PyGithub==2.8.1
|
|
24
|
+
Requires-Dist: python-dotenv==1.2.1
|
|
24
25
|
Requires-Dist: rich==14.2.0
|
|
25
26
|
Dynamic: license-file
|
|
26
27
|
|
|
@@ -57,6 +58,10 @@ Archives are saved in compressed zip format (`.zip`) by default. You can also
|
|
|
57
58
|
save as tarballs (`.tar.gz`) using the `--format=tar` option, or skip archiving
|
|
58
59
|
using the `--format=none` option.
|
|
59
60
|
|
|
61
|
+
If a repo with history exists from a previous run, it will pull new changes.
|
|
62
|
+
Otherwise, it will clone the repo. Use the `--keep` if you don't want repos
|
|
63
|
+
deleted after an archive is created.
|
|
64
|
+
|
|
60
65
|
## Requirements:
|
|
61
66
|
|
|
62
67
|
- Python 3.12+
|
|
@@ -83,13 +88,24 @@ can set the `GITHUB_TOKEN` environment variable:
|
|
|
83
88
|
$ export GITHUB_TOKEN=<access token>
|
|
84
89
|
```
|
|
85
90
|
|
|
91
|
+
You can also set this in an `.env` file in the current directory. It should
|
|
92
|
+
contain:
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
GITHUB_TOKEN=<access token>
|
|
96
|
+
```
|
|
97
|
+
|
|
86
98
|
If you prefer to be prompted for your token each time you run the program, use
|
|
87
99
|
the `--token` argument.
|
|
88
100
|
|
|
89
101
|
## CLI Options:
|
|
90
102
|
|
|
91
103
|
```
|
|
92
|
-
|
|
104
|
+
$ githubtakeout --help
|
|
105
|
+
usage: githubtakeout [-h] [--dir DIR] [--pattern PATTERN] [--skip_pattern PATTERN]
|
|
106
|
+
[--format {tar,zip,none}] [--gists] [--history]
|
|
107
|
+
[--skip_forks] [--keep] [--list] [--token]
|
|
108
|
+
username
|
|
93
109
|
|
|
94
110
|
positional arguments:
|
|
95
111
|
username github username
|
|
@@ -97,10 +113,13 @@ positional arguments:
|
|
|
97
113
|
options:
|
|
98
114
|
-h, --help show this help message and exit
|
|
99
115
|
--dir DIR output directory (default: .)
|
|
100
|
-
--pattern PATTERN regex matching repo names
|
|
116
|
+
--pattern PATTERN regex matching repo names to include
|
|
117
|
+
--skip_pattern PATTERN regex matching repo names to skip
|
|
101
118
|
--format {tar,zip,none} archive format (default: zip)
|
|
102
119
|
--gists include gists
|
|
103
120
|
--history include commit history and branches (.git directory)
|
|
121
|
+
--skip_forks skip repos that are forks
|
|
122
|
+
--keep keep repos after archiving
|
|
104
123
|
--list list repos only
|
|
105
124
|
--token prompt for auth token
|
|
106
125
|
```
|
|
@@ -8,4 +8,5 @@ src/githubtakeout.egg-info/SOURCES.txt
|
|
|
8
8
|
src/githubtakeout.egg-info/dependency_links.txt
|
|
9
9
|
src/githubtakeout.egg-info/entry_points.txt
|
|
10
10
|
src/githubtakeout.egg-info/requires.txt
|
|
11
|
-
src/githubtakeout.egg-info/top_level.txt
|
|
11
|
+
src/githubtakeout.egg-info/top_level.txt
|
|
12
|
+
tests/test_githubtakeout.py
|
|
@@ -20,6 +20,7 @@ from timeit import default_timer
|
|
|
20
20
|
|
|
21
21
|
import git
|
|
22
22
|
import github
|
|
23
|
+
from dotenv import load_dotenv
|
|
23
24
|
|
|
24
25
|
from progress import GitProgress
|
|
25
26
|
|
|
@@ -89,7 +90,7 @@ def clone(repo_url, local_repo_dir, include_history):
|
|
|
89
90
|
repo = git.Repo.clone_from(
|
|
90
91
|
url=repo_url,
|
|
91
92
|
to_path=local_repo_dir,
|
|
92
|
-
multi_options=["--
|
|
93
|
+
multi_options=["--depth=1"],
|
|
93
94
|
progress=GitProgress(),
|
|
94
95
|
)
|
|
95
96
|
except git.GitCommandError as e:
|
|
@@ -104,7 +105,27 @@ def clone(repo_url, local_repo_dir, include_history):
|
|
|
104
105
|
pass
|
|
105
106
|
|
|
106
107
|
|
|
107
|
-
def
|
|
108
|
+
def pull(local_repo_dir):
|
|
109
|
+
try:
|
|
110
|
+
repo = git.Repo(local_repo_dir)
|
|
111
|
+
origin = repo.remotes.origin
|
|
112
|
+
origin.fetch()
|
|
113
|
+
origin.pull(progress=GitProgress())
|
|
114
|
+
except git.GitCommandError as e:
|
|
115
|
+
logger.error(e)
|
|
116
|
+
sys.exit("error: failed pulling changes in repo")
|
|
117
|
+
finally:
|
|
118
|
+
try:
|
|
119
|
+
# release resources
|
|
120
|
+
repo.close()
|
|
121
|
+
except UnboundLocalError:
|
|
122
|
+
# this occurs if we catch a signal while pulling
|
|
123
|
+
pass
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def get_and_archive_repo(
|
|
127
|
+
repo_url, local_repo_dir, archive_format, include_history, keep, is_gist=False
|
|
128
|
+
):
|
|
108
129
|
def remove_readonly(func, path, _):
|
|
109
130
|
# This is necessary so rmtree() doesn't fail if there are any readonly
|
|
110
131
|
# dirs/files when trying to delete. This seems to happen after cloning on
|
|
@@ -118,19 +139,27 @@ def clone_and_archive_repo(repo_url, local_repo_dir, archive_format, include_his
|
|
|
118
139
|
# try again
|
|
119
140
|
func(path)
|
|
120
141
|
|
|
121
|
-
try:
|
|
122
|
-
# delete repo if it already exists
|
|
123
|
-
shutil.rmtree(local_repo_dir, onexc=remove_readonly)
|
|
124
|
-
except FileNotFoundError:
|
|
125
|
-
pass
|
|
126
142
|
repo_name = urllib.parse.urlparse(repo_url).path.lstrip("/")
|
|
127
|
-
|
|
143
|
+
# we can only pull if the local repo exists and has a .git directory
|
|
144
|
+
if Path(local_repo_dir, ".git").exists():
|
|
145
|
+
needs_clone = False
|
|
146
|
+
else:
|
|
147
|
+
needs_clone = True
|
|
148
|
+
try:
|
|
149
|
+
shutil.rmtree(local_repo_dir, onexc=remove_readonly)
|
|
150
|
+
except FileNotFoundError:
|
|
151
|
+
pass
|
|
128
152
|
start = default_timer()
|
|
129
|
-
|
|
153
|
+
if needs_clone:
|
|
154
|
+
logger.info(f"cloning repo: {repo_name} to: {local_repo_dir}")
|
|
155
|
+
clone(repo_url, local_repo_dir, include_history)
|
|
156
|
+
else:
|
|
157
|
+
logger.info(f"pulling changes from repo: {repo_name} to: {local_repo_dir}")
|
|
158
|
+
pull(local_repo_dir)
|
|
130
159
|
if not include_history:
|
|
131
160
|
# delete the .git directory if we are not saving history
|
|
132
161
|
try:
|
|
133
|
-
git_dir =
|
|
162
|
+
git_dir = Path(local_repo_dir, ".git")
|
|
134
163
|
shutil.rmtree(git_dir, onexc=remove_readonly)
|
|
135
164
|
except FileNotFoundError:
|
|
136
165
|
pass
|
|
@@ -138,12 +167,13 @@ def clone_and_archive_repo(repo_url, local_repo_dir, archive_format, include_his
|
|
|
138
167
|
if archive_path:
|
|
139
168
|
size = convert_size(os.path.getsize(archive_path))
|
|
140
169
|
logger.info(f"archive size: {size}")
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
170
|
+
if not keep:
|
|
171
|
+
logger.info("deleting repo")
|
|
172
|
+
try:
|
|
173
|
+
# delete repo after archive is created
|
|
174
|
+
shutil.rmtree(local_repo_dir, onexc=remove_readonly)
|
|
175
|
+
except FileNotFoundError:
|
|
176
|
+
pass
|
|
147
177
|
elapsed = default_timer() - start
|
|
148
178
|
base_name = os.path.basename(local_repo_dir)
|
|
149
179
|
logger.info(f"successfully backed up '{base_name}' repo in {elapsed:.3f} secs\n")
|
|
@@ -178,16 +208,27 @@ def get_repos(username, token, include_gists):
|
|
|
178
208
|
if include_gists:
|
|
179
209
|
gists = user.get_gists()
|
|
180
210
|
else:
|
|
181
|
-
gists =
|
|
211
|
+
gists = []
|
|
182
212
|
return repos, gists
|
|
183
213
|
|
|
184
214
|
|
|
215
|
+
def filter_repos(repos, pattern, skip_pattern, skip_forks):
|
|
216
|
+
repos = (repo for repo in repos if re.match(pattern, repo.name))
|
|
217
|
+
if skip_pattern:
|
|
218
|
+
repos = (repo for repo in repos if not re.match(skip_pattern, repo.name))
|
|
219
|
+
if skip_forks:
|
|
220
|
+
repos = (repo for repo in repos if not repo.fork)
|
|
221
|
+
return repos
|
|
222
|
+
|
|
223
|
+
|
|
185
224
|
def get_token(prompt_for_token):
|
|
186
225
|
if prompt_for_token:
|
|
187
226
|
token = getpass.getpass("Token:")
|
|
188
227
|
if not token:
|
|
189
228
|
sys.exit("error: auth token cannot be empty")
|
|
190
229
|
else:
|
|
230
|
+
# populate environment variable via .env file if it exists
|
|
231
|
+
load_dotenv()
|
|
191
232
|
token = os.getenv("GITHUB_TOKEN")
|
|
192
233
|
return token
|
|
193
234
|
|
|
@@ -196,39 +237,56 @@ def run(
|
|
|
196
237
|
username,
|
|
197
238
|
base_dir,
|
|
198
239
|
pattern,
|
|
240
|
+
skip_pattern,
|
|
199
241
|
archive_format,
|
|
200
242
|
include_gists,
|
|
201
243
|
include_history,
|
|
244
|
+
skip_forks,
|
|
245
|
+
keep,
|
|
202
246
|
list_only,
|
|
203
247
|
prompt_for_token,
|
|
204
248
|
):
|
|
205
249
|
working_dir = os.path.join(base_dir, "backups")
|
|
206
250
|
token = get_token(prompt_for_token)
|
|
207
|
-
|
|
208
|
-
|
|
251
|
+
|
|
252
|
+
all_repos, gists = get_repos(username, token, include_gists)
|
|
253
|
+
repos = filter_repos(all_repos, pattern, skip_pattern, skip_forks)
|
|
254
|
+
num_repos = sum(1 for repo in repos)
|
|
255
|
+
num_gists = sum(1 for gist in gists)
|
|
256
|
+
|
|
257
|
+
# we have to fetch the repos again because we exhaust the generator
|
|
258
|
+
# when counting repos above
|
|
259
|
+
all_repos, gists = get_repos(username, token, include_gists)
|
|
260
|
+
repos = filter_repos(all_repos, pattern, skip_pattern, skip_forks)
|
|
209
261
|
if not list_only:
|
|
210
262
|
logger.info(f"creating archives in: {working_dir}\n")
|
|
211
263
|
logger.info(f"found {num_repos} repos for user '{username}':\n")
|
|
212
264
|
for repo in repos:
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
265
|
+
local_repo_dir = os.path.join(working_dir, repo.name)
|
|
266
|
+
url = add_creds(repo.clone_url, username, token)
|
|
267
|
+
if list_only:
|
|
268
|
+
logger.info(f"{username}/{repo.name}")
|
|
269
|
+
else:
|
|
270
|
+
get_and_archive_repo(
|
|
271
|
+
url, local_repo_dir, archive_format, include_history, keep
|
|
272
|
+
)
|
|
273
|
+
if include_gists:
|
|
222
274
|
logger.info("")
|
|
223
275
|
logger.info(f"found {num_gists} gists for user '{username}':\n")
|
|
224
276
|
for gist in gists:
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
277
|
+
local_repo_dir = os.path.join(working_dir, gist.id)
|
|
278
|
+
url = add_creds(gist.git_pull_url, username, token)
|
|
279
|
+
if list_only:
|
|
280
|
+
logger.info(f"{username}/{gist.id}\n - {gist.description}")
|
|
281
|
+
else:
|
|
282
|
+
get_and_archive_repo(
|
|
283
|
+
url,
|
|
284
|
+
local_repo_dir,
|
|
285
|
+
archive_format,
|
|
286
|
+
include_history,
|
|
287
|
+
keep,
|
|
288
|
+
is_gist=True,
|
|
289
|
+
)
|
|
232
290
|
|
|
233
291
|
|
|
234
292
|
def main():
|
|
@@ -248,7 +306,12 @@ def main():
|
|
|
248
306
|
parser.add_argument(
|
|
249
307
|
"--pattern",
|
|
250
308
|
default=".*",
|
|
251
|
-
help="regex matching repo names
|
|
309
|
+
help="regex matching repo names to include",
|
|
310
|
+
)
|
|
311
|
+
parser.add_argument(
|
|
312
|
+
"--skip_pattern",
|
|
313
|
+
metavar="PATTERN",
|
|
314
|
+
help="regex matching repo names to skip",
|
|
252
315
|
)
|
|
253
316
|
parser.add_argument(
|
|
254
317
|
"--format",
|
|
@@ -256,24 +319,42 @@ def main():
|
|
|
256
319
|
default="zip",
|
|
257
320
|
help="archive format (default: %(default)s)",
|
|
258
321
|
)
|
|
259
|
-
parser.add_argument(
|
|
322
|
+
parser.add_argument(
|
|
323
|
+
"--gists", action="store_true", default=False, help="include gists"
|
|
324
|
+
)
|
|
260
325
|
parser.add_argument(
|
|
261
326
|
"--history",
|
|
262
327
|
action="store_true",
|
|
263
328
|
default=False,
|
|
264
329
|
help="include commit history and branches (.git directory)",
|
|
265
330
|
)
|
|
266
|
-
parser.add_argument(
|
|
267
|
-
|
|
331
|
+
parser.add_argument(
|
|
332
|
+
"--skip_forks",
|
|
333
|
+
action="store_true",
|
|
334
|
+
default=False,
|
|
335
|
+
help="skip repos that are forks",
|
|
336
|
+
)
|
|
337
|
+
parser.add_argument(
|
|
338
|
+
"--keep", action="store_true", default=False, help="keep repos after archiving"
|
|
339
|
+
)
|
|
340
|
+
parser.add_argument(
|
|
341
|
+
"--list", action="store_true", default=False, help="list repos only"
|
|
342
|
+
)
|
|
343
|
+
parser.add_argument(
|
|
344
|
+
"--token", action="store_true", default=False, help="prompt for auth token"
|
|
345
|
+
)
|
|
268
346
|
args = parser.parse_args()
|
|
269
347
|
try:
|
|
270
348
|
run(
|
|
271
349
|
username=args.username,
|
|
272
350
|
base_dir=args.dir,
|
|
273
351
|
pattern=args.pattern,
|
|
352
|
+
skip_pattern=args.skip_pattern,
|
|
274
353
|
archive_format=args.format,
|
|
275
354
|
include_gists=args.gists,
|
|
276
355
|
include_history=args.history,
|
|
356
|
+
skip_forks=args.skip_forks,
|
|
357
|
+
keep=args.keep,
|
|
277
358
|
list_only=args.list,
|
|
278
359
|
prompt_for_token=args.token,
|
|
279
360
|
)
|
|
@@ -20,7 +20,9 @@ class GitProgress(git.RemoteProgress):
|
|
|
20
20
|
"RESOLVING",
|
|
21
21
|
"WRITING",
|
|
22
22
|
]
|
|
23
|
-
OP_CODE_MAP = {
|
|
23
|
+
OP_CODE_MAP = {
|
|
24
|
+
getattr(git.RemoteProgress, _op_code): _op_code for _op_code in OP_CODES
|
|
25
|
+
}
|
|
24
26
|
|
|
25
27
|
def __init__(self):
|
|
26
28
|
super().__init__()
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
from dotenv import load_dotenv
|
|
7
|
+
|
|
8
|
+
import githubtakeout
|
|
9
|
+
|
|
10
|
+
USER = "cgoldberg"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@pytest.fixture(autouse=True, scope="session")
|
|
14
|
+
def require_token():
|
|
15
|
+
load_dotenv()
|
|
16
|
+
required = "GITHUB_TOKEN"
|
|
17
|
+
if not os.getenv(required):
|
|
18
|
+
pytest.skip(
|
|
19
|
+
reason=f"{required} environment variable or .env file is "
|
|
20
|
+
"required to avoid rate limiting"
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@pytest.mark.skipif(
|
|
25
|
+
os.getenv("GITHUB_TOKEN"), reason="GITHUB_TOKEN required to avoid rate limiting"
|
|
26
|
+
)
|
|
27
|
+
def test_run_list_1_match(tmp_path, caplog):
|
|
28
|
+
caplog.set_level("INFO")
|
|
29
|
+
repo = "githubtakeout"
|
|
30
|
+
githubtakeout.run(
|
|
31
|
+
username=USER,
|
|
32
|
+
base_dir=tmp_path,
|
|
33
|
+
pattern=repo,
|
|
34
|
+
skip_pattern=None,
|
|
35
|
+
archive_format="none",
|
|
36
|
+
include_gists=False,
|
|
37
|
+
include_history=False,
|
|
38
|
+
skip_forks=False,
|
|
39
|
+
keep=False,
|
|
40
|
+
list_only=True,
|
|
41
|
+
prompt_for_token=False,
|
|
42
|
+
)
|
|
43
|
+
assert "creating archives" not in caplog.text
|
|
44
|
+
assert f"found 1 repos for user '{USER}'" in caplog.text
|
|
45
|
+
assert not re.search(r"found \d* gists", caplog.text)
|
|
46
|
+
assert f"{USER}/{repo}" in caplog.text
|
|
47
|
+
assert not Path(tmp_path / "backups" / repo).exists()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_run_list_0_match(tmp_path, caplog):
|
|
51
|
+
caplog.set_level("INFO")
|
|
52
|
+
repo = "this_repo_does_not_exist"
|
|
53
|
+
githubtakeout.run(
|
|
54
|
+
username=USER,
|
|
55
|
+
base_dir=tmp_path,
|
|
56
|
+
pattern=repo,
|
|
57
|
+
skip_pattern=None,
|
|
58
|
+
archive_format="none",
|
|
59
|
+
include_gists=False,
|
|
60
|
+
include_history=False,
|
|
61
|
+
skip_forks=False,
|
|
62
|
+
keep=False,
|
|
63
|
+
list_only=True,
|
|
64
|
+
prompt_for_token=False,
|
|
65
|
+
)
|
|
66
|
+
assert "creating archives" not in caplog.text
|
|
67
|
+
assert "found 0 repos" in caplog.text
|
|
68
|
+
assert not re.search(r"found \d* gists", caplog.text)
|
|
69
|
+
assert not Path(tmp_path / "backups" / repo).exists()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_run_list_0_match_with_gists(tmp_path, caplog):
|
|
73
|
+
caplog.set_level("INFO")
|
|
74
|
+
repo = "this_repo_does_not_exist"
|
|
75
|
+
githubtakeout.run(
|
|
76
|
+
username=USER,
|
|
77
|
+
base_dir=tmp_path,
|
|
78
|
+
pattern=repo,
|
|
79
|
+
skip_pattern=None,
|
|
80
|
+
archive_format="none",
|
|
81
|
+
include_gists=True,
|
|
82
|
+
include_history=False,
|
|
83
|
+
skip_forks=False,
|
|
84
|
+
keep=False,
|
|
85
|
+
list_only=True,
|
|
86
|
+
prompt_for_token=False,
|
|
87
|
+
)
|
|
88
|
+
assert "creating archives" not in caplog.text
|
|
89
|
+
assert "found 0 repos" in caplog.text
|
|
90
|
+
assert re.search(r"found \d* gists", caplog.text)
|
|
91
|
+
assert not Path(tmp_path / "backups" / repo).exists()
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def test_run_list_skip_all(tmp_path, caplog):
|
|
95
|
+
caplog.set_level("INFO")
|
|
96
|
+
githubtakeout.run(
|
|
97
|
+
username=USER,
|
|
98
|
+
base_dir=tmp_path,
|
|
99
|
+
pattern=".*",
|
|
100
|
+
skip_pattern=".*",
|
|
101
|
+
archive_format="none",
|
|
102
|
+
include_gists=False,
|
|
103
|
+
include_history=False,
|
|
104
|
+
skip_forks=False,
|
|
105
|
+
keep=False,
|
|
106
|
+
list_only=True,
|
|
107
|
+
prompt_for_token=False,
|
|
108
|
+
)
|
|
109
|
+
assert "creating archives" not in caplog.text
|
|
110
|
+
assert "found 0 repos" in caplog.text
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@pytest.mark.parametrize("archive_format", ["zip", "tar"])
|
|
114
|
+
def test_run_archive_1_match(archive_format, tmp_path, caplog):
|
|
115
|
+
caplog.set_level("INFO")
|
|
116
|
+
backup_dir = "backups"
|
|
117
|
+
repo = "githubtakeout"
|
|
118
|
+
githubtakeout.run(
|
|
119
|
+
username=USER,
|
|
120
|
+
base_dir=tmp_path,
|
|
121
|
+
pattern=repo,
|
|
122
|
+
skip_pattern=None,
|
|
123
|
+
archive_format=archive_format,
|
|
124
|
+
include_gists=False,
|
|
125
|
+
include_history=False,
|
|
126
|
+
skip_forks=False,
|
|
127
|
+
keep=False,
|
|
128
|
+
list_only=False,
|
|
129
|
+
prompt_for_token=False,
|
|
130
|
+
)
|
|
131
|
+
assert re.search(f"creating archives in: .*{backup_dir}", caplog.text)
|
|
132
|
+
assert f"found 1 repos for user '{USER}'" in caplog.text
|
|
133
|
+
assert "gists" not in caplog.text
|
|
134
|
+
assert re.search(f"cloning repo: {USER}/{repo}.git to: .*{backup_dir}", caplog.text)
|
|
135
|
+
assert "creating archive:" in caplog.text
|
|
136
|
+
assert "archive size:" in caplog.text
|
|
137
|
+
assert "deleting repo" in caplog.text
|
|
138
|
+
assert f"successfully backed up '{repo}' repo" in caplog.text
|
|
139
|
+
assert not Path(tmp_path / backup_dir / repo).exists()
|
|
140
|
+
if archive_format == "tar":
|
|
141
|
+
extension = "tar.gz"
|
|
142
|
+
else:
|
|
143
|
+
extension = archive_format
|
|
144
|
+
assert Path(tmp_path / backup_dir / f"{repo}.{extension}").exists()
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def test_run_no_archive_1_match_with_history(tmp_path, caplog):
|
|
148
|
+
caplog.set_level("INFO")
|
|
149
|
+
backup_dir = "backups"
|
|
150
|
+
repo = "githubtakeout"
|
|
151
|
+
githubtakeout.run(
|
|
152
|
+
username=USER,
|
|
153
|
+
base_dir=tmp_path,
|
|
154
|
+
pattern=repo,
|
|
155
|
+
skip_pattern=None,
|
|
156
|
+
archive_format="none",
|
|
157
|
+
include_gists=False,
|
|
158
|
+
include_history=True,
|
|
159
|
+
skip_forks=False,
|
|
160
|
+
keep=False,
|
|
161
|
+
list_only=False,
|
|
162
|
+
prompt_for_token=False,
|
|
163
|
+
)
|
|
164
|
+
assert re.search(f"creating archives in: .*{backup_dir}", caplog.text)
|
|
165
|
+
assert f"found 1 repos for user '{USER}'" in caplog.text
|
|
166
|
+
assert "gists" not in caplog.text
|
|
167
|
+
assert re.search(f"cloning repo: {USER}/{repo}.git to: .*{backup_dir}", caplog.text)
|
|
168
|
+
assert "creating archive:" not in caplog.text
|
|
169
|
+
assert "archive size:" not in caplog.text
|
|
170
|
+
assert "deleting repo" not in caplog.text
|
|
171
|
+
assert f"successfully backed up '{repo}' repo" in caplog.text
|
|
172
|
+
assert Path(tmp_path / backup_dir / repo).exists()
|
|
173
|
+
assert Path(tmp_path / backup_dir / repo / ".git").exists()
|
|
174
|
+
for extension in ("tar.gz", "zip"):
|
|
175
|
+
assert not Path(tmp_path / backup_dir / f"{repo}.{extension}").exists()
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def test_run_no_archive_1_match_without_history(tmp_path, caplog):
|
|
179
|
+
caplog.set_level("INFO")
|
|
180
|
+
backup_dir = "backups"
|
|
181
|
+
repo = "githubtakeout"
|
|
182
|
+
githubtakeout.run(
|
|
183
|
+
username=USER,
|
|
184
|
+
base_dir=tmp_path,
|
|
185
|
+
pattern=repo,
|
|
186
|
+
skip_pattern=None,
|
|
187
|
+
archive_format="none",
|
|
188
|
+
include_gists=False,
|
|
189
|
+
include_history=False,
|
|
190
|
+
skip_forks=False,
|
|
191
|
+
keep=False, # this doesn't matter when archive_format is "none"
|
|
192
|
+
list_only=False,
|
|
193
|
+
prompt_for_token=False,
|
|
194
|
+
)
|
|
195
|
+
assert re.search("creating archives in: .*backups", caplog.text)
|
|
196
|
+
assert f"found 1 repos for user '{USER}'" in caplog.text
|
|
197
|
+
assert "gists" not in caplog.text
|
|
198
|
+
assert re.search(f"cloning repo: {USER}/{repo}.git to: .*backups", caplog.text)
|
|
199
|
+
assert "creating archive:" not in caplog.text
|
|
200
|
+
assert "archive size:" not in caplog.text
|
|
201
|
+
assert "deleting repo" not in caplog.text
|
|
202
|
+
assert f"successfully backed up '{repo}' repo" in caplog.text
|
|
203
|
+
assert Path(tmp_path / backup_dir / repo).exists()
|
|
204
|
+
assert not Path(tmp_path / backup_dir / repo / ".git").exists()
|
|
205
|
+
for extension in ("tar.gz", "zip"):
|
|
206
|
+
assert not Path(tmp_path / backup_dir / f"{repo}.{extension}").exists()
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def test_run_archive_1_match_with_history_pull_with_keep(tmp_path, caplog):
|
|
210
|
+
caplog.set_level("INFO")
|
|
211
|
+
backup_dir = "backups"
|
|
212
|
+
repo = "githubtakeout"
|
|
213
|
+
githubtakeout.run(
|
|
214
|
+
username=USER,
|
|
215
|
+
base_dir=tmp_path,
|
|
216
|
+
pattern=repo,
|
|
217
|
+
skip_pattern=None,
|
|
218
|
+
archive_format="none",
|
|
219
|
+
include_gists=False,
|
|
220
|
+
include_history=True,
|
|
221
|
+
skip_forks=False,
|
|
222
|
+
keep=True,
|
|
223
|
+
list_only=False,
|
|
224
|
+
prompt_for_token=False,
|
|
225
|
+
)
|
|
226
|
+
assert f"found 1 repos for user '{USER}'" in caplog.text
|
|
227
|
+
assert "gists" not in caplog.text
|
|
228
|
+
assert re.search(f"cloning repo: {USER}/{repo}.git to: .*backups", caplog.text)
|
|
229
|
+
assert "pulling changes" not in caplog.text
|
|
230
|
+
assert "creating archive:" not in caplog.text
|
|
231
|
+
assert "archive size:" not in caplog.text
|
|
232
|
+
assert "deleting repo" not in caplog.text
|
|
233
|
+
assert f"successfully backed up '{repo}' repo" in caplog.text
|
|
234
|
+
assert Path(tmp_path / backup_dir / repo).exists()
|
|
235
|
+
assert Path(tmp_path / backup_dir / repo / ".git").exists()
|
|
236
|
+
caplog.clear()
|
|
237
|
+
githubtakeout.run(
|
|
238
|
+
username=USER,
|
|
239
|
+
base_dir=tmp_path,
|
|
240
|
+
pattern=repo,
|
|
241
|
+
skip_pattern=None,
|
|
242
|
+
archive_format="zip",
|
|
243
|
+
include_gists=False,
|
|
244
|
+
include_history=True,
|
|
245
|
+
skip_forks=False,
|
|
246
|
+
keep=True,
|
|
247
|
+
list_only=False,
|
|
248
|
+
prompt_for_token=False,
|
|
249
|
+
)
|
|
250
|
+
assert f"found 1 repos for user '{USER}'" in caplog.text
|
|
251
|
+
assert "gists" not in caplog.text
|
|
252
|
+
assert not re.search(f"cloning repo: {USER}/{repo}.git to: .*backups", caplog.text)
|
|
253
|
+
assert re.search(
|
|
254
|
+
f"pulling changes from repo: {USER}/{repo}.git to: .*backups", caplog.text
|
|
255
|
+
)
|
|
256
|
+
assert "creating archive:" in caplog.text
|
|
257
|
+
assert "archive size:" in caplog.text
|
|
258
|
+
assert "deleting repo" not in caplog.text
|
|
259
|
+
assert f"successfully backed up '{repo}' repo" in caplog.text
|
|
260
|
+
assert Path(tmp_path / backup_dir / repo).exists()
|
|
261
|
+
assert Path(tmp_path / backup_dir / repo / ".git").exists()
|
|
262
|
+
assert Path(tmp_path / backup_dir / f"{repo}.zip").exists()
|
|
File without changes
|
|
File without changes
|
{githubtakeout-0.1.15 → githubtakeout-0.1.17}/src/githubtakeout.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|