githubtakeout 0.1.14__tar.gz → 0.1.16__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {githubtakeout-0.1.14/src/githubtakeout.egg-info → githubtakeout-0.1.16}/PKG-INFO +31 -13
- {githubtakeout-0.1.14 → githubtakeout-0.1.16}/README.md +29 -12
- {githubtakeout-0.1.14 → githubtakeout-0.1.16}/pyproject.toml +9 -1
- {githubtakeout-0.1.14 → githubtakeout-0.1.16/src/githubtakeout.egg-info}/PKG-INFO +31 -13
- {githubtakeout-0.1.14 → githubtakeout-0.1.16}/src/githubtakeout.egg-info/SOURCES.txt +2 -1
- {githubtakeout-0.1.14 → githubtakeout-0.1.16}/src/githubtakeout.egg-info/requires.txt +1 -0
- {githubtakeout-0.1.14 → githubtakeout-0.1.16}/src/githubtakeout.py +74 -22
- githubtakeout-0.1.16/tests/test_githubtakeout.py +246 -0
- {githubtakeout-0.1.14 → githubtakeout-0.1.16}/LICENSE +0 -0
- {githubtakeout-0.1.14 → githubtakeout-0.1.16}/setup.cfg +0 -0
- {githubtakeout-0.1.14 → githubtakeout-0.1.16}/src/githubtakeout.egg-info/dependency_links.txt +0 -0
- {githubtakeout-0.1.14 → githubtakeout-0.1.16}/src/githubtakeout.egg-info/entry_points.txt +0 -0
- {githubtakeout-0.1.14 → githubtakeout-0.1.16}/src/githubtakeout.egg-info/top_level.txt +0 -0
- {githubtakeout-0.1.14 → githubtakeout-0.1.16}/src/progress.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: githubtakeout
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.16
|
|
4
4
|
Summary: Backup and archive Git Repos and Gists from GitHub
|
|
5
5
|
Author: Corey Goldberg
|
|
6
6
|
Maintainer: Corey Goldberg
|
|
@@ -21,15 +21,18 @@ Description-Content-Type: text/markdown
|
|
|
21
21
|
License-File: LICENSE
|
|
22
22
|
Requires-Dist: GitPython==3.1.45
|
|
23
23
|
Requires-Dist: PyGithub==2.8.1
|
|
24
|
+
Requires-Dist: python-dotenv==1.2.1
|
|
24
25
|
Requires-Dist: rich==14.2.0
|
|
25
26
|
Dynamic: license-file
|
|
26
27
|
|
|
27
28
|
# githubtakeout
|
|
28
29
|
|
|
29
|
-
|
|
30
|
+
### Backup and archive Git Repos and Gists from GitHub
|
|
30
31
|
|
|
31
32
|
---
|
|
32
33
|
|
|
34
|
+
[](https://pypi.org/project/githubtakeout)
|
|
35
|
+
|
|
33
36
|
- Copyright (c) 2015-2025 [Corey Goldberg][github-home]
|
|
34
37
|
- Development: [GitHub][github-repo]
|
|
35
38
|
- Download/Install: [PyPI][pypi-githubtakeout]
|
|
@@ -55,6 +58,10 @@ Archives are saved in compressed zip format (`.zip`) by default. You can also
|
|
|
55
58
|
save as tarballs (`.tar.gz`) using the `--format=tar` option, or skip archiving
|
|
56
59
|
using the `--format=none` option.
|
|
57
60
|
|
|
61
|
+
If a repo with history exists from a previous run, it will pull new changes.
|
|
62
|
+
Otherwise, it will clone the repo. Use the `--keep` if you don't want repos
|
|
63
|
+
deleted after an archive is created.
|
|
64
|
+
|
|
58
65
|
## Requirements:
|
|
59
66
|
|
|
60
67
|
- Python 3.12+
|
|
@@ -81,26 +88,37 @@ can set the `GITHUB_TOKEN` environment variable:
|
|
|
81
88
|
$ export GITHUB_TOKEN=<access token>
|
|
82
89
|
```
|
|
83
90
|
|
|
91
|
+
You can also set this in an `.env` file in the current directory. It should
|
|
92
|
+
contain:
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
GITHUB_TOKEN=<access token>
|
|
96
|
+
```
|
|
97
|
+
|
|
84
98
|
If you prefer to be prompted for your token each time you run the program, use
|
|
85
99
|
the `--token` argument.
|
|
86
100
|
|
|
87
101
|
## CLI Options:
|
|
88
102
|
|
|
89
103
|
```
|
|
90
|
-
usage: githubtakeout [-h] [--dir DIR] [--pattern PATTERN] [--format {tar,zip,none}]
|
|
104
|
+
usage: githubtakeout [-h] [--dir DIR] [--pattern PATTERN] [--format {tar,zip,none}]
|
|
105
|
+
[--gists] [--history] [--keep] [--list] [--token]
|
|
106
|
+
username
|
|
91
107
|
|
|
92
108
|
positional arguments:
|
|
93
|
-
username
|
|
109
|
+
username github username
|
|
94
110
|
|
|
95
111
|
options:
|
|
96
|
-
-h, --help
|
|
97
|
-
--dir DIR
|
|
98
|
-
--pattern PATTERN
|
|
99
|
-
--
|
|
100
|
-
--
|
|
101
|
-
--
|
|
102
|
-
--
|
|
103
|
-
--
|
|
112
|
+
-h, --help show this help message and exit
|
|
113
|
+
--dir DIR output directory (default: .)
|
|
114
|
+
--pattern PATTERN regex matching repo names to include
|
|
115
|
+
--skip_pattern PATTERN regex matching repo names to skip
|
|
116
|
+
--format {tar,zip,none} archive format (default: zip)
|
|
117
|
+
--gists include gists
|
|
118
|
+
--history include commit history and branches (.git directory)
|
|
119
|
+
--keep keep repos after archiving
|
|
120
|
+
--list list repos only
|
|
121
|
+
--token prompt for auth token
|
|
104
122
|
```
|
|
105
123
|
|
|
106
124
|
## Usage Examples:
|
|
@@ -124,7 +142,7 @@ githubtakeout <github username>
|
|
|
124
142
|
```
|
|
125
143
|
|
|
126
144
|
[github-home]: https://github.com/cgoldberg
|
|
127
|
-
[github-repo]: https://github.com/cgoldberg/
|
|
145
|
+
[github-repo]: https://github.com/cgoldberg/githubtakeout
|
|
128
146
|
[github-pat]: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens
|
|
129
147
|
[pypi-githubtakeout]: https://pypi.org/project/githubtakeout
|
|
130
148
|
[mit-license]: https://raw.githubusercontent.com/cgoldberg/githubtakeout/refs/heads/master/LICENSE
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
# githubtakeout
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
### Backup and archive Git Repos and Gists from GitHub
|
|
4
4
|
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
+
[](https://pypi.org/project/githubtakeout)
|
|
8
|
+
|
|
7
9
|
- Copyright (c) 2015-2025 [Corey Goldberg][github-home]
|
|
8
10
|
- Development: [GitHub][github-repo]
|
|
9
11
|
- Download/Install: [PyPI][pypi-githubtakeout]
|
|
@@ -29,6 +31,10 @@ Archives are saved in compressed zip format (`.zip`) by default. You can also
|
|
|
29
31
|
save as tarballs (`.tar.gz`) using the `--format=tar` option, or skip archiving
|
|
30
32
|
using the `--format=none` option.
|
|
31
33
|
|
|
34
|
+
If a repo with history exists from a previous run, it will pull new changes.
|
|
35
|
+
Otherwise, it will clone the repo. Use the `--keep` if you don't want repos
|
|
36
|
+
deleted after an archive is created.
|
|
37
|
+
|
|
32
38
|
## Requirements:
|
|
33
39
|
|
|
34
40
|
- Python 3.12+
|
|
@@ -55,26 +61,37 @@ can set the `GITHUB_TOKEN` environment variable:
|
|
|
55
61
|
$ export GITHUB_TOKEN=<access token>
|
|
56
62
|
```
|
|
57
63
|
|
|
64
|
+
You can also set this in an `.env` file in the current directory. It should
|
|
65
|
+
contain:
|
|
66
|
+
|
|
67
|
+
```
|
|
68
|
+
GITHUB_TOKEN=<access token>
|
|
69
|
+
```
|
|
70
|
+
|
|
58
71
|
If you prefer to be prompted for your token each time you run the program, use
|
|
59
72
|
the `--token` argument.
|
|
60
73
|
|
|
61
74
|
## CLI Options:
|
|
62
75
|
|
|
63
76
|
```
|
|
64
|
-
usage: githubtakeout [-h] [--dir DIR] [--pattern PATTERN] [--format {tar,zip,none}]
|
|
77
|
+
usage: githubtakeout [-h] [--dir DIR] [--pattern PATTERN] [--format {tar,zip,none}]
|
|
78
|
+
[--gists] [--history] [--keep] [--list] [--token]
|
|
79
|
+
username
|
|
65
80
|
|
|
66
81
|
positional arguments:
|
|
67
|
-
username
|
|
82
|
+
username github username
|
|
68
83
|
|
|
69
84
|
options:
|
|
70
|
-
-h, --help
|
|
71
|
-
--dir DIR
|
|
72
|
-
--pattern PATTERN
|
|
73
|
-
--
|
|
74
|
-
--
|
|
75
|
-
--
|
|
76
|
-
--
|
|
77
|
-
--
|
|
85
|
+
-h, --help show this help message and exit
|
|
86
|
+
--dir DIR output directory (default: .)
|
|
87
|
+
--pattern PATTERN regex matching repo names to include
|
|
88
|
+
--skip_pattern PATTERN regex matching repo names to skip
|
|
89
|
+
--format {tar,zip,none} archive format (default: zip)
|
|
90
|
+
--gists include gists
|
|
91
|
+
--history include commit history and branches (.git directory)
|
|
92
|
+
--keep keep repos after archiving
|
|
93
|
+
--list list repos only
|
|
94
|
+
--token prompt for auth token
|
|
78
95
|
```
|
|
79
96
|
|
|
80
97
|
## Usage Examples:
|
|
@@ -98,7 +115,7 @@ githubtakeout <github username>
|
|
|
98
115
|
```
|
|
99
116
|
|
|
100
117
|
[github-home]: https://github.com/cgoldberg
|
|
101
|
-
[github-repo]: https://github.com/cgoldberg/
|
|
118
|
+
[github-repo]: https://github.com/cgoldberg/githubtakeout
|
|
102
119
|
[github-pat]: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens
|
|
103
120
|
[pypi-githubtakeout]: https://pypi.org/project/githubtakeout
|
|
104
121
|
[mit-license]: https://raw.githubusercontent.com/cgoldberg/githubtakeout/refs/heads/master/LICENSE
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "githubtakeout"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.16"
|
|
8
8
|
description = "Backup and archive Git Repos and Gists from GitHub"
|
|
9
9
|
license = "MIT"
|
|
10
10
|
license-files = ["LICENSE"]
|
|
@@ -25,6 +25,7 @@ classifiers = [
|
|
|
25
25
|
dependencies = [
|
|
26
26
|
"GitPython==3.1.45",
|
|
27
27
|
"PyGithub==2.8.1",
|
|
28
|
+
"python-dotenv==1.2.1",
|
|
28
29
|
"rich==14.2.0",
|
|
29
30
|
]
|
|
30
31
|
|
|
@@ -45,6 +46,9 @@ lint = [
|
|
|
45
46
|
"refurb",
|
|
46
47
|
"ruff",
|
|
47
48
|
]
|
|
49
|
+
test = [
|
|
50
|
+
"pytest",
|
|
51
|
+
]
|
|
48
52
|
validate = [
|
|
49
53
|
"packaging",
|
|
50
54
|
"validate-pyproject",
|
|
@@ -53,6 +57,10 @@ validate = [
|
|
|
53
57
|
[tool.setuptools]
|
|
54
58
|
package-dir = {"" = "src"}
|
|
55
59
|
|
|
60
|
+
[tool.pytest.ini_options]
|
|
61
|
+
addopts = "-vv" # extra verbose
|
|
62
|
+
tmp_path_retention_policy = "none"
|
|
63
|
+
|
|
56
64
|
[tool.black]
|
|
57
65
|
line-length = 120
|
|
58
66
|
target-version = [
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: githubtakeout
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.16
|
|
4
4
|
Summary: Backup and archive Git Repos and Gists from GitHub
|
|
5
5
|
Author: Corey Goldberg
|
|
6
6
|
Maintainer: Corey Goldberg
|
|
@@ -21,15 +21,18 @@ Description-Content-Type: text/markdown
|
|
|
21
21
|
License-File: LICENSE
|
|
22
22
|
Requires-Dist: GitPython==3.1.45
|
|
23
23
|
Requires-Dist: PyGithub==2.8.1
|
|
24
|
+
Requires-Dist: python-dotenv==1.2.1
|
|
24
25
|
Requires-Dist: rich==14.2.0
|
|
25
26
|
Dynamic: license-file
|
|
26
27
|
|
|
27
28
|
# githubtakeout
|
|
28
29
|
|
|
29
|
-
|
|
30
|
+
### Backup and archive Git Repos and Gists from GitHub
|
|
30
31
|
|
|
31
32
|
---
|
|
32
33
|
|
|
34
|
+
[](https://pypi.org/project/githubtakeout)
|
|
35
|
+
|
|
33
36
|
- Copyright (c) 2015-2025 [Corey Goldberg][github-home]
|
|
34
37
|
- Development: [GitHub][github-repo]
|
|
35
38
|
- Download/Install: [PyPI][pypi-githubtakeout]
|
|
@@ -55,6 +58,10 @@ Archives are saved in compressed zip format (`.zip`) by default. You can also
|
|
|
55
58
|
save as tarballs (`.tar.gz`) using the `--format=tar` option, or skip archiving
|
|
56
59
|
using the `--format=none` option.
|
|
57
60
|
|
|
61
|
+
If a repo with history exists from a previous run, it will pull new changes.
|
|
62
|
+
Otherwise, it will clone the repo. Use the `--keep` if you don't want repos
|
|
63
|
+
deleted after an archive is created.
|
|
64
|
+
|
|
58
65
|
## Requirements:
|
|
59
66
|
|
|
60
67
|
- Python 3.12+
|
|
@@ -81,26 +88,37 @@ can set the `GITHUB_TOKEN` environment variable:
|
|
|
81
88
|
$ export GITHUB_TOKEN=<access token>
|
|
82
89
|
```
|
|
83
90
|
|
|
91
|
+
You can also set this in an `.env` file in the current directory. It should
|
|
92
|
+
contain:
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
GITHUB_TOKEN=<access token>
|
|
96
|
+
```
|
|
97
|
+
|
|
84
98
|
If you prefer to be prompted for your token each time you run the program, use
|
|
85
99
|
the `--token` argument.
|
|
86
100
|
|
|
87
101
|
## CLI Options:
|
|
88
102
|
|
|
89
103
|
```
|
|
90
|
-
usage: githubtakeout [-h] [--dir DIR] [--pattern PATTERN] [--format {tar,zip,none}]
|
|
104
|
+
usage: githubtakeout [-h] [--dir DIR] [--pattern PATTERN] [--format {tar,zip,none}]
|
|
105
|
+
[--gists] [--history] [--keep] [--list] [--token]
|
|
106
|
+
username
|
|
91
107
|
|
|
92
108
|
positional arguments:
|
|
93
|
-
username
|
|
109
|
+
username github username
|
|
94
110
|
|
|
95
111
|
options:
|
|
96
|
-
-h, --help
|
|
97
|
-
--dir DIR
|
|
98
|
-
--pattern PATTERN
|
|
99
|
-
--
|
|
100
|
-
--
|
|
101
|
-
--
|
|
102
|
-
--
|
|
103
|
-
--
|
|
112
|
+
-h, --help show this help message and exit
|
|
113
|
+
--dir DIR output directory (default: .)
|
|
114
|
+
--pattern PATTERN regex matching repo names to include
|
|
115
|
+
--skip_pattern PATTERN regex matching repo names to skip
|
|
116
|
+
--format {tar,zip,none} archive format (default: zip)
|
|
117
|
+
--gists include gists
|
|
118
|
+
--history include commit history and branches (.git directory)
|
|
119
|
+
--keep keep repos after archiving
|
|
120
|
+
--list list repos only
|
|
121
|
+
--token prompt for auth token
|
|
104
122
|
```
|
|
105
123
|
|
|
106
124
|
## Usage Examples:
|
|
@@ -124,7 +142,7 @@ githubtakeout <github username>
|
|
|
124
142
|
```
|
|
125
143
|
|
|
126
144
|
[github-home]: https://github.com/cgoldberg
|
|
127
|
-
[github-repo]: https://github.com/cgoldberg/
|
|
145
|
+
[github-repo]: https://github.com/cgoldberg/githubtakeout
|
|
128
146
|
[github-pat]: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens
|
|
129
147
|
[pypi-githubtakeout]: https://pypi.org/project/githubtakeout
|
|
130
148
|
[mit-license]: https://raw.githubusercontent.com/cgoldberg/githubtakeout/refs/heads/master/LICENSE
|
|
@@ -8,4 +8,5 @@ src/githubtakeout.egg-info/SOURCES.txt
|
|
|
8
8
|
src/githubtakeout.egg-info/dependency_links.txt
|
|
9
9
|
src/githubtakeout.egg-info/entry_points.txt
|
|
10
10
|
src/githubtakeout.egg-info/requires.txt
|
|
11
|
-
src/githubtakeout.egg-info/top_level.txt
|
|
11
|
+
src/githubtakeout.egg-info/top_level.txt
|
|
12
|
+
tests/test_githubtakeout.py
|
|
@@ -20,6 +20,7 @@ from timeit import default_timer
|
|
|
20
20
|
|
|
21
21
|
import git
|
|
22
22
|
import github
|
|
23
|
+
from dotenv import load_dotenv
|
|
23
24
|
|
|
24
25
|
from progress import GitProgress
|
|
25
26
|
|
|
@@ -104,7 +105,25 @@ def clone(repo_url, local_repo_dir, include_history):
|
|
|
104
105
|
pass
|
|
105
106
|
|
|
106
107
|
|
|
107
|
-
def
|
|
108
|
+
def pull(local_repo_dir):
|
|
109
|
+
try:
|
|
110
|
+
repo = git.Repo(local_repo_dir)
|
|
111
|
+
origin = repo.remotes.origin
|
|
112
|
+
origin.fetch()
|
|
113
|
+
origin.pull(progress=GitProgress())
|
|
114
|
+
except git.GitCommandError as e:
|
|
115
|
+
logger.error(e)
|
|
116
|
+
sys.exit("error: failed pulling changes in repo")
|
|
117
|
+
finally:
|
|
118
|
+
try:
|
|
119
|
+
# release resources
|
|
120
|
+
repo.close()
|
|
121
|
+
except UnboundLocalError:
|
|
122
|
+
# this occurs if we catch a signal while pulling
|
|
123
|
+
pass
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def get_and_archive_repo(repo_url, local_repo_dir, archive_format, include_history, keep, is_gist=False):
|
|
108
127
|
def remove_readonly(func, path, _):
|
|
109
128
|
# This is necessary so rmtree() doesn't fail if there are any readonly
|
|
110
129
|
# dirs/files when trying to delete. This seems to happen after cloning on
|
|
@@ -118,19 +137,27 @@ def clone_and_archive_repo(repo_url, local_repo_dir, archive_format, include_his
|
|
|
118
137
|
# try again
|
|
119
138
|
func(path)
|
|
120
139
|
|
|
121
|
-
try:
|
|
122
|
-
# delete repo if it already exists
|
|
123
|
-
shutil.rmtree(local_repo_dir, onexc=remove_readonly)
|
|
124
|
-
except FileNotFoundError:
|
|
125
|
-
pass
|
|
126
140
|
repo_name = urllib.parse.urlparse(repo_url).path.lstrip("/")
|
|
127
|
-
|
|
141
|
+
# we can only pull if the local repo exists and has a .git directory
|
|
142
|
+
if Path(local_repo_dir, ".git").exists():
|
|
143
|
+
needs_clone = False
|
|
144
|
+
else:
|
|
145
|
+
needs_clone = True
|
|
146
|
+
try:
|
|
147
|
+
shutil.rmtree(local_repo_dir, onexc=remove_readonly)
|
|
148
|
+
except FileNotFoundError:
|
|
149
|
+
pass
|
|
128
150
|
start = default_timer()
|
|
129
|
-
|
|
151
|
+
if needs_clone:
|
|
152
|
+
logger.info(f"cloning repo: {repo_name} to: {local_repo_dir}")
|
|
153
|
+
clone(repo_url, local_repo_dir, include_history)
|
|
154
|
+
else:
|
|
155
|
+
logger.info(f"pulling changes from repo: {repo_name} to: {local_repo_dir}")
|
|
156
|
+
pull(local_repo_dir)
|
|
130
157
|
if not include_history:
|
|
131
158
|
# delete the .git directory if we are not saving history
|
|
132
159
|
try:
|
|
133
|
-
git_dir =
|
|
160
|
+
git_dir = Path(local_repo_dir, ".git")
|
|
134
161
|
shutil.rmtree(git_dir, onexc=remove_readonly)
|
|
135
162
|
except FileNotFoundError:
|
|
136
163
|
pass
|
|
@@ -138,12 +165,13 @@ def clone_and_archive_repo(repo_url, local_repo_dir, archive_format, include_his
|
|
|
138
165
|
if archive_path:
|
|
139
166
|
size = convert_size(os.path.getsize(archive_path))
|
|
140
167
|
logger.info(f"archive size: {size}")
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
168
|
+
if not keep:
|
|
169
|
+
logger.info("deleting repo")
|
|
170
|
+
try:
|
|
171
|
+
# delete repo after archive is created
|
|
172
|
+
shutil.rmtree(local_repo_dir, onexc=remove_readonly)
|
|
173
|
+
except FileNotFoundError:
|
|
174
|
+
pass
|
|
147
175
|
elapsed = default_timer() - start
|
|
148
176
|
base_name = os.path.basename(local_repo_dir)
|
|
149
177
|
logger.info(f"successfully backed up '{base_name}' repo in {elapsed:.3f} secs\n")
|
|
@@ -188,6 +216,8 @@ def get_token(prompt_for_token):
|
|
|
188
216
|
if not token:
|
|
189
217
|
sys.exit("error: auth token cannot be empty")
|
|
190
218
|
else:
|
|
219
|
+
# populate environment variable via .env file if it exists
|
|
220
|
+
load_dotenv()
|
|
191
221
|
token = os.getenv("GITHUB_TOKEN")
|
|
192
222
|
return token
|
|
193
223
|
|
|
@@ -196,39 +226,53 @@ def run(
|
|
|
196
226
|
username,
|
|
197
227
|
base_dir,
|
|
198
228
|
pattern,
|
|
229
|
+
skip_pattern,
|
|
199
230
|
archive_format,
|
|
200
231
|
include_gists,
|
|
201
232
|
include_history,
|
|
233
|
+
keep,
|
|
202
234
|
list_only,
|
|
203
235
|
prompt_for_token,
|
|
204
236
|
):
|
|
205
237
|
working_dir = os.path.join(base_dir, "backups")
|
|
206
238
|
token = get_token(prompt_for_token)
|
|
207
239
|
repos, gists = get_repos(username, token, include_gists)
|
|
208
|
-
num_repos = len(
|
|
240
|
+
num_repos = len(
|
|
241
|
+
[
|
|
242
|
+
repo
|
|
243
|
+
for repo in repos
|
|
244
|
+
if re.match(pattern, repo.name) and not (re.match(skip_pattern, repo.name) if skip_pattern else False)
|
|
245
|
+
]
|
|
246
|
+
)
|
|
209
247
|
if not list_only:
|
|
210
248
|
logger.info(f"creating archives in: {working_dir}\n")
|
|
211
249
|
logger.info(f"found {num_repos} repos for user '{username}':\n")
|
|
212
250
|
for repo in repos:
|
|
213
|
-
if re.match(pattern, repo.name):
|
|
251
|
+
if re.match(pattern, repo.name) and not (re.match(skip_pattern, repo.name) if skip_pattern else False):
|
|
214
252
|
local_repo_dir = os.path.join(working_dir, repo.name)
|
|
215
253
|
url = add_creds(repo.clone_url, username, token)
|
|
216
254
|
if list_only:
|
|
217
255
|
logger.info(f"{username}/{repo.name}")
|
|
218
256
|
else:
|
|
219
|
-
|
|
257
|
+
get_and_archive_repo(url, local_repo_dir, archive_format, include_history, keep)
|
|
220
258
|
if gists is not None:
|
|
221
|
-
num_gists = len(
|
|
259
|
+
num_gists = len(
|
|
260
|
+
[
|
|
261
|
+
gist
|
|
262
|
+
for gist in gists
|
|
263
|
+
if re.match(pattern, gist.id) and not (re.match(skip_pattern, gist.id) if skip_pattern else False)
|
|
264
|
+
]
|
|
265
|
+
)
|
|
222
266
|
logger.info("")
|
|
223
267
|
logger.info(f"found {num_gists} gists for user '{username}':\n")
|
|
224
268
|
for gist in gists:
|
|
225
|
-
if re.match(pattern, gist.id):
|
|
269
|
+
if re.match(pattern, gist.id) and not (re.match(skip_pattern, gist.id) if skip_pattern else False):
|
|
226
270
|
local_repo_dir = os.path.join(working_dir, gist.id)
|
|
227
271
|
url = add_creds(gist.git_pull_url, username, token)
|
|
228
272
|
if list_only:
|
|
229
273
|
logger.info(f"{username}/{gist.id}\n - {gist.description}")
|
|
230
274
|
else:
|
|
231
|
-
|
|
275
|
+
get_and_archive_repo(url, local_repo_dir, archive_format, include_history, keep, is_gist=True)
|
|
232
276
|
|
|
233
277
|
|
|
234
278
|
def main():
|
|
@@ -248,7 +292,12 @@ def main():
|
|
|
248
292
|
parser.add_argument(
|
|
249
293
|
"--pattern",
|
|
250
294
|
default=".*",
|
|
251
|
-
help="regex matching repo names
|
|
295
|
+
help="regex matching repo names to include",
|
|
296
|
+
)
|
|
297
|
+
parser.add_argument(
|
|
298
|
+
"--skip_pattern",
|
|
299
|
+
metavar="PATTERN",
|
|
300
|
+
help="regex matching repo names to skip",
|
|
252
301
|
)
|
|
253
302
|
parser.add_argument(
|
|
254
303
|
"--format",
|
|
@@ -263,6 +312,7 @@ def main():
|
|
|
263
312
|
default=False,
|
|
264
313
|
help="include commit history and branches (.git directory)",
|
|
265
314
|
)
|
|
315
|
+
parser.add_argument("--keep", action="store_true", default=False, help="keep repos after archiving")
|
|
266
316
|
parser.add_argument("--list", action="store_true", default=False, help="list repos only")
|
|
267
317
|
parser.add_argument("--token", action="store_true", default=False, help="prompt for auth token")
|
|
268
318
|
args = parser.parse_args()
|
|
@@ -271,9 +321,11 @@ def main():
|
|
|
271
321
|
username=args.username,
|
|
272
322
|
base_dir=args.dir,
|
|
273
323
|
pattern=args.pattern,
|
|
324
|
+
skip_pattern=args.skip_pattern,
|
|
274
325
|
archive_format=args.format,
|
|
275
326
|
include_gists=args.gists,
|
|
276
327
|
include_history=args.history,
|
|
328
|
+
keep=args.keep,
|
|
277
329
|
list_only=args.list,
|
|
278
330
|
prompt_for_token=args.token,
|
|
279
331
|
)
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
from dotenv import load_dotenv
|
|
7
|
+
|
|
8
|
+
import githubtakeout
|
|
9
|
+
|
|
10
|
+
USER = "cgoldberg"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@pytest.fixture(autouse=True, scope="session")
|
|
14
|
+
def require_token():
|
|
15
|
+
load_dotenv()
|
|
16
|
+
required = "GITHUB_TOKEN"
|
|
17
|
+
if not os.getenv(required):
|
|
18
|
+
pytest.skip(reason=f"{required} environment variable or .env file is required to avoid rate limiting")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@pytest.mark.skipif(os.getenv("GITHUB_TOKEN"), reason="GITHUB_TOKEN required to avoid rate limiting")
|
|
22
|
+
def test_run_list_1_match(tmp_path, caplog):
|
|
23
|
+
caplog.set_level("INFO")
|
|
24
|
+
repo = "githubtakeout"
|
|
25
|
+
githubtakeout.run(
|
|
26
|
+
username=USER,
|
|
27
|
+
base_dir=tmp_path,
|
|
28
|
+
pattern=repo,
|
|
29
|
+
skip_pattern=None,
|
|
30
|
+
archive_format="none",
|
|
31
|
+
include_gists=False,
|
|
32
|
+
include_history=False,
|
|
33
|
+
keep=False,
|
|
34
|
+
list_only=True,
|
|
35
|
+
prompt_for_token=False,
|
|
36
|
+
)
|
|
37
|
+
assert "creating archives" not in caplog.text
|
|
38
|
+
assert f"found 1 repos for user '{USER}'" in caplog.text
|
|
39
|
+
assert "gists" not in caplog.text
|
|
40
|
+
assert f"{USER}/{repo}" in caplog.text
|
|
41
|
+
assert not Path(tmp_path / "backups" / repo).exists()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_run_list_0_match(tmp_path, caplog):
|
|
45
|
+
caplog.set_level("INFO")
|
|
46
|
+
repo = "this_repo_does_not_exist"
|
|
47
|
+
githubtakeout.run(
|
|
48
|
+
username=USER,
|
|
49
|
+
base_dir=tmp_path,
|
|
50
|
+
pattern=repo,
|
|
51
|
+
skip_pattern=None,
|
|
52
|
+
archive_format="none",
|
|
53
|
+
include_gists=False,
|
|
54
|
+
include_history=False,
|
|
55
|
+
keep=False,
|
|
56
|
+
list_only=True,
|
|
57
|
+
prompt_for_token=False,
|
|
58
|
+
)
|
|
59
|
+
assert "creating archives" not in caplog.text
|
|
60
|
+
assert f"found 0 repos for user '{USER}'" in caplog.text
|
|
61
|
+
assert "gists" not in caplog.text
|
|
62
|
+
assert not Path(tmp_path / "backups" / repo).exists()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_run_list_0_match_with_gists(tmp_path, caplog):
|
|
66
|
+
caplog.set_level("INFO")
|
|
67
|
+
repo = "this_repo_does_not_exist"
|
|
68
|
+
githubtakeout.run(
|
|
69
|
+
username=USER,
|
|
70
|
+
base_dir=tmp_path,
|
|
71
|
+
pattern=repo,
|
|
72
|
+
skip_pattern=None,
|
|
73
|
+
archive_format="none",
|
|
74
|
+
include_gists=True,
|
|
75
|
+
include_history=False,
|
|
76
|
+
keep=False,
|
|
77
|
+
list_only=True,
|
|
78
|
+
prompt_for_token=False,
|
|
79
|
+
)
|
|
80
|
+
assert "creating archives" not in caplog.text
|
|
81
|
+
assert f"found 0 repos for user '{USER}'" in caplog.text
|
|
82
|
+
assert f"found 0 gists for user '{USER}'" in caplog.text
|
|
83
|
+
assert not Path(tmp_path / "backups" / repo).exists()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_run_list_skip_all(tmp_path, caplog):
|
|
87
|
+
caplog.set_level("INFO")
|
|
88
|
+
githubtakeout.run(
|
|
89
|
+
username=USER,
|
|
90
|
+
base_dir=tmp_path,
|
|
91
|
+
pattern=".*",
|
|
92
|
+
skip_pattern=".",
|
|
93
|
+
archive_format="none",
|
|
94
|
+
include_gists=False,
|
|
95
|
+
include_history=False,
|
|
96
|
+
keep=False,
|
|
97
|
+
list_only=True,
|
|
98
|
+
prompt_for_token=False,
|
|
99
|
+
)
|
|
100
|
+
assert "creating archives" not in caplog.text
|
|
101
|
+
assert f"found 0 repos for user '{USER}'" in caplog.text
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@pytest.mark.parametrize("archive_format", ["zip", "tar"])
|
|
105
|
+
def test_run_archive_1_match(archive_format, tmp_path, caplog):
|
|
106
|
+
caplog.set_level("INFO")
|
|
107
|
+
backup_dir = "backups"
|
|
108
|
+
repo = "githubtakeout"
|
|
109
|
+
githubtakeout.run(
|
|
110
|
+
username=USER,
|
|
111
|
+
base_dir=tmp_path,
|
|
112
|
+
pattern=repo,
|
|
113
|
+
skip_pattern=None,
|
|
114
|
+
archive_format=archive_format,
|
|
115
|
+
include_gists=False,
|
|
116
|
+
include_history=False,
|
|
117
|
+
keep=False,
|
|
118
|
+
list_only=False,
|
|
119
|
+
prompt_for_token=False,
|
|
120
|
+
)
|
|
121
|
+
assert re.search(f"creating archives in: .*{backup_dir}", caplog.text)
|
|
122
|
+
assert f"found 1 repos for user '{USER}'" in caplog.text
|
|
123
|
+
assert "gists" not in caplog.text
|
|
124
|
+
assert re.search(f"cloning repo: {USER}/{repo}.git to: .*{backup_dir}", caplog.text)
|
|
125
|
+
assert "creating archive:" in caplog.text
|
|
126
|
+
assert "archive size:" in caplog.text
|
|
127
|
+
assert "deleting repo" in caplog.text
|
|
128
|
+
assert f"successfully backed up '{repo}' repo" in caplog.text
|
|
129
|
+
assert not Path(tmp_path / backup_dir / repo).exists()
|
|
130
|
+
if archive_format == "tar":
|
|
131
|
+
extension = "tar.gz"
|
|
132
|
+
else:
|
|
133
|
+
extension = archive_format
|
|
134
|
+
assert Path(tmp_path / backup_dir / f"{repo}.{extension}").exists()
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def test_run_no_archive_1_match_with_history(tmp_path, caplog):
|
|
138
|
+
caplog.set_level("INFO")
|
|
139
|
+
backup_dir = "backups"
|
|
140
|
+
repo = "githubtakeout"
|
|
141
|
+
githubtakeout.run(
|
|
142
|
+
username=USER,
|
|
143
|
+
base_dir=tmp_path,
|
|
144
|
+
pattern=repo,
|
|
145
|
+
skip_pattern=None,
|
|
146
|
+
archive_format="none",
|
|
147
|
+
include_gists=False,
|
|
148
|
+
include_history=True,
|
|
149
|
+
keep=False,
|
|
150
|
+
list_only=False,
|
|
151
|
+
prompt_for_token=False,
|
|
152
|
+
)
|
|
153
|
+
assert re.search(f"creating archives in: .*{backup_dir}", caplog.text)
|
|
154
|
+
assert f"found 1 repos for user '{USER}'" in caplog.text
|
|
155
|
+
assert "gists" not in caplog.text
|
|
156
|
+
assert re.search(f"cloning repo: {USER}/{repo}.git to: .*{backup_dir}", caplog.text)
|
|
157
|
+
assert "creating archive:" not in caplog.text
|
|
158
|
+
assert "archive size:" not in caplog.text
|
|
159
|
+
assert "deleting repo" not in caplog.text
|
|
160
|
+
assert f"successfully backed up '{repo}' repo" in caplog.text
|
|
161
|
+
assert Path(tmp_path / backup_dir / repo).exists()
|
|
162
|
+
assert Path(tmp_path / backup_dir / repo / ".git").exists()
|
|
163
|
+
for extension in ("tar.gz", "zip"):
|
|
164
|
+
assert not Path(tmp_path / backup_dir / f"{repo}.{extension}").exists()
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def test_run_no_archive_1_match_without_history(tmp_path, caplog):
|
|
168
|
+
caplog.set_level("INFO")
|
|
169
|
+
backup_dir = "backups"
|
|
170
|
+
repo = "githubtakeout"
|
|
171
|
+
githubtakeout.run(
|
|
172
|
+
username=USER,
|
|
173
|
+
base_dir=tmp_path,
|
|
174
|
+
pattern=repo,
|
|
175
|
+
skip_pattern=None,
|
|
176
|
+
archive_format="none",
|
|
177
|
+
include_gists=False,
|
|
178
|
+
include_history=False,
|
|
179
|
+
keep=False, # this doesn't matter when archive_format is "none"
|
|
180
|
+
list_only=False,
|
|
181
|
+
prompt_for_token=False,
|
|
182
|
+
)
|
|
183
|
+
assert re.search("creating archives in: .*backups", caplog.text)
|
|
184
|
+
assert f"found 1 repos for user '{USER}'" in caplog.text
|
|
185
|
+
assert "gists" not in caplog.text
|
|
186
|
+
assert re.search(f"cloning repo: {USER}/{repo}.git to: .*backups", caplog.text)
|
|
187
|
+
assert "creating archive:" not in caplog.text
|
|
188
|
+
assert "archive size:" not in caplog.text
|
|
189
|
+
assert "deleting repo" not in caplog.text
|
|
190
|
+
assert f"successfully backed up '{repo}' repo" in caplog.text
|
|
191
|
+
assert Path(tmp_path / backup_dir / repo).exists()
|
|
192
|
+
assert not Path(tmp_path / backup_dir / repo / ".git").exists()
|
|
193
|
+
for extension in ("tar.gz", "zip"):
|
|
194
|
+
assert not Path(tmp_path / backup_dir / f"{repo}.{extension}").exists()
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def test_run_archive_1_match_with_history_pull_with_keep(tmp_path, caplog):
|
|
198
|
+
caplog.set_level("INFO")
|
|
199
|
+
backup_dir = "backups"
|
|
200
|
+
repo = "githubtakeout"
|
|
201
|
+
githubtakeout.run(
|
|
202
|
+
username=USER,
|
|
203
|
+
base_dir=tmp_path,
|
|
204
|
+
pattern=repo,
|
|
205
|
+
skip_pattern=None,
|
|
206
|
+
archive_format="none",
|
|
207
|
+
include_gists=False,
|
|
208
|
+
include_history=True,
|
|
209
|
+
keep=True,
|
|
210
|
+
list_only=False,
|
|
211
|
+
prompt_for_token=False,
|
|
212
|
+
)
|
|
213
|
+
assert f"found 1 repos for user '{USER}'" in caplog.text
|
|
214
|
+
assert "gists" not in caplog.text
|
|
215
|
+
assert re.search(f"cloning repo: {USER}/{repo}.git to: .*backups", caplog.text)
|
|
216
|
+
assert "pulling changes" not in caplog.text
|
|
217
|
+
assert "creating archive:" not in caplog.text
|
|
218
|
+
assert "archive size:" not in caplog.text
|
|
219
|
+
assert "deleting repo" not in caplog.text
|
|
220
|
+
assert f"successfully backed up '{repo}' repo" in caplog.text
|
|
221
|
+
assert Path(tmp_path / backup_dir / repo).exists()
|
|
222
|
+
assert Path(tmp_path / backup_dir / repo / ".git").exists()
|
|
223
|
+
caplog.clear()
|
|
224
|
+
githubtakeout.run(
|
|
225
|
+
username=USER,
|
|
226
|
+
base_dir=tmp_path,
|
|
227
|
+
pattern=repo,
|
|
228
|
+
skip_pattern=None,
|
|
229
|
+
archive_format="zip",
|
|
230
|
+
include_gists=False,
|
|
231
|
+
include_history=True,
|
|
232
|
+
keep=True,
|
|
233
|
+
list_only=False,
|
|
234
|
+
prompt_for_token=False,
|
|
235
|
+
)
|
|
236
|
+
assert f"found 1 repos for user '{USER}'" in caplog.text
|
|
237
|
+
assert "gists" not in caplog.text
|
|
238
|
+
assert not re.search(f"cloning repo: {USER}/{repo}.git to: .*backups", caplog.text)
|
|
239
|
+
assert re.search(f"pulling changes from repo: {USER}/{repo}.git to: .*backups", caplog.text)
|
|
240
|
+
assert "creating archive:" in caplog.text
|
|
241
|
+
assert "archive size:" in caplog.text
|
|
242
|
+
assert "deleting repo" not in caplog.text
|
|
243
|
+
assert f"successfully backed up '{repo}' repo" in caplog.text
|
|
244
|
+
assert Path(tmp_path / backup_dir / repo).exists()
|
|
245
|
+
assert Path(tmp_path / backup_dir / repo / ".git").exists()
|
|
246
|
+
assert Path(tmp_path / backup_dir / f"{repo}.zip").exists()
|
|
File without changes
|
|
File without changes
|
{githubtakeout-0.1.14 → githubtakeout-0.1.16}/src/githubtakeout.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|