batchfetch 1.0.4__tar.gz → 1.0.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {batchfetch-1.0.4/batchfetch.egg-info → batchfetch-1.0.5}/PKG-INFO +5 -9
- {batchfetch-1.0.4 → batchfetch-1.0.5}/README.md +4 -4
- {batchfetch-1.0.4 → batchfetch-1.0.5}/batchfetch/__init__.py +0 -5
- {batchfetch-1.0.4 → batchfetch-1.0.5}/batchfetch/batchfetch_base.py +3 -9
- {batchfetch-1.0.4 → batchfetch-1.0.5}/batchfetch/batchfetch_cli.py +76 -66
- {batchfetch-1.0.4 → batchfetch-1.0.5}/batchfetch/batchfetch_git.py +53 -38
- {batchfetch-1.0.4 → batchfetch-1.0.5/batchfetch.egg-info}/PKG-INFO +5 -9
- {batchfetch-1.0.4 → batchfetch-1.0.5}/batchfetch.egg-info/SOURCES.txt +1 -6
- batchfetch-1.0.5/batchfetch.egg-info/entry_points.txt +2 -0
- {batchfetch-1.0.4 → batchfetch-1.0.5}/setup.py +2 -2
- batchfetch-1.0.4/.gitignore +0 -138
- batchfetch-1.0.4/batchfetch.egg-info/entry_points.txt +0 -2
- batchfetch-1.0.4/run_tests.sh +0 -23
- batchfetch-1.0.4/tests/data/test-md5sum.txt +0 -3
- batchfetch-1.0.4/tests/data/test-run_simple.sh +0 -5
- batchfetch-1.0.4/tests/test_helpers.py +0 -56
- {batchfetch-1.0.4 → batchfetch-1.0.5}/LICENSE +0 -0
- {batchfetch-1.0.4 → batchfetch-1.0.5}/batchfetch/helpers.py +0 -0
- {batchfetch-1.0.4 → batchfetch-1.0.5}/batchfetch.egg-info/dependency_links.txt +0 -0
- {batchfetch-1.0.4 → batchfetch-1.0.5}/batchfetch.egg-info/requires.txt +1 -1
- {batchfetch-1.0.4 → batchfetch-1.0.5}/batchfetch.egg-info/top_level.txt +0 -0
- {batchfetch-1.0.4 → batchfetch-1.0.5}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: batchfetch
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.5
|
|
4
4
|
Summary: Efficiently clone and pull multiple Git repositories.
|
|
5
5
|
Home-page: https://github.com/jamescherti/batchfetch
|
|
6
6
|
Author: James Cherti
|
|
@@ -17,10 +17,6 @@ Classifier: Topic :: Utilities
|
|
|
17
17
|
Requires-Python: >=3.6, <4
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: LICENSE
|
|
20
|
-
Requires-Dist: colorama
|
|
21
|
-
Requires-Dist: schema
|
|
22
|
-
Requires-Dist: setproctitle
|
|
23
|
-
Requires-Dist: PyYAML
|
|
24
20
|
|
|
25
21
|
# Batchfetch - Efficiently clone or pull multiple Git repositories in parallel
|
|
26
22
|
|
|
@@ -61,12 +57,12 @@ tasks:
|
|
|
61
57
|
# Clone the tag 1.5 of the consult repository to the './consult'
|
|
62
58
|
# directory
|
|
63
59
|
- git: https://github.com/minad/consult
|
|
64
|
-
|
|
60
|
+
reference: "1.5"
|
|
65
61
|
|
|
66
62
|
# Clone the s.el repository to the './another-name.el' directory
|
|
67
63
|
- git: https://github.com/magnars/s.el
|
|
68
64
|
path: another-name.el
|
|
69
|
-
|
|
65
|
+
reference: dda84d38fffdaf0c9b12837b504b402af910d01d
|
|
70
66
|
|
|
71
67
|
# Delete './impatient-mode'
|
|
72
68
|
- git: https://github.com/skeeto/impatient-mode
|
|
@@ -86,12 +82,12 @@ Command line interface.
|
|
|
86
82
|
|
|
87
83
|
options:
|
|
88
84
|
-h, --help show this help message and exit
|
|
89
|
-
-j JOBS, --jobs JOBS
|
|
90
|
-
Run up to N Number of parallel git processes (Default: 5).
|
|
85
|
+
-j JOBS, --jobs JOBS Run up to N Number of parallel processes (Default: 5).
|
|
91
86
|
-v, --verbose Enable verbose mode.
|
|
92
87
|
-f BATCHFETCH_FILE, --batchfetch-file BATCHFETCH_FILE
|
|
93
88
|
Specify the batchfetch YAML file (default: './batchfetch.yaml').
|
|
94
89
|
```
|
|
90
|
+
|
|
95
91
|
## License
|
|
96
92
|
|
|
97
93
|
Copyright (c) [James Cherti](https://www.jamescherti.com)
|
|
@@ -37,12 +37,12 @@ tasks:
|
|
|
37
37
|
# Clone the tag 1.5 of the consult repository to the './consult'
|
|
38
38
|
# directory
|
|
39
39
|
- git: https://github.com/minad/consult
|
|
40
|
-
|
|
40
|
+
reference: "1.5"
|
|
41
41
|
|
|
42
42
|
# Clone the s.el repository to the './another-name.el' directory
|
|
43
43
|
- git: https://github.com/magnars/s.el
|
|
44
44
|
path: another-name.el
|
|
45
|
-
|
|
45
|
+
reference: dda84d38fffdaf0c9b12837b504b402af910d01d
|
|
46
46
|
|
|
47
47
|
# Delete './impatient-mode'
|
|
48
48
|
- git: https://github.com/skeeto/impatient-mode
|
|
@@ -62,12 +62,12 @@ Command line interface.
|
|
|
62
62
|
|
|
63
63
|
options:
|
|
64
64
|
-h, --help show this help message and exit
|
|
65
|
-
-j JOBS, --jobs JOBS
|
|
66
|
-
Run up to N Number of parallel git processes (Default: 5).
|
|
65
|
+
-j JOBS, --jobs JOBS Run up to N Number of parallel processes (Default: 5).
|
|
67
66
|
-v, --verbose Enable verbose mode.
|
|
68
67
|
-f BATCHFETCH_FILE, --batchfetch-file BATCHFETCH_FILE
|
|
69
68
|
Specify the batchfetch YAML file (default: './batchfetch.yaml').
|
|
70
69
|
```
|
|
70
|
+
|
|
71
71
|
## License
|
|
72
72
|
|
|
73
73
|
Copyright (c) [James Cherti](https://www.jamescherti.com)
|
|
@@ -31,19 +31,13 @@ class BatchFetchError(Exception):
|
|
|
31
31
|
"""Exception raised by Downloader()."""
|
|
32
32
|
|
|
33
33
|
|
|
34
|
-
class GitBranchDoesNotExist(Exception):
|
|
35
|
-
"""Exception raised by Name()."""
|
|
36
|
-
|
|
37
|
-
|
|
38
34
|
class BatchFetchBase:
|
|
39
35
|
"""Plugin downloader base class."""
|
|
40
36
|
|
|
41
|
-
env = os.environ.copy()
|
|
42
|
-
env["GIT_TERMINAL_PROMPT"] = "0"
|
|
43
|
-
|
|
44
|
-
indent = 4
|
|
45
|
-
|
|
46
37
|
def __init__(self, data: Dict[str, Any], options: Dict[str, Any]):
|
|
38
|
+
self.indent = 4
|
|
39
|
+
self.env = os.environ.copy()
|
|
40
|
+
|
|
47
41
|
# Default
|
|
48
42
|
self.global_options_schema: Dict[Any, Any] = {
|
|
49
43
|
Optional("pre_exec"): Or([str], str),
|
|
@@ -24,7 +24,7 @@ import subprocess
|
|
|
24
24
|
import sys
|
|
25
25
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
26
26
|
from pathlib import Path
|
|
27
|
-
from typing import Set
|
|
27
|
+
from typing import Any, Dict, Set, TypeVar
|
|
28
28
|
|
|
29
29
|
import colorama
|
|
30
30
|
import yaml # type: ignore
|
|
@@ -32,31 +32,13 @@ from colorama import Fore
|
|
|
32
32
|
from schema import Optional, Or, Schema, SchemaError
|
|
33
33
|
from setproctitle import setproctitle
|
|
34
34
|
|
|
35
|
-
from .batchfetch_base import BatchFetchError
|
|
35
|
+
from .batchfetch_base import BatchFetchBase, BatchFetchError
|
|
36
36
|
from .batchfetch_git import BatchFetchGit
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
class BatchFetchCli:
|
|
40
40
|
"""Command-line-interface that downloads."""
|
|
41
41
|
|
|
42
|
-
main_key = "git"
|
|
43
|
-
|
|
44
|
-
# TODO: Make this automatic
|
|
45
|
-
empty_downloader_git = BatchFetchGit(
|
|
46
|
-
data={main_key: "http://www.domain.com"},
|
|
47
|
-
options={},
|
|
48
|
-
)
|
|
49
|
-
empty_downloader_git.validate_schema() # Gen structure
|
|
50
|
-
|
|
51
|
-
cfg_schema = {
|
|
52
|
-
Optional("options"):
|
|
53
|
-
empty_downloader_git.global_options_schema, # type: ignore
|
|
54
|
-
# TODO: Make this added automatically
|
|
55
|
-
Optional("tasks"): [
|
|
56
|
-
Or(str, empty_downloader_git.item_schema)
|
|
57
|
-
]
|
|
58
|
-
}
|
|
59
|
-
|
|
60
42
|
def __init__(self, max_workers: int, verbose: bool = False):
|
|
61
43
|
self.cfg: dict = {}
|
|
62
44
|
self.folder = Path(".")
|
|
@@ -66,16 +48,57 @@ class BatchFetchCli:
|
|
|
66
48
|
self._logger = logging.getLogger(self.__class__.__name__)
|
|
67
49
|
self.dirs_relative_to_batchfetch: Set[str] = set()
|
|
68
50
|
|
|
51
|
+
self.main_key = "git"
|
|
52
|
+
|
|
53
|
+
# Plugin
|
|
54
|
+
self.batchfetch_schemas: Dict[Any, Any] = {}
|
|
55
|
+
self.batchfetch_classes: Dict[str, BatchFetchBase] = {}
|
|
56
|
+
self.cfg_schema: Dict[Any, Any] = {}
|
|
57
|
+
self._plugin_add("git", BatchFetchGit)
|
|
58
|
+
|
|
59
|
+
def _plugin_add(self, keyword: str, batchfetch_class: Any):
|
|
60
|
+
batchfetch_instance = batchfetch_class(data={},
|
|
61
|
+
options={})
|
|
62
|
+
batchfetch_instance.validate_schema()
|
|
63
|
+
|
|
64
|
+
self.batchfetch_schemas[keyword] = batchfetch_instance.item_schema
|
|
65
|
+
self.batchfetch_classes[keyword] = batchfetch_class
|
|
66
|
+
self.cfg_schema = {
|
|
67
|
+
Optional("options"):
|
|
68
|
+
batchfetch_instance.global_options_schema, # type: ignore
|
|
69
|
+
Optional("tasks"): [
|
|
70
|
+
Or(*list(self.batchfetch_schemas.values()))
|
|
71
|
+
]
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
def _plugin_get(self, raw_data: dict) -> str:
|
|
75
|
+
keyword_found = None
|
|
76
|
+
for keyword in self.batchfetch_classes:
|
|
77
|
+
if keyword in raw_data:
|
|
78
|
+
if keyword_found is not None:
|
|
79
|
+
err_str = (f"The keywords {keyword} and {keyword_found} "
|
|
80
|
+
f"are mutually exclusive. Error in: {raw_data}")
|
|
81
|
+
raise BatchFetchError(err_str)
|
|
82
|
+
keyword_found = keyword
|
|
83
|
+
|
|
84
|
+
if not keyword_found:
|
|
85
|
+
err_str = "None of the keywords " + \
|
|
86
|
+
", ".join(self.batchfetch_classes.keys()) + \
|
|
87
|
+
f" have been found in: {raw_data}"
|
|
88
|
+
raise BatchFetchError(err_str)
|
|
89
|
+
|
|
90
|
+
return keyword_found
|
|
91
|
+
|
|
69
92
|
def load(self, path: Path):
|
|
70
93
|
try:
|
|
71
94
|
with open(path, "r", encoding="utf-8") as fhandler:
|
|
72
95
|
yaml_dict = yaml.load(fhandler, Loader=yaml.FullLoader)
|
|
73
|
-
self.
|
|
96
|
+
self._loads(dict(yaml_dict))
|
|
74
97
|
except OSError as err:
|
|
75
98
|
raise BatchFetchError(str(err)) from err
|
|
76
99
|
|
|
77
|
-
def
|
|
78
|
-
schema = Schema(
|
|
100
|
+
def _loads(self, data: dict):
|
|
101
|
+
schema = Schema(self.cfg_schema)
|
|
79
102
|
try:
|
|
80
103
|
schema.validate(data)
|
|
81
104
|
except SchemaError as err:
|
|
@@ -84,47 +107,46 @@ class BatchFetchCli:
|
|
|
84
107
|
|
|
85
108
|
self.cfg = {
|
|
86
109
|
"options": {"clone_args": []},
|
|
87
|
-
# TODO: Make this added automatically
|
|
88
110
|
"tasks": [],
|
|
89
111
|
}
|
|
90
112
|
|
|
91
113
|
if "options" in data:
|
|
92
114
|
self.cfg["options"].update(data["options"])
|
|
93
115
|
|
|
94
|
-
|
|
95
|
-
self._loads_git(data)
|
|
116
|
+
self._loads_tasks(data)
|
|
96
117
|
|
|
97
|
-
def
|
|
118
|
+
def _loads_tasks(self, data: dict):
|
|
98
119
|
if "tasks" not in data:
|
|
99
120
|
return
|
|
100
121
|
|
|
101
122
|
dict_local_dir = {} # type: ignore
|
|
102
|
-
for
|
|
103
|
-
|
|
104
|
-
|
|
123
|
+
for task in data["tasks"]:
|
|
124
|
+
keyword = self._plugin_get(task)
|
|
125
|
+
batchfetch_class = self.batchfetch_classes[keyword]
|
|
105
126
|
|
|
106
127
|
try:
|
|
107
|
-
|
|
108
|
-
data=
|
|
128
|
+
batchfetch_instance = batchfetch_class( # type: ignore
|
|
129
|
+
data=task,
|
|
109
130
|
options=self.cfg["options"],
|
|
110
131
|
)
|
|
111
|
-
self.cfg["tasks"].append(
|
|
132
|
+
self.cfg["tasks"].append(batchfetch_instance)
|
|
112
133
|
except SchemaError as err:
|
|
113
134
|
print(f"Schema error: {err}.", file=sys.stderr)
|
|
114
135
|
sys.exit(1)
|
|
115
136
|
|
|
116
|
-
if
|
|
117
|
-
err_str = ("more than one repository
|
|
118
|
-
"
|
|
119
|
-
str(
|
|
120
|
-
str(
|
|
121
|
-
str(dict_local_dir[
|
|
137
|
+
if batchfetch_instance["path"] in dict_local_dir:
|
|
138
|
+
err_str = ("more than one repository have the " +
|
|
139
|
+
"destination path '" +
|
|
140
|
+
str(batchfetch_instance["path"]) + "' (" +
|
|
141
|
+
str(task[keyword]) + " and " +
|
|
142
|
+
str(dict_local_dir[batchfetch_instance["path"]]) +
|
|
122
143
|
")")
|
|
123
144
|
raise BatchFetchError(err_str)
|
|
124
|
-
dict_local_dir[plugin_downloader_git["path"]] = \
|
|
125
|
-
plugin_downloader_git[BatchFetchCli.main_key]
|
|
126
145
|
|
|
127
|
-
|
|
146
|
+
dict_local_dir[batchfetch_instance["path"]] = \
|
|
147
|
+
batchfetch_instance[keyword]
|
|
148
|
+
|
|
149
|
+
def run_tasks(self) -> bool:
|
|
128
150
|
failed = []
|
|
129
151
|
error = False
|
|
130
152
|
threads = []
|
|
@@ -136,13 +158,12 @@ class BatchFetchCli:
|
|
|
136
158
|
try:
|
|
137
159
|
self.dirs_relative_to_batchfetch = set()
|
|
138
160
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
threads.append(executor_update.submit(download_item.update))
|
|
161
|
+
all_tasks = self.cfg["tasks"]
|
|
162
|
+
for task in all_tasks:
|
|
163
|
+
self.dirs_relative_to_batchfetch.add(str(task["path"]))
|
|
164
|
+
if not task["delete"]:
|
|
165
|
+
self.managed_filenames.add(task["path"])
|
|
166
|
+
threads.append(executor_update.submit(task.update))
|
|
146
167
|
|
|
147
168
|
for future in as_completed(threads):
|
|
148
169
|
data = future.result()
|
|
@@ -174,9 +195,8 @@ class BatchFetchCli:
|
|
|
174
195
|
if error:
|
|
175
196
|
if failed:
|
|
176
197
|
print("Failed:")
|
|
177
|
-
for
|
|
178
|
-
print(" -
|
|
179
|
-
print(" dir:", git_update_result["path"])
|
|
198
|
+
for failed_result in failed:
|
|
199
|
+
print(" -", failed_result["path"])
|
|
180
200
|
else:
|
|
181
201
|
print("Failed.")
|
|
182
202
|
|
|
@@ -189,14 +209,6 @@ class BatchFetchCli:
|
|
|
189
209
|
|
|
190
210
|
return True
|
|
191
211
|
|
|
192
|
-
# def check_managed_directories(self):
|
|
193
|
-
# # managed_downloads = {Path(item).resolve()
|
|
194
|
-
# # for item in self.managed_filenames}
|
|
195
|
-
# # managed_directories = {item.parent
|
|
196
|
-
# # for item in managed_downloads}
|
|
197
|
-
# # TODO Implement checker
|
|
198
|
-
# pass
|
|
199
|
-
|
|
200
212
|
|
|
201
213
|
def parse_args():
|
|
202
214
|
"""Parse the command line arguments."""
|
|
@@ -206,7 +218,7 @@ def parse_args():
|
|
|
206
218
|
|
|
207
219
|
parser.add_argument(
|
|
208
220
|
"-j", "--jobs", default="5", required=False,
|
|
209
|
-
help="Run up to N Number of parallel
|
|
221
|
+
help="Run up to N Number of parallel processes (Default: 5).",
|
|
210
222
|
)
|
|
211
223
|
|
|
212
224
|
parser.add_argument(
|
|
@@ -243,14 +255,14 @@ def command_line_interface():
|
|
|
243
255
|
sys.argv[1:]))
|
|
244
256
|
|
|
245
257
|
args = parse_args()
|
|
246
|
-
|
|
258
|
+
batchfetch_cli = BatchFetchCli(verbose=args.verbose,
|
|
247
259
|
max_workers=int(args.jobs))
|
|
248
260
|
|
|
249
|
-
|
|
261
|
+
batchfetch_cli.load(args.batchfetch_file)
|
|
250
262
|
os.chdir(os.path.dirname(args.batchfetch_file))
|
|
251
263
|
|
|
252
264
|
try:
|
|
253
|
-
if not
|
|
265
|
+
if not batchfetch_cli.run_tasks():
|
|
254
266
|
errno = 1
|
|
255
267
|
except KeyboardInterrupt:
|
|
256
268
|
print("Interrupted.", file=sys.stderr)
|
|
@@ -258,7 +270,5 @@ def command_line_interface():
|
|
|
258
270
|
except BatchFetchError as err:
|
|
259
271
|
print(f"Error: {err}.", file=sys.stderr)
|
|
260
272
|
errno = 1
|
|
261
|
-
# else:
|
|
262
|
-
# downloader_cli.check_managed_directories()
|
|
263
273
|
|
|
264
274
|
sys.exit(errno)
|
|
@@ -27,25 +27,28 @@ from typing import List, Union
|
|
|
27
27
|
|
|
28
28
|
from schema import Optional
|
|
29
29
|
|
|
30
|
-
from .batchfetch_base import
|
|
31
|
-
GitBranchDoesNotExist)
|
|
30
|
+
from .batchfetch_base import BatchFetchBase, BatchFetchError
|
|
32
31
|
from .helpers import run_simple
|
|
33
32
|
|
|
34
33
|
|
|
34
|
+
class GitReferenceDoesNotExist(Exception):
|
|
35
|
+
"""Exception raised by Name()."""
|
|
36
|
+
|
|
37
|
+
|
|
35
38
|
class BatchFetchGit(BatchFetchBase):
|
|
36
39
|
"""Clone or update a Git repository."""
|
|
37
40
|
|
|
38
|
-
indent_spaces = " " * BatchFetchBase.indent
|
|
39
|
-
main_key = "git"
|
|
40
|
-
|
|
41
41
|
def __init__(self, *args, **kwargs):
|
|
42
42
|
super().__init__(*args, **kwargs)
|
|
43
|
+
self.env["GIT_TERMINAL_PROMPT"] = "0"
|
|
44
|
+
self.indent_spaces = " " * self.indent
|
|
45
|
+
self.main_key = "git"
|
|
43
46
|
|
|
44
47
|
# Schema
|
|
45
48
|
self.item_schema.update({
|
|
46
49
|
# Local options
|
|
47
|
-
|
|
48
|
-
Optional("
|
|
50
|
+
self.main_key: str,
|
|
51
|
+
Optional("reference"): str,
|
|
49
52
|
|
|
50
53
|
# Same as global options
|
|
51
54
|
Optional("clone_args"): [str],
|
|
@@ -63,8 +66,8 @@ class BatchFetchGit(BatchFetchBase):
|
|
|
63
66
|
"git_pull": True})
|
|
64
67
|
|
|
65
68
|
self.item_default_values.update({
|
|
66
|
-
|
|
67
|
-
"
|
|
69
|
+
self.main_key: "",
|
|
70
|
+
"reference": "",
|
|
68
71
|
"delete": False,
|
|
69
72
|
})
|
|
70
73
|
|
|
@@ -75,16 +78,19 @@ class BatchFetchGit(BatchFetchBase):
|
|
|
75
78
|
def _initialize_data(self):
|
|
76
79
|
super()._initialize_data()
|
|
77
80
|
|
|
78
|
-
self.values[
|
|
79
|
-
self.values[
|
|
81
|
+
self.values[self.main_key] = \
|
|
82
|
+
self.values[self.main_key].rstrip("/")
|
|
80
83
|
|
|
81
84
|
if "path" not in self.values:
|
|
82
85
|
self.values["path"] = \
|
|
83
86
|
posixpath.basename(
|
|
84
|
-
self.values[
|
|
87
|
+
self.values[self.main_key]) # type: ignore
|
|
85
88
|
|
|
86
89
|
def _git_ref(self, cwd: Union[None, Path] = None) -> str:
|
|
87
|
-
"""Get the commit reference of HEAD.
|
|
90
|
+
"""Get the commit reference of HEAD.
|
|
91
|
+
|
|
92
|
+
The command will fail if the branch is detached.
|
|
93
|
+
"""
|
|
88
94
|
cmd = "git show-ref --head --verify HEAD"
|
|
89
95
|
try:
|
|
90
96
|
stdout, _ = run_simple(cmd, cwd=cwd, env=self.env)
|
|
@@ -112,8 +118,8 @@ class BatchFetchGit(BatchFetchBase):
|
|
|
112
118
|
|
|
113
119
|
self.add_output(
|
|
114
120
|
f"[GIT {update_type}] {self[self.main_key]}" +
|
|
115
|
-
(f" (
|
|
116
|
-
if self["
|
|
121
|
+
(f" (Reference: {self['reference']})"
|
|
122
|
+
if self["reference"] else "") + "\n"
|
|
117
123
|
)
|
|
118
124
|
|
|
119
125
|
try:
|
|
@@ -188,7 +194,7 @@ class BatchFetchGit(BatchFetchBase):
|
|
|
188
194
|
git_clone_args += ["--recurse-submodules"]
|
|
189
195
|
|
|
190
196
|
cmd = ["git", "clone"] + git_clone_args + \
|
|
191
|
-
[self[
|
|
197
|
+
[self[self.main_key], str(self.git_local_dir)]
|
|
192
198
|
self._run(cmd, env=self.env)
|
|
193
199
|
self.set_changed(True)
|
|
194
200
|
|
|
@@ -204,23 +210,32 @@ class BatchFetchGit(BatchFetchBase):
|
|
|
204
210
|
ignore_git_pull = False
|
|
205
211
|
if not self["git_pull"]:
|
|
206
212
|
ignore_git_pull = True
|
|
207
|
-
|
|
213
|
+
|
|
214
|
+
if self["reference"]:
|
|
215
|
+
ignore_git_pull = False
|
|
208
216
|
# Check if the new branch exists
|
|
209
217
|
try:
|
|
210
|
-
self._git_tags(self["
|
|
211
|
-
except
|
|
218
|
+
self._git_tags(self["reference"])
|
|
219
|
+
except GitReferenceDoesNotExist:
|
|
212
220
|
pass
|
|
213
221
|
else:
|
|
214
|
-
# The branch exists
|
|
215
|
-
|
|
216
|
-
#
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
222
|
+
# The branch exists:
|
|
223
|
+
# 1. Ignore Git pull when the git reference is the same
|
|
224
|
+
# as the "branch:" key
|
|
225
|
+
try:
|
|
226
|
+
commit_ref = self._git_ref(cwd=self.git_local_dir)
|
|
227
|
+
except subprocess.CalledProcessError:
|
|
228
|
+
# Ignore git pull because the head is detached
|
|
229
|
+
pass
|
|
230
|
+
else:
|
|
231
|
+
if (self.current_branch and
|
|
232
|
+
(commit_ref == self["reference"] or
|
|
233
|
+
self.current_branch == self["reference"])):
|
|
234
|
+
ignore_git_pull = True
|
|
235
|
+
|
|
236
|
+
# 2. Ignore Git pull if it is not a local branch
|
|
237
|
+
if (not ignore_git_pull and
|
|
238
|
+
not self._git_is_local_branch(self["reference"])):
|
|
224
239
|
ignore_git_pull = True
|
|
225
240
|
|
|
226
241
|
if ignore_git_pull:
|
|
@@ -272,15 +287,15 @@ class BatchFetchGit(BatchFetchBase):
|
|
|
272
287
|
env=self.env,
|
|
273
288
|
cwd=self.git_local_dir)
|
|
274
289
|
except subprocess.CalledProcessError as err:
|
|
275
|
-
raise
|
|
276
|
-
f"The
|
|
290
|
+
raise GitReferenceDoesNotExist(
|
|
291
|
+
f"The reference '{branch}' does not exist.") from err
|
|
277
292
|
|
|
278
293
|
return stdout
|
|
279
294
|
|
|
280
295
|
def _repo_fix_branch(self) -> bool:
|
|
281
296
|
git_ref_after_merge = self._git_ref(cwd=self.git_local_dir)
|
|
282
297
|
branch_changed = False
|
|
283
|
-
if self["
|
|
298
|
+
if self["reference"]:
|
|
284
299
|
# We also need tags because sometimes, a branch
|
|
285
300
|
# returns a different commit reference
|
|
286
301
|
git_tags, _ = run_simple(
|
|
@@ -292,18 +307,18 @@ class BatchFetchGit(BatchFetchBase):
|
|
|
292
307
|
# Also check the commit reference in case
|
|
293
308
|
# branch is a commit reference instead of a tag
|
|
294
309
|
try:
|
|
295
|
-
git_ref_branch = self._git_tags(self["
|
|
296
|
-
except
|
|
310
|
+
git_ref_branch = self._git_tags(self["reference"])[0]
|
|
311
|
+
except GitReferenceDoesNotExist as err:
|
|
297
312
|
raise BatchFetchError(f"The branch '{self['branch']}' "
|
|
298
313
|
"does not exist.") from err
|
|
299
314
|
|
|
300
315
|
if git_ref_after_merge != git_ref_branch and \
|
|
301
|
-
self["
|
|
316
|
+
self["reference"] not in git_tags:
|
|
302
317
|
# Update the branch
|
|
303
|
-
self._run(["git", "checkout"] + [self["
|
|
318
|
+
self._run(["git", "checkout"] + [self["reference"]],
|
|
304
319
|
cwd=str(self.git_local_dir), env=self.env)
|
|
305
320
|
self.add_output(self.indent_spaces + "# Branch changed to " +
|
|
306
|
-
self["
|
|
321
|
+
self["reference"] + "\n")
|
|
307
322
|
self.set_changed(True)
|
|
308
323
|
branch_changed = True
|
|
309
324
|
|
|
@@ -339,7 +354,7 @@ class BatchFetchGit(BatchFetchBase):
|
|
|
339
354
|
except IndexError:
|
|
340
355
|
origin_url = ""
|
|
341
356
|
|
|
342
|
-
if origin_url != self[
|
|
357
|
+
if origin_url != self[self.main_key]:
|
|
343
358
|
self.set_error(True)
|
|
344
359
|
self.add_output(
|
|
345
360
|
self.indent_spaces +
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: batchfetch
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.5
|
|
4
4
|
Summary: Efficiently clone and pull multiple Git repositories.
|
|
5
5
|
Home-page: https://github.com/jamescherti/batchfetch
|
|
6
6
|
Author: James Cherti
|
|
@@ -17,10 +17,6 @@ Classifier: Topic :: Utilities
|
|
|
17
17
|
Requires-Python: >=3.6, <4
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: LICENSE
|
|
20
|
-
Requires-Dist: colorama
|
|
21
|
-
Requires-Dist: schema
|
|
22
|
-
Requires-Dist: setproctitle
|
|
23
|
-
Requires-Dist: PyYAML
|
|
24
20
|
|
|
25
21
|
# Batchfetch - Efficiently clone or pull multiple Git repositories in parallel
|
|
26
22
|
|
|
@@ -61,12 +57,12 @@ tasks:
|
|
|
61
57
|
# Clone the tag 1.5 of the consult repository to the './consult'
|
|
62
58
|
# directory
|
|
63
59
|
- git: https://github.com/minad/consult
|
|
64
|
-
|
|
60
|
+
reference: "1.5"
|
|
65
61
|
|
|
66
62
|
# Clone the s.el repository to the './another-name.el' directory
|
|
67
63
|
- git: https://github.com/magnars/s.el
|
|
68
64
|
path: another-name.el
|
|
69
|
-
|
|
65
|
+
reference: dda84d38fffdaf0c9b12837b504b402af910d01d
|
|
70
66
|
|
|
71
67
|
# Delete './impatient-mode'
|
|
72
68
|
- git: https://github.com/skeeto/impatient-mode
|
|
@@ -86,12 +82,12 @@ Command line interface.
|
|
|
86
82
|
|
|
87
83
|
options:
|
|
88
84
|
-h, --help show this help message and exit
|
|
89
|
-
-j JOBS, --jobs JOBS
|
|
90
|
-
Run up to N Number of parallel git processes (Default: 5).
|
|
85
|
+
-j JOBS, --jobs JOBS Run up to N Number of parallel processes (Default: 5).
|
|
91
86
|
-v, --verbose Enable verbose mode.
|
|
92
87
|
-f BATCHFETCH_FILE, --batchfetch-file BATCHFETCH_FILE
|
|
93
88
|
Specify the batchfetch YAML file (default: './batchfetch.yaml').
|
|
94
89
|
```
|
|
90
|
+
|
|
95
91
|
## License
|
|
96
92
|
|
|
97
93
|
Copyright (c) [James Cherti](https://www.jamescherti.com)
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
.gitignore
|
|
2
1
|
LICENSE
|
|
3
2
|
README.md
|
|
4
|
-
run_tests.sh
|
|
5
3
|
setup.py
|
|
6
4
|
batchfetch/__init__.py
|
|
7
5
|
batchfetch/batchfetch_base.py
|
|
@@ -13,7 +11,4 @@ batchfetch.egg-info/SOURCES.txt
|
|
|
13
11
|
batchfetch.egg-info/dependency_links.txt
|
|
14
12
|
batchfetch.egg-info/entry_points.txt
|
|
15
13
|
batchfetch.egg-info/requires.txt
|
|
16
|
-
batchfetch.egg-info/top_level.txt
|
|
17
|
-
tests/test_helpers.py
|
|
18
|
-
tests/data/test-md5sum.txt
|
|
19
|
-
tests/data/test-run_simple.sh
|
|
14
|
+
batchfetch.egg-info/top_level.txt
|
|
@@ -22,7 +22,7 @@ from setuptools import find_packages, setup
|
|
|
22
22
|
|
|
23
23
|
setup(
|
|
24
24
|
name="batchfetch",
|
|
25
|
-
version="1.0.
|
|
25
|
+
version="1.0.5",
|
|
26
26
|
packages=find_packages(),
|
|
27
27
|
description="Efficiently clone and pull multiple Git repositories.",
|
|
28
28
|
license="GPLv3",
|
|
@@ -51,7 +51,7 @@ setup(
|
|
|
51
51
|
],
|
|
52
52
|
entry_points={
|
|
53
53
|
"console_scripts": [
|
|
54
|
-
"batchfetch=batchfetch.
|
|
54
|
+
"batchfetch=batchfetch.batchfetch_cli:command_line_interface",
|
|
55
55
|
],
|
|
56
56
|
},
|
|
57
57
|
)
|
batchfetch-1.0.4/.gitignore
DELETED
|
@@ -1,138 +0,0 @@
|
|
|
1
|
-
# Byte-compiled / optimized / DLL files
|
|
2
|
-
__pycache__/
|
|
3
|
-
*.py[cod]
|
|
4
|
-
*$py.class
|
|
5
|
-
|
|
6
|
-
# C extensions
|
|
7
|
-
*.so
|
|
8
|
-
|
|
9
|
-
# Distribution / packaging
|
|
10
|
-
.Python
|
|
11
|
-
build/
|
|
12
|
-
develop-eggs/
|
|
13
|
-
dist/
|
|
14
|
-
downloads/
|
|
15
|
-
eggs/
|
|
16
|
-
.eggs/
|
|
17
|
-
lib/
|
|
18
|
-
lib64/
|
|
19
|
-
parts/
|
|
20
|
-
sdist/
|
|
21
|
-
var/
|
|
22
|
-
wheels/
|
|
23
|
-
share/python-wheels/
|
|
24
|
-
*.egg-info/
|
|
25
|
-
.installed.cfg
|
|
26
|
-
*.egg
|
|
27
|
-
MANIFEST
|
|
28
|
-
|
|
29
|
-
# PyInstaller
|
|
30
|
-
# Usually these files are written by a python script from a template
|
|
31
|
-
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
-
*.manifest
|
|
33
|
-
*.spec
|
|
34
|
-
|
|
35
|
-
# Installer logs
|
|
36
|
-
pip-log.txt
|
|
37
|
-
pip-delete-this-directory.txt
|
|
38
|
-
|
|
39
|
-
# Unit test / coverage reports
|
|
40
|
-
htmlcov/
|
|
41
|
-
.tox/
|
|
42
|
-
.nox/
|
|
43
|
-
.coverage
|
|
44
|
-
.coverage.*
|
|
45
|
-
.cache
|
|
46
|
-
nosetests.xml
|
|
47
|
-
coverage.xml
|
|
48
|
-
*.cover
|
|
49
|
-
*.py,cover
|
|
50
|
-
.hypothesis/
|
|
51
|
-
.pytest_cache/
|
|
52
|
-
cover/
|
|
53
|
-
|
|
54
|
-
# Translations
|
|
55
|
-
*.mo
|
|
56
|
-
*.pot
|
|
57
|
-
|
|
58
|
-
# Django stuff:
|
|
59
|
-
*.log
|
|
60
|
-
local_settings.py
|
|
61
|
-
db.sqlite3
|
|
62
|
-
db.sqlite3-journal
|
|
63
|
-
|
|
64
|
-
# Flask stuff:
|
|
65
|
-
instance/
|
|
66
|
-
.webassets-cache
|
|
67
|
-
|
|
68
|
-
# Scrapy stuff:
|
|
69
|
-
.scrapy
|
|
70
|
-
|
|
71
|
-
# Sphinx documentation
|
|
72
|
-
docs/_build/
|
|
73
|
-
|
|
74
|
-
# PyBuilder
|
|
75
|
-
.pybuilder/
|
|
76
|
-
target/
|
|
77
|
-
|
|
78
|
-
# Jupyter Notebook
|
|
79
|
-
.ipynb_checkpoints
|
|
80
|
-
|
|
81
|
-
# IPython
|
|
82
|
-
profile_default/
|
|
83
|
-
ipython_config.py
|
|
84
|
-
|
|
85
|
-
# pyenv
|
|
86
|
-
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
-
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
-
# .python-version
|
|
89
|
-
|
|
90
|
-
# pipenv
|
|
91
|
-
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
-
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
-
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
-
# install all needed dependencies.
|
|
95
|
-
#Pipfile.lock
|
|
96
|
-
|
|
97
|
-
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
|
98
|
-
__pypackages__/
|
|
99
|
-
|
|
100
|
-
# Celery stuff
|
|
101
|
-
celerybeat-schedule
|
|
102
|
-
celerybeat.pid
|
|
103
|
-
|
|
104
|
-
# SageMath parsed files
|
|
105
|
-
*.sage.py
|
|
106
|
-
|
|
107
|
-
# Environments
|
|
108
|
-
.env
|
|
109
|
-
.venv
|
|
110
|
-
env/
|
|
111
|
-
venv/
|
|
112
|
-
ENV/
|
|
113
|
-
env.bak/
|
|
114
|
-
venv.bak/
|
|
115
|
-
|
|
116
|
-
# Spyder project settings
|
|
117
|
-
.spyderproject
|
|
118
|
-
.spyproject
|
|
119
|
-
|
|
120
|
-
# Rope project settings
|
|
121
|
-
.ropeproject
|
|
122
|
-
|
|
123
|
-
# mkdocs documentation
|
|
124
|
-
/site
|
|
125
|
-
|
|
126
|
-
# mypy
|
|
127
|
-
.mypy_cache/
|
|
128
|
-
.dmypy.json
|
|
129
|
-
dmypy.json
|
|
130
|
-
|
|
131
|
-
# Pyre type checker
|
|
132
|
-
.pyre/
|
|
133
|
-
|
|
134
|
-
# pytype static type analyzer
|
|
135
|
-
.pytype/
|
|
136
|
-
|
|
137
|
-
# Cython debug symbols
|
|
138
|
-
cython_debug/
|
batchfetch-1.0.4/run_tests.sh
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
#
|
|
3
|
-
# Copyright (c) James Cherti
|
|
4
|
-
# URL: https://github.com/jamescherti/batchfetch
|
|
5
|
-
#
|
|
6
|
-
# This program is free software: you can redistribute it and/or modify it under
|
|
7
|
-
# the terms of the GNU General Public License as published by the Free Software
|
|
8
|
-
# Foundation, either version 3 of the License, or (at your option) any later
|
|
9
|
-
# version.
|
|
10
|
-
#
|
|
11
|
-
# This program is distributed in the hope that it will be useful, but WITHOUT
|
|
12
|
-
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
13
|
-
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
|
14
|
-
#
|
|
15
|
-
# You should have received a copy of the GNU General Public License along with
|
|
16
|
-
# this program. If not, see <https://www.gnu.org/licenses/>.
|
|
17
|
-
#
|
|
18
|
-
cd "$(dirname "${BASH_SOURCE[0]}")"
|
|
19
|
-
export PYTHONPATH="$(pwd)"
|
|
20
|
-
exec pytest -v -v --cov=batchfetch --cov=tests \
|
|
21
|
-
--cov-report=term \
|
|
22
|
-
--cov-report=html:htmlcov \
|
|
23
|
-
tests/test_*py
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python
|
|
2
|
-
#
|
|
3
|
-
# Copyright (c) James Cherti
|
|
4
|
-
# URL: https://github.com/jamescherti/batchfetch
|
|
5
|
-
#
|
|
6
|
-
# This program is free software: you can redistribute it and/or modify it under
|
|
7
|
-
# the terms of the GNU General Public License as published by the Free Software
|
|
8
|
-
# Foundation, either version 3 of the License, or (at your option) any later
|
|
9
|
-
# version.
|
|
10
|
-
#
|
|
11
|
-
# This program is distributed in the hope that it will be useful, but WITHOUT
|
|
12
|
-
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
13
|
-
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
|
14
|
-
#
|
|
15
|
-
# You should have received a copy of the GNU General Public License along with
|
|
16
|
-
# this program. If not, see <https://www.gnu.org/licenses/>.
|
|
17
|
-
#
|
|
18
|
-
"""Unit tests."""
|
|
19
|
-
|
|
20
|
-
from pathlib import Path
|
|
21
|
-
|
|
22
|
-
from batchfetch import helpers
|
|
23
|
-
|
|
24
|
-
DATA_PATH = Path(".").joinpath("tests", "data").absolute()
|
|
25
|
-
SCRIPT_RUN_SIMPLE = DATA_PATH / "test-run_simple.sh"
|
|
26
|
-
TEST_MD5SUM_FILE = DATA_PATH / "test-md5sum.txt"
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def test_md5sum():
|
|
30
|
-
md5sum = helpers.md5sum(TEST_MD5SUM_FILE)
|
|
31
|
-
assert md5sum == "f31e127edc87a6aa2eb01b7d94d2ec58"
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def test_run_simple():
|
|
35
|
-
# Stdout
|
|
36
|
-
stdout_lines, stderr_lines = helpers.run_simple(str(SCRIPT_RUN_SIMPLE))
|
|
37
|
-
assert stdout_lines == ['Test 1.',
|
|
38
|
-
'Test 2.',
|
|
39
|
-
'Test 3.']
|
|
40
|
-
assert stderr_lines == ['Test 3.']
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def test_indent_raw_output():
|
|
44
|
-
list_str = \
|
|
45
|
-
helpers.indent_raw_output(["Test 1.", "Test 2."])
|
|
46
|
-
|
|
47
|
-
assert list_str == [" Test 1.", " Test 2."]
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
def test_run_indent():
|
|
51
|
-
stdout_lines, stderr_lines = helpers.run_indent(str(SCRIPT_RUN_SIMPLE))
|
|
52
|
-
assert stdout_lines == [f' [RUN] {str(SCRIPT_RUN_SIMPLE)}',
|
|
53
|
-
' Test 1.',
|
|
54
|
-
' Test 2.',
|
|
55
|
-
' Test 3.']
|
|
56
|
-
assert stderr_lines == [' Test 3.']
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|