batchfetch 1.0.9__tar.gz → 1.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {batchfetch-1.0.9/batchfetch.egg-info → batchfetch-1.1.1}/PKG-INFO +1 -1
- {batchfetch-1.0.9 → batchfetch-1.1.1}/batchfetch/batchfetch_base.py +71 -55
- {batchfetch-1.0.9 → batchfetch-1.1.1}/batchfetch/batchfetch_cli.py +2 -2
- {batchfetch-1.0.9 → batchfetch-1.1.1}/batchfetch/batchfetch_git.py +49 -40
- {batchfetch-1.0.9 → batchfetch-1.1.1/batchfetch.egg-info}/PKG-INFO +1 -1
- {batchfetch-1.0.9 → batchfetch-1.1.1}/setup.py +1 -1
- {batchfetch-1.0.9 → batchfetch-1.1.1}/.gitignore +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/LICENSE +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/README.md +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/batchfetch/__init__.py +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/batchfetch/helpers.py +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/batchfetch.egg-info/SOURCES.txt +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/batchfetch.egg-info/dependency_links.txt +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/batchfetch.egg-info/entry_points.txt +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/batchfetch.egg-info/requires.txt +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/batchfetch.egg-info/top_level.txt +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/run_tests.sh +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/setup.cfg +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/tests/data/test-md5sum.txt +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/tests/data/test-run_simple.sh +0 -0
- {batchfetch-1.0.9 → batchfetch-1.1.1}/tests/test_helpers.py +0 -0
|
@@ -32,39 +32,20 @@ class BatchFetchError(Exception):
|
|
|
32
32
|
"""Exception raised by Downloader()."""
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
class
|
|
36
|
-
"""
|
|
37
|
-
|
|
38
|
-
def __init__(self, data: Dict[str, Any], options: Dict[str, Any]):
|
|
39
|
-
self.indent = 4
|
|
40
|
-
self.env = os.environ.copy()
|
|
41
|
-
|
|
42
|
-
# Default
|
|
43
|
-
self.global_options_schema: Dict[Any, Any] = {
|
|
44
|
-
Optional("exec_before"): Or([str], str),
|
|
45
|
-
Optional("exec_after"): Or([str], str),
|
|
46
|
-
}
|
|
35
|
+
class DataAlreadyInitialized(Exception):
|
|
36
|
+
"""Data already initialized."""
|
|
47
37
|
|
|
48
|
-
self.item_schema: Dict[Any, Any] = {
|
|
49
|
-
Optional("path"): str,
|
|
50
|
-
Optional("delete"): bool,
|
|
51
38
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
}
|
|
39
|
+
class TaskBase:
|
|
40
|
+
def __init__(self, data: Dict[str, Any], options: Dict[str, Any]):
|
|
41
|
+
self.global_options_schema: Dict[Any, Any] = {}
|
|
42
|
+
self.global_options_values: Dict[str, Any] = {}
|
|
55
43
|
|
|
56
|
-
self.
|
|
57
|
-
|
|
58
|
-
"exec_after": [],
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
self.item_default_values: Dict[str, Any] = {
|
|
62
|
-
# Optional items
|
|
63
|
-
"delete": False,
|
|
64
|
-
}
|
|
44
|
+
self.task_schema: Dict[Any, Any] = {}
|
|
45
|
+
self.task_default_values: Dict[str, Any] = {}
|
|
65
46
|
|
|
66
47
|
self._item_values = data
|
|
67
|
-
self.
|
|
48
|
+
self._item_options = options
|
|
68
49
|
|
|
69
50
|
# Variables
|
|
70
51
|
self.values: Dict[str, Any] = {}
|
|
@@ -73,20 +54,20 @@ class BatchFetchBase:
|
|
|
73
54
|
|
|
74
55
|
def _initialize_data(self):
|
|
75
56
|
if self._values_initialized:
|
|
76
|
-
|
|
57
|
+
raise DataAlreadyInitialized
|
|
77
58
|
|
|
78
59
|
# Options
|
|
79
60
|
self.options = {}
|
|
80
|
-
self.options.update(deepcopy(self.
|
|
61
|
+
self.options.update(deepcopy(self.global_options_values)) # Default
|
|
62
|
+
self.options.update(deepcopy(self._item_options))
|
|
81
63
|
schema = Schema(self.global_options_schema)
|
|
82
64
|
schema.validate(self.options)
|
|
83
65
|
|
|
84
66
|
# Data
|
|
85
67
|
self.values = {}
|
|
86
|
-
self.values.update(deepcopy(self.
|
|
68
|
+
self.values.update(deepcopy(self.task_default_values))
|
|
87
69
|
self.values.update(deepcopy(self._item_values))
|
|
88
|
-
|
|
89
|
-
schema = Schema(self.item_schema)
|
|
70
|
+
schema = Schema(self.task_schema)
|
|
90
71
|
schema.validate(self.values)
|
|
91
72
|
|
|
92
73
|
self.values["result"] = {
|
|
@@ -95,14 +76,66 @@ class BatchFetchBase:
|
|
|
95
76
|
"changed": False,
|
|
96
77
|
}
|
|
97
78
|
|
|
98
|
-
|
|
79
|
+
def validate_schema(self):
|
|
80
|
+
self._initialize_data()
|
|
81
|
+
|
|
82
|
+
def __getitem__(self, key):
|
|
83
|
+
self._initialize_data()
|
|
84
|
+
|
|
85
|
+
if key in self.values:
|
|
86
|
+
return self.values[key]
|
|
99
87
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
88
|
+
if key in self.options:
|
|
89
|
+
return self.options[key]
|
|
90
|
+
|
|
91
|
+
if key in self.global_options_values:
|
|
92
|
+
return self.global_options_values[key]
|
|
93
|
+
|
|
94
|
+
raise KeyError(f"The item '{key}' was not found in '{self.values}")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class BatchFetchBase(TaskBase):
|
|
98
|
+
"""Plugin downloader base class."""
|
|
99
|
+
|
|
100
|
+
def __init__(self, data: Dict[str, Any], options: Dict[str, Any]):
|
|
101
|
+
super().__init__(data=data, options=options)
|
|
102
|
+
self.indent = 4
|
|
103
|
+
self.indent_spaces = " " * self.indent
|
|
104
|
+
self.env = os.environ.copy()
|
|
105
|
+
|
|
106
|
+
# Default
|
|
107
|
+
self.global_options_schema: Dict[Any, Any] = {
|
|
108
|
+
Optional("exec_before"): Or([str], str),
|
|
109
|
+
Optional("exec_after"): Or([str], str),
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
self.task_schema: Dict[Any, Any] = {
|
|
113
|
+
Optional("path"): str,
|
|
114
|
+
Optional("delete"): bool,
|
|
115
|
+
|
|
116
|
+
Optional("exec_before"): Or([str], str),
|
|
117
|
+
Optional("exec_after"): Or([str], str),
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
self.global_options_values: Dict[str, Any] = {
|
|
121
|
+
"exec_before": [],
|
|
122
|
+
"exec_after": [],
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
self.task_default_values: Dict[str, Any] = {
|
|
126
|
+
# Optional items
|
|
127
|
+
"delete": False,
|
|
104
128
|
}
|
|
105
129
|
|
|
130
|
+
def _initialize_data(self):
|
|
131
|
+
try:
|
|
132
|
+
super()._initialize_data()
|
|
133
|
+
except DataAlreadyInitialized:
|
|
134
|
+
return
|
|
135
|
+
|
|
136
|
+
# Mark values as initialized
|
|
137
|
+
self._values_initialized = True
|
|
138
|
+
|
|
106
139
|
def _run(self, *args, **kwargs):
|
|
107
140
|
stdout = run_indent_str(*args, **kwargs)
|
|
108
141
|
|
|
@@ -131,23 +164,6 @@ class BatchFetchBase:
|
|
|
131
164
|
self._run(post_exec, cwd=str(cwd), env=self.env,
|
|
132
165
|
spaces=self.indent)
|
|
133
166
|
|
|
134
|
-
def __getitem__(self, key):
|
|
135
|
-
self._initialize_data()
|
|
136
|
-
|
|
137
|
-
if key in self.values:
|
|
138
|
-
return self.values[key]
|
|
139
|
-
|
|
140
|
-
if key in self.options:
|
|
141
|
-
return self.options[key]
|
|
142
|
-
|
|
143
|
-
if key in self.global_options_values:
|
|
144
|
-
return self.global_options_values[key]
|
|
145
|
-
|
|
146
|
-
raise KeyError(f"The item '{key}' was not found in '{self.values}")
|
|
147
|
-
|
|
148
|
-
def validate_schema(self):
|
|
149
|
-
self._initialize_data()
|
|
150
|
-
|
|
151
167
|
def is_changed(self) -> bool:
|
|
152
168
|
self._initialize_data()
|
|
153
169
|
return bool(self.values["result"]["changed"])
|
|
@@ -60,7 +60,7 @@ class BatchFetchCli:
|
|
|
60
60
|
options={})
|
|
61
61
|
batchfetch_instance.validate_schema()
|
|
62
62
|
|
|
63
|
-
self.batchfetch_schemas[keyword] = batchfetch_instance.
|
|
63
|
+
self.batchfetch_schemas[keyword] = batchfetch_instance.task_schema
|
|
64
64
|
self.batchfetch_classes[keyword] = batchfetch_class
|
|
65
65
|
self.cfg_schema = {
|
|
66
66
|
Optional("options"):
|
|
@@ -105,7 +105,7 @@ class BatchFetchCli:
|
|
|
105
105
|
sys.exit(1)
|
|
106
106
|
|
|
107
107
|
self.cfg = {
|
|
108
|
-
"options": {"
|
|
108
|
+
"options": {"git_clone_args": []},
|
|
109
109
|
"tasks": [],
|
|
110
110
|
}
|
|
111
111
|
|
|
@@ -42,31 +42,31 @@ class BatchFetchGit(BatchFetchBase):
|
|
|
42
42
|
def __init__(self, *args, **kwargs):
|
|
43
43
|
super().__init__(*args, **kwargs)
|
|
44
44
|
self.env["GIT_TERMINAL_PROMPT"] = "0"
|
|
45
|
-
self.
|
|
45
|
+
self.env["GIT_PAGER"] = ""
|
|
46
46
|
self.main_key = "git"
|
|
47
47
|
|
|
48
48
|
# Schema
|
|
49
|
-
self.
|
|
49
|
+
self.task_schema.update({
|
|
50
50
|
# Local options
|
|
51
51
|
self.main_key: str,
|
|
52
52
|
Optional("reference"): str,
|
|
53
53
|
|
|
54
54
|
# Same as global options
|
|
55
|
-
Optional("
|
|
55
|
+
Optional("git_clone_args"): [str],
|
|
56
56
|
Optional("git_pull"): bool,
|
|
57
57
|
})
|
|
58
58
|
|
|
59
59
|
self.global_options_schema.update({
|
|
60
60
|
# Global options
|
|
61
|
-
Optional("
|
|
61
|
+
Optional("git_clone_args"): [str],
|
|
62
62
|
Optional("git_pull"): bool,
|
|
63
63
|
})
|
|
64
64
|
|
|
65
65
|
# Data
|
|
66
|
-
self.global_options_values.update({"
|
|
66
|
+
self.global_options_values.update({"git_clone_args": [],
|
|
67
67
|
"git_pull": True})
|
|
68
68
|
|
|
69
|
-
self.
|
|
69
|
+
self.task_default_values.update({
|
|
70
70
|
self.main_key: "",
|
|
71
71
|
"reference": "",
|
|
72
72
|
"delete": False,
|
|
@@ -191,7 +191,7 @@ class BatchFetchGit(BatchFetchBase):
|
|
|
191
191
|
self.set_changed(True)
|
|
192
192
|
|
|
193
193
|
def _repo_clone(self):
|
|
194
|
-
git_clone_args = self["
|
|
194
|
+
git_clone_args = self["git_clone_args"]
|
|
195
195
|
git_clone_args += ["--recurse-submodules"]
|
|
196
196
|
|
|
197
197
|
cmd = ["git", "clone"] + git_clone_args + \
|
|
@@ -209,57 +209,65 @@ class BatchFetchGit(BatchFetchBase):
|
|
|
209
209
|
|
|
210
210
|
# Merge
|
|
211
211
|
do_git_pull = self["git_pull"]
|
|
212
|
-
disable_merge = False
|
|
213
212
|
if self["reference"]:
|
|
213
|
+
do_git_pull = False
|
|
214
214
|
commit_ref = None
|
|
215
|
+
|
|
215
216
|
try:
|
|
216
217
|
# Returns the commit ref of the branch or commit
|
|
217
|
-
|
|
218
|
+
#
|
|
219
|
+
# If the tag is annotated, it points to a tag object, not
|
|
220
|
+
# directly to a commit. You need to resolve it to the commit it
|
|
221
|
+
# points to. Using `git rev-parse <tagname>^{commit}` allows
|
|
222
|
+
# getting the right reference.
|
|
223
|
+
commit_ref = self._git_tags(self["reference"] + "^{commit}")[0]
|
|
218
224
|
except GitReferenceDoesNotExist:
|
|
219
|
-
# The reference does not exist. We should git pull
|
|
220
|
-
#
|
|
225
|
+
# The wanted reference does not exist. We should git pull in
|
|
226
|
+
# case we can get the reference
|
|
221
227
|
do_git_pull = True
|
|
222
228
|
else: # The reference exists
|
|
223
|
-
if not self._git_is_local_branch(self["reference"]):
|
|
224
|
-
# This is not a real branch where we can merge
|
|
225
|
-
disable_merge = True
|
|
226
|
-
|
|
227
229
|
try:
|
|
228
|
-
#
|
|
229
|
-
|
|
230
|
+
# If the tag is annotated, it points to a tag object, not
|
|
231
|
+
# directly to a commit. You need to resolve it to the
|
|
232
|
+
# commit it points to. Using `git rev-parse
|
|
233
|
+
# <tagname>^{commit}` allows getting the right reference.
|
|
234
|
+
commit_ref_head = self._git_tags("HEAD^{commit}")[0]
|
|
230
235
|
except GitReferenceDoesNotExist:
|
|
231
236
|
# HEAD is detached
|
|
232
237
|
commit_ref_head = None
|
|
233
238
|
|
|
234
239
|
# The wanted commit reference does not exist
|
|
235
240
|
# Or the commit ref of HEAD hasn't changed
|
|
236
|
-
if
|
|
237
|
-
do_git_pull = True
|
|
238
|
-
else:
|
|
241
|
+
if commit_ref and commit_ref_head == commit_ref:
|
|
239
242
|
do_git_pull = False
|
|
243
|
+
else:
|
|
244
|
+
do_git_pull = True
|
|
240
245
|
|
|
241
246
|
if not do_git_pull:
|
|
242
|
-
self.add_output(self.indent_spaces +
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
+
self.add_output(self.indent_spaces + "[INFO] git pull ignored\n")
|
|
248
|
+
return git_merge
|
|
249
|
+
|
|
250
|
+
# Fetch
|
|
251
|
+
cmd = ["git", "fetch", "origin"]
|
|
252
|
+
self._run(cmd, cwd=str(self.git_local_dir), env=self.env)
|
|
247
253
|
|
|
254
|
+
# Merge
|
|
255
|
+
real_branch = self._git_is_local_branch("HEAD")
|
|
256
|
+
if real_branch:
|
|
248
257
|
# TODO: only merge when difference from upstream
|
|
249
258
|
commit_ref_head = self._git_ref(cwd=self.git_local_dir)
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
env=self.env)
|
|
259
|
+
self._run(["git", "merge", "--ff-only"],
|
|
260
|
+
cwd=str(self.git_local_dir), env=self.env)
|
|
261
|
+
git_ref_after_merge = self._git_ref(cwd=self.git_local_dir)
|
|
262
|
+
if commit_ref_head != git_ref_after_merge:
|
|
263
|
+
git_merge = True
|
|
264
|
+
self.set_changed(True)
|
|
265
|
+
self._run(["git", "log",
|
|
266
|
+
'--pretty=format:"%h %ad %s [%cn]"',
|
|
267
|
+
"--decorate", "--date=short",
|
|
268
|
+
f"{commit_ref_head}..{git_ref_after_merge}"],
|
|
269
|
+
cwd=str(self.git_local_dir),
|
|
270
|
+
env=self.env)
|
|
263
271
|
|
|
264
272
|
return git_merge
|
|
265
273
|
|
|
@@ -307,9 +315,10 @@ class BatchFetchGit(BatchFetchBase):
|
|
|
307
315
|
# Also check the commit reference in case
|
|
308
316
|
# branch is a commit reference instead of a tag
|
|
309
317
|
try:
|
|
310
|
-
git_ref_branch = self._git_tags(self["reference"]
|
|
318
|
+
git_ref_branch = self._git_tags(self["reference"] +
|
|
319
|
+
"^{commit}")[0]
|
|
311
320
|
except GitReferenceDoesNotExist as err:
|
|
312
|
-
raise BatchFetchError(f"The branch '{self['
|
|
321
|
+
raise BatchFetchError(f"The branch '{self['reference']}' "
|
|
313
322
|
"does not exist.") from err
|
|
314
323
|
|
|
315
324
|
if git_ref_after_merge != git_ref_branch and \
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|