batchfetch 1.2.6__tar.gz → 1.2.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- batchfetch-1.2.8/PKG-INFO +212 -0
- batchfetch-1.2.8/README.md +192 -0
- {batchfetch-1.2.6 → batchfetch-1.2.8}/batchfetch/batchfetch_base.py +3 -3
- {batchfetch-1.2.6 → batchfetch-1.2.8}/batchfetch/batchfetch_cli.py +120 -38
- {batchfetch-1.2.6 → batchfetch-1.2.8}/batchfetch/batchfetch_git.py +61 -108
- {batchfetch-1.2.6 → batchfetch-1.2.8}/batchfetch/helpers.py +1 -1
- batchfetch-1.2.8/batchfetch.egg-info/PKG-INFO +212 -0
- {batchfetch-1.2.6 → batchfetch-1.2.8}/setup.py +1 -1
- batchfetch-1.2.6/PKG-INFO +0 -130
- batchfetch-1.2.6/README.md +0 -110
- batchfetch-1.2.6/batchfetch.egg-info/PKG-INFO +0 -130
- {batchfetch-1.2.6 → batchfetch-1.2.8}/LICENSE +0 -0
- {batchfetch-1.2.6 → batchfetch-1.2.8}/batchfetch/__init__.py +0 -0
- {batchfetch-1.2.6 → batchfetch-1.2.8}/batchfetch.egg-info/SOURCES.txt +0 -0
- {batchfetch-1.2.6 → batchfetch-1.2.8}/batchfetch.egg-info/dependency_links.txt +0 -0
- {batchfetch-1.2.6 → batchfetch-1.2.8}/batchfetch.egg-info/entry_points.txt +0 -0
- {batchfetch-1.2.6 → batchfetch-1.2.8}/batchfetch.egg-info/requires.txt +0 -0
- {batchfetch-1.2.6 → batchfetch-1.2.8}/batchfetch.egg-info/top_level.txt +0 -0
- {batchfetch-1.2.6 → batchfetch-1.2.8}/setup.cfg +0 -0
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: batchfetch
|
|
3
|
+
Version: 1.2.8
|
|
4
|
+
Summary: Efficiently clone and pull multiple Git repositories.
|
|
5
|
+
Home-page: https://github.com/jamescherti/batchfetch
|
|
6
|
+
Author: James Cherti
|
|
7
|
+
License: GPLv3
|
|
8
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
9
|
+
Classifier: License :: OSI Approved :: GNU General Public License (GPL)
|
|
10
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
13
|
+
Classifier: Operating System :: POSIX :: Other
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Software Development :: Version Control :: Git
|
|
16
|
+
Classifier: Topic :: Utilities
|
|
17
|
+
Requires-Python: >=3.6, <4
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
|
|
21
|
+
# Batchfetch - Efficiently clone or pull multiple Git repositories in parallel
|
|
22
|
+
|
|
23
|
+
## Introduction
|
|
24
|
+
|
|
25
|
+
Batchfetch is a command-line tool designed to clone, fetch, and merge multiple Git repositories simultaneously. With Batchfetch, you no longer need to manually manage each repository one by one. It automates the tedious aspects of repository management, freeing you up to focus on what truly matters: your workflow.
|
|
26
|
+
|
|
27
|
+
But why use Batchfetch? Because it is extremely fast, cloning repositories quickly by running Git operations in parallel. It intelligently detects whether a `git fetch` is needed, further speeding up the process of downloading data from repositories. Additionally, it allows specifying the revision (for Git), ensuring that the cloned repository matches the exact version you require.
|
|
28
|
+
|
|
29
|
+
Batchfetch is ideal for quickly cloning or pulling multiple Git repositories. It is also useful for cloning various addons, such as Vim plugins, Emacs packages, Ansible roles, Ansible collections, and other addons available on websites like GitHub, Codeberg, and GitLab.
|
|
30
|
+
|
|
31
|
+
## Installation
|
|
32
|
+
|
|
33
|
+
Here is how to install *batchfetch* using [pip](https://pypi.org/project/pip/):
|
|
34
|
+
```
|
|
35
|
+
pip install --user batchfetch
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The pip command above installs the *batchfetch* executable in the `~/.local/bin/` directory. Omitting the `--user` flag will install it system-wide.
|
|
39
|
+
|
|
40
|
+
## Usage
|
|
41
|
+
|
|
42
|
+
### Example of a `batchfetch.yaml` file
|
|
43
|
+
|
|
44
|
+
Here is an example of a `batchfetch.yaml` file:
|
|
45
|
+
|
|
46
|
+
```yaml
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
tasks:
|
|
50
|
+
# Clone the default branch of the general.el repository to the
|
|
51
|
+
# './general.el' directory
|
|
52
|
+
- git: https://github.com/jamescherti/compile-angel.el
|
|
53
|
+
|
|
54
|
+
# Clone the tag 1.5 of the consult repository to the './consult'
|
|
55
|
+
# directory
|
|
56
|
+
- git: https://github.com/jamescherti/outline-indent.el
|
|
57
|
+
revision: "1.1.0"
|
|
58
|
+
|
|
59
|
+
# Clone the s.el repository to the './another-name.el' directory
|
|
60
|
+
- git: https://github.com/jamescherti/easysession.el
|
|
61
|
+
path: easysession
|
|
62
|
+
revision: b9c6d9b6134b4981760893254f804a371ffbc899
|
|
63
|
+
|
|
64
|
+
# Delete the local copy of the following repository
|
|
65
|
+
- git: https://github.com/jamescherti/dir-config.el
|
|
66
|
+
delete: true
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Execute the `batchfetch` command from the same directory as `batchfetch.yml` to make it clone or update the local copies of the repositories above.
|
|
70
|
+
|
|
71
|
+
## Command-line options
|
|
72
|
+
|
|
73
|
+
Here are the various options that `batchfetch` provides, along with descriptions of their usage:
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
usage: batchfetch [--option] [TARGET]
|
|
77
|
+
|
|
78
|
+
Efficiently clone/pull multiple Git repositories in parallel.
|
|
79
|
+
|
|
80
|
+
positional arguments:
|
|
81
|
+
target This is a target path that batchfetch is supposed to
|
|
82
|
+
handle. When no target is specified, execute the tasks
|
|
83
|
+
of all target paths defined in the batchfetch.yml list
|
|
84
|
+
of tasks.
|
|
85
|
+
|
|
86
|
+
options:
|
|
87
|
+
-h, --help show this help message and exit
|
|
88
|
+
-f FILE, --file FILE Specify the batchfetch YAML file (default:
|
|
89
|
+
'./batchfetch.yaml').
|
|
90
|
+
-C DIRECTORY, --directory DIRECTORY
|
|
91
|
+
Change the working directory before reading the
|
|
92
|
+
batchfetch.yaml file. If not specified, the directory
|
|
93
|
+
is set to the parent directory of the batchfetch.yaml
|
|
94
|
+
file.
|
|
95
|
+
-j JOBS, --jobs JOBS Run up to N parallel processes (default: 5).
|
|
96
|
+
Alternatively, the BATCHFETCH_JOBS environment
|
|
97
|
+
variable can be used to configure the number of jobs.
|
|
98
|
+
-v, --verbose Enable verbose mode.
|
|
99
|
+
-u, --check-untracked
|
|
100
|
+
Abort if untracked files or directories exist.
|
|
101
|
+
Alternatively, set the BATCHFETCH_CHECK_UNTRACKED=1
|
|
102
|
+
environment variable to enable this check.
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## Features
|
|
106
|
+
- Git Clone and Fetch/Merge: Clones the repositories and their submodules, ensuring that all the repositories are always up-to-date by fetching and merging changes.
|
|
107
|
+
- Parallel Operations: Utilizes threads to simultaneously Git clone or pull multiple repositories, dramatically reducing wait times.
|
|
108
|
+
- User-Friendly Interface: Provides simple and straightforward command-line options that make it easy to get started and effectively manage your repositories.
|
|
109
|
+
- Custom Configuration: Allows the use of a YAML configuration file to specify and manage the repositories you interact with, enabling repeatable setups and consistent environments.
|
|
110
|
+
- Detect files that should not be present in directories managed by batchfetch, known as untracked files.
|
|
111
|
+
|
|
112
|
+
## Frequently Asked Questions
|
|
113
|
+
|
|
114
|
+
### What are untracked files?
|
|
115
|
+
|
|
116
|
+
The parent directory of the "path:" value defines the managed directory, where the directory of each path is considered as the managed directory.
|
|
117
|
+
|
|
118
|
+
For example, if the "path:" value is `file/my-project`, the managed directory will be `file/`. Any file within `file/` that is not managed by batchfetch will be considered an untracked file.
|
|
119
|
+
|
|
120
|
+
When *batchfetch* encounters an untracked file, it displays an error message to inform users about paths that are not managed by the system. The message provides clear instructions on how to handle these paths by adding them to the `options.ignore_untracked_paths` list, enabling users to manage untracked files effectively.
|
|
121
|
+
|
|
122
|
+
Here is an example of a *batchfetch.yaml* file that enables *batchfetch* to accept a list of untracked files:
|
|
123
|
+
|
|
124
|
+
``` yaml
|
|
125
|
+
options:
|
|
126
|
+
ignore_untracked_paths:
|
|
127
|
+
- ./test
|
|
128
|
+
- /absolute/path
|
|
129
|
+
- ../relative/path
|
|
130
|
+
|
|
131
|
+
tasks:
|
|
132
|
+
- git: https://github.com/user/project
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
By default, *batchfetch.yaml* is the only untracked file that is ignored. The user does not need to add it to the *ignore_untracked_paths* option.
|
|
136
|
+
|
|
137
|
+
### How is the Git local paths handled?
|
|
138
|
+
|
|
139
|
+
When "path:" is specified, that's the path that is used.
|
|
140
|
+
|
|
141
|
+
When "path:" is not specified, Batchfetch attempts to determine the path name by extracting the repository name from the URI (e.g., `https://domain.com/repo` becomes `repo`). If the URL ends with a `.git` extension, it removes the extension (e.g., `https://domain.com/repo.git` becomes `repo`).
|
|
142
|
+
|
|
143
|
+
### How does Batchfetch detect when a git fetch is necessary?
|
|
144
|
+
|
|
145
|
+
Batchfetch is fast, not only because it runs Git commands in parallel, but also because it intelligently detects whether a `git fetch` is needed, further speeding up the process of downloading data from repositories.
|
|
146
|
+
|
|
147
|
+
When the user has specifies a revision (branch or commit reference), Batchfetch only performs a `git fetch` if that revision does not exist locally. If the revision is already up to date, it simply proceeds to the next repository in the queue.
|
|
148
|
+
|
|
149
|
+
That's why it is highly recommended to always specify the revision to speed up Batchfetch, if speed is important to you. Here is an example of a `batchfetch.yaml` file where the branch (`1.1.0`) or commit reference (`b9c6d9b6134b4981760893254f804a371ffbc899`) is specified:
|
|
150
|
+
``` yaml
|
|
151
|
+
tasks:
|
|
152
|
+
- git: https://github.com/jamescherti/outline-indent.el
|
|
153
|
+
revision: "1.1.0"
|
|
154
|
+
|
|
155
|
+
- git: https://github.com/jamescherti/easysession.el
|
|
156
|
+
path: easysession
|
|
157
|
+
revision: b9c6d9b6134b4981760893254f804a371ffbc899
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### How to execute a command before and after a task?
|
|
161
|
+
|
|
162
|
+
To execute a command both before and after a specific task, you can define the `exec_before` and `exec_after` directives within the task configuration. These directives specify commands to be executed at the respective stages of the task lifecycle.
|
|
163
|
+
|
|
164
|
+
Here is an example:
|
|
165
|
+
``` yaml
|
|
166
|
+
---
|
|
167
|
+
tasks:
|
|
168
|
+
- git: https://github.com/jamescherti/easysession.el
|
|
169
|
+
path: easysession
|
|
170
|
+
exec_before: ["sh", "-c", "echo exec_before_task"]
|
|
171
|
+
exec_after: ["sh", "-c", "echo exec_after_task"]
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
### How to make batchfetch handle only one path?
|
|
175
|
+
|
|
176
|
+
To configure `batchfetch` to handle a specific path, you can define your tasks in a `batchfetch.yml` file and pass the desired path as an argument to the `batchfetch` command.
|
|
177
|
+
|
|
178
|
+
#### Example `batchfetch.yml` file:
|
|
179
|
+
|
|
180
|
+
In the following example, the `easysession` task clones two Git repositories:
|
|
181
|
+
```yaml
|
|
182
|
+
---
|
|
183
|
+
tasks:
|
|
184
|
+
- git: https://github.com/jamescherti/easysession.el
|
|
185
|
+
path: easysession
|
|
186
|
+
|
|
187
|
+
- git: https://github.com/jamescherti/outline-indent.el
|
|
188
|
+
revision: "1.1.0"
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
To make `batchfetch` clone only `easysession`, pass its path as an argument:
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
batchfetch easysession
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
This will execute only the task corresponding to the `easysession` path, skipping all others in the `batchfetch.yml` file.
|
|
198
|
+
|
|
199
|
+
## License
|
|
200
|
+
|
|
201
|
+
Copyright (C) 2024 [James Cherti](https://www.jamescherti.com)
|
|
202
|
+
|
|
203
|
+
This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
|
|
204
|
+
|
|
205
|
+
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
|
206
|
+
|
|
207
|
+
You should have received a copy of the GNU General Public License along with this program.
|
|
208
|
+
|
|
209
|
+
## Links
|
|
210
|
+
|
|
211
|
+
- [batchfetch @GitHub](https://github.com/jamescherti/batchfetch)
|
|
212
|
+
- [batchfetch @Pypi](https://pypi.org/project/batchfetch/)
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
# Batchfetch - Efficiently clone or pull multiple Git repositories in parallel
|
|
2
|
+
|
|
3
|
+
## Introduction
|
|
4
|
+
|
|
5
|
+
Batchfetch is a command-line tool designed to clone, fetch, and merge multiple Git repositories simultaneously. With Batchfetch, you no longer need to manually manage each repository one by one. It automates the tedious aspects of repository management, freeing you up to focus on what truly matters: your workflow.
|
|
6
|
+
|
|
7
|
+
But why use Batchfetch? Because it is extremely fast, cloning repositories quickly by running Git operations in parallel. It intelligently detects whether a `git fetch` is needed, further speeding up the process of downloading data from repositories. Additionally, it allows specifying the revision (for Git), ensuring that the cloned repository matches the exact version you require.
|
|
8
|
+
|
|
9
|
+
Batchfetch is ideal for quickly cloning or pulling multiple Git repositories. It is also useful for cloning various addons, such as Vim plugins, Emacs packages, Ansible roles, Ansible collections, and other addons available on websites like GitHub, Codeberg, and GitLab.
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
Here is how to install *batchfetch* using [pip](https://pypi.org/project/pip/):
|
|
14
|
+
```
|
|
15
|
+
pip install --user batchfetch
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
The pip command above installs the *batchfetch* executable in the `~/.local/bin/` directory. Omitting the `--user` flag will install it system-wide.
|
|
19
|
+
|
|
20
|
+
## Usage
|
|
21
|
+
|
|
22
|
+
### Example of a `batchfetch.yaml` file
|
|
23
|
+
|
|
24
|
+
Here is an example of a `batchfetch.yaml` file:
|
|
25
|
+
|
|
26
|
+
```yaml
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
tasks:
|
|
30
|
+
# Clone the default branch of the general.el repository to the
|
|
31
|
+
# './general.el' directory
|
|
32
|
+
- git: https://github.com/jamescherti/compile-angel.el
|
|
33
|
+
|
|
34
|
+
# Clone the tag 1.5 of the consult repository to the './consult'
|
|
35
|
+
# directory
|
|
36
|
+
- git: https://github.com/jamescherti/outline-indent.el
|
|
37
|
+
revision: "1.1.0"
|
|
38
|
+
|
|
39
|
+
# Clone the s.el repository to the './another-name.el' directory
|
|
40
|
+
- git: https://github.com/jamescherti/easysession.el
|
|
41
|
+
path: easysession
|
|
42
|
+
revision: b9c6d9b6134b4981760893254f804a371ffbc899
|
|
43
|
+
|
|
44
|
+
# Delete the local copy of the following repository
|
|
45
|
+
- git: https://github.com/jamescherti/dir-config.el
|
|
46
|
+
delete: true
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Execute the `batchfetch` command from the same directory as `batchfetch.yml` to make it clone or update the local copies of the repositories above.
|
|
50
|
+
|
|
51
|
+
## Command-line options
|
|
52
|
+
|
|
53
|
+
Here are the various options that `batchfetch` provides, along with descriptions of their usage:
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
usage: batchfetch [--option] [TARGET]
|
|
57
|
+
|
|
58
|
+
Efficiently clone/pull multiple Git repositories in parallel.
|
|
59
|
+
|
|
60
|
+
positional arguments:
|
|
61
|
+
target This is a target path that batchfetch is supposed to
|
|
62
|
+
handle. When no target is specified, execute the tasks
|
|
63
|
+
of all target paths defined in the batchfetch.yml list
|
|
64
|
+
of tasks.
|
|
65
|
+
|
|
66
|
+
options:
|
|
67
|
+
-h, --help show this help message and exit
|
|
68
|
+
-f FILE, --file FILE Specify the batchfetch YAML file (default:
|
|
69
|
+
'./batchfetch.yaml').
|
|
70
|
+
-C DIRECTORY, --directory DIRECTORY
|
|
71
|
+
Change the working directory before reading the
|
|
72
|
+
batchfetch.yaml file. If not specified, the directory
|
|
73
|
+
is set to the parent directory of the batchfetch.yaml
|
|
74
|
+
file.
|
|
75
|
+
-j JOBS, --jobs JOBS Run up to N parallel processes (default: 5).
|
|
76
|
+
Alternatively, the BATCHFETCH_JOBS environment
|
|
77
|
+
variable can be used to configure the number of jobs.
|
|
78
|
+
-v, --verbose Enable verbose mode.
|
|
79
|
+
-u, --check-untracked
|
|
80
|
+
Abort if untracked files or directories exist.
|
|
81
|
+
Alternatively, set the BATCHFETCH_CHECK_UNTRACKED=1
|
|
82
|
+
environment variable to enable this check.
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Features
|
|
86
|
+
- Git Clone and Fetch/Merge: Clones the repositories and their submodules, ensuring that all the repositories are always up-to-date by fetching and merging changes.
|
|
87
|
+
- Parallel Operations: Utilizes threads to simultaneously Git clone or pull multiple repositories, dramatically reducing wait times.
|
|
88
|
+
- User-Friendly Interface: Provides simple and straightforward command-line options that make it easy to get started and effectively manage your repositories.
|
|
89
|
+
- Custom Configuration: Allows the use of a YAML configuration file to specify and manage the repositories you interact with, enabling repeatable setups and consistent environments.
|
|
90
|
+
- Detect files that should not be present in directories managed by batchfetch, known as untracked files.
|
|
91
|
+
|
|
92
|
+
## Frequently Asked Questions
|
|
93
|
+
|
|
94
|
+
### What are untracked files?
|
|
95
|
+
|
|
96
|
+
The parent directory of the "path:" value defines the managed directory, where the directory of each path is considered as the managed directory.
|
|
97
|
+
|
|
98
|
+
For example, if the "path:" value is `file/my-project`, the managed directory will be `file/`. Any file within `file/` that is not managed by batchfetch will be considered an untracked file.
|
|
99
|
+
|
|
100
|
+
When *batchfetch* encounters an untracked file, it displays an error message to inform users about paths that are not managed by the system. The message provides clear instructions on how to handle these paths by adding them to the `options.ignore_untracked_paths` list, enabling users to manage untracked files effectively.
|
|
101
|
+
|
|
102
|
+
Here is an example of a *batchfetch.yaml* file that enables *batchfetch* to accept a list of untracked files:
|
|
103
|
+
|
|
104
|
+
``` yaml
|
|
105
|
+
options:
|
|
106
|
+
ignore_untracked_paths:
|
|
107
|
+
- ./test
|
|
108
|
+
- /absolute/path
|
|
109
|
+
- ../relative/path
|
|
110
|
+
|
|
111
|
+
tasks:
|
|
112
|
+
- git: https://github.com/user/project
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
By default, *batchfetch.yaml* is the only untracked file that is ignored. The user does not need to add it to the *ignore_untracked_paths* option.
|
|
116
|
+
|
|
117
|
+
### How is the Git local paths handled?
|
|
118
|
+
|
|
119
|
+
When "path:" is specified, that's the path that is used.
|
|
120
|
+
|
|
121
|
+
When "path:" is not specified, Batchfetch attempts to determine the path name by extracting the repository name from the URI (e.g., `https://domain.com/repo` becomes `repo`). If the URL ends with a `.git` extension, it removes the extension (e.g., `https://domain.com/repo.git` becomes `repo`).
|
|
122
|
+
|
|
123
|
+
### How does Batchfetch detect when a git fetch is necessary?
|
|
124
|
+
|
|
125
|
+
Batchfetch is fast, not only because it runs Git commands in parallel, but also because it intelligently detects whether a `git fetch` is needed, further speeding up the process of downloading data from repositories.
|
|
126
|
+
|
|
127
|
+
When the user has specifies a revision (branch or commit reference), Batchfetch only performs a `git fetch` if that revision does not exist locally. If the revision is already up to date, it simply proceeds to the next repository in the queue.
|
|
128
|
+
|
|
129
|
+
That's why it is highly recommended to always specify the revision to speed up Batchfetch, if speed is important to you. Here is an example of a `batchfetch.yaml` file where the branch (`1.1.0`) or commit reference (`b9c6d9b6134b4981760893254f804a371ffbc899`) is specified:
|
|
130
|
+
``` yaml
|
|
131
|
+
tasks:
|
|
132
|
+
- git: https://github.com/jamescherti/outline-indent.el
|
|
133
|
+
revision: "1.1.0"
|
|
134
|
+
|
|
135
|
+
- git: https://github.com/jamescherti/easysession.el
|
|
136
|
+
path: easysession
|
|
137
|
+
revision: b9c6d9b6134b4981760893254f804a371ffbc899
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
### How to execute a command before and after a task?
|
|
141
|
+
|
|
142
|
+
To execute a command both before and after a specific task, you can define the `exec_before` and `exec_after` directives within the task configuration. These directives specify commands to be executed at the respective stages of the task lifecycle.
|
|
143
|
+
|
|
144
|
+
Here is an example:
|
|
145
|
+
``` yaml
|
|
146
|
+
---
|
|
147
|
+
tasks:
|
|
148
|
+
- git: https://github.com/jamescherti/easysession.el
|
|
149
|
+
path: easysession
|
|
150
|
+
exec_before: ["sh", "-c", "echo exec_before_task"]
|
|
151
|
+
exec_after: ["sh", "-c", "echo exec_after_task"]
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
### How to make batchfetch handle only one path?
|
|
155
|
+
|
|
156
|
+
To configure `batchfetch` to handle a specific path, you can define your tasks in a `batchfetch.yml` file and pass the desired path as an argument to the `batchfetch` command.
|
|
157
|
+
|
|
158
|
+
#### Example `batchfetch.yml` file:
|
|
159
|
+
|
|
160
|
+
In the following example, the `easysession` task clones two Git repositories:
|
|
161
|
+
```yaml
|
|
162
|
+
---
|
|
163
|
+
tasks:
|
|
164
|
+
- git: https://github.com/jamescherti/easysession.el
|
|
165
|
+
path: easysession
|
|
166
|
+
|
|
167
|
+
- git: https://github.com/jamescherti/outline-indent.el
|
|
168
|
+
revision: "1.1.0"
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
To make `batchfetch` clone only `easysession`, pass its path as an argument:
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
batchfetch easysession
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
This will execute only the task corresponding to the `easysession` path, skipping all others in the `batchfetch.yml` file.
|
|
178
|
+
|
|
179
|
+
## License
|
|
180
|
+
|
|
181
|
+
Copyright (C) 2024 [James Cherti](https://www.jamescherti.com)
|
|
182
|
+
|
|
183
|
+
This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
|
|
184
|
+
|
|
185
|
+
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
|
186
|
+
|
|
187
|
+
You should have received a copy of the GNU General Public License along with this program.
|
|
188
|
+
|
|
189
|
+
## Links
|
|
190
|
+
|
|
191
|
+
- [batchfetch @GitHub](https://github.com/jamescherti/batchfetch)
|
|
192
|
+
- [batchfetch @Pypi](https://pypi.org/project/batchfetch/)
|
|
@@ -138,7 +138,7 @@ class TaskBatchFetch(TaskBase):
|
|
|
138
138
|
# Mark values as initialized
|
|
139
139
|
self._values_initialized = True
|
|
140
140
|
|
|
141
|
-
def
|
|
141
|
+
def _pre_run(self, *args, **kwargs):
|
|
142
142
|
stdout = run_indent_str(*args, **kwargs)
|
|
143
143
|
|
|
144
144
|
if not stdout.endswith("\n"):
|
|
@@ -151,7 +151,7 @@ class TaskBatchFetch(TaskBase):
|
|
|
151
151
|
if self["delete"] or not cmd:
|
|
152
152
|
return
|
|
153
153
|
|
|
154
|
-
self.
|
|
154
|
+
self._pre_run(cmd, cwd=str(cwd), env=self.env, spaces=self.indent)
|
|
155
155
|
|
|
156
156
|
def _run_post_exec(self, cwd: os.PathLike = Path(".")):
|
|
157
157
|
self._initialize_data()
|
|
@@ -159,7 +159,7 @@ class TaskBatchFetch(TaskBase):
|
|
|
159
159
|
if not cmd or self["delete"] or not self.is_changed():
|
|
160
160
|
return
|
|
161
161
|
|
|
162
|
-
self.
|
|
162
|
+
self._pre_run(cmd, cwd=str(cwd), env=self.env, spaces=self.indent)
|
|
163
163
|
|
|
164
164
|
def is_changed(self) -> bool:
|
|
165
165
|
self._initialize_data()
|
|
@@ -25,7 +25,7 @@ import subprocess
|
|
|
25
25
|
import sys
|
|
26
26
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
27
27
|
from pathlib import Path
|
|
28
|
-
from typing import Any, Dict, Set
|
|
28
|
+
from typing import Any, Dict, List, Set, Union
|
|
29
29
|
|
|
30
30
|
import colorama
|
|
31
31
|
import yaml # type: ignore
|
|
@@ -40,14 +40,16 @@ from .batchfetch_git import BatchFetchGit
|
|
|
40
40
|
class BatchFetchCli:
|
|
41
41
|
"""Command-line-interface that downloads."""
|
|
42
42
|
|
|
43
|
-
def __init__(self, max_workers: int, verbose: bool
|
|
43
|
+
def __init__(self, max_workers: int, verbose: bool, check_untracked: bool,
|
|
44
|
+
targets: List[str]):
|
|
45
|
+
self._logger = logging.getLogger(self.__class__.__name__)
|
|
46
|
+
self.targets = {Path(item).absolute() for item in targets}
|
|
44
47
|
self.cfg: dict = {}
|
|
45
|
-
self.
|
|
48
|
+
self.check_untracked = check_untracked
|
|
46
49
|
self.tracked_paths: Dict[Path, Set[str]] = {}
|
|
47
50
|
self.ignore_untracked_paths: Set[Path] = set()
|
|
48
51
|
self.verbose = verbose
|
|
49
52
|
self.max_workers = max_workers
|
|
50
|
-
self._logger = logging.getLogger(self.__class__.__name__)
|
|
51
53
|
self.dirs_relative_to_batchfetch: Set[str] = set()
|
|
52
54
|
|
|
53
55
|
# Plugin
|
|
@@ -101,7 +103,8 @@ class BatchFetchCli:
|
|
|
101
103
|
self.ignore_untracked_paths.add(Path(path).absolute())
|
|
102
104
|
self.ignore_untracked_paths.add(Path(path).resolve())
|
|
103
105
|
untracked_paths = None
|
|
104
|
-
if "options" in yaml_dict
|
|
106
|
+
if "options" in yaml_dict and \
|
|
107
|
+
"ignore_untracked_paths" in yaml_dict["options"]:
|
|
105
108
|
untracked_paths = \
|
|
106
109
|
yaml_dict["options"]["ignore_untracked_paths"]
|
|
107
110
|
|
|
@@ -155,10 +158,12 @@ class BatchFetchCli:
|
|
|
155
158
|
print(f"Schema error: {err}.", file=sys.stderr)
|
|
156
159
|
sys.exit(1)
|
|
157
160
|
|
|
158
|
-
dest_path = Path(batchfetch_instance["path"]).
|
|
159
|
-
|
|
161
|
+
dest_path = Path(batchfetch_instance["path"]).absolute()
|
|
162
|
+
dest_path2 = Path(batchfetch_instance["path"]).resolve()
|
|
163
|
+
if str(dest_path) in dict_local_dir \
|
|
164
|
+
or str(dest_path2) in dict_local_dir:
|
|
160
165
|
err_str = ("More than one task have the " +
|
|
161
|
-
f"destination path
|
|
166
|
+
f"destination path (" +
|
|
162
167
|
str(task[keyword]) + " and " +
|
|
163
168
|
str(dict_local_dir[(str(dest_path))]) +
|
|
164
169
|
")")
|
|
@@ -179,10 +184,31 @@ class BatchFetchCli:
|
|
|
179
184
|
self.dirs_relative_to_batchfetch = set()
|
|
180
185
|
|
|
181
186
|
all_tasks = self.cfg["tasks"]
|
|
187
|
+
|
|
188
|
+
# Check targets
|
|
189
|
+
if self.targets:
|
|
190
|
+
targets_executed = set()
|
|
191
|
+
for task in all_tasks:
|
|
192
|
+
full_path = Path(task["path"]).absolute()
|
|
193
|
+
if full_path not in self.targets:
|
|
194
|
+
continue
|
|
195
|
+
|
|
196
|
+
targets_executed.add(full_path)
|
|
197
|
+
|
|
198
|
+
targets_not_found = self.targets - targets_executed
|
|
199
|
+
if targets_not_found:
|
|
200
|
+
err_str = \
|
|
201
|
+
f"Error: Target(s) not found: "
|
|
202
|
+
err_str += ", ".join([str(item)
|
|
203
|
+
for item in targets_not_found])
|
|
204
|
+
print(f"Error: {err_str}", file=sys.stderr)
|
|
205
|
+
sys.exit(1)
|
|
206
|
+
|
|
182
207
|
for task in all_tasks:
|
|
208
|
+
full_path = Path(task["path"]).absolute()
|
|
209
|
+
|
|
183
210
|
self.dirs_relative_to_batchfetch.add(str(task["path"]))
|
|
184
211
|
if not task["delete"]:
|
|
185
|
-
full_path = Path(task["path"]).absolute()
|
|
186
212
|
base_path = full_path.parent
|
|
187
213
|
try:
|
|
188
214
|
self.tracked_paths[base_path]
|
|
@@ -191,6 +217,8 @@ class BatchFetchCli:
|
|
|
191
217
|
|
|
192
218
|
self.tracked_paths[base_path].add(full_path.name)
|
|
193
219
|
|
|
220
|
+
if self.targets and full_path not in self.targets:
|
|
221
|
+
continue
|
|
194
222
|
threads.append(executor_update.submit(task.update))
|
|
195
223
|
|
|
196
224
|
for future in as_completed(threads):
|
|
@@ -232,7 +260,8 @@ class BatchFetchCli:
|
|
|
232
260
|
|
|
233
261
|
return False
|
|
234
262
|
else:
|
|
235
|
-
self.
|
|
263
|
+
if self.check_untracked:
|
|
264
|
+
self._find_untracked_paths()
|
|
236
265
|
|
|
237
266
|
if num_success == 0:
|
|
238
267
|
print("Nothing to do.")
|
|
@@ -269,10 +298,34 @@ class BatchFetchCli:
|
|
|
269
298
|
|
|
270
299
|
def parse_args():
|
|
271
300
|
"""Parse the command line arguments."""
|
|
272
|
-
|
|
273
|
-
|
|
301
|
+
# Jobs
|
|
302
|
+
try:
|
|
303
|
+
jobs = os.environ["BATCHFETCH_JOBS"]
|
|
304
|
+
except KeyError:
|
|
305
|
+
jobs = 5
|
|
306
|
+
else:
|
|
307
|
+
if jobs:
|
|
308
|
+
jobs = int(jobs)
|
|
309
|
+
|
|
310
|
+
# Check untracked
|
|
311
|
+
try:
|
|
312
|
+
check_untracked = os.environ["BATCHFETCH_CHECK_UNTRACKED"]
|
|
313
|
+
except KeyError:
|
|
314
|
+
check_untracked = False
|
|
315
|
+
else:
|
|
316
|
+
check_untracked = bool(check_untracked)
|
|
317
|
+
|
|
318
|
+
desc = "Efficiently clone/pull multiple Git repositories in parallel."
|
|
319
|
+
usage = "%(prog)s [--option] [TARGET]"
|
|
274
320
|
parser = argparse.ArgumentParser(description=desc, usage=usage)
|
|
275
321
|
|
|
322
|
+
parser.add_argument(
|
|
323
|
+
"targets", metavar="target", type=str, nargs="*",
|
|
324
|
+
help=("This is a target path that batchfetch is supposed to handle. "
|
|
325
|
+
"When no target is specified, execute the tasks of all target "
|
|
326
|
+
"paths defined in the batchfetch.yml list of tasks."),
|
|
327
|
+
)
|
|
328
|
+
|
|
276
329
|
parser.add_argument("-f",
|
|
277
330
|
"--file",
|
|
278
331
|
default=None,
|
|
@@ -281,15 +334,40 @@ def parse_args():
|
|
|
281
334
|
"(default: './batchfetch.yaml')."))
|
|
282
335
|
|
|
283
336
|
parser.add_argument(
|
|
284
|
-
"-
|
|
285
|
-
|
|
337
|
+
"-C",
|
|
338
|
+
"--directory",
|
|
339
|
+
default=None,
|
|
340
|
+
required=False,
|
|
341
|
+
help=("Change the working directory before reading the "
|
|
342
|
+
"batchfetch.yaml file. If not specified, the directory is "
|
|
343
|
+
"set to the parent directory of the batchfetch.yaml file.")
|
|
286
344
|
)
|
|
287
345
|
|
|
346
|
+
parser.add_argument(
|
|
347
|
+
"-j",
|
|
348
|
+
"--jobs",
|
|
349
|
+
default=jobs,
|
|
350
|
+
type=int,
|
|
351
|
+
required=False,
|
|
352
|
+
help=("Run up to N parallel processes (default: 5). "
|
|
353
|
+
"Alternatively, the BATCHFETCH_JOBS environment variable can be "
|
|
354
|
+
"used to configure the number of jobs."))
|
|
355
|
+
|
|
288
356
|
parser.add_argument(
|
|
289
357
|
"-v", "--verbose", action="store_true", default=False,
|
|
290
358
|
help="Enable verbose mode.",
|
|
291
359
|
)
|
|
292
360
|
|
|
361
|
+
parser.add_argument(
|
|
362
|
+
"-u",
|
|
363
|
+
"--check-untracked",
|
|
364
|
+
default=check_untracked,
|
|
365
|
+
action="store_true",
|
|
366
|
+
required=False,
|
|
367
|
+
help=("Abort if untracked files or directories exist. "
|
|
368
|
+
"Alternatively, set the BATCHFETCH_CHECK_UNTRACKED=1 "
|
|
369
|
+
"environment variable to enable this check."))
|
|
370
|
+
|
|
293
371
|
args = parser.parse_args()
|
|
294
372
|
|
|
295
373
|
if not args.file:
|
|
@@ -303,26 +381,6 @@ def parse_args():
|
|
|
303
381
|
return args
|
|
304
382
|
|
|
305
383
|
|
|
306
|
-
def run_batchfetch_procedure(file: Path, args) -> int:
|
|
307
|
-
errno = 0
|
|
308
|
-
batchfetch_cli = BatchFetchCli(verbose=args.verbose,
|
|
309
|
-
max_workers=int(args.jobs))
|
|
310
|
-
os.chdir(file.parent)
|
|
311
|
-
batchfetch_cli.load(file)
|
|
312
|
-
|
|
313
|
-
try:
|
|
314
|
-
if not batchfetch_cli.run_tasks():
|
|
315
|
-
errno = 1
|
|
316
|
-
except KeyboardInterrupt:
|
|
317
|
-
print("Interrupted.", file=sys.stderr)
|
|
318
|
-
errno = 1
|
|
319
|
-
except BatchFetchError as err:
|
|
320
|
-
print(f"Error: {err}.", file=sys.stderr)
|
|
321
|
-
errno = 1
|
|
322
|
-
|
|
323
|
-
return errno
|
|
324
|
-
|
|
325
|
-
|
|
326
384
|
def command_line_interface():
|
|
327
385
|
"""Command line interface."""
|
|
328
386
|
try:
|
|
@@ -336,16 +394,40 @@ def command_line_interface():
|
|
|
336
394
|
|
|
337
395
|
args = parse_args()
|
|
338
396
|
done = []
|
|
339
|
-
file = Path(args.file)
|
|
340
|
-
file_resolved = file.
|
|
397
|
+
file = Path(args.file).absolute()
|
|
398
|
+
file_resolved = file.absolute()
|
|
341
399
|
if not file_resolved:
|
|
342
400
|
print(f"Error: cannot resolve the path {file}",
|
|
343
401
|
file=sys.stderr)
|
|
344
402
|
sys.exit(1)
|
|
345
403
|
|
|
346
404
|
done.append(file_resolved)
|
|
347
|
-
|
|
405
|
+
|
|
406
|
+
args.directory = args.directory if args.directory else file.parent
|
|
407
|
+
if args.verbose and args.jobs:
|
|
408
|
+
print(f"[FILE] {file}")
|
|
409
|
+
print(f"[DIR] {args.directory}")
|
|
410
|
+
print(f"[JOBS] {args.jobs}")
|
|
411
|
+
print(f"[CHECK UNTRACKED] {args.check_untracked}")
|
|
412
|
+
print()
|
|
413
|
+
|
|
414
|
+
os.chdir(args.directory)
|
|
415
|
+
batchfetch_cli = BatchFetchCli(verbose=args.verbose,
|
|
416
|
+
max_workers=args.jobs,
|
|
417
|
+
check_untracked=args.check_untracked,
|
|
418
|
+
targets=args.targets)
|
|
419
|
+
batchfetch_cli.load(file)
|
|
420
|
+
|
|
421
|
+
try:
|
|
422
|
+
if not batchfetch_cli.run_tasks():
|
|
423
|
+
errno = 1
|
|
424
|
+
except KeyboardInterrupt:
|
|
425
|
+
print("Interrupted.", file=sys.stderr)
|
|
426
|
+
errno = 1
|
|
427
|
+
except BatchFetchError as err:
|
|
428
|
+
print(f"Error: {err}.", file=sys.stderr)
|
|
429
|
+
errno = 1
|
|
348
430
|
|
|
349
431
|
sys.exit(errno)
|
|
350
432
|
except BrokenPipeError:
|
|
351
|
-
|
|
433
|
+
sys.exit(1)
|