close1 0.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- close1-0.7.1/LICENSE +21 -0
- close1-0.7.1/PKG-INFO +118 -0
- close1-0.7.1/README.md +93 -0
- close1-0.7.1/pyproject.toml +44 -0
- close1-0.7.1/setup.cfg +4 -0
- close1-0.7.1/src/close1/__init__.py +3 -0
- close1-0.7.1/src/close1/__main__.py +3 -0
- close1-0.7.1/src/close1/backup.py +214 -0
- close1-0.7.1/src/close1/cli.py +168 -0
- close1-0.7.1/src/close1/config.py +271 -0
- close1-0.7.1/src/close1/messages.py +38 -0
- close1-0.7.1/src/close1/rotate.py +21 -0
- close1-0.7.1/src/close1/ssh.py +199 -0
- close1-0.7.1/src/close1.egg-info/PKG-INFO +118 -0
- close1-0.7.1/src/close1.egg-info/SOURCES.txt +21 -0
- close1-0.7.1/src/close1.egg-info/dependency_links.txt +1 -0
- close1-0.7.1/src/close1.egg-info/entry_points.txt +2 -0
- close1-0.7.1/src/close1.egg-info/requires.txt +4 -0
- close1-0.7.1/src/close1.egg-info/top_level.txt +1 -0
- close1-0.7.1/tests/test_backup.py +170 -0
- close1-0.7.1/tests/test_config.py +103 -0
- close1-0.7.1/tests/test_rotate.py +16 -0
- close1-0.7.1/tests/test_ssh.py +69 -0
close1-0.7.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Yaroslav Polyakov
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
close1-0.7.1/PKG-INFO
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: close1
|
|
3
|
+
Version: 0.7.1
|
|
4
|
+
Summary: Simple backups from servers over SSH
|
|
5
|
+
Author: Yaroslav Polyakov
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/yaroslaff/close1
|
|
8
|
+
Project-URL: Repository, https://github.com/yaroslaff/close1
|
|
9
|
+
Project-URL: Issues, https://github.com/yaroslaff/close1/issues
|
|
10
|
+
Keywords: backup,ssh,cron,pull,sqlite,mysql
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: System Administrators
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Topic :: System :: Archiving :: Backup
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: PyYAML>=5.4
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: pytest; extra == "dev"
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# close1
|
|
27
|
+
|
|
28
|
+
A simple tool for daily backups of files and databases over SSH.
|
|
29
|
+
|
|
30
|
+
## Features
|
|
31
|
+
- Pulls files and directories (.tar.gz/.zip) from the servers to your machine; nothing is installed on the servers.
|
|
32
|
+
- Runs commands on the servers (for example, `mysqldump`, `.backup` in sqlite3, or stopping a web server for the duration of the backup and starting it again afterwards).
|
|
33
|
+
- Good SSH integration: uses your key through the SSH agent (even if it is passphrase-protected). ControlMaster for speed and for reliability on lossy networks.
|
|
34
|
+
- Tolerates missed cron runs (e.g. when the computer was off). You can run close1 from cron every hour and it still makes one backup per day.
|
|
35
|
+
- Careful rotation: you set how many backups to keep (say, 7). Rotation removes the oldest ones, but it will not delete a 10-day-old backup if there were no newer ones. Even in the worst case you still have some backups.
|
|
36
|
+
|
|
37
|
+
## Install
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
pipx install . # or: pip install -e '.[dev]' in a venv
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
```sh
|
|
46
|
+
close1 --check # SSH agent, keys, host reachability (copies nothing)
|
|
47
|
+
close1 # back up all hosts from the config
|
|
48
|
+
close1 blog -f -v # only blog, overwrite today's backup, verbose
|
|
49
|
+
close1 log # show the log (-v: print its path first)
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Config: `-c path`, otherwise the first one found of `$CLOSE1_CONFIG`, `~/.config/close1/close1.yml`, `/etc/close1.yml`
|
|
53
|
+
(no merging). See `close1-example.yml` (short) and `close1-full-example.yml` (every option explained).
|
|
54
|
+
Log: `~/.local/state/close1/close1.log` (root: `/var/log/close1.log`).
|
|
55
|
+
|
|
56
|
+
## Config
|
|
57
|
+
|
|
58
|
+
```yaml
|
|
59
|
+
_default:
|
|
60
|
+
dest: ~/backups # -> ~/backups/<host>/<YYYY-MM-DD>/
|
|
61
|
+
keep: 7
|
|
62
|
+
|
|
63
|
+
blog:
|
|
64
|
+
pre:
|
|
65
|
+
- mysqldump blog | gzip > /tmp/blog-${DATE}.sql.gz
|
|
66
|
+
files:
|
|
67
|
+
- blog.sql.gz: /tmp/blog-${DATE}.sql.gz
|
|
68
|
+
- /var/www/blog/uploads
|
|
69
|
+
- /etc/nginx/sites-enabled
|
|
70
|
+
post:
|
|
71
|
+
- rm /tmp/blog-${DATE}.sql.gz
|
|
72
|
+
|
|
73
|
+
"backup@bot:2222":
|
|
74
|
+
files:
|
|
75
|
+
- /opt/bot/config.yml
|
|
76
|
+
stdout:
|
|
77
|
+
packages.txt: dpkg -l
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
All options with explanations: [close1-full-example.yml](close1-full-example.yml).
|
|
81
|
+
|
|
82
|
+
### SQLite
|
|
83
|
+
|
|
84
|
+
Do not copy a live SQLite file (in WAL mode part of the data lives in `-wal`). Take a snapshot in `pre` and fetch it:
|
|
85
|
+
|
|
86
|
+
```yaml
|
|
87
|
+
pre:
|
|
88
|
+
- umask 077 && sqlite3 /opt/app/app.db ".backup /tmp/app-${DATE}.db" && gzip -f /tmp/app-${DATE}.db
|
|
89
|
+
files:
|
|
90
|
+
- app.db.gz: /tmp/app-${DATE}.db.gz
|
|
91
|
+
post:
|
|
92
|
+
- rm -f /tmp/app-${DATE}.db.gz
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Unreliable connections
|
|
96
|
+
|
|
97
|
+
All commands to a host go through one SSH connection (ControlMaster), so a backup needs a single handshake.
|
|
98
|
+
After a connection error (ssh exit code 255) close1 retries the connection check, file checks and downloads
|
|
99
|
+
(`retries` times, `retry_delay` seconds apart). `pre`, `post` and `stdout` commands are not retried: after a dropped
|
|
100
|
+
connection it is unknown whether they ran. If they fail, the backup fails and the next cron run starts it again.
|
|
101
|
+
|
|
102
|
+
## cron and ssh-agent
|
|
103
|
+
|
|
104
|
+
close1 authenticates through ssh-agent. If `SSH_AUTH_SOCK` is not set (as under cron), it looks for the socket in
|
|
105
|
+
`/run/user/$UID/` (`openssh_agent`, `ssh-agent.socket`, `gcr/ssh`, `keyring/ssh`, `gnupg/S.gpg-agent.ssh`). ssh is always
|
|
106
|
+
called with `BatchMode=yes` and `ForwardAgent=no`. While you are not logged in or the key is not loaded into the agent,
|
|
107
|
+
the backup is skipped and retried on the next run.
|
|
108
|
+
|
|
109
|
+
By default the first agent that holds any key is used. If you have several agents (e.g. ssh-agent and gnome-keyring)
|
|
110
|
+
with different keys, set `ssh_key: SHA256:...` (globally or per host): close1 then uses the agent that holds this key
|
|
111
|
+
and passes it to ssh as `IdentityAgent`. If the key is in no agent, the host is skipped. `close1 --check` lists all
|
|
112
|
+
agents with their key fingerprints. The key comment (`user@host`) is not used for matching: it is arbitrary text.
|
|
113
|
+
|
|
114
|
+
```cron
|
|
115
|
+
17 * * * * $HOME/.local/bin/close1
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Only errors go to stderr (and so to cron mail); everything else goes to the log.
|
close1-0.7.1/README.md
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# close1
|
|
2
|
+
|
|
3
|
+
A simple tool for daily backups of files and databases over SSH.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
- Pulls files and directories (.tar.gz/.zip) from the servers to your machine; nothing is installed on the servers.
|
|
7
|
+
- Runs commands on the servers (for example, `mysqldump`, `.backup` in sqlite3, or stopping a web server for the duration of the backup and starting it again afterwards).
|
|
8
|
+
- Good SSH integration: uses your key through the SSH agent (even if it is passphrase-protected). ControlMaster for speed and for reliability on lossy networks.
|
|
9
|
+
- Tolerates missed cron runs (e.g. when the computer was off). You can run close1 from cron every hour and it still makes one backup per day.
|
|
10
|
+
- Careful rotation: you set how many backups to keep (say, 7). Rotation removes the oldest ones, but it will not delete a 10-day-old backup if there were no newer ones. Even in the worst case you still have some backups.
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
pipx install . # or: pip install -e '.[dev]' in a venv
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Usage
|
|
19
|
+
|
|
20
|
+
```sh
|
|
21
|
+
close1 --check # SSH agent, keys, host reachability (copies nothing)
|
|
22
|
+
close1 # back up all hosts from the config
|
|
23
|
+
close1 blog -f -v # only blog, overwrite today's backup, verbose
|
|
24
|
+
close1 log # show the log (-v: print its path first)
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Config: `-c path`, otherwise the first one found of `$CLOSE1_CONFIG`, `~/.config/close1/close1.yml`, `/etc/close1.yml`
|
|
28
|
+
(no merging). See `close1-example.yml` (short) and `close1-full-example.yml` (every option explained).
|
|
29
|
+
Log: `~/.local/state/close1/close1.log` (root: `/var/log/close1.log`).
|
|
30
|
+
|
|
31
|
+
## Config
|
|
32
|
+
|
|
33
|
+
```yaml
|
|
34
|
+
_default:
|
|
35
|
+
dest: ~/backups # -> ~/backups/<host>/<YYYY-MM-DD>/
|
|
36
|
+
keep: 7
|
|
37
|
+
|
|
38
|
+
blog:
|
|
39
|
+
pre:
|
|
40
|
+
- mysqldump blog | gzip > /tmp/blog-${DATE}.sql.gz
|
|
41
|
+
files:
|
|
42
|
+
- blog.sql.gz: /tmp/blog-${DATE}.sql.gz
|
|
43
|
+
- /var/www/blog/uploads
|
|
44
|
+
- /etc/nginx/sites-enabled
|
|
45
|
+
post:
|
|
46
|
+
- rm /tmp/blog-${DATE}.sql.gz
|
|
47
|
+
|
|
48
|
+
"backup@bot:2222":
|
|
49
|
+
files:
|
|
50
|
+
- /opt/bot/config.yml
|
|
51
|
+
stdout:
|
|
52
|
+
packages.txt: dpkg -l
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
All options with explanations: [close1-full-example.yml](close1-full-example.yml).
|
|
56
|
+
|
|
57
|
+
### SQLite
|
|
58
|
+
|
|
59
|
+
Do not copy a live SQLite file (in WAL mode part of the data lives in `-wal`). Take a snapshot in `pre` and fetch it:
|
|
60
|
+
|
|
61
|
+
```yaml
|
|
62
|
+
pre:
|
|
63
|
+
- umask 077 && sqlite3 /opt/app/app.db ".backup /tmp/app-${DATE}.db" && gzip -f /tmp/app-${DATE}.db
|
|
64
|
+
files:
|
|
65
|
+
- app.db.gz: /tmp/app-${DATE}.db.gz
|
|
66
|
+
post:
|
|
67
|
+
- rm -f /tmp/app-${DATE}.db.gz
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Unreliable connections
|
|
71
|
+
|
|
72
|
+
All commands to a host go through one SSH connection (ControlMaster), so a backup needs a single handshake.
|
|
73
|
+
After a connection error (ssh exit code 255) close1 retries the connection check, file checks and downloads
|
|
74
|
+
(`retries` times, `retry_delay` seconds apart). `pre`, `post` and `stdout` commands are not retried: after a dropped
|
|
75
|
+
connection it is unknown whether they ran. If they fail, the backup fails and the next cron run starts it again.
|
|
76
|
+
|
|
77
|
+
## cron and ssh-agent
|
|
78
|
+
|
|
79
|
+
close1 authenticates through ssh-agent. If `SSH_AUTH_SOCK` is not set (as under cron), it looks for the socket in
|
|
80
|
+
`/run/user/$UID/` (`openssh_agent`, `ssh-agent.socket`, `gcr/ssh`, `keyring/ssh`, `gnupg/S.gpg-agent.ssh`). ssh is always
|
|
81
|
+
called with `BatchMode=yes` and `ForwardAgent=no`. While you are not logged in or the key is not loaded into the agent,
|
|
82
|
+
the backup is skipped and retried on the next run.
|
|
83
|
+
|
|
84
|
+
By default the first agent that holds any key is used. If you have several agents (e.g. ssh-agent and gnome-keyring)
|
|
85
|
+
with different keys, set `ssh_key: SHA256:...` (globally or per host): close1 then uses the agent that holds this key
|
|
86
|
+
and passes it to ssh as `IdentityAgent`. If the key is in no agent, the host is skipped. `close1 --check` lists all
|
|
87
|
+
agents with their key fingerprints. The key comment (`user@host`) is not used for matching: it is arbitrary text.
|
|
88
|
+
|
|
89
|
+
```cron
|
|
90
|
+
17 * * * * $HOME/.local/bin/close1
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Only errors go to stderr (and so to cron mail); everything else goes to the log.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "close1"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Simple backups from servers over SSH"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Yaroslav Polyakov" }]
|
|
14
|
+
keywords = ["backup", "ssh", "cron", "pull", "sqlite", "mysql"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Intended Audience :: System Administrators",
|
|
20
|
+
"Operating System :: POSIX :: Linux",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Topic :: System :: Archiving :: Backup",
|
|
23
|
+
]
|
|
24
|
+
dependencies = ["PyYAML>=5.4"]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://github.com/yaroslaff/close1"
|
|
28
|
+
Repository = "https://github.com/yaroslaff/close1"
|
|
29
|
+
Issues = "https://github.com/yaroslaff/close1/issues"
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
dev = ["pytest"]
|
|
33
|
+
|
|
34
|
+
[project.scripts]
|
|
35
|
+
close1 = "close1.cli:main"
|
|
36
|
+
|
|
37
|
+
[tool.setuptools.dynamic]
|
|
38
|
+
version = { attr = "close1.__version__" }
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.packages.find]
|
|
41
|
+
where = ["src"]
|
|
42
|
+
|
|
43
|
+
[tool.pytest.ini_options]
|
|
44
|
+
testpaths = ["tests"]
|
close1-0.7.1/setup.cfg
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""Backing up a single server."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime as dt
|
|
6
|
+
import logging
|
|
7
|
+
import posixpath
|
|
8
|
+
import re
|
|
9
|
+
import shutil
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from close1.config import FileItem, HostConfig
|
|
14
|
+
from close1.messages import t
|
|
15
|
+
from close1.rotate import to_delete
|
|
16
|
+
from close1.ssh import Remote, RemoteError, remote_path
|
|
17
|
+
|
|
18
|
+
log = logging.getLogger("close1")
|
|
19
|
+
|
|
20
|
+
_DATE_RE = re.compile(r"\$\{DATE\}|\$DATE\b")
|
|
21
|
+
ARCHIVE_EXT = {"tar.gz": ".tar.gz", "zip": ".zip"}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class BackupError(Exception):
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def substitute(text: str, date: str) -> str:
|
|
29
|
+
"""$DATE and ${DATE} -> 2026-12-31."""
|
|
30
|
+
return _DATE_RE.sub(date, text)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def archive_type(name: str | None, default: str) -> str:
|
|
34
|
+
"""Archive format for a directory: from the explicit name's extension, else the compress option."""
|
|
35
|
+
if name:
|
|
36
|
+
if name.endswith(".zip"):
|
|
37
|
+
return "zip"
|
|
38
|
+
if name.endswith((".tar.gz", ".tgz")):
|
|
39
|
+
return "tar.gz"
|
|
40
|
+
return default
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class Plan:
|
|
45
|
+
"""What to copy and under which local name."""
|
|
46
|
+
|
|
47
|
+
remote: str
|
|
48
|
+
local: str
|
|
49
|
+
archive: str | None # None for a regular file
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def plan_files(items: list[tuple[FileItem, bool]], compress: str, reserved: set[str]) -> list[Plan]:
|
|
53
|
+
"""Choose local names. items are (item, is_dir); reserved are names taken by stdout.
|
|
54
|
+
|
|
55
|
+
Explicit names are claimed first; the rest get the basename, or the full path joined with '-' on collision.
|
|
56
|
+
"""
|
|
57
|
+
used = set(reserved)
|
|
58
|
+
locals_: list[str | None] = []
|
|
59
|
+
for item, is_dir in items:
|
|
60
|
+
local = None
|
|
61
|
+
if item.name:
|
|
62
|
+
local = item.name
|
|
63
|
+
archive = archive_type(item.name, compress) if is_dir else None
|
|
64
|
+
if archive and not local.endswith((".zip", ".tar.gz", ".tgz")):
|
|
65
|
+
local += ARCHIVE_EXT[archive]
|
|
66
|
+
if local in used:
|
|
67
|
+
raise BackupError(t("name_collision", path=item.remote))
|
|
68
|
+
used.add(local)
|
|
69
|
+
locals_.append(local)
|
|
70
|
+
|
|
71
|
+
plans = []
|
|
72
|
+
for (item, is_dir), local in zip(items, locals_):
|
|
73
|
+
archive = archive_type(item.name, compress) if is_dir else None
|
|
74
|
+
if local is None:
|
|
75
|
+
ext = ARCHIVE_EXT[archive] if archive else ""
|
|
76
|
+
stripped = item.remote.strip("/")
|
|
77
|
+
base = posixpath.basename(stripped) or "root"
|
|
78
|
+
full = stripped.replace("/", "-") or "root"
|
|
79
|
+
local = next((c + ext for c in (base, full) if c + ext not in used), None)
|
|
80
|
+
if local is None:
|
|
81
|
+
raise BackupError(t("name_collision", path=item.remote))
|
|
82
|
+
used.add(local)
|
|
83
|
+
plans.append(Plan(remote=item.remote, local=local, archive=archive))
|
|
84
|
+
return plans
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def remote_kind(remote: Remote, path: str) -> str:
|
|
88
|
+
q = remote_path(path)
|
|
89
|
+
out = remote.run(f"if [ -d {q} ]; then echo dir; elif [ -e {q} ]; then echo file; else echo none; fi", retry=True)
|
|
90
|
+
return out.strip()
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _require_tool(remote: Remote, tool: str) -> None:
|
|
94
|
+
if remote.run(f"command -v {tool} >/dev/null 2>&1 && echo yes || echo no", retry=True).strip() != "yes":
|
|
95
|
+
raise BackupError(t("tool_missing", tool=tool))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def fetch(remote: Remote, plan: Plan, target: Path) -> None:
|
|
99
|
+
q = remote_path(plan.remote)
|
|
100
|
+
if plan.archive == "tar.gz":
|
|
101
|
+
cmd = f"tar -C {q} -czf - ."
|
|
102
|
+
elif plan.archive == "zip":
|
|
103
|
+
cmd = f"cd {q} && zip -qr - ."
|
|
104
|
+
else:
|
|
105
|
+
cmd = f"cat -- {q}"
|
|
106
|
+
with open(target, "wb") as f:
|
|
107
|
+
remote.run(cmd, stdout=f, retry=True)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class HostBackup:
|
|
111
|
+
def __init__(self, host: HostConfig, agent_sock: str | None, force: bool = False,
|
|
112
|
+
today: dt.date | None = None):
|
|
113
|
+
self.host = host
|
|
114
|
+
self.force = force
|
|
115
|
+
self.today = today or dt.date.today()
|
|
116
|
+
self.date = self.today.isoformat()
|
|
117
|
+
self.remote = Remote(host, agent_sock)
|
|
118
|
+
self.host_dir = host.dest / host.name
|
|
119
|
+
self.final = self.host_dir / self.date
|
|
120
|
+
self.partial = self.host_dir / f"{self.date}.partial"
|
|
121
|
+
|
|
122
|
+
def run(self) -> bool:
|
|
123
|
+
"""True if the backup was made or is not needed, False on error."""
|
|
124
|
+
name = self.host.section
|
|
125
|
+
if self.final.exists() and not self.force:
|
|
126
|
+
log.info(t("already_done", name=name, date=self.date))
|
|
127
|
+
return True
|
|
128
|
+
if self.host.ssh_key and not self.remote.agent_sock:
|
|
129
|
+
log.warning(t("key_not_found", name=name, fp=self.host.ssh_key))
|
|
130
|
+
return True # not an error: the key may be loaded later
|
|
131
|
+
with self.remote:
|
|
132
|
+
try:
|
|
133
|
+
self.remote.check()
|
|
134
|
+
except RemoteError as e:
|
|
135
|
+
log.warning(t("unreachable", name=name, error=e.stderr or e.code))
|
|
136
|
+
return True # not an error: retry on the next run
|
|
137
|
+
|
|
138
|
+
try:
|
|
139
|
+
self._backup()
|
|
140
|
+
except (RemoteError, BackupError, OSError) as e:
|
|
141
|
+
log.error(t("failed", name=name, error=e))
|
|
142
|
+
return False
|
|
143
|
+
log.info(t("done", name=name, path=self.final))
|
|
144
|
+
self._rotate()
|
|
145
|
+
return True
|
|
146
|
+
|
|
147
|
+
def _run_stage(self, stage: str, commands: list[str]) -> None:
|
|
148
|
+
for cmd in commands:
|
|
149
|
+
cmd = substitute(cmd, self.date)
|
|
150
|
+
log.info(t("run_cmd", name=self.host.section, stage=stage, cmd=cmd))
|
|
151
|
+
out = self.remote.run(cmd)
|
|
152
|
+
if out.strip():
|
|
153
|
+
log.debug(out.rstrip())
|
|
154
|
+
|
|
155
|
+
def _backup(self) -> None:
|
|
156
|
+
h = self.host
|
|
157
|
+
if self.partial.exists():
|
|
158
|
+
shutil.rmtree(self.partial)
|
|
159
|
+
self.partial.mkdir(parents=True)
|
|
160
|
+
log.info(t("start", name=h.section, path=self.final))
|
|
161
|
+
|
|
162
|
+
try:
|
|
163
|
+
self._run_stage("pre", h.pre)
|
|
164
|
+
self._copy_files()
|
|
165
|
+
for local, cmd in h.stdout.items():
|
|
166
|
+
cmd = substitute(cmd, self.date)
|
|
167
|
+
log.info(t("run_cmd", name=h.section, stage="stdout", cmd=f"{cmd} > {local}"))
|
|
168
|
+
with open(self.partial / local, "wb") as f:
|
|
169
|
+
self.remote.run(cmd, stdout=f)
|
|
170
|
+
except BaseException:
|
|
171
|
+
# run post even on failure, e.g. to turn maintenance mode off
|
|
172
|
+
try:
|
|
173
|
+
self._run_stage("post", h.post)
|
|
174
|
+
except RemoteError as e:
|
|
175
|
+
log.error(t("failed", name=h.section, error=e))
|
|
176
|
+
raise
|
|
177
|
+
self._run_stage("post", h.post)
|
|
178
|
+
|
|
179
|
+
if self.final.exists():
|
|
180
|
+
old = self.host_dir / f"{self.date}.old"
|
|
181
|
+
if old.exists():
|
|
182
|
+
shutil.rmtree(old)
|
|
183
|
+
self.final.rename(old)
|
|
184
|
+
self.partial.rename(self.final)
|
|
185
|
+
shutil.rmtree(old)
|
|
186
|
+
else:
|
|
187
|
+
self.partial.rename(self.final)
|
|
188
|
+
|
|
189
|
+
def _copy_files(self) -> None:
|
|
190
|
+
h = self.host
|
|
191
|
+
items = []
|
|
192
|
+
for item in h.files:
|
|
193
|
+
path = substitute(item.remote, self.date)
|
|
194
|
+
kind = remote_kind(self.remote, path)
|
|
195
|
+
if kind == "none":
|
|
196
|
+
raise BackupError(t("remote_missing", path=path))
|
|
197
|
+
items.append((FileItem(remote=path, name=item.name), kind == "dir"))
|
|
198
|
+
|
|
199
|
+
tools = {"tar" if a == "tar.gz" else "zip"
|
|
200
|
+
for a in (archive_type(i.name, h.compress) for i, d in items if d)}
|
|
201
|
+
for tool in sorted(tools):
|
|
202
|
+
_require_tool(self.remote, tool)
|
|
203
|
+
|
|
204
|
+
for plan in plan_files(items, h.compress, set(h.stdout)):
|
|
205
|
+
log.info(t("copy", name=h.section, remote=plan.remote, local=plan.local))
|
|
206
|
+
fetch(self.remote, plan, self.partial / plan.local)
|
|
207
|
+
|
|
208
|
+
def _rotate(self) -> None:
|
|
209
|
+
for p in self.host_dir.glob("*.partial"):
|
|
210
|
+
if p.is_dir():
|
|
211
|
+
shutil.rmtree(p)
|
|
212
|
+
for p in to_delete(self.host_dir, self.host.keep):
|
|
213
|
+
log.info(t("rotate_delete", name=self.host.section, path=p))
|
|
214
|
+
shutil.rmtree(p)
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""close1 command line."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import fcntl
|
|
7
|
+
import logging
|
|
8
|
+
import os
|
|
9
|
+
import shutil
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from close1 import __version__
|
|
14
|
+
from close1.backup import HostBackup
|
|
15
|
+
from close1.config import Config, ConfigError, config_search_paths, find_config, is_root, load_config
|
|
16
|
+
from close1.messages import t
|
|
17
|
+
from close1.ssh import Remote, RemoteError, find_agent, list_agents
|
|
18
|
+
|
|
19
|
+
log = logging.getLogger("close1")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def default_log_path() -> Path:
|
|
23
|
+
if is_root():
|
|
24
|
+
return Path("/var/log/close1.log")
|
|
25
|
+
base = os.environ.get("XDG_STATE_HOME") or "~/.local/state"
|
|
26
|
+
return Path(base).expanduser() / "close1" / "close1.log"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def setup_logging(log_path: Path, verbose: bool) -> None:
|
|
30
|
+
log.setLevel(logging.DEBUG)
|
|
31
|
+
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
32
|
+
fh = logging.FileHandler(log_path, encoding="utf-8")
|
|
33
|
+
fh.setLevel(logging.DEBUG if verbose else logging.INFO)
|
|
34
|
+
fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
|
35
|
+
log.addHandler(fh)
|
|
36
|
+
|
|
37
|
+
# Under cron (no tty) only errors go to stderr, so cron mails only on real problems.
|
|
38
|
+
sh = logging.StreamHandler(sys.stderr)
|
|
39
|
+
if verbose:
|
|
40
|
+
sh.setLevel(logging.DEBUG)
|
|
41
|
+
else:
|
|
42
|
+
sh.setLevel(logging.WARNING if sys.stderr.isatty() else logging.ERROR)
|
|
43
|
+
sh.setFormatter(logging.Formatter("%(levelname)s %(message)s"))
|
|
44
|
+
log.addHandler(sh)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
48
|
+
p = argparse.ArgumentParser(prog="close1", description="Back up files from servers over SSH")
|
|
49
|
+
p.add_argument("hosts", nargs="*", metavar="HOST", help="config sections to back up (default: all), or 'log' to show the log")
|
|
50
|
+
p.add_argument("-c", "--config", type=Path, help="config file (default: first of $CLOSE1_CONFIG, ~/.config/close1/close1.yml, /etc/close1.yml)")
|
|
51
|
+
p.add_argument("-f", "--force", action="store_true", help="overwrite today's backup")
|
|
52
|
+
p.add_argument("-v", "--verbose", action="store_true", help="verbose output")
|
|
53
|
+
p.add_argument("--check", action="store_true", help="check the SSH agent and host reachability, copy nothing")
|
|
54
|
+
p.add_argument("--version", action="version", version=f"close1 {__version__}")
|
|
55
|
+
return p
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def select_hosts(cfg: Config, names: list[str]):
|
|
59
|
+
if not names:
|
|
60
|
+
return cfg.hosts
|
|
61
|
+
by_name = {h.section: h for h in cfg.hosts}
|
|
62
|
+
missing = [n for n in names if n not in by_name]
|
|
63
|
+
if missing:
|
|
64
|
+
raise ConfigError(t("unknown_host", name=", ".join(missing)))
|
|
65
|
+
return [by_name[n] for n in names]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def check(cfg: Config, hosts, sock: str | None, log_path: Path) -> int:
|
|
69
|
+
print(t("check_header", version=__version__, config=cfg.path, log=log_path))
|
|
70
|
+
agents = list_agents(cfg.ssh_auth_sock)
|
|
71
|
+
for agent_sock, keys in agents:
|
|
72
|
+
print(t("agent_found", sock=agent_sock, count=len(keys)))
|
|
73
|
+
for k in keys:
|
|
74
|
+
print(" " + k)
|
|
75
|
+
if not agents:
|
|
76
|
+
print(t("agent_not_found"))
|
|
77
|
+
rc = 0
|
|
78
|
+
for h in hosts:
|
|
79
|
+
host_sock = sock
|
|
80
|
+
if h.ssh_key:
|
|
81
|
+
host_sock = find_agent(cfg.ssh_auth_sock, h.ssh_key)[0]
|
|
82
|
+
if not host_sock:
|
|
83
|
+
print(t("check_fail", name=h.section, error=t("check_key_missing", fp=h.ssh_key)))
|
|
84
|
+
rc = 1
|
|
85
|
+
continue
|
|
86
|
+
print(t("check_agent", name=h.section, sock=host_sock))
|
|
87
|
+
try:
|
|
88
|
+
with Remote(h, host_sock) as remote:
|
|
89
|
+
remote.check()
|
|
90
|
+
print(t("check_ok", name=h.section))
|
|
91
|
+
except RemoteError as e:
|
|
92
|
+
print(t("check_fail", name=h.section, error=e.stderr or e.code))
|
|
93
|
+
rc = 1
|
|
94
|
+
return rc
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def show_log(log_path: Path, verbose: bool) -> int:
|
|
98
|
+
"""close1 log: print the log file (with -v, first a '# path' line)."""
|
|
99
|
+
if verbose:
|
|
100
|
+
print(f"# {log_path}", flush=True)
|
|
101
|
+
try:
|
|
102
|
+
with open(log_path, "rb") as f:
|
|
103
|
+
shutil.copyfileobj(f, sys.stdout.buffer)
|
|
104
|
+
sys.stdout.flush()
|
|
105
|
+
except FileNotFoundError:
|
|
106
|
+
print(t("log_missing", path=log_path), file=sys.stderr)
|
|
107
|
+
return 1
|
|
108
|
+
except BrokenPipeError: # e.g. close1 log | head
|
|
109
|
+
sys.stderr.close()
|
|
110
|
+
return 0
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def main(argv: list[str] | None = None) -> int:
|
|
114
|
+
args = build_parser().parse_args(argv)
|
|
115
|
+
log_cmd = args.hosts == ["log"]
|
|
116
|
+
if log_cmd:
|
|
117
|
+
args.hosts = []
|
|
118
|
+
|
|
119
|
+
if args.config:
|
|
120
|
+
cfg_path = args.config
|
|
121
|
+
if not cfg_path.is_file():
|
|
122
|
+
print(t("config_not_found", path=cfg_path), file=sys.stderr)
|
|
123
|
+
return 2
|
|
124
|
+
else:
|
|
125
|
+
cfg_path = find_config()
|
|
126
|
+
if cfg_path is None:
|
|
127
|
+
paths = "\n".join(f" {p}" for p in config_search_paths())
|
|
128
|
+
print(t("config_not_found_search", paths=paths), file=sys.stderr)
|
|
129
|
+
return 2
|
|
130
|
+
try:
|
|
131
|
+
cfg = load_config(cfg_path)
|
|
132
|
+
hosts = select_hosts(cfg, args.hosts)
|
|
133
|
+
except ConfigError as e:
|
|
134
|
+
print(t("config_error", path=cfg_path, error=e), file=sys.stderr)
|
|
135
|
+
return 2
|
|
136
|
+
|
|
137
|
+
log_path = Path(cfg.log).expanduser() if cfg.log else default_log_path()
|
|
138
|
+
if log_cmd:
|
|
139
|
+
return show_log(log_path, args.verbose)
|
|
140
|
+
setup_logging(log_path, args.verbose)
|
|
141
|
+
log.debug(t("log_file", version=__version__, config=cfg_path, path=log_path))
|
|
142
|
+
|
|
143
|
+
sock, keys = find_agent(cfg.ssh_auth_sock)
|
|
144
|
+
if args.check:
|
|
145
|
+
return check(cfg, hosts, sock, log_path)
|
|
146
|
+
|
|
147
|
+
lock_path = log_path.with_suffix(".lock")
|
|
148
|
+
with open(lock_path, "w") as lock:
|
|
149
|
+
try:
|
|
150
|
+
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
151
|
+
except BlockingIOError:
|
|
152
|
+
log.info(t("already_running", path=lock_path))
|
|
153
|
+
return 0
|
|
154
|
+
|
|
155
|
+
if sock:
|
|
156
|
+
log.debug(t("agent_found", sock=sock, count=len(keys)))
|
|
157
|
+
else:
|
|
158
|
+
log.warning(t("agent_not_found"))
|
|
159
|
+
|
|
160
|
+
ok = True
|
|
161
|
+
for h in hosts:
|
|
162
|
+
host_sock = find_agent(cfg.ssh_auth_sock, h.ssh_key)[0] if h.ssh_key else sock
|
|
163
|
+
ok = HostBackup(h, host_sock, force=args.force).run() and ok
|
|
164
|
+
return 0 if ok else 1
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
if __name__ == "__main__":
|
|
168
|
+
raise SystemExit(main())
|