salla-gitpuller 1.0.2__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- salla_gitpuller-1.1.0/LICENSE +21 -0
- salla_gitpuller-1.1.0/PKG-INFO +261 -0
- salla_gitpuller-1.1.0/README.md +249 -0
- {salla_gitpuller-1.0.2 → salla_gitpuller-1.1.0}/gitpuller/__init__.py +6 -1
- {salla_gitpuller-1.0.2 → salla_gitpuller-1.1.0}/gitpuller/alert_manager.py +29 -14
- salla_gitpuller-1.1.0/gitpuller/gitpull.py +324 -0
- {salla_gitpuller-1.0.2 → salla_gitpuller-1.1.0}/gitpuller/slack_notifier.py +23 -10
- {salla_gitpuller-1.0.2 → salla_gitpuller-1.1.0}/gitpuller/state_manager.py +21 -4
- {salla_gitpuller-1.0.2 → salla_gitpuller-1.1.0}/pyproject.toml +5 -2
- salla_gitpuller-1.1.0/salla_gitpuller.egg-info/PKG-INFO +261 -0
- {salla_gitpuller-1.0.2 → salla_gitpuller-1.1.0}/salla_gitpuller.egg-info/SOURCES.txt +1 -0
- salla_gitpuller-1.1.0/salla_gitpuller.egg-info/requires.txt +1 -0
- {salla_gitpuller-1.0.2 → salla_gitpuller-1.1.0}/setup.cfg +4 -4
- salla_gitpuller-1.0.2/LICENSE +0 -0
- salla_gitpuller-1.0.2/PKG-INFO +0 -35
- salla_gitpuller-1.0.2/README.md +0 -24
- salla_gitpuller-1.0.2/gitpuller/gitpull.py +0 -194
- salla_gitpuller-1.0.2/salla_gitpuller.egg-info/PKG-INFO +0 -35
- {salla_gitpuller-1.0.2 → salla_gitpuller-1.1.0}/gitpuller/utils.py +0 -0
- {salla_gitpuller-1.0.2 → salla_gitpuller-1.1.0}/salla_gitpuller.egg-info/dependency_links.txt +0 -0
- {salla_gitpuller-1.0.2 → salla_gitpuller-1.1.0}/salla_gitpuller.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Mohammed Junaid, Muhammad Zahid
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: salla_gitpuller
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: A lightweight utility to git pull a repository using SSH deploy keys stored in environment variables
|
|
5
|
+
Author-email: Mohammed Junaid <safijunaid.ss@gmail.com>, Muhammad Zahid <zahidmuhammad127@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Zahid07/gitpuller
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: requests>=2.20
|
|
11
|
+
Dynamic: license-file
|
|
12
|
+
|
|
13
|
+
# gitpuller
|
|
14
|
+
|
|
15
|
+
A lightweight utility to keep a git repository in sync with its remote, designed
|
|
16
|
+
to run **inside Mage AI pipelines** as an auto-pull step. It authenticates with an
|
|
17
|
+
SSH deploy key stored in an environment variable, forces the local checkout to
|
|
18
|
+
match the remote branch (even if someone manually edited files on the runner),
|
|
19
|
+
and sends a de-duplicated Slack alert if anything goes wrong.
|
|
20
|
+
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
## Why this exists
|
|
24
|
+
|
|
25
|
+
Mage runners are long-lived boxes. If anyone manually edits a tracked file,
|
|
26
|
+
adds a stray file, or commits locally, a plain `git pull` fails with an opaque
|
|
27
|
+
error like:
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
Command '['git', '-c', 'core.sshCommand=...', 'pull', 'git@github.com:...', 'master']'
|
|
31
|
+
returned non-zero exit status 1.
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
gitpuller solves three problems at once:
|
|
35
|
+
|
|
36
|
+
1. **Clear errors** — surfaces the actual git output, not the wrapper message.
|
|
37
|
+
2. **Self-healing sync** — discards local drift so the pull can't be blocked.
|
|
38
|
+
3. **Alerting without spam** — pings Slack on failure, but suppresses repeats of
|
|
39
|
+
the same error within a configurable window.
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## Installation
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install salla_gitpuller
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Dependency: [`requests`](https://pypi.org/project/requests/) (installed
|
|
50
|
+
automatically). Optional: `mage-ai` — only needed if you want alert-suppression
|
|
51
|
+
state to persist across pipeline runs (see [State management](#state-management)).
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## Quick start
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from gitpuller import GitPullExecutor
|
|
59
|
+
|
|
60
|
+
executor = GitPullExecutor(
|
|
61
|
+
slack_webhook_url="https://hooks.slack.com/services/XXX", # or set CDM_SLACK_WEBHOOK_URL
|
|
62
|
+
use_mage_ai=True, # persist alert state via Mage
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
result = executor.execute_with_alerting(
|
|
66
|
+
repo_path="/home/src/my-repo",
|
|
67
|
+
git_url="git@github.com:my-org/my-repo.git",
|
|
68
|
+
workspace_name="myworkspace", # reads the private key from {workspace_name}_SSHKEY
|
|
69
|
+
# branch omitted -> defaults to "master"
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
print(result["git_pull_status"]) # "success"
|
|
73
|
+
print(result["discarded_changes"]) # what local drift (if any) was wiped
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
On failure, `execute_with_alerting` sends a Slack alert (subject to suppression)
|
|
77
|
+
and then **re-raises**, so the Mage pipeline still fails loudly.
|
|
78
|
+
|
|
79
|
+
---
|
|
80
|
+
|
|
81
|
+
## How it works
|
|
82
|
+
|
|
83
|
+
`execute_git_pull` does **not** run `git pull`. Instead it forces the local repo
|
|
84
|
+
to exactly match the remote, which is robust against manual edits *and* divergent
|
|
85
|
+
history (local commits / rewritten history) that a stash-based approach can't
|
|
86
|
+
handle:
|
|
87
|
+
|
|
88
|
+
1. **Prepare the SSH key** — the private deploy key is read from an env var,
|
|
89
|
+
normalized (strips wrapping quotes, converts literal `\n` to real newlines,
|
|
90
|
+
ensures a trailing newline), and written to `~/.ssh` with strict `0600`
|
|
91
|
+
permissions. It is injected per-command via
|
|
92
|
+
`git -c core.sshCommand="ssh -i <key> -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new"`.
|
|
93
|
+
2. **`git fetch <url> <branch>`** — `FETCH_HEAD` now points at the remote tip.
|
|
94
|
+
3. **Snapshot + log local drift** — before anything is discarded, it records:
|
|
95
|
+
- **working tree changes** — uncommitted edits and untracked files
|
|
96
|
+
(`git status --porcelain`),
|
|
97
|
+
- **local-only commits** — commits on the runner but not the remote
|
|
98
|
+
(`FETCH_HEAD..HEAD`),
|
|
99
|
+
and prints them to the pipeline log so you have a record of what was wiped.
|
|
100
|
+
4. **`git reset --hard FETCH_HEAD`** — makes the working tree and branch pointer
|
|
101
|
+
match the remote exactly.
|
|
102
|
+
5. **`git clean -fd`** — removes untracked files/directories so the tree truly
|
|
103
|
+
matches remote. **Ignored files are preserved** (no `-x`), so runner-local
|
|
104
|
+
`.env` files and deploy keys survive.
|
|
105
|
+
6. **Cleanup** — the key file is removed and the working directory is restored,
|
|
106
|
+
even on failure (`finally`).
|
|
107
|
+
|
|
108
|
+
Any failing step raises a `RuntimeError` containing the **real git stdout/stderr
|
|
109
|
+
and exit code**, which becomes the Slack alert body and the pipeline error.
|
|
110
|
+
|
|
111
|
+
> ⚠️ **This is destructive by design.** Local changes on the runner are treated as
|
|
112
|
+
> contamination and discarded. Don't point gitpuller at a repo where the runner
|
|
113
|
+
> holds work you intend to keep.
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
117
|
+
## API
|
|
118
|
+
|
|
119
|
+
### `GitPullExecutor(slack_webhook_url=None, use_mage_ai=False, state_manager=None)`
|
|
120
|
+
|
|
121
|
+
| Param | Description |
|
|
122
|
+
|-------|-------------|
|
|
123
|
+
| `slack_webhook_url` | Slack incoming-webhook URL. Falls back to the `CDM_SLACK_WEBHOOK_URL` env var. Required (one of the two must be set). |
|
|
124
|
+
| `use_mage_ai` | If `True`, persist alert-suppression state via Mage global variables (falls back to in-memory if Mage isn't installed). |
|
|
125
|
+
| `state_manager` | Inject a custom `StateManager`; overrides `use_mage_ai`. |
|
|
126
|
+
|
|
127
|
+
### `execute_with_alerting(...)` → `dict`
|
|
128
|
+
|
|
129
|
+
Runs the sync and, on failure, alerts Slack (with suppression) then re-raises.
|
|
130
|
+
|
|
131
|
+
| Param | Default | Description |
|
|
132
|
+
|-------|---------|-------------|
|
|
133
|
+
| `repo_path` | — | Absolute path to the local repo (must exist). |
|
|
134
|
+
| `git_url` | — | SSH remote URL, e.g. `git@github.com:Org/repo.git`. |
|
|
135
|
+
| `branch` | `"master"` | Branch to sync to. **Note: defaults to `master`, not `main`.** |
|
|
136
|
+
| `ssh_key` | `None` | Private key material. If omitted, read from `{workspace_name}_SSHKEY`. |
|
|
137
|
+
| `workspace_name` | `None` | Used to locate the key env var and name the key file. |
|
|
138
|
+
| `pipeline_uuid` | `"auto_git_pull"` | Key under which alert state is stored. |
|
|
139
|
+
| `suppression_hours` | `1` | Don't re-alert on the *same* error within this many hours. |
|
|
140
|
+
| `key_filename` | `None` | Override the on-disk key filename. |
|
|
141
|
+
| `ssh_dir` | `"/home/src/.ssh"` | Directory to write the key into. |
|
|
142
|
+
|
|
143
|
+
### `execute_git_pull(...)` → `dict`
|
|
144
|
+
|
|
145
|
+
Same signature as above (minus `pipeline_uuid` / `suppression_hours`). Performs
|
|
146
|
+
the sync **without** alerting — use this if you handle errors yourself.
|
|
147
|
+
|
|
148
|
+
### Return value
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
{
|
|
152
|
+
"workspace": "myworkspace",
|
|
153
|
+
"repo_path": "/home/src/my-repo",
|
|
154
|
+
"git_pull_status": "success", # or raises on error
|
|
155
|
+
"git_pull_output": "HEAD is now at <sha> <subject>",
|
|
156
|
+
"discarded_changes": {
|
|
157
|
+
"working_tree_changes": "?? stray.txt", # git status --porcelain output
|
|
158
|
+
"local_commits": "949688e local-only commit" # FETCH_HEAD..HEAD output
|
|
159
|
+
},
|
|
160
|
+
"key_env_var_used": "myworkspace_SSHKEY",
|
|
161
|
+
}
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
---
|
|
165
|
+
|
|
166
|
+
## SSH key setup
|
|
167
|
+
|
|
168
|
+
Provide the **private** deploy key as an environment variable named
|
|
169
|
+
`{workspace_name}_SSHKEY` (e.g. `myworkspace_SSHKEY`), or pass `ssh_key=` directly.
|
|
170
|
+
The matching public key must be registered as a deploy key on the GitHub repo.
|
|
171
|
+
|
|
172
|
+
The key may be stored with literal `\n` (single-line) or real newlines — both are
|
|
173
|
+
handled. Wrapping quotes are stripped automatically.
|
|
174
|
+
|
|
175
|
+
### Use a read-only deploy key
|
|
176
|
+
|
|
177
|
+
gitpuller is a one-way mirror (remote → runner) and **never pushes**. Its only
|
|
178
|
+
remote operation is `git fetch`; the reset and clean steps are local. So the
|
|
179
|
+
deploy key only needs **read access** — leave GitHub's *"Allow write access"*
|
|
180
|
+
checkbox **unchecked**. This is the least-privilege setup and means the runner
|
|
181
|
+
can never push its discarded local changes back upstream.
|
|
182
|
+
|
|
183
|
+
Setup steps:
|
|
184
|
+
|
|
185
|
+
1. Generate a dedicated key pair: `ssh-keygen -t ed25519 -f deploy_key -N ""`.
|
|
186
|
+
2. On the GitHub repo: **Settings → Deploy keys → Add deploy key**, paste
|
|
187
|
+
`deploy_key.pub`, and leave **Allow write access unchecked**.
|
|
188
|
+
3. Store the **private** key (`deploy_key`) in the `{workspace_name}_SSHKEY` env var.
|
|
189
|
+
|
|
190
|
+
**Note:** GitHub deploy keys are **per-repository** — each repo you sync needs its
|
|
191
|
+
own key pair and its own `{workspace_name}_SSHKEY` env var.
|
|
192
|
+
|
|
193
|
+
On first connection the remote host key is auto-accepted
|
|
194
|
+
(`StrictHostKeyChecking=accept-new`), i.e. trust-on-first-use rather than a
|
|
195
|
+
pre-pinned fingerprint.
|
|
196
|
+
|
|
197
|
+
---
|
|
198
|
+
|
|
199
|
+
## State management
|
|
200
|
+
|
|
201
|
+
Alert suppression needs to remember the last error and when it was alerted:
|
|
202
|
+
|
|
203
|
+
- **`InMemoryStateManager`** (default) — process-local; suppression only works
|
|
204
|
+
within a single run.
|
|
205
|
+
- **`MageAIStateManager`** (`use_mage_ai=True`) — persists across runs via Mage
|
|
206
|
+
global variables, so repeated failures across scheduled runs stay de-duplicated.
|
|
207
|
+
- **`StateManager`** — subclass it to plug in your own backend (e.g. Redis, a DB).
|
|
208
|
+
|
|
209
|
+
---
|
|
210
|
+
|
|
211
|
+
## Build & release
|
|
212
|
+
|
|
213
|
+
```bash
|
|
214
|
+
rm -rf build dist *.egg-info
|
|
215
|
+
python -m build
|
|
216
|
+
# then upload to PyPI (twine upload dist/*) and bump the version in pyproject.toml
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
Keep the version in sync in **both** `pyproject.toml` and `gitpuller/__init__.py`.
|
|
220
|
+
|
|
221
|
+
---
|
|
222
|
+
|
|
223
|
+
## Changelog
|
|
224
|
+
|
|
225
|
+
### 1.1.0 (current)
|
|
226
|
+
|
|
227
|
+
Reliability and clarity overhaul.
|
|
228
|
+
|
|
229
|
+
- **Self-healing sync.** Replaced `git pull` with `git fetch` →
|
|
230
|
+
`git reset --hard FETCH_HEAD` → `git clean -fd`. Manual edits, stray files, and
|
|
231
|
+
even local commits / divergent history on the runner no longer break the sync.
|
|
232
|
+
Ignored files (`.env`, keys) are preserved.
|
|
233
|
+
- **Clear error messages.** Failures now raise with the real git stdout/stderr and
|
|
234
|
+
exit code instead of the opaque
|
|
235
|
+
`Command '[...]' returned non-zero exit status 1.` wrapper. The same detail flows
|
|
236
|
+
into the Slack alert.
|
|
237
|
+
- **Audit log of discarded changes.** Before resetting, the working-tree drift and
|
|
238
|
+
any local-only commits are logged and returned under `discarded_changes`, so
|
|
239
|
+
there's always a record of what was wiped.
|
|
240
|
+
- **Packaging fixes.** Declared the previously-missing `requests` dependency; synced
|
|
241
|
+
the version between `pyproject.toml` and `__init__.py`.
|
|
242
|
+
- **Docs & comments.** Full README and inline documentation across all modules.
|
|
243
|
+
|
|
244
|
+
> **Migration note:** `git_pull_output` now reflects `reset --hard` output
|
|
245
|
+
> (`HEAD is now at <sha> <subject>`) rather than pull's `Updating x..y` /
|
|
246
|
+
> `Already up to date`. The result key `recovery_steps` (briefly present during
|
|
247
|
+
> development) is replaced by `discarded_changes`. Update any code that parses
|
|
248
|
+
> these. The public method signatures are unchanged.
|
|
249
|
+
|
|
250
|
+
### 1.0.x (previous)
|
|
251
|
+
|
|
252
|
+
- Initial release. Ran a plain `git pull <url> <branch>` over an SSH deploy key.
|
|
253
|
+
- Slack alerting with same-error suppression (`AlertManager` + `StateManager`,
|
|
254
|
+
in-memory or Mage-backed).
|
|
255
|
+
- **Limitations addressed in 1.1.0:** any manual change on the runner caused the
|
|
256
|
+
pull to fail; errors were opaque wrapper messages; `requests` was imported but
|
|
257
|
+
not declared as a dependency.
|
|
258
|
+
|
|
259
|
+
---
|
|
260
|
+
|
|
261
|
+
<sub>Created and maintained by Mohammed Junaid and Muhammad Zahid.</sub>
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
# gitpuller
|
|
2
|
+
|
|
3
|
+
A lightweight utility to keep a git repository in sync with its remote, designed
|
|
4
|
+
to run **inside Mage AI pipelines** as an auto-pull step. It authenticates with an
|
|
5
|
+
SSH deploy key stored in an environment variable, forces the local checkout to
|
|
6
|
+
match the remote branch (even if someone manually edited files on the runner),
|
|
7
|
+
and sends a de-duplicated Slack alert if anything goes wrong.
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## Why this exists
|
|
12
|
+
|
|
13
|
+
Mage runners are long-lived boxes. If anyone manually edits a tracked file,
|
|
14
|
+
adds a stray file, or commits locally, a plain `git pull` fails with an opaque
|
|
15
|
+
error like:
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
Command '['git', '-c', 'core.sshCommand=...', 'pull', 'git@github.com:...', 'master']'
|
|
19
|
+
returned non-zero exit status 1.
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
gitpuller solves three problems at once:
|
|
23
|
+
|
|
24
|
+
1. **Clear errors** — surfaces the actual git output, not the wrapper message.
|
|
25
|
+
2. **Self-healing sync** — discards local drift so the pull can't be blocked.
|
|
26
|
+
3. **Alerting without spam** — pings Slack on failure, but suppresses repeats of
|
|
27
|
+
the same error within a configurable window.
|
|
28
|
+
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
## Installation
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install salla_gitpuller
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Dependency: [`requests`](https://pypi.org/project/requests/) (installed
|
|
38
|
+
automatically). Optional: `mage-ai` — only needed if you want alert-suppression
|
|
39
|
+
state to persist across pipeline runs (see [State management](#state-management)).
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## Quick start
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from gitpuller import GitPullExecutor
|
|
47
|
+
|
|
48
|
+
executor = GitPullExecutor(
|
|
49
|
+
slack_webhook_url="https://hooks.slack.com/services/XXX", # or set CDM_SLACK_WEBHOOK_URL
|
|
50
|
+
use_mage_ai=True, # persist alert state via Mage
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
result = executor.execute_with_alerting(
|
|
54
|
+
repo_path="/home/src/my-repo",
|
|
55
|
+
git_url="git@github.com:my-org/my-repo.git",
|
|
56
|
+
workspace_name="myworkspace", # reads the private key from {workspace_name}_SSHKEY
|
|
57
|
+
# branch omitted -> defaults to "master"
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
print(result["git_pull_status"]) # "success"
|
|
61
|
+
print(result["discarded_changes"]) # what local drift (if any) was wiped
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
On failure, `execute_with_alerting` sends a Slack alert (subject to suppression)
|
|
65
|
+
and then **re-raises**, so the Mage pipeline still fails loudly.
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## How it works
|
|
70
|
+
|
|
71
|
+
`execute_git_pull` does **not** run `git pull`. Instead it forces the local repo
|
|
72
|
+
to exactly match the remote, which is robust against manual edits *and* divergent
|
|
73
|
+
history (local commits / rewritten history) that a stash-based approach can't
|
|
74
|
+
handle:
|
|
75
|
+
|
|
76
|
+
1. **Prepare the SSH key** — the private deploy key is read from an env var,
|
|
77
|
+
normalized (strips wrapping quotes, converts literal `\n` to real newlines,
|
|
78
|
+
ensures a trailing newline), and written to `~/.ssh` with strict `0600`
|
|
79
|
+
permissions. It is injected per-command via
|
|
80
|
+
`git -c core.sshCommand="ssh -i <key> -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new"`.
|
|
81
|
+
2. **`git fetch <url> <branch>`** — `FETCH_HEAD` now points at the remote tip.
|
|
82
|
+
3. **Snapshot + log local drift** — before anything is discarded, it records:
|
|
83
|
+
- **working tree changes** — uncommitted edits and untracked files
|
|
84
|
+
(`git status --porcelain`),
|
|
85
|
+
- **local-only commits** — commits on the runner but not the remote
|
|
86
|
+
(`FETCH_HEAD..HEAD`),
|
|
87
|
+
and prints them to the pipeline log so you have a record of what was wiped.
|
|
88
|
+
4. **`git reset --hard FETCH_HEAD`** — makes the working tree and branch pointer
|
|
89
|
+
match the remote exactly.
|
|
90
|
+
5. **`git clean -fd`** — removes untracked files/directories so the tree truly
|
|
91
|
+
matches remote. **Ignored files are preserved** (no `-x`), so runner-local
|
|
92
|
+
`.env` files and deploy keys survive.
|
|
93
|
+
6. **Cleanup** — the key file is removed and the working directory is restored,
|
|
94
|
+
even on failure (`finally`).
|
|
95
|
+
|
|
96
|
+
Any failing step raises a `RuntimeError` containing the **real git stdout/stderr
|
|
97
|
+
and exit code**, which becomes the Slack alert body and the pipeline error.
|
|
98
|
+
|
|
99
|
+
> ⚠️ **This is destructive by design.** Local changes on the runner are treated as
|
|
100
|
+
> contamination and discarded. Don't point gitpuller at a repo where the runner
|
|
101
|
+
> holds work you intend to keep.
|
|
102
|
+
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
## API
|
|
106
|
+
|
|
107
|
+
### `GitPullExecutor(slack_webhook_url=None, use_mage_ai=False, state_manager=None)`
|
|
108
|
+
|
|
109
|
+
| Param | Description |
|
|
110
|
+
|-------|-------------|
|
|
111
|
+
| `slack_webhook_url` | Slack incoming-webhook URL. Falls back to the `CDM_SLACK_WEBHOOK_URL` env var. Required (one of the two must be set). |
|
|
112
|
+
| `use_mage_ai` | If `True`, persist alert-suppression state via Mage global variables (falls back to in-memory if Mage isn't installed). |
|
|
113
|
+
| `state_manager` | Inject a custom `StateManager`; overrides `use_mage_ai`. |
|
|
114
|
+
|
|
115
|
+
### `execute_with_alerting(...)` → `dict`
|
|
116
|
+
|
|
117
|
+
Runs the sync and, on failure, alerts Slack (with suppression) then re-raises.
|
|
118
|
+
|
|
119
|
+
| Param | Default | Description |
|
|
120
|
+
|-------|---------|-------------|
|
|
121
|
+
| `repo_path` | — | Absolute path to the local repo (must exist). |
|
|
122
|
+
| `git_url` | — | SSH remote URL, e.g. `git@github.com:Org/repo.git`. |
|
|
123
|
+
| `branch` | `"master"` | Branch to sync to. **Note: defaults to `master`, not `main`.** |
|
|
124
|
+
| `ssh_key` | `None` | Private key material. If omitted, read from `{workspace_name}_SSHKEY`. |
|
|
125
|
+
| `workspace_name` | `None` | Used to locate the key env var and name the key file. |
|
|
126
|
+
| `pipeline_uuid` | `"auto_git_pull"` | Key under which alert state is stored. |
|
|
127
|
+
| `suppression_hours` | `1` | Don't re-alert on the *same* error within this many hours. |
|
|
128
|
+
| `key_filename` | `None` | Override the on-disk key filename. |
|
|
129
|
+
| `ssh_dir` | `"/home/src/.ssh"` | Directory to write the key into. |
|
|
130
|
+
|
|
131
|
+
### `execute_git_pull(...)` → `dict`
|
|
132
|
+
|
|
133
|
+
Same signature as above (minus `pipeline_uuid` / `suppression_hours`). Performs
|
|
134
|
+
the sync **without** alerting — use this if you handle errors yourself.
|
|
135
|
+
|
|
136
|
+
### Return value
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
{
|
|
140
|
+
"workspace": "myworkspace",
|
|
141
|
+
"repo_path": "/home/src/my-repo",
|
|
142
|
+
"git_pull_status": "success", # or raises on error
|
|
143
|
+
"git_pull_output": "HEAD is now at <sha> <subject>",
|
|
144
|
+
"discarded_changes": {
|
|
145
|
+
"working_tree_changes": "?? stray.txt", # git status --porcelain output
|
|
146
|
+
"local_commits": "949688e local-only commit" # FETCH_HEAD..HEAD output
|
|
147
|
+
},
|
|
148
|
+
"key_env_var_used": "myworkspace_SSHKEY",
|
|
149
|
+
}
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
---
|
|
153
|
+
|
|
154
|
+
## SSH key setup
|
|
155
|
+
|
|
156
|
+
Provide the **private** deploy key as an environment variable named
|
|
157
|
+
`{workspace_name}_SSHKEY` (e.g. `myworkspace_SSHKEY`), or pass `ssh_key=` directly.
|
|
158
|
+
The matching public key must be registered as a deploy key on the GitHub repo.
|
|
159
|
+
|
|
160
|
+
The key may be stored with literal `\n` (single-line) or real newlines — both are
|
|
161
|
+
handled. Wrapping quotes are stripped automatically.
|
|
162
|
+
|
|
163
|
+
### Use a read-only deploy key
|
|
164
|
+
|
|
165
|
+
gitpuller is a one-way mirror (remote → runner) and **never pushes**. Its only
|
|
166
|
+
remote operation is `git fetch`; the reset and clean steps are local. So the
|
|
167
|
+
deploy key only needs **read access** — leave GitHub's *"Allow write access"*
|
|
168
|
+
checkbox **unchecked**. This is the least-privilege setup and means the runner
|
|
169
|
+
can never push its discarded local changes back upstream.
|
|
170
|
+
|
|
171
|
+
Setup steps:
|
|
172
|
+
|
|
173
|
+
1. Generate a dedicated key pair: `ssh-keygen -t ed25519 -f deploy_key -N ""`.
|
|
174
|
+
2. On the GitHub repo: **Settings → Deploy keys → Add deploy key**, paste
|
|
175
|
+
`deploy_key.pub`, and leave **Allow write access unchecked**.
|
|
176
|
+
3. Store the **private** key (`deploy_key`) in the `{workspace_name}_SSHKEY` env var.
|
|
177
|
+
|
|
178
|
+
**Note:** GitHub deploy keys are **per-repository** — each repo you sync needs its
|
|
179
|
+
own key pair and its own `{workspace_name}_SSHKEY` env var.
|
|
180
|
+
|
|
181
|
+
On first connection the remote host key is auto-accepted
|
|
182
|
+
(`StrictHostKeyChecking=accept-new`), i.e. trust-on-first-use rather than a
|
|
183
|
+
pre-pinned fingerprint.
|
|
184
|
+
|
|
185
|
+
---
|
|
186
|
+
|
|
187
|
+
## State management
|
|
188
|
+
|
|
189
|
+
Alert suppression needs to remember the last error and when it was alerted:
|
|
190
|
+
|
|
191
|
+
- **`InMemoryStateManager`** (default) — process-local; suppression only works
|
|
192
|
+
within a single run.
|
|
193
|
+
- **`MageAIStateManager`** (`use_mage_ai=True`) — persists across runs via Mage
|
|
194
|
+
global variables, so repeated failures across scheduled runs stay de-duplicated.
|
|
195
|
+
- **`StateManager`** — subclass it to plug in your own backend (e.g. Redis, a DB).
|
|
196
|
+
|
|
197
|
+
---
|
|
198
|
+
|
|
199
|
+
## Build & release
|
|
200
|
+
|
|
201
|
+
```bash
|
|
202
|
+
rm -rf build dist *.egg-info
|
|
203
|
+
python -m build
|
|
204
|
+
# then upload to PyPI (twine upload dist/*) and bump the version in pyproject.toml
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
Keep the version in sync in **both** `pyproject.toml` and `gitpuller/__init__.py`.
|
|
208
|
+
|
|
209
|
+
---
|
|
210
|
+
|
|
211
|
+
## Changelog
|
|
212
|
+
|
|
213
|
+
### 1.1.0 (current)
|
|
214
|
+
|
|
215
|
+
Reliability and clarity overhaul.
|
|
216
|
+
|
|
217
|
+
- **Self-healing sync.** Replaced `git pull` with `git fetch` →
|
|
218
|
+
`git reset --hard FETCH_HEAD` → `git clean -fd`. Manual edits, stray files, and
|
|
219
|
+
even local commits / divergent history on the runner no longer break the sync.
|
|
220
|
+
Ignored files (`.env`, keys) are preserved.
|
|
221
|
+
- **Clear error messages.** Failures now raise with the real git stdout/stderr and
|
|
222
|
+
exit code instead of the opaque
|
|
223
|
+
`Command '[...]' returned non-zero exit status 1.` wrapper. The same detail flows
|
|
224
|
+
into the Slack alert.
|
|
225
|
+
- **Audit log of discarded changes.** Before resetting, the working-tree drift and
|
|
226
|
+
any local-only commits are logged and returned under `discarded_changes`, so
|
|
227
|
+
there's always a record of what was wiped.
|
|
228
|
+
- **Packaging fixes.** Declared the previously-missing `requests` dependency; synced
|
|
229
|
+
the version between `pyproject.toml` and `__init__.py`.
|
|
230
|
+
- **Docs & comments.** Full README and inline documentation across all modules.
|
|
231
|
+
|
|
232
|
+
> **Migration note:** `git_pull_output` now reflects `reset --hard` output
|
|
233
|
+
> (`HEAD is now at <sha> <subject>`) rather than pull's `Updating x..y` /
|
|
234
|
+
> `Already up to date`. The result key `recovery_steps` (briefly present during
|
|
235
|
+
> development) is replaced by `discarded_changes`. Update any code that parses
|
|
236
|
+
> these. The public method signatures are unchanged.
|
|
237
|
+
|
|
238
|
+
### 1.0.x (previous)
|
|
239
|
+
|
|
240
|
+
- Initial release. Ran a plain `git pull <url> <branch>` over an SSH deploy key.
|
|
241
|
+
- Slack alerting with same-error suppression (`AlertManager` + `StateManager`,
|
|
242
|
+
in-memory or Mage-backed).
|
|
243
|
+
- **Limitations addressed in 1.1.0:** any manual change on the runner caused the
|
|
244
|
+
pull to fail; errors were opaque wrapper messages; `requests` was imported but
|
|
245
|
+
not declared as a dependency.
|
|
246
|
+
|
|
247
|
+
---
|
|
248
|
+
|
|
249
|
+
<sub>Created and maintained by Mohammed Junaid and Muhammad Zahid.</sub>
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
"""gitpuller — auto-pull a git repo over SSH inside Mage pipelines.
|
|
2
|
+
|
|
3
|
+
Public API is re-exported here so callers can ``from gitpuller import ...``.
|
|
4
|
+
"""
|
|
5
|
+
|
|
1
6
|
from .gitpull import GitPullExecutor
|
|
2
7
|
from .alert_manager import AlertManager
|
|
3
8
|
from .state_manager import StateManager, InMemoryStateManager, MageAIStateManager
|
|
@@ -5,7 +10,7 @@ from .slack_notifier import SlackNotifier
|
|
|
5
10
|
from .utils import transform_custom, get_repo_path, get_env_base_path
|
|
6
11
|
|
|
7
12
|
|
|
8
|
-
__version__ = "1.
|
|
13
|
+
__version__ = "1.1.0"
|
|
9
14
|
__all__ = [
|
|
10
15
|
"GitPullExecutor",
|
|
11
16
|
"AlertManager",
|
|
@@ -1,12 +1,21 @@
|
|
|
1
|
+
"""Alert de-duplication: decides whether a failure should page Slack."""
|
|
2
|
+
|
|
1
3
|
from datetime import datetime, timedelta
|
|
2
4
|
from typing import Any, Dict, Optional, Tuple
|
|
3
5
|
from .state_manager import StateManager, InMemoryStateManager, MageAIStateManager
|
|
4
6
|
|
|
5
|
-
|
|
7
|
+
|
|
8
|
+
class AlertManager:
|
|
9
|
+
"""Wraps a ``StateManager`` to suppress repeated identical alerts."""
|
|
10
|
+
|
|
6
11
|
def __init__(self, state_manager: Optional[StateManager] = None, use_mage_ai: bool = False):
|
|
12
|
+
# Pick the backing store for "last alert" bookkeeping:
|
|
7
13
|
if state_manager is not None:
|
|
14
|
+
# Caller supplied one explicitly.
|
|
8
15
|
self.state_manager = state_manager
|
|
9
16
|
elif use_mage_ai:
|
|
17
|
+
# Persist across runs via Mage global variables when available,
|
|
18
|
+
# otherwise degrade gracefully to in-memory (per-process) state.
|
|
10
19
|
try:
|
|
11
20
|
self.state_manager = MageAIStateManager()
|
|
12
21
|
except ImportError:
|
|
@@ -14,37 +23,43 @@ class AlertManager:
|
|
|
14
23
|
self.state_manager = InMemoryStateManager()
|
|
15
24
|
else:
|
|
16
25
|
self.state_manager = InMemoryStateManager()
|
|
17
|
-
|
|
26
|
+
|
|
18
27
|
def should_send_alert(
|
|
19
|
-
self,
|
|
20
|
-
pipeline_uuid: str,
|
|
21
|
-
current_error: str,
|
|
28
|
+
self,
|
|
29
|
+
pipeline_uuid: str,
|
|
30
|
+
current_error: str,
|
|
22
31
|
suppression_hours: int = 1
|
|
23
32
|
) -> Tuple[bool, Dict[str, Any]]:
|
|
24
|
-
|
|
33
|
+
"""
|
|
34
|
+
Decide whether to alert for ``current_error``.
|
|
35
|
+
|
|
36
|
+
Rule: alert immediately on a first-ever or *changed* error; for an
|
|
37
|
+
unchanged error, only re-alert once ``suppression_hours`` has elapsed.
|
|
38
|
+
Returns ``(should_send, previous_state)``.
|
|
39
|
+
"""
|
|
25
40
|
state = self.state_manager.load_alert_state(pipeline_uuid)
|
|
26
41
|
last_error = state.get("last_error_message")
|
|
27
42
|
last_alert_time_str = state.get("last_alert_time")
|
|
28
|
-
|
|
29
|
-
#
|
|
43
|
+
|
|
44
|
+
# No prior alert on record — always send.
|
|
30
45
|
if not last_error or not last_alert_time_str:
|
|
31
46
|
return True, {"last_error_message": None, "last_alert_time": None}
|
|
32
|
-
|
|
33
|
-
#
|
|
47
|
+
|
|
48
|
+
# A different error than last time — always send.
|
|
34
49
|
if last_error != current_error:
|
|
35
50
|
return True, {"last_error_message": last_error, "last_alert_time": last_alert_time_str}
|
|
36
|
-
|
|
37
|
-
# Same error -
|
|
51
|
+
|
|
52
|
+
# Same error as last time — only re-send once the window has elapsed.
|
|
38
53
|
try:
|
|
39
54
|
last_alert_time = datetime.fromisoformat(last_alert_time_str)
|
|
40
55
|
time_since_last_alert = datetime.now() - last_alert_time
|
|
41
|
-
|
|
56
|
+
|
|
42
57
|
if time_since_last_alert >= timedelta(hours=suppression_hours):
|
|
43
58
|
return True, {"last_error_message": last_error, "last_alert_time": last_alert_time_str}
|
|
44
59
|
else:
|
|
45
60
|
return False, {"last_error_message": last_error, "last_alert_time": last_alert_time_str}
|
|
46
61
|
except (ValueError, TypeError):
|
|
47
|
-
#
|
|
62
|
+
# Unparseable timestamp — fail open and alert rather than stay silent.
|
|
48
63
|
return True, {"last_error_message": last_error, "last_alert_time": last_alert_time_str}
|
|
49
64
|
|
|
50
65
|
def save_alert_state(
|