fetch2gmail 1.1.9__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fetch2gmail-1.1.9/src/fetch2gmail.egg-info → fetch2gmail-1.3.0}/PKG-INFO +73 -30
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/README.md +71 -29
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/pyproject.toml +2 -1
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0/src/fetch2gmail.egg-info}/PKG-INFO +73 -30
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetch2gmail.egg-info/SOURCES.txt +11 -1
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetch2gmail.egg-info/requires.txt +1 -0
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/auth_server.py +2 -4
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/cli.py +71 -26
- fetch2gmail-1.3.0/src/fetcher/config.py +341 -0
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/env_file.py +23 -1
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/gmail_client.py +77 -18
- fetch2gmail-1.3.0/src/fetcher/idle_watcher.py +774 -0
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/imap_client.py +117 -11
- fetch2gmail-1.3.0/src/fetcher/run.py +849 -0
- fetch2gmail-1.3.0/src/fetcher/state.py +514 -0
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/web_ui.py +935 -225
- fetch2gmail-1.3.0/tests/test_config_imap_accounts.py +396 -0
- fetch2gmail-1.3.0/tests/test_config_password_env_collision.py +264 -0
- fetch2gmail-1.3.0/tests/test_deliver_destinations.py +325 -0
- fetch2gmail-1.3.0/tests/test_env_credential_refresh.py +40 -0
- fetch2gmail-1.3.0/tests/test_gmail_token_atomic.py +100 -0
- fetch2gmail-1.3.0/tests/test_idle_gate_per_account.py +37 -0
- fetch2gmail-1.3.0/tests/test_idle_watcher.py +309 -0
- fetch2gmail-1.3.0/tests/test_run_delivery_error.py +74 -0
- fetch2gmail-1.3.0/tests/test_state_account_id.py +161 -0
- fetch2gmail-1.1.9/src/fetcher/config.py +0 -68
- fetch2gmail-1.1.9/src/fetcher/run.py +0 -482
- fetch2gmail-1.1.9/src/fetcher/state.py +0 -152
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/LICENSE +0 -0
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/setup.cfg +0 -0
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetch2gmail.egg-info/dependency_links.txt +0 -0
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetch2gmail.egg-info/entry_points.txt +0 -0
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetch2gmail.egg-info/top_level.txt +0 -0
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/__init__.py +0 -0
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/auth_ui.py +0 -0
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/log_buffer.py +0 -0
- {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/ui_auth.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fetch2gmail
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Self-hosted email fetcher: IMAP to Gmail API import with idempotent state tracking
|
|
5
5
|
Author: Fetch2Gmail Contributors
|
|
6
6
|
License-Expression: MIT
|
|
@@ -24,6 +24,7 @@ Requires-Dist: python-dotenv>=1.0.0
|
|
|
24
24
|
Requires-Dist: python-multipart>=0.0.6
|
|
25
25
|
Requires-Dist: cryptography>=42.0.0
|
|
26
26
|
Requires-Dist: bcrypt>=4.0.0
|
|
27
|
+
Requires-Dist: imapclient>=3.0.0
|
|
27
28
|
Provides-Extra: dev
|
|
28
29
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
29
30
|
Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
|
|
@@ -33,8 +34,9 @@ Dynamic: license-file
|
|
|
33
34
|
|
|
34
35
|
Self-hosted email fetcher: **IMAP (ISP mailbox) → Gmail API import**. Replaces Gmail’s deprecated POP3 fetch.
|
|
35
36
|
|
|
36
|
-
-
|
|
37
|
-
- Imports messages with **Gmail API** (not SMTP); applies a Gmail label, preserves headers and date.
|
|
37
|
+
- Watches one or more ISP mailboxes via **IMAPS** (port 993), using **IMAP IDLE** for near-realtime delivery when the server supports it (falls back to polling otherwise).
|
|
38
|
+
- Imports messages with **Gmail API** (not SMTP); applies a Gmail label, preserves headers and date. By default uses **`neverMarkSpam`** so Gmail’s spam classifier does not bury ISP→Gmail imports.
|
|
39
|
+
- Supports **multiple IMAP accounts in one process**, with optional **per-IMAP routing** to one or more Gmail accounts.
|
|
38
40
|
- Deletes from ISP only after Gmail confirms import. **Idempotent** (UID + message hash).
|
|
39
41
|
|
|
40
42
|
**Requirements:** Python 3.11+, IMAP credentials, Google Cloud project with Gmail API and OAuth2 (Web application).
|
|
@@ -211,9 +213,17 @@ Your Google OAuth client must use **Web application** (not Desktop) and have the
|
|
|
211
213
|
|
|
212
214
|
Google does not allow redirect URIs that use an IP address. That is why the token must be obtained on a machine where the app can use localhost (Use case 1 steps 1–4 on a PC; Use case 2 on the same machine).
|
|
213
215
|
|
|
214
|
-
###
|
|
216
|
+
### IMAP IDLE and poll interval
|
|
215
217
|
|
|
216
|
-
The
|
|
218
|
+
The background watcher in the web UI process prefers **IMAP IDLE** (RFC 2177) when the server advertises the `IDLE` capability. New mail can trigger an import immediately instead of waiting for the next poll.
|
|
219
|
+
|
|
220
|
+
- **`imap.use_idle`** (default `true`): enable IDLE when supported. Disable to force polling.
|
|
221
|
+
- **`imap.idle_refresh_minutes`** (default `25`): leave IDLE, send `NOOP`, and re-enter IDLE before the typical ~29-minute server timeout. Also enables TCP `SO_KEEPALIVE` and reconnects with exponential backoff on silent drops.
|
|
222
|
+
- **`poll_interval_minutes`** (default `5`): used when IDLE is missing or disabled.
|
|
223
|
+
|
|
224
|
+
All three can be changed in the web UI (Config) or in **config.json**.
|
|
225
|
+
|
|
226
|
+
**Manual smoke checks:** with IDLE supported, send yourself mail and confirm an import in the UI logs within seconds (look for `IDLE: mailbox update`). Set `use_idle` to `false` (or use a server without IDLE) and confirm fetches still run on the poll interval.
|
|
217
227
|
|
|
218
228
|
### Data directory
|
|
219
229
|
|
|
@@ -225,39 +235,72 @@ All app files live in **one directory**: **config.json**, **credentials.json**,
|
|
|
225
235
|
- The UI is protected by a username and password in **.ui_auth** (when set). Get **token.json** via **`fetch2gmail auth`** (CLI only; there is no "Sign in with Google" in the UI).
|
|
226
236
|
- The Gmail scope requested is **gmail.modify** (read and modify labels/messages only).
|
|
227
237
|
|
|
228
|
-
### Multiple accounts (
|
|
238
|
+
### Multiple IMAP accounts (recommended: one process)
|
|
239
|
+
|
|
240
|
+
One Fetch2Gmail process can watch **multiple IMAP mailboxes** and import into one or more Gmail accounts. Configure this in the UI (**IMAP accounts** section) or in **config.json** via `imap_accounts`.
|
|
241
|
+
|
|
242
|
+
**Example: 3 IMAP → 1 Gmail**
|
|
243
|
+
|
|
244
|
+
```json
|
|
245
|
+
{
|
|
246
|
+
"imap_accounts": [
|
|
247
|
+
{
|
|
248
|
+
"id": "isp1",
|
|
249
|
+
"host": "imap.isp1.example",
|
|
250
|
+
"port": 993,
|
|
251
|
+
"username": "you@isp1.example",
|
|
252
|
+
"password_env": "IMAP_PASSWORD_ISP1",
|
|
253
|
+
"mailbox": "INBOX",
|
|
254
|
+
"use_ssl": true,
|
|
255
|
+
"delete_after_import": true,
|
|
256
|
+
"gmail_token_paths": []
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
"id": "isp2",
|
|
260
|
+
"host": "imap.isp2.example",
|
|
261
|
+
"username": "you@isp2.example",
|
|
262
|
+
"password_env": "IMAP_PASSWORD_ISP2",
|
|
263
|
+
"mailbox": "INBOX"
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
"id": "isp3",
|
|
267
|
+
"host": "imap.isp3.example",
|
|
268
|
+
"username": "you@isp3.example",
|
|
269
|
+
"password_env": "IMAP_PASSWORD_ISP3",
|
|
270
|
+
"mailbox": "INBOX"
|
|
271
|
+
}
|
|
272
|
+
],
|
|
273
|
+
"gmail_accounts": [
|
|
274
|
+
{
|
|
275
|
+
"label": "ISP Mail",
|
|
276
|
+
"credentials_path": "credentials.json",
|
|
277
|
+
"token_path": "token.json"
|
|
278
|
+
}
|
|
279
|
+
]
|
|
280
|
+
}
|
|
281
|
+
```
|
|
229
282
|
|
|
230
|
-
|
|
283
|
+
- Leave `gmail_token_paths` empty (or omit it) so that IMAP account imports into **all** configured Gmail accounts.
|
|
284
|
+
- To route one IMAP account to a subset of Gmail accounts, set `gmail_token_paths` to the matching `token_path` values (e.g. `["token.json"]`).
|
|
285
|
+
- Passwords are stored encrypted in **.env** under each account’s `password_env` key (default `IMAP_PASSWORD` for id `default`, otherwise `IMAP_PASSWORD_<ID>`).
|
|
286
|
+
- Legacy configs with a single top-level `"imap"` object still work (treated as one account with id `default`).
|
|
231
287
|
|
|
232
|
-
|
|
288
|
+
In the UI you can add/remove IMAP accounts and pick Gmail destinations per account. One systemd service / one UI port is enough.
|
|
233
289
|
|
|
234
|
-
|
|
290
|
+
### Multiple instances (optional isolation)
|
|
235
291
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
```
|
|
292
|
+
You can still run **multiple processes** (separate data directories and ports) when you want isolation across machines or fully separate configs. That is optional now that multi-IMAP works in one process.
|
|
293
|
+
|
|
294
|
+
**Second instance (example):**
|
|
240
295
|
|
|
241
|
-
|
|
296
|
+
1. Create a second data directory (e.g. `/opt/fetch2gmail2`) with its own **config.json**, credentials, and tokens.
|
|
297
|
+
2. Generate a second systemd unit and bind a different port:
|
|
242
298
|
```bash
|
|
299
|
+
fetch2gmail install-service --user YOUR_USER --dir /opt/fetch2gmail2 -o /tmp/fetch2gmail2.service
|
|
243
300
|
sudo mv /tmp/fetch2gmail2.service /etc/systemd/system/fetch2gmail2.service
|
|
244
|
-
sudo nano /etc/systemd/system/fetch2gmail2.service
|
|
245
|
-
```
|
|
246
|
-
In the editor, find the `ExecStart=` line and make sure it includes a port flag, for example:
|
|
247
301
|
```
|
|
248
|
-
ExecStart
|
|
249
|
-
|
|
250
|
-
Also set **WorkingDirectory** and **Environment=FETCH2GMAIL_CONFIG** to the second directory (e.g. `/opt/fetch2gmail2`, `/opt/fetch2gmail2/config.json`). Then reload and enable:
|
|
251
|
-
```bash
|
|
252
|
-
sudo systemctl daemon-reload
|
|
253
|
-
sudo systemctl enable fetch2gmail2
|
|
254
|
-
sudo systemctl start fetch2gmail2
|
|
255
|
-
```
|
|
256
|
-
Run **`fetch2gmail set-ui-password`** once from the second directory (or with **FETCH2GMAIL_CONFIG** set to the second config) so the second UI is protected.
|
|
257
|
-
|
|
258
|
-
4. Open the first UI at **http://server:8765** and the second at **http://server:8766**. Each has its own config, token, and poller.
|
|
259
|
-
|
|
260
|
-
**You need a different port per instance** — each `fetch2gmail serve` binds to one port, so a second instance must use `--port 8766` (or another free port).
|
|
302
|
+
Edit `ExecStart=` to include `--port 8766`, then enable/start `fetch2gmail2`.
|
|
303
|
+
3. Open the first UI at **http://server:8765** and the second at **http://server:8766**.
|
|
261
304
|
|
|
262
305
|
### Troubleshooting
|
|
263
306
|
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
Self-hosted email fetcher: **IMAP (ISP mailbox) → Gmail API import**. Replaces Gmail’s deprecated POP3 fetch.
|
|
4
4
|
|
|
5
|
-
-
|
|
6
|
-
- Imports messages with **Gmail API** (not SMTP); applies a Gmail label, preserves headers and date.
|
|
5
|
+
- Watches one or more ISP mailboxes via **IMAPS** (port 993), using **IMAP IDLE** for near-realtime delivery when the server supports it (falls back to polling otherwise).
|
|
6
|
+
- Imports messages with **Gmail API** (not SMTP); applies a Gmail label, preserves headers and date. By default uses **`neverMarkSpam`** so Gmail’s spam classifier does not bury ISP→Gmail imports.
|
|
7
|
+
- Supports **multiple IMAP accounts in one process**, with optional **per-IMAP routing** to one or more Gmail accounts.
|
|
7
8
|
- Deletes from ISP only after Gmail confirms import. **Idempotent** (UID + message hash).
|
|
8
9
|
|
|
9
10
|
**Requirements:** Python 3.11+, IMAP credentials, Google Cloud project with Gmail API and OAuth2 (Web application).
|
|
@@ -180,9 +181,17 @@ Your Google OAuth client must use **Web application** (not Desktop) and have the
|
|
|
180
181
|
|
|
181
182
|
Google does not allow redirect URIs that use an IP address. That is why the token must be obtained on a machine where the app can use localhost (Use case 1 steps 1–4 on a PC; Use case 2 on the same machine).
|
|
182
183
|
|
|
183
|
-
###
|
|
184
|
+
### IMAP IDLE and poll interval
|
|
184
185
|
|
|
185
|
-
The
|
|
186
|
+
The background watcher in the web UI process prefers **IMAP IDLE** (RFC 2177) when the server advertises the `IDLE` capability. New mail can trigger an import immediately instead of waiting for the next poll.
|
|
187
|
+
|
|
188
|
+
- **`imap.use_idle`** (default `true`): enable IDLE when supported. Disable to force polling.
|
|
189
|
+
- **`imap.idle_refresh_minutes`** (default `25`): leave IDLE, send `NOOP`, and re-enter IDLE before the typical ~29-minute server timeout. Also enables TCP `SO_KEEPALIVE` and reconnects with exponential backoff on silent drops.
|
|
190
|
+
- **`poll_interval_minutes`** (default `5`): used when IDLE is missing or disabled.
|
|
191
|
+
|
|
192
|
+
All three can be changed in the web UI (Config) or in **config.json**.
|
|
193
|
+
|
|
194
|
+
**Manual smoke checks:** with IDLE supported, send yourself mail and confirm an import in the UI logs within seconds (look for `IDLE: mailbox update`). Set `use_idle` to `false` (or use a server without IDLE) and confirm fetches still run on the poll interval.
|
|
186
195
|
|
|
187
196
|
### Data directory
|
|
188
197
|
|
|
@@ -194,39 +203,72 @@ All app files live in **one directory**: **config.json**, **credentials.json**,
|
|
|
194
203
|
- The UI is protected by a username and password in **.ui_auth** (when set). Get **token.json** via **`fetch2gmail auth`** (CLI only; there is no "Sign in with Google" in the UI).
|
|
195
204
|
- The Gmail scope requested is **gmail.modify** (read and modify labels/messages only).
|
|
196
205
|
|
|
197
|
-
### Multiple accounts (
|
|
206
|
+
### Multiple IMAP accounts (recommended: one process)
|
|
207
|
+
|
|
208
|
+
One Fetch2Gmail process can watch **multiple IMAP mailboxes** and import into one or more Gmail accounts. Configure this in the UI (**IMAP accounts** section) or in **config.json** via `imap_accounts`.
|
|
209
|
+
|
|
210
|
+
**Example: 3 IMAP → 1 Gmail**
|
|
211
|
+
|
|
212
|
+
```json
|
|
213
|
+
{
|
|
214
|
+
"imap_accounts": [
|
|
215
|
+
{
|
|
216
|
+
"id": "isp1",
|
|
217
|
+
"host": "imap.isp1.example",
|
|
218
|
+
"port": 993,
|
|
219
|
+
"username": "you@isp1.example",
|
|
220
|
+
"password_env": "IMAP_PASSWORD_ISP1",
|
|
221
|
+
"mailbox": "INBOX",
|
|
222
|
+
"use_ssl": true,
|
|
223
|
+
"delete_after_import": true,
|
|
224
|
+
"gmail_token_paths": []
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
"id": "isp2",
|
|
228
|
+
"host": "imap.isp2.example",
|
|
229
|
+
"username": "you@isp2.example",
|
|
230
|
+
"password_env": "IMAP_PASSWORD_ISP2",
|
|
231
|
+
"mailbox": "INBOX"
|
|
232
|
+
},
|
|
233
|
+
{
|
|
234
|
+
"id": "isp3",
|
|
235
|
+
"host": "imap.isp3.example",
|
|
236
|
+
"username": "you@isp3.example",
|
|
237
|
+
"password_env": "IMAP_PASSWORD_ISP3",
|
|
238
|
+
"mailbox": "INBOX"
|
|
239
|
+
}
|
|
240
|
+
],
|
|
241
|
+
"gmail_accounts": [
|
|
242
|
+
{
|
|
243
|
+
"label": "ISP Mail",
|
|
244
|
+
"credentials_path": "credentials.json",
|
|
245
|
+
"token_path": "token.json"
|
|
246
|
+
}
|
|
247
|
+
]
|
|
248
|
+
}
|
|
249
|
+
```
|
|
198
250
|
|
|
199
|
-
|
|
251
|
+
- Leave `gmail_token_paths` empty (or omit it) so that IMAP account imports into **all** configured Gmail accounts.
|
|
252
|
+
- To route one IMAP account to a subset of Gmail accounts, set `gmail_token_paths` to the matching `token_path` values (e.g. `["token.json"]`).
|
|
253
|
+
- Passwords are stored encrypted in **.env** under each account’s `password_env` key (default `IMAP_PASSWORD` for id `default`, otherwise `IMAP_PASSWORD_<ID>`).
|
|
254
|
+
- Legacy configs with a single top-level `"imap"` object still work (treated as one account with id `default`).
|
|
200
255
|
|
|
201
|
-
|
|
256
|
+
In the UI you can add/remove IMAP accounts and pick Gmail destinations per account. One systemd service / one UI port is enough.
|
|
202
257
|
|
|
203
|
-
|
|
258
|
+
### Multiple instances (optional isolation)
|
|
204
259
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
```
|
|
260
|
+
You can still run **multiple processes** (separate data directories and ports) when you want isolation across machines or fully separate configs. That is optional now that multi-IMAP works in one process.
|
|
261
|
+
|
|
262
|
+
**Second instance (example):**
|
|
209
263
|
|
|
210
|
-
|
|
264
|
+
1. Create a second data directory (e.g. `/opt/fetch2gmail2`) with its own **config.json**, credentials, and tokens.
|
|
265
|
+
2. Generate a second systemd unit and bind a different port:
|
|
211
266
|
```bash
|
|
267
|
+
fetch2gmail install-service --user YOUR_USER --dir /opt/fetch2gmail2 -o /tmp/fetch2gmail2.service
|
|
212
268
|
sudo mv /tmp/fetch2gmail2.service /etc/systemd/system/fetch2gmail2.service
|
|
213
|
-
sudo nano /etc/systemd/system/fetch2gmail2.service
|
|
214
|
-
```
|
|
215
|
-
In the editor, find the `ExecStart=` line and make sure it includes a port flag, for example:
|
|
216
269
|
```
|
|
217
|
-
ExecStart
|
|
218
|
-
|
|
219
|
-
Also set **WorkingDirectory** and **Environment=FETCH2GMAIL_CONFIG** to the second directory (e.g. `/opt/fetch2gmail2`, `/opt/fetch2gmail2/config.json`). Then reload and enable:
|
|
220
|
-
```bash
|
|
221
|
-
sudo systemctl daemon-reload
|
|
222
|
-
sudo systemctl enable fetch2gmail2
|
|
223
|
-
sudo systemctl start fetch2gmail2
|
|
224
|
-
```
|
|
225
|
-
Run **`fetch2gmail set-ui-password`** once from the second directory (or with **FETCH2GMAIL_CONFIG** set to the second config) so the second UI is protected.
|
|
226
|
-
|
|
227
|
-
4. Open the first UI at **http://server:8765** and the second at **http://server:8766**. Each has its own config, token, and poller.
|
|
228
|
-
|
|
229
|
-
**You need a different port per instance** — each `fetch2gmail serve` binds to one port, so a second instance must use `--port 8766` (or another free port).
|
|
270
|
+
Edit `ExecStart=` to include `--port 8766`, then enable/start `fetch2gmail2`.
|
|
271
|
+
3. Open the first UI at **http://server:8765** and the second at **http://server:8766**.
|
|
230
272
|
|
|
231
273
|
### Troubleshooting
|
|
232
274
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "fetch2gmail"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.3.0"
|
|
8
8
|
description = "Self-hosted email fetcher: IMAP to Gmail API import with idempotent state tracking"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -31,6 +31,7 @@ dependencies = [
|
|
|
31
31
|
"python-multipart>=0.0.6",
|
|
32
32
|
"cryptography>=42.0.0",
|
|
33
33
|
"bcrypt>=4.0.0",
|
|
34
|
+
"imapclient>=3.0.0",
|
|
34
35
|
]
|
|
35
36
|
|
|
36
37
|
[project.optional-dependencies]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fetch2gmail
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Self-hosted email fetcher: IMAP to Gmail API import with idempotent state tracking
|
|
5
5
|
Author: Fetch2Gmail Contributors
|
|
6
6
|
License-Expression: MIT
|
|
@@ -24,6 +24,7 @@ Requires-Dist: python-dotenv>=1.0.0
|
|
|
24
24
|
Requires-Dist: python-multipart>=0.0.6
|
|
25
25
|
Requires-Dist: cryptography>=42.0.0
|
|
26
26
|
Requires-Dist: bcrypt>=4.0.0
|
|
27
|
+
Requires-Dist: imapclient>=3.0.0
|
|
27
28
|
Provides-Extra: dev
|
|
28
29
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
29
30
|
Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
|
|
@@ -33,8 +34,9 @@ Dynamic: license-file
|
|
|
33
34
|
|
|
34
35
|
Self-hosted email fetcher: **IMAP (ISP mailbox) → Gmail API import**. Replaces Gmail’s deprecated POP3 fetch.
|
|
35
36
|
|
|
36
|
-
-
|
|
37
|
-
- Imports messages with **Gmail API** (not SMTP); applies a Gmail label, preserves headers and date.
|
|
37
|
+
- Watches one or more ISP mailboxes via **IMAPS** (port 993), using **IMAP IDLE** for near-realtime delivery when the server supports it (falls back to polling otherwise).
|
|
38
|
+
- Imports messages with **Gmail API** (not SMTP); applies a Gmail label, preserves headers and date. By default uses **`neverMarkSpam`** so Gmail’s spam classifier does not bury ISP→Gmail imports.
|
|
39
|
+
- Supports **multiple IMAP accounts in one process**, with optional **per-IMAP routing** to one or more Gmail accounts.
|
|
38
40
|
- Deletes from ISP only after Gmail confirms import. **Idempotent** (UID + message hash).
|
|
39
41
|
|
|
40
42
|
**Requirements:** Python 3.11+, IMAP credentials, Google Cloud project with Gmail API and OAuth2 (Web application).
|
|
@@ -211,9 +213,17 @@ Your Google OAuth client must use **Web application** (not Desktop) and have the
|
|
|
211
213
|
|
|
212
214
|
Google does not allow redirect URIs that use an IP address. That is why the token must be obtained on a machine where the app can use localhost (Use case 1 steps 1–4 on a PC; Use case 2 on the same machine).
|
|
213
215
|
|
|
214
|
-
###
|
|
216
|
+
### IMAP IDLE and poll interval
|
|
215
217
|
|
|
216
|
-
The
|
|
218
|
+
The background watcher in the web UI process prefers **IMAP IDLE** (RFC 2177) when the server advertises the `IDLE` capability. New mail can trigger an import immediately instead of waiting for the next poll.
|
|
219
|
+
|
|
220
|
+
- **`imap.use_idle`** (default `true`): enable IDLE when supported. Disable to force polling.
|
|
221
|
+
- **`imap.idle_refresh_minutes`** (default `25`): leave IDLE, send `NOOP`, and re-enter IDLE before the typical ~29-minute server timeout. Also enables TCP `SO_KEEPALIVE` and reconnects with exponential backoff on silent drops.
|
|
222
|
+
- **`poll_interval_minutes`** (default `5`): used when IDLE is missing or disabled.
|
|
223
|
+
|
|
224
|
+
All three can be changed in the web UI (Config) or in **config.json**.
|
|
225
|
+
|
|
226
|
+
**Manual smoke checks:** with IDLE supported, send yourself mail and confirm an import in the UI logs within seconds (look for `IDLE: mailbox update`). Set `use_idle` to `false` (or use a server without IDLE) and confirm fetches still run on the poll interval.
|
|
217
227
|
|
|
218
228
|
### Data directory
|
|
219
229
|
|
|
@@ -225,39 +235,72 @@ All app files live in **one directory**: **config.json**, **credentials.json**,
|
|
|
225
235
|
- The UI is protected by a username and password in **.ui_auth** (when set). Get **token.json** via **`fetch2gmail auth`** (CLI only; there is no "Sign in with Google" in the UI).
|
|
226
236
|
- The Gmail scope requested is **gmail.modify** (read and modify labels/messages only).
|
|
227
237
|
|
|
228
|
-
### Multiple accounts (
|
|
238
|
+
### Multiple IMAP accounts (recommended: one process)
|
|
239
|
+
|
|
240
|
+
One Fetch2Gmail process can watch **multiple IMAP mailboxes** and import into one or more Gmail accounts. Configure this in the UI (**IMAP accounts** section) or in **config.json** via `imap_accounts`.
|
|
241
|
+
|
|
242
|
+
**Example: 3 IMAP → 1 Gmail**
|
|
243
|
+
|
|
244
|
+
```json
|
|
245
|
+
{
|
|
246
|
+
"imap_accounts": [
|
|
247
|
+
{
|
|
248
|
+
"id": "isp1",
|
|
249
|
+
"host": "imap.isp1.example",
|
|
250
|
+
"port": 993,
|
|
251
|
+
"username": "you@isp1.example",
|
|
252
|
+
"password_env": "IMAP_PASSWORD_ISP1",
|
|
253
|
+
"mailbox": "INBOX",
|
|
254
|
+
"use_ssl": true,
|
|
255
|
+
"delete_after_import": true,
|
|
256
|
+
"gmail_token_paths": []
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
"id": "isp2",
|
|
260
|
+
"host": "imap.isp2.example",
|
|
261
|
+
"username": "you@isp2.example",
|
|
262
|
+
"password_env": "IMAP_PASSWORD_ISP2",
|
|
263
|
+
"mailbox": "INBOX"
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
"id": "isp3",
|
|
267
|
+
"host": "imap.isp3.example",
|
|
268
|
+
"username": "you@isp3.example",
|
|
269
|
+
"password_env": "IMAP_PASSWORD_ISP3",
|
|
270
|
+
"mailbox": "INBOX"
|
|
271
|
+
}
|
|
272
|
+
],
|
|
273
|
+
"gmail_accounts": [
|
|
274
|
+
{
|
|
275
|
+
"label": "ISP Mail",
|
|
276
|
+
"credentials_path": "credentials.json",
|
|
277
|
+
"token_path": "token.json"
|
|
278
|
+
}
|
|
279
|
+
]
|
|
280
|
+
}
|
|
281
|
+
```
|
|
229
282
|
|
|
230
|
-
|
|
283
|
+
- Leave `gmail_token_paths` empty (or omit it) so that IMAP account imports into **all** configured Gmail accounts.
|
|
284
|
+
- To route one IMAP account to a subset of Gmail accounts, set `gmail_token_paths` to the matching `token_path` values (e.g. `["token.json"]`).
|
|
285
|
+
- Passwords are stored encrypted in **.env** under each account’s `password_env` key (default `IMAP_PASSWORD` for id `default`, otherwise `IMAP_PASSWORD_<ID>`).
|
|
286
|
+
- Legacy configs with a single top-level `"imap"` object still work (treated as one account with id `default`).
|
|
231
287
|
|
|
232
|
-
|
|
288
|
+
In the UI you can add/remove IMAP accounts and pick Gmail destinations per account. One systemd service / one UI port is enough.
|
|
233
289
|
|
|
234
|
-
|
|
290
|
+
### Multiple instances (optional isolation)
|
|
235
291
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
```
|
|
292
|
+
You can still run **multiple processes** (separate data directories and ports) when you want isolation across machines or fully separate configs. That is optional now that multi-IMAP works in one process.
|
|
293
|
+
|
|
294
|
+
**Second instance (example):**
|
|
240
295
|
|
|
241
|
-
|
|
296
|
+
1. Create a second data directory (e.g. `/opt/fetch2gmail2`) with its own **config.json**, credentials, and tokens.
|
|
297
|
+
2. Generate a second systemd unit and bind a different port:
|
|
242
298
|
```bash
|
|
299
|
+
fetch2gmail install-service --user YOUR_USER --dir /opt/fetch2gmail2 -o /tmp/fetch2gmail2.service
|
|
243
300
|
sudo mv /tmp/fetch2gmail2.service /etc/systemd/system/fetch2gmail2.service
|
|
244
|
-
sudo nano /etc/systemd/system/fetch2gmail2.service
|
|
245
|
-
```
|
|
246
|
-
In the editor, find the `ExecStart=` line and make sure it includes a port flag, for example:
|
|
247
301
|
```
|
|
248
|
-
ExecStart
|
|
249
|
-
|
|
250
|
-
Also set **WorkingDirectory** and **Environment=FETCH2GMAIL_CONFIG** to the second directory (e.g. `/opt/fetch2gmail2`, `/opt/fetch2gmail2/config.json`). Then reload and enable:
|
|
251
|
-
```bash
|
|
252
|
-
sudo systemctl daemon-reload
|
|
253
|
-
sudo systemctl enable fetch2gmail2
|
|
254
|
-
sudo systemctl start fetch2gmail2
|
|
255
|
-
```
|
|
256
|
-
Run **`fetch2gmail set-ui-password`** once from the second directory (or with **FETCH2GMAIL_CONFIG** set to the second config) so the second UI is protected.
|
|
257
|
-
|
|
258
|
-
4. Open the first UI at **http://server:8765** and the second at **http://server:8766**. Each has its own config, token, and poller.
|
|
259
|
-
|
|
260
|
-
**You need a different port per instance** — each `fetch2gmail serve` binds to one port, so a second instance must use `--port 8766` (or another free port).
|
|
302
|
+
Edit `ExecStart=` to include `--port 8766`, then enable/start `fetch2gmail2`.
|
|
303
|
+
3. Open the first UI at **http://server:8765** and the second at **http://server:8766**.
|
|
261
304
|
|
|
262
305
|
### Troubleshooting
|
|
263
306
|
|
|
@@ -14,9 +14,19 @@ src/fetcher/cli.py
|
|
|
14
14
|
src/fetcher/config.py
|
|
15
15
|
src/fetcher/env_file.py
|
|
16
16
|
src/fetcher/gmail_client.py
|
|
17
|
+
src/fetcher/idle_watcher.py
|
|
17
18
|
src/fetcher/imap_client.py
|
|
18
19
|
src/fetcher/log_buffer.py
|
|
19
20
|
src/fetcher/run.py
|
|
20
21
|
src/fetcher/state.py
|
|
21
22
|
src/fetcher/ui_auth.py
|
|
22
|
-
src/fetcher/web_ui.py
|
|
23
|
+
src/fetcher/web_ui.py
|
|
24
|
+
tests/test_config_imap_accounts.py
|
|
25
|
+
tests/test_config_password_env_collision.py
|
|
26
|
+
tests/test_deliver_destinations.py
|
|
27
|
+
tests/test_env_credential_refresh.py
|
|
28
|
+
tests/test_gmail_token_atomic.py
|
|
29
|
+
tests/test_idle_gate_per_account.py
|
|
30
|
+
tests/test_idle_watcher.py
|
|
31
|
+
tests/test_run_delivery_error.py
|
|
32
|
+
tests/test_state_account_id.py
|
|
@@ -11,7 +11,7 @@ from pathlib import Path
|
|
|
11
11
|
from fastapi import FastAPI, Request
|
|
12
12
|
from fastapi.responses import HTMLResponse, RedirectResponse
|
|
13
13
|
|
|
14
|
-
from .gmail_client import SCOPES
|
|
14
|
+
from .gmail_client import SCOPES, atomic_write_token
|
|
15
15
|
|
|
16
16
|
# Single-purpose app: no config, no session, just OAuth -> save token
|
|
17
17
|
app = FastAPI(
|
|
@@ -91,9 +91,7 @@ def _auth_callback(
|
|
|
91
91
|
flow.redirect_uri = f"{callback_base}/auth/gmail/callback"
|
|
92
92
|
flow.fetch_token(code=code)
|
|
93
93
|
creds = flow.credentials
|
|
94
|
-
token_path.
|
|
95
|
-
with open(token_path, "w", encoding="utf-8") as f:
|
|
96
|
-
f.write(creds.to_json())
|
|
94
|
+
atomic_write_token(token_path, creds.to_json())
|
|
97
95
|
|
|
98
96
|
cred_abs = cred_path.resolve()
|
|
99
97
|
tok_abs = token_path.resolve()
|
|
@@ -10,8 +10,8 @@ import sys
|
|
|
10
10
|
from pathlib import Path
|
|
11
11
|
|
|
12
12
|
from . import __version__
|
|
13
|
-
from .config import get_config_path, load_config
|
|
14
|
-
from .run import
|
|
13
|
+
from .config import get_config_path, get_imap_accounts, load_config
|
|
14
|
+
from .run import run_all_accounts, setup_logging
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
def main() -> None:
|
|
@@ -95,7 +95,8 @@ def _cmd_run(args: argparse.Namespace) -> None:
|
|
|
95
95
|
setup_logging()
|
|
96
96
|
config_path = args.config or str(get_config_path())
|
|
97
97
|
try:
|
|
98
|
-
|
|
98
|
+
# Run all IMAP accounts in one CLI invocation.
|
|
99
|
+
result = run_all_accounts(config_path=config_path, dry_run=args.dry_run)
|
|
99
100
|
except FileNotFoundError as e:
|
|
100
101
|
print(f"Config not found: {e}", file=sys.stderr)
|
|
101
102
|
sys.exit(1)
|
|
@@ -111,6 +112,18 @@ def _cmd_run(args: argparse.Namespace) -> None:
|
|
|
111
112
|
)
|
|
112
113
|
if result.get("last_fetch_time"):
|
|
113
114
|
print(f"Last fetch: {result['last_fetch_time']}")
|
|
115
|
+
accounts = result.get("accounts") or []
|
|
116
|
+
if len(accounts) > 1:
|
|
117
|
+
for acct in accounts:
|
|
118
|
+
aid = acct.get("account_id", "?")
|
|
119
|
+
err = acct.get("error")
|
|
120
|
+
if err:
|
|
121
|
+
print(f" [{aid}] error: {err}", file=sys.stderr)
|
|
122
|
+
else:
|
|
123
|
+
print(
|
|
124
|
+
f" [{aid}] imported={acct.get('imported', 0)} "
|
|
125
|
+
f"skipped={acct.get('skipped_duplicate', 0)} deleted={acct.get('deleted', 0)}"
|
|
126
|
+
)
|
|
114
127
|
|
|
115
128
|
|
|
116
129
|
def _cmd_config(args: argparse.Namespace) -> None:
|
|
@@ -135,7 +148,11 @@ def _cmd_config(args: argparse.Namespace) -> None:
|
|
|
135
148
|
if args.validate:
|
|
136
149
|
config_path = args.config or str(get_config_path())
|
|
137
150
|
try:
|
|
138
|
-
|
|
151
|
+
from .config import get_gmail_destinations_for_imap
|
|
152
|
+
|
|
153
|
+
cfg = load_config(config_path, resolve_password=False)
|
|
154
|
+
for acct in get_imap_accounts(cfg):
|
|
155
|
+
get_gmail_destinations_for_imap(cfg, acct)
|
|
139
156
|
print("Config OK.")
|
|
140
157
|
except Exception as e:
|
|
141
158
|
print(f"Invalid config: {e}", file=sys.stderr)
|
|
@@ -276,18 +293,32 @@ def _cmd_auth(args: argparse.Namespace) -> None:
|
|
|
276
293
|
def _write_default_config(dest: Path) -> None:
|
|
277
294
|
"""Write a default config.json to dest."""
|
|
278
295
|
cfg = {
|
|
279
|
-
"
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
296
|
+
"imap_accounts": [
|
|
297
|
+
{
|
|
298
|
+
"id": "default",
|
|
299
|
+
"host": "imap.example.com",
|
|
300
|
+
"port": 993,
|
|
301
|
+
"username": "your-isp-email@example.com",
|
|
302
|
+
"password_env": "IMAP_PASSWORD",
|
|
303
|
+
"mailbox": "INBOX",
|
|
304
|
+
"use_ssl": True,
|
|
305
|
+
"delete_after_import": True,
|
|
306
|
+
"never_mark_spam": True,
|
|
307
|
+
"use_idle": True,
|
|
308
|
+
"idle_refresh_minutes": 25,
|
|
309
|
+
"since_date": None,
|
|
310
|
+
"gmail_token_paths": [],
|
|
311
|
+
}
|
|
312
|
+
],
|
|
289
313
|
# Canonical: gmail_accounts (multi-account). Single-account setups will have one entry.
|
|
290
|
-
"gmail_accounts": [
|
|
314
|
+
"gmail_accounts": [
|
|
315
|
+
{
|
|
316
|
+
"use_label": False,
|
|
317
|
+
"label": "ISP Mail",
|
|
318
|
+
"credentials_path": "credentials.json",
|
|
319
|
+
"token_path": "token.json",
|
|
320
|
+
}
|
|
321
|
+
],
|
|
291
322
|
"state": {"db_path": "state.db"},
|
|
292
323
|
"ui": {"host": "127.0.0.1", "port": 8765},
|
|
293
324
|
"poll_interval_minutes": 5,
|
|
@@ -310,17 +341,31 @@ def config_wizard_interactive() -> None:
|
|
|
310
341
|
cred_path = input("Gmail OAuth credentials.json path [credentials.json]: ").strip() or "credentials.json"
|
|
311
342
|
token_path = input("Gmail OAuth token path [token.json]: ").strip() or "token.json"
|
|
312
343
|
cfg = {
|
|
313
|
-
"
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
344
|
+
"imap_accounts": [
|
|
345
|
+
{
|
|
346
|
+
"id": "default",
|
|
347
|
+
"host": host,
|
|
348
|
+
"port": int(port),
|
|
349
|
+
"username": user,
|
|
350
|
+
"password_env": password_env,
|
|
351
|
+
"mailbox": mailbox,
|
|
352
|
+
"use_ssl": True,
|
|
353
|
+
"delete_after_import": True,
|
|
354
|
+
"never_mark_spam": True,
|
|
355
|
+
"use_idle": True,
|
|
356
|
+
"idle_refresh_minutes": 25,
|
|
357
|
+
"since_date": None,
|
|
358
|
+
"gmail_token_paths": [],
|
|
359
|
+
}
|
|
360
|
+
],
|
|
361
|
+
"gmail_accounts": [
|
|
362
|
+
{
|
|
363
|
+
"use_label": bool(label),
|
|
364
|
+
"label": label or "ISP Mail",
|
|
365
|
+
"credentials_path": cred_path,
|
|
366
|
+
"token_path": token_path,
|
|
367
|
+
}
|
|
368
|
+
],
|
|
324
369
|
"state": {"db_path": db_path},
|
|
325
370
|
"ui": {"host": "127.0.0.1", "port": 8765},
|
|
326
371
|
"poll_interval_minutes": 5,
|