fetch2gmail 1.1.9__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. {fetch2gmail-1.1.9/src/fetch2gmail.egg-info → fetch2gmail-1.3.0}/PKG-INFO +73 -30
  2. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/README.md +71 -29
  3. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/pyproject.toml +2 -1
  4. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0/src/fetch2gmail.egg-info}/PKG-INFO +73 -30
  5. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetch2gmail.egg-info/SOURCES.txt +11 -1
  6. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetch2gmail.egg-info/requires.txt +1 -0
  7. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/auth_server.py +2 -4
  8. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/cli.py +71 -26
  9. fetch2gmail-1.3.0/src/fetcher/config.py +341 -0
  10. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/env_file.py +23 -1
  11. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/gmail_client.py +77 -18
  12. fetch2gmail-1.3.0/src/fetcher/idle_watcher.py +774 -0
  13. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/imap_client.py +117 -11
  14. fetch2gmail-1.3.0/src/fetcher/run.py +849 -0
  15. fetch2gmail-1.3.0/src/fetcher/state.py +514 -0
  16. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/web_ui.py +935 -225
  17. fetch2gmail-1.3.0/tests/test_config_imap_accounts.py +396 -0
  18. fetch2gmail-1.3.0/tests/test_config_password_env_collision.py +264 -0
  19. fetch2gmail-1.3.0/tests/test_deliver_destinations.py +325 -0
  20. fetch2gmail-1.3.0/tests/test_env_credential_refresh.py +40 -0
  21. fetch2gmail-1.3.0/tests/test_gmail_token_atomic.py +100 -0
  22. fetch2gmail-1.3.0/tests/test_idle_gate_per_account.py +37 -0
  23. fetch2gmail-1.3.0/tests/test_idle_watcher.py +309 -0
  24. fetch2gmail-1.3.0/tests/test_run_delivery_error.py +74 -0
  25. fetch2gmail-1.3.0/tests/test_state_account_id.py +161 -0
  26. fetch2gmail-1.1.9/src/fetcher/config.py +0 -68
  27. fetch2gmail-1.1.9/src/fetcher/run.py +0 -482
  28. fetch2gmail-1.1.9/src/fetcher/state.py +0 -152
  29. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/LICENSE +0 -0
  30. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/setup.cfg +0 -0
  31. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetch2gmail.egg-info/dependency_links.txt +0 -0
  32. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetch2gmail.egg-info/entry_points.txt +0 -0
  33. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetch2gmail.egg-info/top_level.txt +0 -0
  34. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/__init__.py +0 -0
  35. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/auth_ui.py +0 -0
  36. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/log_buffer.py +0 -0
  37. {fetch2gmail-1.1.9 → fetch2gmail-1.3.0}/src/fetcher/ui_auth.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fetch2gmail
3
- Version: 1.1.9
3
+ Version: 1.3.0
4
4
  Summary: Self-hosted email fetcher: IMAP to Gmail API import with idempotent state tracking
5
5
  Author: Fetch2Gmail Contributors
6
6
  License-Expression: MIT
@@ -24,6 +24,7 @@ Requires-Dist: python-dotenv>=1.0.0
24
24
  Requires-Dist: python-multipart>=0.0.6
25
25
  Requires-Dist: cryptography>=42.0.0
26
26
  Requires-Dist: bcrypt>=4.0.0
27
+ Requires-Dist: imapclient>=3.0.0
27
28
  Provides-Extra: dev
28
29
  Requires-Dist: pytest>=7.0; extra == "dev"
29
30
  Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
@@ -33,8 +34,9 @@ Dynamic: license-file
33
34
 
34
35
  Self-hosted email fetcher: **IMAP (ISP mailbox) → Gmail API import**. Replaces Gmail’s deprecated POP3 fetch.
35
36
 
36
- - Polls an ISP mailbox via **IMAPS** (port 993).
37
- - Imports messages with **Gmail API** (not SMTP); applies a Gmail label, preserves headers and date.
37
+ - Watches one or more ISP mailboxes via **IMAPS** (port 993), using **IMAP IDLE** for near-realtime delivery when the server supports it (falls back to polling otherwise).
38
+ - Imports messages with **Gmail API** (not SMTP); applies a Gmail label, preserves headers and date. By default uses **`neverMarkSpam`** so Gmail’s spam classifier does not bury ISP→Gmail imports.
39
+ - Supports **multiple IMAP accounts in one process**, with optional **per-IMAP routing** to one or more Gmail accounts.
38
40
  - Deletes from ISP only after Gmail confirms import. **Idempotent** (UID + message hash).
39
41
 
40
42
  **Requirements:** Python 3.11+, IMAP credentials, Google Cloud project with Gmail API and OAuth2 (Web application).
@@ -211,9 +213,17 @@ Your Google OAuth client must use **Web application** (not Desktop) and have the
211
213
 
212
214
  Google does not allow redirect URIs that use an IP address. That is why the token must be obtained on a machine where the app can use localhost (Use case 1 steps 1–4 on a PC; Use case 2 on the same machine).
213
215
 
214
- ### Poll interval
216
+ ### IMAP IDLE and poll interval
215
217
 
216
- The app runs the fetch every 5 minutes by default (background poller in the same process as the web UI). You can change the interval in the web UI (Config) or in **config.json** (`poll_interval_minutes`).
218
+ The background watcher in the web UI process prefers **IMAP IDLE** (RFC 2177) when the server advertises the `IDLE` capability. New mail can trigger an import immediately instead of waiting for the next poll.
219
+
220
+ - **`imap.use_idle`** (default `true`): enable IDLE when supported. Disable to force polling.
221
+ - **`imap.idle_refresh_minutes`** (default `25`): leave IDLE, send `NOOP`, and re-enter IDLE before the typical ~29-minute server timeout. Also enables TCP `SO_KEEPALIVE` and reconnects with exponential backoff on silent drops.
222
+ - **`poll_interval_minutes`** (default `5`): used when IDLE is missing or disabled.
223
+
224
+ All three can be changed in the web UI (Config) or in **config.json**.
225
+
226
+ **Manual smoke checks:** with IDLE supported, send yourself mail and confirm an import in the UI logs within seconds (look for `IDLE: mailbox update`). Set `use_idle` to `false` (or use a server without IDLE) and confirm fetches still run on the poll interval.
217
227
 
218
228
  ### Data directory
219
229
 
@@ -225,39 +235,72 @@ All app files live in **one directory**: **config.json**, **credentials.json**,
225
235
  - The UI is protected by a username and password in **.ui_auth** (when set). Get **token.json** via **`fetch2gmail auth`** (CLI only; there is no "Sign in with Google" in the UI).
226
236
  - The Gmail scope requested is **gmail.modify** (read and modify labels/messages only).
227
237
 
228
- ### Multiple accounts (multiple instances)
238
+ ### Multiple IMAP accounts (recommended: one process)
239
+
240
+ One Fetch2Gmail process can watch **multiple IMAP mailboxes** and import into one or more Gmail accounts. Configure this in the UI (**IMAP accounts** section) or in **config.json** via `imap_accounts`.
241
+
242
+ **Example: 3 IMAP → 1 Gmail**
243
+
244
+ ```json
245
+ {
246
+ "imap_accounts": [
247
+ {
248
+ "id": "isp1",
249
+ "host": "imap.isp1.example",
250
+ "port": 993,
251
+ "username": "you@isp1.example",
252
+ "password_env": "IMAP_PASSWORD_ISP1",
253
+ "mailbox": "INBOX",
254
+ "use_ssl": true,
255
+ "delete_after_import": true,
256
+ "gmail_token_paths": []
257
+ },
258
+ {
259
+ "id": "isp2",
260
+ "host": "imap.isp2.example",
261
+ "username": "you@isp2.example",
262
+ "password_env": "IMAP_PASSWORD_ISP2",
263
+ "mailbox": "INBOX"
264
+ },
265
+ {
266
+ "id": "isp3",
267
+ "host": "imap.isp3.example",
268
+ "username": "you@isp3.example",
269
+ "password_env": "IMAP_PASSWORD_ISP3",
270
+ "mailbox": "INBOX"
271
+ }
272
+ ],
273
+ "gmail_accounts": [
274
+ {
275
+ "label": "ISP Mail",
276
+ "credentials_path": "credentials.json",
277
+ "token_path": "token.json"
278
+ }
279
+ ]
280
+ }
281
+ ```
229
282
 
230
- One instance = one IMAP mailbox → one Gmail account. To run **multiple accounts** (e.g. two ISP mailboxes forwarding to two Gmail addresses), run **multiple instances**, each with its own data directory and **its own port** so the UIs don’t conflict.
283
+ - Leave `gmail_token_paths` empty (or omit it) so that IMAP account imports into **all** configured Gmail accounts.
284
+ - To route one IMAP account to a subset of Gmail accounts, set `gmail_token_paths` to the matching `token_path` values (e.g. `["token.json"]`).
285
+ - Passwords are stored encrypted in **.env** under each account’s `password_env` key (default `IMAP_PASSWORD` for id `default`, otherwise `IMAP_PASSWORD_<ID>`).
286
+ - Legacy configs with a single top-level `"imap"` object still work (treated as one account with id `default`).
231
287
 
232
- **Second instance (example):**
288
+ In the UI you can add/remove IMAP accounts and pick Gmail destinations per account. One systemd service / one UI port is enough.
233
289
 
234
- 1. Create a second data directory and put config and credentials there (e.g. `/opt/fetch2gmail2` with its own **config.json**, **credentials.json**, **token.json**, **.env**). Use the same GCP **credentials.json** if you like; get a separate **token.json** for the second Gmail account (run `fetch2gmail auth` and sign in with the second Gmail, then copy that **token.json** into the second directory).
290
+ ### Multiple instances (optional isolation)
235
291
 
236
- 2. Generate a second systemd unit with a **different port** and unit name:
237
- ```bash
238
- fetch2gmail install-service --user YOUR_USER --dir /opt/fetch2gmail2 -o /tmp/fetch2gmail2.service
239
- ```
292
+ You can still run **multiple processes** (separate data directories and ports) when you want isolation across machines or fully separate configs. That is optional now that multi-IMAP works in one process.
293
+
294
+ **Second instance (example):**
240
295
 
241
- 3. Install the second unit under a different name so both can run, **then edit its ExecStart to add a port**:
296
+ 1. Create a second data directory (e.g. `/opt/fetch2gmail2`) with its own **config.json**, credentials, and tokens.
297
+ 2. Generate a second systemd unit and bind a different port:
242
298
  ```bash
299
+ fetch2gmail install-service --user YOUR_USER --dir /opt/fetch2gmail2 -o /tmp/fetch2gmail2.service
243
300
  sudo mv /tmp/fetch2gmail2.service /etc/systemd/system/fetch2gmail2.service
244
- sudo nano /etc/systemd/system/fetch2gmail2.service
245
- ```
246
- In the editor, find the `ExecStart=` line and make sure it includes a port flag, for example:
247
301
  ```
248
- ExecStart=/home/YOUR_USER/.local/pipx/venvs/fetch2gmail/bin/fetch2gmail serve --host 0.0.0.0 --port 8766
249
- ```
250
- Also set **WorkingDirectory** and **Environment=FETCH2GMAIL_CONFIG** to the second directory (e.g. `/opt/fetch2gmail2`, `/opt/fetch2gmail2/config.json`). Then reload and enable:
251
- ```bash
252
- sudo systemctl daemon-reload
253
- sudo systemctl enable fetch2gmail2
254
- sudo systemctl start fetch2gmail2
255
- ```
256
- Run **`fetch2gmail set-ui-password`** once from the second directory (or with **FETCH2GMAIL_CONFIG** set to the second config) so the second UI is protected.
257
-
258
- 4. Open the first UI at **http://server:8765** and the second at **http://server:8766**. Each has its own config, token, and poller.
259
-
260
- **You need a different port per instance** — each `fetch2gmail serve` binds to one port, so a second instance must use `--port 8766` (or another free port).
302
+ Edit `ExecStart=` to include `--port 8766`, then enable/start `fetch2gmail2`.
303
+ 3. Open the first UI at **http://server:8765** and the second at **http://server:8766**.
261
304
 
262
305
  ### Troubleshooting
263
306
 
@@ -2,8 +2,9 @@
2
2
 
3
3
  Self-hosted email fetcher: **IMAP (ISP mailbox) → Gmail API import**. Replaces Gmail’s deprecated POP3 fetch.
4
4
 
5
- - Polls an ISP mailbox via **IMAPS** (port 993).
6
- - Imports messages with **Gmail API** (not SMTP); applies a Gmail label, preserves headers and date.
5
+ - Watches one or more ISP mailboxes via **IMAPS** (port 993), using **IMAP IDLE** for near-realtime delivery when the server supports it (falls back to polling otherwise).
6
+ - Imports messages with **Gmail API** (not SMTP); applies a Gmail label, preserves headers and date. By default uses **`neverMarkSpam`** so Gmail’s spam classifier does not bury ISP→Gmail imports.
7
+ - Supports **multiple IMAP accounts in one process**, with optional **per-IMAP routing** to one or more Gmail accounts.
7
8
  - Deletes from ISP only after Gmail confirms import. **Idempotent** (UID + message hash).
8
9
 
9
10
  **Requirements:** Python 3.11+, IMAP credentials, Google Cloud project with Gmail API and OAuth2 (Web application).
@@ -180,9 +181,17 @@ Your Google OAuth client must use **Web application** (not Desktop) and have the
180
181
 
181
182
  Google does not allow redirect URIs that use an IP address. That is why the token must be obtained on a machine where the app can use localhost (Use case 1 steps 1–4 on a PC; Use case 2 on the same machine).
182
183
 
183
- ### Poll interval
184
+ ### IMAP IDLE and poll interval
184
185
 
185
- The app runs the fetch every 5 minutes by default (background poller in the same process as the web UI). You can change the interval in the web UI (Config) or in **config.json** (`poll_interval_minutes`).
186
+ The background watcher in the web UI process prefers **IMAP IDLE** (RFC 2177) when the server advertises the `IDLE` capability. New mail can trigger an import immediately instead of waiting for the next poll.
187
+
188
+ - **`imap.use_idle`** (default `true`): enable IDLE when supported. Disable to force polling.
189
+ - **`imap.idle_refresh_minutes`** (default `25`): leave IDLE, send `NOOP`, and re-enter IDLE before the typical ~29-minute server timeout. Also enables TCP `SO_KEEPALIVE` and reconnects with exponential backoff on silent drops.
190
+ - **`poll_interval_minutes`** (default `5`): used when IDLE is missing or disabled.
191
+
192
+ All three can be changed in the web UI (Config) or in **config.json**.
193
+
194
+ **Manual smoke checks:** with IDLE supported, send yourself mail and confirm an import in the UI logs within seconds (look for `IDLE: mailbox update`). Set `use_idle` to `false` (or use a server without IDLE) and confirm fetches still run on the poll interval.
186
195
 
187
196
  ### Data directory
188
197
 
@@ -194,39 +203,72 @@ All app files live in **one directory**: **config.json**, **credentials.json**,
194
203
  - The UI is protected by a username and password in **.ui_auth** (when set). Get **token.json** via **`fetch2gmail auth`** (CLI only; there is no "Sign in with Google" in the UI).
195
204
  - The Gmail scope requested is **gmail.modify** (read and modify labels/messages only).
196
205
 
197
- ### Multiple accounts (multiple instances)
206
+ ### Multiple IMAP accounts (recommended: one process)
207
+
208
+ One Fetch2Gmail process can watch **multiple IMAP mailboxes** and import into one or more Gmail accounts. Configure this in the UI (**IMAP accounts** section) or in **config.json** via `imap_accounts`.
209
+
210
+ **Example: 3 IMAP → 1 Gmail**
211
+
212
+ ```json
213
+ {
214
+ "imap_accounts": [
215
+ {
216
+ "id": "isp1",
217
+ "host": "imap.isp1.example",
218
+ "port": 993,
219
+ "username": "you@isp1.example",
220
+ "password_env": "IMAP_PASSWORD_ISP1",
221
+ "mailbox": "INBOX",
222
+ "use_ssl": true,
223
+ "delete_after_import": true,
224
+ "gmail_token_paths": []
225
+ },
226
+ {
227
+ "id": "isp2",
228
+ "host": "imap.isp2.example",
229
+ "username": "you@isp2.example",
230
+ "password_env": "IMAP_PASSWORD_ISP2",
231
+ "mailbox": "INBOX"
232
+ },
233
+ {
234
+ "id": "isp3",
235
+ "host": "imap.isp3.example",
236
+ "username": "you@isp3.example",
237
+ "password_env": "IMAP_PASSWORD_ISP3",
238
+ "mailbox": "INBOX"
239
+ }
240
+ ],
241
+ "gmail_accounts": [
242
+ {
243
+ "label": "ISP Mail",
244
+ "credentials_path": "credentials.json",
245
+ "token_path": "token.json"
246
+ }
247
+ ]
248
+ }
249
+ ```
198
250
 
199
- One instance = one IMAP mailbox → one Gmail account. To run **multiple accounts** (e.g. two ISP mailboxes forwarding to two Gmail addresses), run **multiple instances**, each with its own data directory and **its own port** so the UIs don’t conflict.
251
+ - Leave `gmail_token_paths` empty (or omit it) so that IMAP account imports into **all** configured Gmail accounts.
252
+ - To route one IMAP account to a subset of Gmail accounts, set `gmail_token_paths` to the matching `token_path` values (e.g. `["token.json"]`).
253
+ - Passwords are stored encrypted in **.env** under each account’s `password_env` key (default `IMAP_PASSWORD` for id `default`, otherwise `IMAP_PASSWORD_<ID>`).
254
+ - Legacy configs with a single top-level `"imap"` object still work (treated as one account with id `default`).
200
255
 
201
- **Second instance (example):**
256
+ In the UI you can add/remove IMAP accounts and pick Gmail destinations per account. One systemd service / one UI port is enough.
202
257
 
203
- 1. Create a second data directory and put config and credentials there (e.g. `/opt/fetch2gmail2` with its own **config.json**, **credentials.json**, **token.json**, **.env**). Use the same GCP **credentials.json** if you like; get a separate **token.json** for the second Gmail account (run `fetch2gmail auth` and sign in with the second Gmail, then copy that **token.json** into the second directory).
258
+ ### Multiple instances (optional isolation)
204
259
 
205
- 2. Generate a second systemd unit with a **different port** and unit name:
206
- ```bash
207
- fetch2gmail install-service --user YOUR_USER --dir /opt/fetch2gmail2 -o /tmp/fetch2gmail2.service
208
- ```
260
+ You can still run **multiple processes** (separate data directories and ports) when you want isolation across machines or fully separate configs. That is optional now that multi-IMAP works in one process.
261
+
262
+ **Second instance (example):**
209
263
 
210
- 3. Install the second unit under a different name so both can run, **then edit its ExecStart to add a port**:
264
+ 1. Create a second data directory (e.g. `/opt/fetch2gmail2`) with its own **config.json**, credentials, and tokens.
265
+ 2. Generate a second systemd unit and bind a different port:
211
266
  ```bash
267
+ fetch2gmail install-service --user YOUR_USER --dir /opt/fetch2gmail2 -o /tmp/fetch2gmail2.service
212
268
  sudo mv /tmp/fetch2gmail2.service /etc/systemd/system/fetch2gmail2.service
213
- sudo nano /etc/systemd/system/fetch2gmail2.service
214
- ```
215
- In the editor, find the `ExecStart=` line and make sure it includes a port flag, for example:
216
269
  ```
217
- ExecStart=/home/YOUR_USER/.local/pipx/venvs/fetch2gmail/bin/fetch2gmail serve --host 0.0.0.0 --port 8766
218
- ```
219
- Also set **WorkingDirectory** and **Environment=FETCH2GMAIL_CONFIG** to the second directory (e.g. `/opt/fetch2gmail2`, `/opt/fetch2gmail2/config.json`). Then reload and enable:
220
- ```bash
221
- sudo systemctl daemon-reload
222
- sudo systemctl enable fetch2gmail2
223
- sudo systemctl start fetch2gmail2
224
- ```
225
- Run **`fetch2gmail set-ui-password`** once from the second directory (or with **FETCH2GMAIL_CONFIG** set to the second config) so the second UI is protected.
226
-
227
- 4. Open the first UI at **http://server:8765** and the second at **http://server:8766**. Each has its own config, token, and poller.
228
-
229
- **You need a different port per instance** — each `fetch2gmail serve` binds to one port, so a second instance must use `--port 8766` (or another free port).
270
+ Edit `ExecStart=` to include `--port 8766`, then enable/start `fetch2gmail2`.
271
+ 3. Open the first UI at **http://server:8765** and the second at **http://server:8766**.
230
272
 
231
273
  ### Troubleshooting
232
274
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "fetch2gmail"
7
- version = "1.1.9"
7
+ version = "1.3.0"
8
8
  description = "Self-hosted email fetcher: IMAP to Gmail API import with idempotent state tracking"
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -31,6 +31,7 @@ dependencies = [
31
31
  "python-multipart>=0.0.6",
32
32
  "cryptography>=42.0.0",
33
33
  "bcrypt>=4.0.0",
34
+ "imapclient>=3.0.0",
34
35
  ]
35
36
 
36
37
  [project.optional-dependencies]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fetch2gmail
3
- Version: 1.1.9
3
+ Version: 1.3.0
4
4
  Summary: Self-hosted email fetcher: IMAP to Gmail API import with idempotent state tracking
5
5
  Author: Fetch2Gmail Contributors
6
6
  License-Expression: MIT
@@ -24,6 +24,7 @@ Requires-Dist: python-dotenv>=1.0.0
24
24
  Requires-Dist: python-multipart>=0.0.6
25
25
  Requires-Dist: cryptography>=42.0.0
26
26
  Requires-Dist: bcrypt>=4.0.0
27
+ Requires-Dist: imapclient>=3.0.0
27
28
  Provides-Extra: dev
28
29
  Requires-Dist: pytest>=7.0; extra == "dev"
29
30
  Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
@@ -33,8 +34,9 @@ Dynamic: license-file
33
34
 
34
35
  Self-hosted email fetcher: **IMAP (ISP mailbox) → Gmail API import**. Replaces Gmail’s deprecated POP3 fetch.
35
36
 
36
- - Polls an ISP mailbox via **IMAPS** (port 993).
37
- - Imports messages with **Gmail API** (not SMTP); applies a Gmail label, preserves headers and date.
37
+ - Watches one or more ISP mailboxes via **IMAPS** (port 993), using **IMAP IDLE** for near-realtime delivery when the server supports it (falls back to polling otherwise).
38
+ - Imports messages with **Gmail API** (not SMTP); applies a Gmail label, preserves headers and date. By default uses **`neverMarkSpam`** so Gmail’s spam classifier does not bury ISP→Gmail imports.
39
+ - Supports **multiple IMAP accounts in one process**, with optional **per-IMAP routing** to one or more Gmail accounts.
38
40
  - Deletes from ISP only after Gmail confirms import. **Idempotent** (UID + message hash).
39
41
 
40
42
  **Requirements:** Python 3.11+, IMAP credentials, Google Cloud project with Gmail API and OAuth2 (Web application).
@@ -211,9 +213,17 @@ Your Google OAuth client must use **Web application** (not Desktop) and have the
211
213
 
212
214
  Google does not allow redirect URIs that use an IP address. That is why the token must be obtained on a machine where the app can use localhost (Use case 1 steps 1–4 on a PC; Use case 2 on the same machine).
213
215
 
214
- ### Poll interval
216
+ ### IMAP IDLE and poll interval
215
217
 
216
- The app runs the fetch every 5 minutes by default (background poller in the same process as the web UI). You can change the interval in the web UI (Config) or in **config.json** (`poll_interval_minutes`).
218
+ The background watcher in the web UI process prefers **IMAP IDLE** (RFC 2177) when the server advertises the `IDLE` capability. New mail can trigger an import immediately instead of waiting for the next poll.
219
+
220
+ - **`imap.use_idle`** (default `true`): enable IDLE when supported. Disable to force polling.
221
+ - **`imap.idle_refresh_minutes`** (default `25`): leave IDLE, send `NOOP`, and re-enter IDLE before the typical ~29-minute server timeout. Also enables TCP `SO_KEEPALIVE` and reconnects with exponential backoff on silent drops.
222
+ - **`poll_interval_minutes`** (default `5`): used when IDLE is missing or disabled.
223
+
224
+ All three can be changed in the web UI (Config) or in **config.json**.
225
+
226
+ **Manual smoke checks:** with IDLE supported, send yourself mail and confirm an import in the UI logs within seconds (look for `IDLE: mailbox update`). Set `use_idle` to `false` (or use a server without IDLE) and confirm fetches still run on the poll interval.
217
227
 
218
228
  ### Data directory
219
229
 
@@ -225,39 +235,72 @@ All app files live in **one directory**: **config.json**, **credentials.json**,
225
235
  - The UI is protected by a username and password in **.ui_auth** (when set). Get **token.json** via **`fetch2gmail auth`** (CLI only; there is no "Sign in with Google" in the UI).
226
236
  - The Gmail scope requested is **gmail.modify** (read and modify labels/messages only).
227
237
 
228
- ### Multiple accounts (multiple instances)
238
+ ### Multiple IMAP accounts (recommended: one process)
239
+
240
+ One Fetch2Gmail process can watch **multiple IMAP mailboxes** and import into one or more Gmail accounts. Configure this in the UI (**IMAP accounts** section) or in **config.json** via `imap_accounts`.
241
+
242
+ **Example: 3 IMAP → 1 Gmail**
243
+
244
+ ```json
245
+ {
246
+ "imap_accounts": [
247
+ {
248
+ "id": "isp1",
249
+ "host": "imap.isp1.example",
250
+ "port": 993,
251
+ "username": "you@isp1.example",
252
+ "password_env": "IMAP_PASSWORD_ISP1",
253
+ "mailbox": "INBOX",
254
+ "use_ssl": true,
255
+ "delete_after_import": true,
256
+ "gmail_token_paths": []
257
+ },
258
+ {
259
+ "id": "isp2",
260
+ "host": "imap.isp2.example",
261
+ "username": "you@isp2.example",
262
+ "password_env": "IMAP_PASSWORD_ISP2",
263
+ "mailbox": "INBOX"
264
+ },
265
+ {
266
+ "id": "isp3",
267
+ "host": "imap.isp3.example",
268
+ "username": "you@isp3.example",
269
+ "password_env": "IMAP_PASSWORD_ISP3",
270
+ "mailbox": "INBOX"
271
+ }
272
+ ],
273
+ "gmail_accounts": [
274
+ {
275
+ "label": "ISP Mail",
276
+ "credentials_path": "credentials.json",
277
+ "token_path": "token.json"
278
+ }
279
+ ]
280
+ }
281
+ ```
229
282
 
230
- One instance = one IMAP mailbox → one Gmail account. To run **multiple accounts** (e.g. two ISP mailboxes forwarding to two Gmail addresses), run **multiple instances**, each with its own data directory and **its own port** so the UIs don’t conflict.
283
+ - Leave `gmail_token_paths` empty (or omit it) so that IMAP account imports into **all** configured Gmail accounts.
284
+ - To route one IMAP account to a subset of Gmail accounts, set `gmail_token_paths` to the matching `token_path` values (e.g. `["token.json"]`).
285
+ - Passwords are stored encrypted in **.env** under each account’s `password_env` key (default `IMAP_PASSWORD` for id `default`, otherwise `IMAP_PASSWORD_<ID>`).
286
+ - Legacy configs with a single top-level `"imap"` object still work (treated as one account with id `default`).
231
287
 
232
- **Second instance (example):**
288
+ In the UI you can add/remove IMAP accounts and pick Gmail destinations per account. One systemd service / one UI port is enough.
233
289
 
234
- 1. Create a second data directory and put config and credentials there (e.g. `/opt/fetch2gmail2` with its own **config.json**, **credentials.json**, **token.json**, **.env**). Use the same GCP **credentials.json** if you like; get a separate **token.json** for the second Gmail account (run `fetch2gmail auth` and sign in with the second Gmail, then copy that **token.json** into the second directory).
290
+ ### Multiple instances (optional isolation)
235
291
 
236
- 2. Generate a second systemd unit with a **different port** and unit name:
237
- ```bash
238
- fetch2gmail install-service --user YOUR_USER --dir /opt/fetch2gmail2 -o /tmp/fetch2gmail2.service
239
- ```
292
+ You can still run **multiple processes** (separate data directories and ports) when you want isolation across machines or fully separate configs. That is optional now that multi-IMAP works in one process.
293
+
294
+ **Second instance (example):**
240
295
 
241
- 3. Install the second unit under a different name so both can run, **then edit its ExecStart to add a port**:
296
+ 1. Create a second data directory (e.g. `/opt/fetch2gmail2`) with its own **config.json**, credentials, and tokens.
297
+ 2. Generate a second systemd unit and bind a different port:
242
298
  ```bash
299
+ fetch2gmail install-service --user YOUR_USER --dir /opt/fetch2gmail2 -o /tmp/fetch2gmail2.service
243
300
  sudo mv /tmp/fetch2gmail2.service /etc/systemd/system/fetch2gmail2.service
244
- sudo nano /etc/systemd/system/fetch2gmail2.service
245
- ```
246
- In the editor, find the `ExecStart=` line and make sure it includes a port flag, for example:
247
301
  ```
248
- ExecStart=/home/YOUR_USER/.local/pipx/venvs/fetch2gmail/bin/fetch2gmail serve --host 0.0.0.0 --port 8766
249
- ```
250
- Also set **WorkingDirectory** and **Environment=FETCH2GMAIL_CONFIG** to the second directory (e.g. `/opt/fetch2gmail2`, `/opt/fetch2gmail2/config.json`). Then reload and enable:
251
- ```bash
252
- sudo systemctl daemon-reload
253
- sudo systemctl enable fetch2gmail2
254
- sudo systemctl start fetch2gmail2
255
- ```
256
- Run **`fetch2gmail set-ui-password`** once from the second directory (or with **FETCH2GMAIL_CONFIG** set to the second config) so the second UI is protected.
257
-
258
- 4. Open the first UI at **http://server:8765** and the second at **http://server:8766**. Each has its own config, token, and poller.
259
-
260
- **You need a different port per instance** — each `fetch2gmail serve` binds to one port, so a second instance must use `--port 8766` (or another free port).
302
+ Edit `ExecStart=` to include `--port 8766`, then enable/start `fetch2gmail2`.
303
+ 3. Open the first UI at **http://server:8765** and the second at **http://server:8766**.
261
304
 
262
305
  ### Troubleshooting
263
306
 
@@ -14,9 +14,19 @@ src/fetcher/cli.py
14
14
  src/fetcher/config.py
15
15
  src/fetcher/env_file.py
16
16
  src/fetcher/gmail_client.py
17
+ src/fetcher/idle_watcher.py
17
18
  src/fetcher/imap_client.py
18
19
  src/fetcher/log_buffer.py
19
20
  src/fetcher/run.py
20
21
  src/fetcher/state.py
21
22
  src/fetcher/ui_auth.py
22
- src/fetcher/web_ui.py
23
+ src/fetcher/web_ui.py
24
+ tests/test_config_imap_accounts.py
25
+ tests/test_config_password_env_collision.py
26
+ tests/test_deliver_destinations.py
27
+ tests/test_env_credential_refresh.py
28
+ tests/test_gmail_token_atomic.py
29
+ tests/test_idle_gate_per_account.py
30
+ tests/test_idle_watcher.py
31
+ tests/test_run_delivery_error.py
32
+ tests/test_state_account_id.py
@@ -7,6 +7,7 @@ python-dotenv>=1.0.0
7
7
  python-multipart>=0.0.6
8
8
  cryptography>=42.0.0
9
9
  bcrypt>=4.0.0
10
+ imapclient>=3.0.0
10
11
 
11
12
  [dev]
12
13
  pytest>=7.0
@@ -11,7 +11,7 @@ from pathlib import Path
11
11
  from fastapi import FastAPI, Request
12
12
  from fastapi.responses import HTMLResponse, RedirectResponse
13
13
 
14
- from .gmail_client import SCOPES
14
+ from .gmail_client import SCOPES, atomic_write_token
15
15
 
16
16
  # Single-purpose app: no config, no session, just OAuth -> save token
17
17
  app = FastAPI(
@@ -91,9 +91,7 @@ def _auth_callback(
91
91
  flow.redirect_uri = f"{callback_base}/auth/gmail/callback"
92
92
  flow.fetch_token(code=code)
93
93
  creds = flow.credentials
94
- token_path.parent.mkdir(parents=True, exist_ok=True)
95
- with open(token_path, "w", encoding="utf-8") as f:
96
- f.write(creds.to_json())
94
+ atomic_write_token(token_path, creds.to_json())
97
95
 
98
96
  cred_abs = cred_path.resolve()
99
97
  tok_abs = token_path.resolve()
@@ -10,8 +10,8 @@ import sys
10
10
  from pathlib import Path
11
11
 
12
12
  from . import __version__
13
- from .config import get_config_path, load_config
14
- from .run import run_once, setup_logging
13
+ from .config import get_config_path, get_imap_accounts, load_config
14
+ from .run import run_all_accounts, setup_logging
15
15
 
16
16
 
17
17
  def main() -> None:
@@ -95,7 +95,8 @@ def _cmd_run(args: argparse.Namespace) -> None:
95
95
  setup_logging()
96
96
  config_path = args.config or str(get_config_path())
97
97
  try:
98
- result = run_once(config_path=config_path, dry_run=args.dry_run)
98
+ # Run all IMAP accounts in one CLI invocation.
99
+ result = run_all_accounts(config_path=config_path, dry_run=args.dry_run)
99
100
  except FileNotFoundError as e:
100
101
  print(f"Config not found: {e}", file=sys.stderr)
101
102
  sys.exit(1)
@@ -111,6 +112,18 @@ def _cmd_run(args: argparse.Namespace) -> None:
111
112
  )
112
113
  if result.get("last_fetch_time"):
113
114
  print(f"Last fetch: {result['last_fetch_time']}")
115
+ accounts = result.get("accounts") or []
116
+ if len(accounts) > 1:
117
+ for acct in accounts:
118
+ aid = acct.get("account_id", "?")
119
+ err = acct.get("error")
120
+ if err:
121
+ print(f" [{aid}] error: {err}", file=sys.stderr)
122
+ else:
123
+ print(
124
+ f" [{aid}] imported={acct.get('imported', 0)} "
125
+ f"skipped={acct.get('skipped_duplicate', 0)} deleted={acct.get('deleted', 0)}"
126
+ )
114
127
 
115
128
 
116
129
  def _cmd_config(args: argparse.Namespace) -> None:
@@ -135,7 +148,11 @@ def _cmd_config(args: argparse.Namespace) -> None:
135
148
  if args.validate:
136
149
  config_path = args.config or str(get_config_path())
137
150
  try:
138
- load_config(config_path)
151
+ from .config import get_gmail_destinations_for_imap
152
+
153
+ cfg = load_config(config_path, resolve_password=False)
154
+ for acct in get_imap_accounts(cfg):
155
+ get_gmail_destinations_for_imap(cfg, acct)
139
156
  print("Config OK.")
140
157
  except Exception as e:
141
158
  print(f"Invalid config: {e}", file=sys.stderr)
@@ -276,18 +293,32 @@ def _cmd_auth(args: argparse.Namespace) -> None:
276
293
  def _write_default_config(dest: Path) -> None:
277
294
  """Write a default config.json to dest."""
278
295
  cfg = {
279
- "imap": {
280
- "host": "imap.example.com",
281
- "port": 993,
282
- "username": "your-isp-email@example.com",
283
- "password_env": "IMAP_PASSWORD",
284
- "mailbox": "INBOX",
285
- "use_ssl": True,
286
- "delete_after_import": True,
287
- "since_date": None,
288
- },
296
+ "imap_accounts": [
297
+ {
298
+ "id": "default",
299
+ "host": "imap.example.com",
300
+ "port": 993,
301
+ "username": "your-isp-email@example.com",
302
+ "password_env": "IMAP_PASSWORD",
303
+ "mailbox": "INBOX",
304
+ "use_ssl": True,
305
+ "delete_after_import": True,
306
+ "never_mark_spam": True,
307
+ "use_idle": True,
308
+ "idle_refresh_minutes": 25,
309
+ "since_date": None,
310
+ "gmail_token_paths": [],
311
+ }
312
+ ],
289
313
  # Canonical: gmail_accounts (multi-account). Single-account setups will have one entry.
290
- "gmail_accounts": [{"use_label": False, "label": "ISP Mail", "credentials_path": "credentials.json", "token_path": "token.json"}],
314
+ "gmail_accounts": [
315
+ {
316
+ "use_label": False,
317
+ "label": "ISP Mail",
318
+ "credentials_path": "credentials.json",
319
+ "token_path": "token.json",
320
+ }
321
+ ],
291
322
  "state": {"db_path": "state.db"},
292
323
  "ui": {"host": "127.0.0.1", "port": 8765},
293
324
  "poll_interval_minutes": 5,
@@ -310,17 +341,31 @@ def config_wizard_interactive() -> None:
310
341
  cred_path = input("Gmail OAuth credentials.json path [credentials.json]: ").strip() or "credentials.json"
311
342
  token_path = input("Gmail OAuth token path [token.json]: ").strip() or "token.json"
312
343
  cfg = {
313
- "imap": {
314
- "host": host,
315
- "port": int(port),
316
- "username": user,
317
- "password_env": password_env,
318
- "mailbox": mailbox,
319
- "use_ssl": True,
320
- "delete_after_import": True,
321
- "since_date": None,
322
- },
323
- "gmail_accounts": [{"use_label": bool(label), "label": label or "ISP Mail", "credentials_path": cred_path, "token_path": token_path}],
344
+ "imap_accounts": [
345
+ {
346
+ "id": "default",
347
+ "host": host,
348
+ "port": int(port),
349
+ "username": user,
350
+ "password_env": password_env,
351
+ "mailbox": mailbox,
352
+ "use_ssl": True,
353
+ "delete_after_import": True,
354
+ "never_mark_spam": True,
355
+ "use_idle": True,
356
+ "idle_refresh_minutes": 25,
357
+ "since_date": None,
358
+ "gmail_token_paths": [],
359
+ }
360
+ ],
361
+ "gmail_accounts": [
362
+ {
363
+ "use_label": bool(label),
364
+ "label": label or "ISP Mail",
365
+ "credentials_path": cred_path,
366
+ "token_path": token_path,
367
+ }
368
+ ],
324
369
  "state": {"db_path": db_path},
325
370
  "ui": {"host": "127.0.0.1", "port": 8765},
326
371
  "poll_interval_minutes": 5,