glean-importer 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of glean-importer might be problematic. Click here for more details.

Files changed (30) hide show
  1. glean_importer-0.1.0/LICENSE +21 -0
  2. glean_importer-0.1.0/PKG-INFO +220 -0
  3. glean_importer-0.1.0/README.md +183 -0
  4. glean_importer-0.1.0/pyproject.toml +124 -0
  5. glean_importer-0.1.0/pyproject.toml.orig +109 -0
  6. glean_importer-0.1.0/src/glean/__init__.py +5 -0
  7. glean_importer-0.1.0/src/glean/__main__.py +5 -0
  8. glean_importer-0.1.0/src/glean/agent.py +287 -0
  9. glean_importer-0.1.0/src/glean/archive.py +95 -0
  10. glean_importer-0.1.0/src/glean/cli.py +292 -0
  11. glean_importer-0.1.0/src/glean/config.py +211 -0
  12. glean_importer-0.1.0/src/glean/connectors/__init__.py +8 -0
  13. glean_importer-0.1.0/src/glean/connectors/base.py +97 -0
  14. glean_importer-0.1.0/src/glean/connectors/mail/__init__.py +206 -0
  15. glean_importer-0.1.0/src/glean/connectors/mail/html.py +18 -0
  16. glean_importer-0.1.0/src/glean/connectors/mail/imap.py +216 -0
  17. glean_importer-0.1.0/src/glean/connectors/mail/instructions.md +55 -0
  18. glean_importer-0.1.0/src/glean/connectors/mail/mailbox.py +80 -0
  19. glean_importer-0.1.0/src/glean/connectors/mail/markdown.py +90 -0
  20. glean_importer-0.1.0/src/glean/connectors/mail/message.py +206 -0
  21. glean_importer-0.1.0/src/glean/connectors/mail/oauth.py +101 -0
  22. glean_importer-0.1.0/src/glean/connectors/mail/options.py +98 -0
  23. glean_importer-0.1.0/src/glean/connectors/mail/skip.py +27 -0
  24. glean_importer-0.1.0/src/glean/console.py +12 -0
  25. glean_importer-0.1.0/src/glean/errors.py +2 -0
  26. glean_importer-0.1.0/src/glean/pipeline.py +170 -0
  27. glean_importer-0.1.0/src/glean/prompts/import.md +60 -0
  28. glean_importer-0.1.0/src/glean/settings.py +23 -0
  29. glean_importer-0.1.0/src/glean/state.py +278 -0
  30. glean_importer-0.1.0/src/glean/wizard.py +64 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Tobias Würth
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,220 @@
1
+ Metadata-Version: 2.4
2
+ Name: glean-importer
3
+ Version: 0.1.0
4
+ Summary: Imports what your sources hold into orbit (a personal CRM) and sift (local RAG search), one sandboxed Claude Code agent run per item. Email works today: IMAP with OAuth2 (Outlook.com, Hotmail) or .eml files.
5
+ Keywords: email,imap,oauth2,outlook,hotmail,crm,personal-crm,rag,mcp,claude-code,ai-agents,local-first
6
+ Author: Tobias Würth
7
+ License-Expression: MIT
8
+ License-File: LICENSE
9
+ Classifier: Development Status :: 3 - Alpha
10
+ Classifier: Environment :: Console
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Intended Audience :: End Users/Desktop
13
+ Classifier: Operating System :: MacOS
14
+ Classifier: Operating System :: Microsoft :: Windows
15
+ Classifier: Operating System :: POSIX :: Linux
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Programming Language :: Python :: 3.14
21
+ Classifier: Programming Language :: Python :: 3.15
22
+ Classifier: Natural Language :: English
23
+ Classifier: Topic :: Communications :: Email
24
+ Classifier: Topic :: Office/Business
25
+ Classifier: Topic :: Text Processing :: Markup :: Markdown
26
+ Requires-Dist: msal>=1.31
27
+ Requires-Dist: msal-extensions>=1.2
28
+ Requires-Dist: inscriptis>=2.5
29
+ Requires-Dist: filelock>=3.16
30
+ Requires-Python: >=3.11
31
+ Project-URL: Homepage, https://github.com/tobiaswuerth/glean
32
+ Project-URL: Documentation, https://github.com/tobiaswuerth/glean#readme
33
+ Project-URL: Source, https://github.com/tobiaswuerth/glean
34
+ Project-URL: Issues, https://github.com/tobiaswuerth/glean/issues
35
+ Project-URL: Changelog, https://github.com/tobiaswuerth/glean/blob/main/CHANGELOG.md
36
+ Description-Content-Type: text/markdown
37
+
38
+ # glean: imports what your sources hold into orbit and sift
39
+
40
+ [![CI](https://github.com/tobiaswuerth/glean/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/tobiaswuerth/glean/actions/workflows/ci.yml)
41
+ [![PyPI](https://img.shields.io/pypi/v/glean-importer)](https://pypi.org/project/glean-importer/)
42
+ [![Python](https://img.shields.io/pypi/pyversions/glean-importer?logo=python&logoColor=white)](https://pypi.org/project/glean-importer/)
43
+ [![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen)](https://github.com/tobiaswuerth/glean/actions/workflows/ci.yml)
44
+ [![License](https://img.shields.io/github/license/tobiaswuerth/glean)](https://github.com/tobiaswuerth/glean/blob/main/LICENSE)
45
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
46
+ [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
47
+
48
+ glean reads information from your sources and records each item in two local stores:
49
+ [orbit](https://github.com/tobiaswuerth/orbit), a personal CRM that holds the structured view (people, organizations,
50
+ interactions, tasks, topics), and [sift](https://github.com/tobiaswuerth/sift), a local search that holds the full
51
+ text. Email works today: from an IMAP mailbox such as Outlook.com or Hotmail, signed in with OAuth2, or from a folder
52
+ of saved `.eml` files.
53
+
54
+ For every item, glean writes a Markdown document, then starts a fresh [Claude Code](https://claude.com/claude-code)
55
+ agent run whose only tools are orbit's and sift's MCP tools. The agent decides who is who, logs the interaction,
56
+ updates people, organizations and tasks, and makes the item searchable. Neither orbit nor sift knows glean exists.
57
+
58
+ - **One run per item, oldest first, one at a time.** Each run starts with a fresh context, so it knows only what orbit
59
+ and sift hold; the people an earlier email created exist when a later one in the thread mentions them.
60
+ - **The source is never changed.** Mail folders are opened read-only and messages fetched without marking them read;
61
+ what glean processed is recorded only in its own state database, which can be inspected and reset.
62
+ - **A sandbox that is checked.** The agent has no shell, file or web tools, and none of orbit's destructive ones
63
+ (merge, rename). Every run's first event lists the tools it really has; a run with anything else is stopped before
64
+ it acts. Items are treated as data, never as instructions
65
+ ([how](https://github.com/tobiaswuerth/glean/blob/main/docs/how-it-works.md#the-agent-run)).
66
+ - **Cheap where it can be.** Newsletters, notifications and automatic replies are skipped without an agent run.
67
+ - **Local.** The archive, the state database and the sign-in tokens stay on your machine; the tokens are encrypted
68
+ with your Windows login (on macOS and Linux, kept in a file only you may read).
69
+
70
+ ## Sources
71
+
72
+ | Source | Commands | Reads |
73
+ |---|---|---|
74
+ | Email | `glean mail …` | An IMAP mailbox with OAuth2 (Outlook.com, Hotmail: every folder but junk, trash and drafts), or a folder of `.eml` files |
75
+
76
+ ## Requirements
77
+
78
+ - Python 3.11 or newer.
79
+ - [Claude Code](https://claude.com/claude-code) 2.1.259 or newer, signed in (`claude auth status`). On Windows, the
80
+ native installer's `claude.exe`.
81
+ - [orbit](https://github.com/tobiaswuerth/orbit) and [sift](https://github.com/tobiaswuerth/sift), each serving MCP
82
+ over HTTP: `orbit serve <vault>` (port 8775) and `sift serve` (port 8765).
83
+ - For an Outlook.com or Hotmail mailbox: an app registered in Microsoft Entra (free, about five minutes; step 1
84
+ below), and IMAP turned on in Outlook's settings.
85
+
86
+ ## Install
87
+
88
+ With [uv](https://docs.astral.sh/uv/):
89
+
90
+ ```bash
91
+ uv tool install glean-importer # installs the `glean` command
92
+ glean --version
93
+ ```
94
+
95
+ Or with pip into a virtual environment: `pip install glean-importer`. The package is called `glean-importer`; the
96
+ command is `glean`. To update: `uv tool upgrade glean-importer`.
97
+
98
+ ## Set up
99
+
100
+ ### 1. Register glean with Microsoft (Outlook.com, Hotmail)
101
+
102
+ Outlook.com and Hotmail accept only OAuth2 for IMAP: Microsoft turned off passwords and app passwords for IMAP on
103
+ personal accounts in September 2024. A program that signs in with OAuth2 needs a client ID, which an app registered in
104
+ Microsoft Entra gives it. The app gives no one but you access: glean still signs in as you, in your browser.
105
+
106
+ 1. Open the [Microsoft Entra admin center](https://entra.microsoft.com) and sign in with your Microsoft account. If it
107
+ says your account has no directory, create a free Azure account first, which makes one.
108
+ 2. **Applications > App registrations > New registration.** Name: `glean`. Supported account types: *Personal
109
+ Microsoft accounts only*. Redirect URI: platform *Public client/native (mobile & desktop)*, `http://localhost`.
110
+ **Register.**
111
+ 3. **API permissions > Add a permission > Microsoft Graph > Delegated permissions:** `IMAP.AccessAsUser.All` and
112
+ `offline_access`. **Add permissions.**
113
+ 4. On the app's **Overview**, copy the **Application (client) ID**. It is not a secret: a public client has none.
114
+ 5. In [Outlook on the web](https://outlook.live.com): **Settings > Mail > Forwarding and IMAP > Let devices and apps
115
+ use IMAP:** on.
116
+
117
+ For a folder of `.eml` files instead of a mailbox, skip this step and set `[mail] eml_folder` after step 3.
118
+ [docs/microsoft-app.md](https://github.com/tobiaswuerth/glean/blob/main/docs/microsoft-app.md) has the same steps
119
+ with troubleshooting.
120
+
121
+ ### 2. Choose glean's home
122
+
123
+ The folder for glean's configuration (`glean.toml`), its state database, its sign-in tokens and, by default, the
124
+ archive of downloaded items. It is `~/.glean` unless `GLEAN_HOME` names another. Keep it out of synced folders: it
125
+ holds your mail. (In a clone of this repository, `data/` is ignored by git for this purpose.)
126
+
127
+ ```powershell
128
+ setx GLEAN_HOME D:\glean-data # Windows, optional; open a new terminal afterwards
129
+ ```
130
+
131
+ ### 3. Run the setup
132
+
133
+ ```bash
134
+ glean setup
135
+ ```
136
+
137
+ It asks for your address, the client ID and your other addresses, writes `glean.toml`, and opens the browser to sign
138
+ you in (`glean mail login` does that again later; `--device-code` signs in without a browser). It also starts by
139
+ itself the first time you use glean. The tokens are kept in glean's home, encrypted with your Windows login (on
140
+ macOS and Linux, in a file only you may read).
141
+
142
+ ### 4. Make the archive and the vault searchable in sift
143
+
144
+ Once: the agent indexes each email's files and the notes it changed, and sift only accepts files inside its
145
+ collections.
146
+
147
+ ```bash
148
+ sift add <GLEAN_HOME>/mail --name mail --title-from name
149
+ sift add <orbit vault> --name crm --title-from name
150
+ ```
151
+
152
+ Other names work too: set them as `[mail] collection` and `[sift] crm_collection` in `glean.toml`.
153
+
154
+ ### 5. Check everything
155
+
156
+ ```bash
157
+ glean doctor
158
+ ```
159
+
160
+ It checks Claude Code and its sign-in, that orbit and sift answer, that an agent run gets exactly the allowed tools,
161
+ and the mail sign-in. Every line should start with `ok`.
162
+
163
+ ## Use
164
+
165
+ ```bash
166
+ glean mail download # fetch what is new; the first time, everything
167
+ glean mail list --latest 20 # the 20 newest not yet imported, in the order run takes them
168
+ glean mail run --latest 20 # import them, oldest first, one agent run each
169
+ glean mail show <id> # one email: its state, files, and the agent's report
170
+ glean mail ignore <id> # never import it
171
+ glean mail reset <id> # import it again
172
+ glean status # counts per source and status, the agents' cost so far
173
+ ```
174
+
175
+ `run` also takes ids, `--retry-failed` for emails whose run failed, `--include-skipped` for those the skip rules set
176
+ aside, and `--dry-run` to write the documents and show what the agent would get, without running it.
177
+
178
+ **Try it on a test vault first.** Point `[orbit] url` at a copy of your vault (`orbit serve D:\crm-test --port 8776`),
179
+ import a few emails, read the reports (`glean mail show <id>`) and the notes, and only then use the real vault. In a
180
+ clone of this repository, [`experiments/`](https://github.com/tobiaswuerth/glean/blob/main/experiments/README.md) imports a made-up mailbox into a new vault with
181
+ the real agent and prints each run's report.
182
+
183
+ ## Configuration
184
+
185
+ `glean.toml` in glean's home; the setup writes it with every setting and a comment on each. The main ones:
186
+
187
+ | Setting | Default | |
188
+ |---|---|---|
189
+ | `[agent] model` | `sonnet` | the model of each run |
190
+ | `[agent] max_turns`, `timeout_minutes`, `max_budget_usd` | 60, 5, 1.0 | limits per item; a run that hits one fails, and is retried with `--retry-failed` |
191
+ | `[orbit] url`, `[sift] url` | ports 8775, 8765 | the MCP servers |
192
+ | `[sift] crm_collection`, `document_collections` | `crm`, all | where the vault is, and where to look for documents an item refers to |
193
+ | `[mail] address`, `client_id` | | the account, and the Entra app's client ID |
194
+ | `[mail] my_addresses` | | your other addresses: mail from them is outbound |
195
+ | `[mail] folders`, `exclude_folders` | all; Junk, Deleted, Drafts, Outbox, Notes | which folders to read |
196
+ | `[mail] archive`, `collection` | `mail` | where mail is kept, and its sift collection |
197
+ | `[mail] skip_bulk`, `skip_senders` | on; `noreply@*`, … | the pre-filter |
198
+ | `[mail] save_attachments` | `.pdf`, `.docx`, `.xlsx`, `.pptx`, … | attachments saved beside the email for sift |
199
+ | `[mail] eml_folder` | | read `.eml` files from this folder instead of the IMAP server |
200
+
201
+ ## How it works
202
+
203
+ [docs/how-it-works.md](https://github.com/tobiaswuerth/glean/blob/main/docs/how-it-works.md) describes the pipeline, the archive and its Markdown documents, the state
204
+ database, the agent's sandbox and how it is checked, and how a source is added.
205
+
206
+ ## Security and privacy
207
+
208
+ Items are untrusted input: an email can carry text written to steer an agent. glean limits what such text can do (no
209
+ dangerous tools, no destructive ones, every run's tools checked, uncertainty turned into a question for you in orbit's
210
+ review queue) and keeps everything local. See [SECURITY.md](https://github.com/tobiaswuerth/glean/blob/main/SECURITY.md).
211
+
212
+ ## Contributing
213
+
214
+ Bug reports and ideas are welcome as [issues](https://github.com/tobiaswuerth/glean/issues);
215
+ [CONTRIBUTING.md](https://github.com/tobiaswuerth/glean/blob/main/CONTRIBUTING.md) describes the setup for development, the code layout and how a release is made.
216
+ The [changelog](https://github.com/tobiaswuerth/glean/blob/main/CHANGELOG.md) lists what changed.
217
+
218
+ ## License
219
+
220
+ [MIT](https://github.com/tobiaswuerth/glean/blob/main/LICENSE)
@@ -0,0 +1,183 @@
1
+ # glean: imports what your sources hold into orbit and sift
2
+
3
+ [![CI](https://github.com/tobiaswuerth/glean/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/tobiaswuerth/glean/actions/workflows/ci.yml)
4
+ [![PyPI](https://img.shields.io/pypi/v/glean-importer)](https://pypi.org/project/glean-importer/)
5
+ [![Python](https://img.shields.io/pypi/pyversions/glean-importer?logo=python&logoColor=white)](https://pypi.org/project/glean-importer/)
6
+ [![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen)](https://github.com/tobiaswuerth/glean/actions/workflows/ci.yml)
7
+ [![License](https://img.shields.io/github/license/tobiaswuerth/glean)](https://github.com/tobiaswuerth/glean/blob/main/LICENSE)
8
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
9
+ [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
10
+
11
+ glean reads information from your sources and records each item in two local stores:
12
+ [orbit](https://github.com/tobiaswuerth/orbit), a personal CRM that holds the structured view (people, organizations,
13
+ interactions, tasks, topics), and [sift](https://github.com/tobiaswuerth/sift), a local search that holds the full
14
+ text. Email works today: from an IMAP mailbox such as Outlook.com or Hotmail, signed in with OAuth2, or from a folder
15
+ of saved `.eml` files.
16
+
17
+ For every item, glean writes a Markdown document, then starts a fresh [Claude Code](https://claude.com/claude-code)
18
+ agent run whose only tools are orbit's and sift's MCP tools. The agent decides who is who, logs the interaction,
19
+ updates people, organizations and tasks, and makes the item searchable. Neither orbit nor sift knows glean exists.
20
+
21
+ - **One run per item, oldest first, one at a time.** Each run starts with a fresh context, so it knows only what orbit
22
+ and sift hold; the people an earlier email created exist when a later one in the thread mentions them.
23
+ - **The source is never changed.** Mail folders are opened read-only and messages fetched without marking them read;
24
+ what glean processed is recorded only in its own state database, which can be inspected and reset.
25
+ - **A sandbox that is checked.** The agent has no shell, file or web tools, and none of orbit's destructive ones
26
+ (merge, rename). Every run's first event lists the tools it really has; a run with anything else is stopped before
27
+ it acts. Items are treated as data, never as instructions
28
+ ([how](https://github.com/tobiaswuerth/glean/blob/main/docs/how-it-works.md#the-agent-run)).
29
+ - **Cheap where it can be.** Newsletters, notifications and automatic replies are skipped without an agent run.
30
+ - **Local.** The archive, the state database and the sign-in tokens stay on your machine; the tokens are encrypted
31
+ with your Windows login (on macOS and Linux, kept in a file only you may read).
32
+
33
+ ## Sources
34
+
35
+ | Source | Commands | Reads |
36
+ |---|---|---|
37
+ | Email | `glean mail …` | An IMAP mailbox with OAuth2 (Outlook.com, Hotmail: every folder but junk, trash and drafts), or a folder of `.eml` files |
38
+
39
+ ## Requirements
40
+
41
+ - Python 3.11 or newer.
42
+ - [Claude Code](https://claude.com/claude-code) 2.1.259 or newer, signed in (`claude auth status`). On Windows, the
43
+ native installer's `claude.exe`.
44
+ - [orbit](https://github.com/tobiaswuerth/orbit) and [sift](https://github.com/tobiaswuerth/sift), each serving MCP
45
+ over HTTP: `orbit serve <vault>` (port 8775) and `sift serve` (port 8765).
46
+ - For an Outlook.com or Hotmail mailbox: an app registered in Microsoft Entra (free, about five minutes; step 1
47
+ below), and IMAP turned on in Outlook's settings.
48
+
49
+ ## Install
50
+
51
+ With [uv](https://docs.astral.sh/uv/):
52
+
53
+ ```bash
54
+ uv tool install glean-importer # installs the `glean` command
55
+ glean --version
56
+ ```
57
+
58
+ Or with pip into a virtual environment: `pip install glean-importer`. The package is called `glean-importer`; the
59
+ command is `glean`. To update: `uv tool upgrade glean-importer`.
60
+
61
+ ## Set up
62
+
63
+ ### 1. Register glean with Microsoft (Outlook.com, Hotmail)
64
+
65
+ Outlook.com and Hotmail accept only OAuth2 for IMAP: Microsoft turned off passwords and app passwords for IMAP on
66
+ personal accounts in September 2024. A program that signs in with OAuth2 needs a client ID, which an app registered in
67
+ Microsoft Entra gives it. The app gives no one but you access: glean still signs in as you, in your browser.
68
+
69
+ 1. Open the [Microsoft Entra admin center](https://entra.microsoft.com) and sign in with your Microsoft account. If it
70
+ says your account has no directory, create a free Azure account first, which makes one.
71
+ 2. **Applications > App registrations > New registration.** Name: `glean`. Supported account types: *Personal
72
+ Microsoft accounts only*. Redirect URI: platform *Public client/native (mobile & desktop)*, `http://localhost`.
73
+ **Register.**
74
+ 3. **API permissions > Add a permission > Microsoft Graph > Delegated permissions:** `IMAP.AccessAsUser.All` and
75
+ `offline_access`. **Add permissions.**
76
+ 4. On the app's **Overview**, copy the **Application (client) ID**. It is not a secret: a public client has none.
77
+ 5. In [Outlook on the web](https://outlook.live.com): **Settings > Mail > Forwarding and IMAP > Let devices and apps
78
+ use IMAP:** on.
79
+
80
+ For a folder of `.eml` files instead of a mailbox, skip this step and set `[mail] eml_folder` after step 3.
81
+ [docs/microsoft-app.md](https://github.com/tobiaswuerth/glean/blob/main/docs/microsoft-app.md) has the same steps
82
+ with troubleshooting.
83
+
84
+ ### 2. Choose glean's home
85
+
86
+ The folder for glean's configuration (`glean.toml`), its state database, its sign-in tokens and, by default, the
87
+ archive of downloaded items. It is `~/.glean` unless `GLEAN_HOME` names another. Keep it out of synced folders: it
88
+ holds your mail. (In a clone of this repository, `data/` is ignored by git for this purpose.)
89
+
90
+ ```powershell
91
+ setx GLEAN_HOME D:\glean-data # Windows, optional; open a new terminal afterwards
92
+ ```
93
+
94
+ ### 3. Run the setup
95
+
96
+ ```bash
97
+ glean setup
98
+ ```
99
+
100
+ It asks for your address, the client ID and your other addresses, writes `glean.toml`, and opens the browser to sign
101
+ you in (`glean mail login` does that again later; `--device-code` signs in without a browser). It also starts by
102
+ itself the first time you use glean. The tokens are kept in glean's home, encrypted with your Windows login (on
103
+ macOS and Linux, in a file only you may read).
104
+
105
+ ### 4. Make the archive and the vault searchable in sift
106
+
107
+ Once: the agent indexes each email's files and the notes it changed, and sift only accepts files inside its
108
+ collections.
109
+
110
+ ```bash
111
+ sift add <GLEAN_HOME>/mail --name mail --title-from name
112
+ sift add <orbit vault> --name crm --title-from name
113
+ ```
114
+
115
+ Other names work too: set them as `[mail] collection` and `[sift] crm_collection` in `glean.toml`.
116
+
117
+ ### 5. Check everything
118
+
119
+ ```bash
120
+ glean doctor
121
+ ```
122
+
123
+ It checks Claude Code and its sign-in, that orbit and sift answer, that an agent run gets exactly the allowed tools,
124
+ and the mail sign-in. Every line should start with `ok`.
125
+
126
+ ## Use
127
+
128
+ ```bash
129
+ glean mail download # fetch what is new; the first time, everything
130
+ glean mail list --latest 20 # the 20 newest not yet imported, in the order run takes them
131
+ glean mail run --latest 20 # import them, oldest first, one agent run each
132
+ glean mail show <id> # one email: its state, files, and the agent's report
133
+ glean mail ignore <id> # never import it
134
+ glean mail reset <id> # import it again
135
+ glean status # counts per source and status, the agents' cost so far
136
+ ```
137
+
138
+ `run` also takes ids, `--retry-failed` for emails whose run failed, `--include-skipped` for those the skip rules set
139
+ aside, and `--dry-run` to write the documents and show what the agent would get, without running it.
140
+
141
+ **Try it on a test vault first.** Point `[orbit] url` at a copy of your vault (`orbit serve D:\crm-test --port 8776`),
142
+ import a few emails, read the reports (`glean mail show <id>`) and the notes, and only then use the real vault. In a
143
+ clone of this repository, [`experiments/`](https://github.com/tobiaswuerth/glean/blob/main/experiments/README.md) imports a made-up mailbox into a new vault with
144
+ the real agent and prints each run's report.
145
+
146
+ ## Configuration
147
+
148
+ `glean.toml` in glean's home; the setup writes it with every setting and a comment on each. The main ones:
149
+
150
+ | Setting | Default | |
151
+ |---|---|---|
152
+ | `[agent] model` | `sonnet` | the model of each run |
153
+ | `[agent] max_turns`, `timeout_minutes`, `max_budget_usd` | 60, 5, 1.0 | limits per item; a run that hits one fails, and is retried with `--retry-failed` |
154
+ | `[orbit] url`, `[sift] url` | ports 8775, 8765 | the MCP servers |
155
+ | `[sift] crm_collection`, `document_collections` | `crm`, all | where the vault is, and where to look for documents an item refers to |
156
+ | `[mail] address`, `client_id` | | the account, and the Entra app's client ID |
157
+ | `[mail] my_addresses` | | your other addresses: mail from them is outbound |
158
+ | `[mail] folders`, `exclude_folders` | all; Junk, Deleted, Drafts, Outbox, Notes | which folders to read |
159
+ | `[mail] archive`, `collection` | `mail` | where mail is kept, and its sift collection |
160
+ | `[mail] skip_bulk`, `skip_senders` | on; `noreply@*`, … | the pre-filter |
161
+ | `[mail] save_attachments` | `.pdf`, `.docx`, `.xlsx`, `.pptx`, … | attachments saved beside the email for sift |
162
+ | `[mail] eml_folder` | | read `.eml` files from this folder instead of the IMAP server |
163
+
164
+ ## How it works
165
+
166
+ [docs/how-it-works.md](https://github.com/tobiaswuerth/glean/blob/main/docs/how-it-works.md) describes the pipeline, the archive and its Markdown documents, the state
167
+ database, the agent's sandbox and how it is checked, and how a source is added.
168
+
169
+ ## Security and privacy
170
+
171
+ Items are untrusted input: an email can carry text written to steer an agent. glean limits what such text can do (no
172
+ dangerous tools, no destructive ones, every run's tools checked, uncertainty turned into a question for you in orbit's
173
+ review queue) and keeps everything local. See [SECURITY.md](https://github.com/tobiaswuerth/glean/blob/main/SECURITY.md).
174
+
175
+ ## Contributing
176
+
177
+ Bug reports and ideas are welcome as [issues](https://github.com/tobiaswuerth/glean/issues);
178
+ [CONTRIBUTING.md](https://github.com/tobiaswuerth/glean/blob/main/CONTRIBUTING.md) describes the setup for development, the code layout and how a release is made.
179
+ The [changelog](https://github.com/tobiaswuerth/glean/blob/main/CHANGELOG.md) lists what changed.
180
+
181
+ ## License
182
+
183
+ [MIT](https://github.com/tobiaswuerth/glean/blob/main/LICENSE)
@@ -0,0 +1,124 @@
1
+ [build-system]
2
+ requires = ["uv_build>=0.11.6,<0.14"]
3
+ build-backend = "uv_build"
4
+
5
+ [project]
6
+ name = "glean-importer"
7
+ version = "0.1.0"
8
+ description = "Imports what your sources hold into orbit (a personal CRM) and sift (local RAG search), one sandboxed Claude Code agent run per item. Email works today: IMAP with OAuth2 (Outlook.com, Hotmail) or .eml files."
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ requires-python = ">=3.11"
13
+ dependencies = [
14
+ "msal>=1.31",
15
+ "msal-extensions>=1.2",
16
+ "inscriptis>=2.5",
17
+ "filelock>=3.16",
18
+ ]
19
+ keywords = [
20
+ "email",
21
+ "imap",
22
+ "oauth2",
23
+ "outlook",
24
+ "hotmail",
25
+ "crm",
26
+ "personal-crm",
27
+ "rag",
28
+ "mcp",
29
+ "claude-code",
30
+ "ai-agents",
31
+ "local-first",
32
+ ]
33
+ classifiers = [
34
+ "Development Status :: 3 - Alpha",
35
+ "Environment :: Console",
36
+ "Intended Audience :: Developers",
37
+ "Intended Audience :: End Users/Desktop",
38
+ "Operating System :: MacOS",
39
+ "Operating System :: Microsoft :: Windows",
40
+ "Operating System :: POSIX :: Linux",
41
+ "Programming Language :: Python :: 3",
42
+ "Programming Language :: Python :: 3.11",
43
+ "Programming Language :: Python :: 3.12",
44
+ "Programming Language :: Python :: 3.13",
45
+ "Programming Language :: Python :: 3.14",
46
+ "Programming Language :: Python :: 3.15",
47
+ "Natural Language :: English",
48
+ "Topic :: Communications :: Email",
49
+ "Topic :: Office/Business",
50
+ "Topic :: Text Processing :: Markup :: Markdown",
51
+ ]
52
+
53
+ [[project.authors]]
54
+ name = "Tobias Würth"
55
+
56
+ [project.urls]
57
+ Homepage = "https://github.com/tobiaswuerth/glean"
58
+ Documentation = "https://github.com/tobiaswuerth/glean#readme"
59
+ Source = "https://github.com/tobiaswuerth/glean"
60
+ Issues = "https://github.com/tobiaswuerth/glean/issues"
61
+ Changelog = "https://github.com/tobiaswuerth/glean/blob/main/CHANGELOG.md"
62
+
63
+ [project.scripts]
64
+ glean = "glean.cli:main"
65
+
66
+ [dependency-groups]
67
+ test = [
68
+ "pytest>=8",
69
+ "pytest-cov>=5",
70
+ "pytest-timeout>=2.3",
71
+ ]
72
+ dev = [
73
+ { include-group = "test" },
74
+ "ruff>=0.15",
75
+ "pre-commit>=4",
76
+ ]
77
+
78
+ [tool.uv.build-backend]
79
+ module-name = "glean"
80
+
81
+ [tool.ruff]
82
+ line-length = 120
83
+ target-version = "py311"
84
+
85
+ [tool.ruff.lint]
86
+ select = [
87
+ "E",
88
+ "F",
89
+ "W",
90
+ "I",
91
+ "B",
92
+ "UP",
93
+ "SIM",
94
+ "C4",
95
+ "PIE",
96
+ "PT",
97
+ "RUF",
98
+ "PERF",
99
+ ]
100
+ ignore = [
101
+ "B905",
102
+ "SIM905",
103
+ ]
104
+
105
+ [tool.ruff.lint.flake8-pytest-style]
106
+ parametrize-names-type = "csv"
107
+
108
+ [tool.pytest.ini_options]
109
+ testpaths = ["tests"]
110
+ addopts = ["--strict-markers"]
111
+ filterwarnings = ["error"]
112
+ timeout = 60
113
+
114
+ [tool.coverage.run]
115
+ source = ["glean"]
116
+ branch = true
117
+
118
+ [tool.coverage.report]
119
+ fail_under = 100
120
+ show_missing = true
121
+ skip_covered = true
122
+
123
+ [tool.codespell]
124
+ ignore-words-list = "ue,foto,passt,ist,sie,ende,oktober,technik,unser"
@@ -0,0 +1,109 @@
1
+ [build-system]
2
+ requires = ["uv_build>=0.11.6,<0.14"]
3
+ build-backend = "uv_build"
4
+
5
+ [project]
6
+ name = "glean-importer"
7
+ version = "0.1.0"
8
+ description = "Imports what your sources hold into orbit (a personal CRM) and sift (local RAG search), one sandboxed Claude Code agent run per item. Email works today: IMAP with OAuth2 (Outlook.com, Hotmail) or .eml files."
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ authors = [{ name = "Tobias Würth" }]
13
+ requires-python = ">=3.11"
14
+ dependencies = [
15
+ "msal>=1.31", # the OAuth2 sign-in to Microsoft, and refreshing its tokens
16
+ "msal-extensions>=1.2", # the token cache, encrypted with the user's Windows login (DPAPI)
17
+ "inscriptis>=2.5", # HTML mail to text, tables and lists kept readable (Apache-2.0)
18
+ "filelock>=3.16", # one glean run at a time per home
19
+ ]
20
+ keywords = [
21
+ "email",
22
+ "imap",
23
+ "oauth2",
24
+ "outlook",
25
+ "hotmail",
26
+ "crm",
27
+ "personal-crm",
28
+ "rag",
29
+ "mcp",
30
+ "claude-code",
31
+ "ai-agents",
32
+ "local-first",
33
+ ]
34
+ classifiers = [
35
+ "Development Status :: 3 - Alpha",
36
+ "Environment :: Console",
37
+ "Intended Audience :: Developers",
38
+ "Intended Audience :: End Users/Desktop",
39
+ "Operating System :: MacOS",
40
+ "Operating System :: Microsoft :: Windows",
41
+ "Operating System :: POSIX :: Linux",
42
+ "Programming Language :: Python :: 3",
43
+ "Programming Language :: Python :: 3.11",
44
+ "Programming Language :: Python :: 3.12",
45
+ "Programming Language :: Python :: 3.13",
46
+ "Programming Language :: Python :: 3.14",
47
+ "Programming Language :: Python :: 3.15",
48
+ "Natural Language :: English",
49
+ "Topic :: Communications :: Email",
50
+ "Topic :: Office/Business",
51
+ "Topic :: Text Processing :: Markup :: Markdown",
52
+ ]
53
+
54
+ [project.urls]
55
+ Homepage = "https://github.com/tobiaswuerth/glean"
56
+ Documentation = "https://github.com/tobiaswuerth/glean#readme"
57
+ Source = "https://github.com/tobiaswuerth/glean"
58
+ Issues = "https://github.com/tobiaswuerth/glean/issues"
59
+ Changelog = "https://github.com/tobiaswuerth/glean/blob/main/CHANGELOG.md"
60
+
61
+ [project.scripts]
62
+ glean = "glean.cli:main"
63
+
64
+ [dependency-groups]
65
+ test = [
66
+ "pytest>=8",
67
+ "pytest-cov>=5",
68
+ "pytest-timeout>=2.3",
69
+ ]
70
+ dev = [
71
+ { include-group = "test" },
72
+ "ruff>=0.15",
73
+ "pre-commit>=4",
74
+ ]
75
+
76
+ [tool.uv.build-backend]
77
+ module-name = "glean"
78
+
79
+ [tool.ruff]
80
+ line-length = 120
81
+ target-version = "py311"
82
+
83
+ [tool.ruff.lint]
84
+ select = ["E", "F", "W", "I", "B", "UP", "SIM", "C4", "PIE", "PT", "RUF", "PERF"]
85
+ ignore = [
86
+ "B905", # zip() without strict=
87
+ "SIM905", # a word list as one split string reads better than a long list literal
88
+ ]
89
+
90
+ [tool.ruff.lint.flake8-pytest-style]
91
+ parametrize-names-type = "csv" # @pytest.mark.parametrize("a, b", ...)
92
+
93
+ [tool.pytest.ini_options]
94
+ testpaths = ["tests"]
95
+ addopts = ["--strict-markers"]
96
+ filterwarnings = ["error"] # a warning today is a failure tomorrow
97
+ timeout = 60 # a test that hangs fails instead
98
+
99
+ [tool.coverage.run]
100
+ source = ["glean"]
101
+ branch = true
102
+
103
+ [tool.coverage.report]
104
+ fail_under = 100
105
+ show_missing = true
106
+ skip_covered = true
107
+
108
+ [tool.codespell]
109
+ ignore-words-list = "ue,foto,passt,ist,sie,ende,oktober,technik,unser" # ü written as ue, and German in made-up mail
@@ -0,0 +1,5 @@
1
+ """glean: imports email into orbit (a personal CRM) and sift (local search), one sandboxed agent run per email."""
2
+
3
+ from importlib.metadata import version
4
+
5
+ __version__ = version("glean-importer")
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())