glean-importer 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of glean-importer might be problematic. Click here for more details.
- glean_importer-0.1.0/LICENSE +21 -0
- glean_importer-0.1.0/PKG-INFO +220 -0
- glean_importer-0.1.0/README.md +183 -0
- glean_importer-0.1.0/pyproject.toml +124 -0
- glean_importer-0.1.0/pyproject.toml.orig +109 -0
- glean_importer-0.1.0/src/glean/__init__.py +5 -0
- glean_importer-0.1.0/src/glean/__main__.py +5 -0
- glean_importer-0.1.0/src/glean/agent.py +287 -0
- glean_importer-0.1.0/src/glean/archive.py +95 -0
- glean_importer-0.1.0/src/glean/cli.py +292 -0
- glean_importer-0.1.0/src/glean/config.py +211 -0
- glean_importer-0.1.0/src/glean/connectors/__init__.py +8 -0
- glean_importer-0.1.0/src/glean/connectors/base.py +97 -0
- glean_importer-0.1.0/src/glean/connectors/mail/__init__.py +206 -0
- glean_importer-0.1.0/src/glean/connectors/mail/html.py +18 -0
- glean_importer-0.1.0/src/glean/connectors/mail/imap.py +216 -0
- glean_importer-0.1.0/src/glean/connectors/mail/instructions.md +55 -0
- glean_importer-0.1.0/src/glean/connectors/mail/mailbox.py +80 -0
- glean_importer-0.1.0/src/glean/connectors/mail/markdown.py +90 -0
- glean_importer-0.1.0/src/glean/connectors/mail/message.py +206 -0
- glean_importer-0.1.0/src/glean/connectors/mail/oauth.py +101 -0
- glean_importer-0.1.0/src/glean/connectors/mail/options.py +98 -0
- glean_importer-0.1.0/src/glean/connectors/mail/skip.py +27 -0
- glean_importer-0.1.0/src/glean/console.py +12 -0
- glean_importer-0.1.0/src/glean/errors.py +2 -0
- glean_importer-0.1.0/src/glean/pipeline.py +170 -0
- glean_importer-0.1.0/src/glean/prompts/import.md +60 -0
- glean_importer-0.1.0/src/glean/settings.py +23 -0
- glean_importer-0.1.0/src/glean/state.py +278 -0
- glean_importer-0.1.0/src/glean/wizard.py +64 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Tobias Würth
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: glean-importer
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Imports what your sources hold into orbit (a personal CRM) and sift (local RAG search), one sandboxed Claude Code agent run per item. Email works today: IMAP with OAuth2 (Outlook.com, Hotmail) or .eml files.
|
|
5
|
+
Keywords: email,imap,oauth2,outlook,hotmail,crm,personal-crm,rag,mcp,claude-code,ai-agents,local-first
|
|
6
|
+
Author: Tobias Würth
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
13
|
+
Classifier: Operating System :: MacOS
|
|
14
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
22
|
+
Classifier: Natural Language :: English
|
|
23
|
+
Classifier: Topic :: Communications :: Email
|
|
24
|
+
Classifier: Topic :: Office/Business
|
|
25
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
26
|
+
Requires-Dist: msal>=1.31
|
|
27
|
+
Requires-Dist: msal-extensions>=1.2
|
|
28
|
+
Requires-Dist: inscriptis>=2.5
|
|
29
|
+
Requires-Dist: filelock>=3.16
|
|
30
|
+
Requires-Python: >=3.11
|
|
31
|
+
Project-URL: Homepage, https://github.com/tobiaswuerth/glean
|
|
32
|
+
Project-URL: Documentation, https://github.com/tobiaswuerth/glean#readme
|
|
33
|
+
Project-URL: Source, https://github.com/tobiaswuerth/glean
|
|
34
|
+
Project-URL: Issues, https://github.com/tobiaswuerth/glean/issues
|
|
35
|
+
Project-URL: Changelog, https://github.com/tobiaswuerth/glean/blob/main/CHANGELOG.md
|
|
36
|
+
Description-Content-Type: text/markdown
|
|
37
|
+
|
|
38
|
+
# glean: imports what your sources hold into orbit and sift
|
|
39
|
+
|
|
40
|
+
[](https://github.com/tobiaswuerth/glean/actions/workflows/ci.yml)
|
|
41
|
+
[](https://pypi.org/project/glean-importer/)
|
|
42
|
+
[](https://pypi.org/project/glean-importer/)
|
|
43
|
+
[](https://github.com/tobiaswuerth/glean/actions/workflows/ci.yml)
|
|
44
|
+
[](https://github.com/tobiaswuerth/glean/blob/main/LICENSE)
|
|
45
|
+
[](https://github.com/astral-sh/ruff)
|
|
46
|
+
[](https://github.com/pre-commit/pre-commit)
|
|
47
|
+
|
|
48
|
+
glean reads information from your sources and records each item in two local stores:
|
|
49
|
+
[orbit](https://github.com/tobiaswuerth/orbit), a personal CRM that holds the structured view (people, organizations,
|
|
50
|
+
interactions, tasks, topics), and [sift](https://github.com/tobiaswuerth/sift), a local search that holds the full
|
|
51
|
+
text. Email works today: from an IMAP mailbox such as Outlook.com or Hotmail, signed in with OAuth2, or from a folder
|
|
52
|
+
of saved `.eml` files.
|
|
53
|
+
|
|
54
|
+
For every item, glean writes a Markdown document, then starts a fresh [Claude Code](https://claude.com/claude-code)
|
|
55
|
+
agent run whose only tools are orbit's and sift's MCP tools. The agent decides who is who, logs the interaction,
|
|
56
|
+
updates people, organizations and tasks, and makes the item searchable. Neither orbit nor sift knows glean exists.
|
|
57
|
+
|
|
58
|
+
- **One run per item, oldest first, one at a time.** Each run starts with a fresh context, so it knows only what orbit
|
|
59
|
+
and sift hold; the people an earlier email created exist when a later one in the thread mentions them.
|
|
60
|
+
- **The source is never changed.** Mail folders are opened read-only and messages fetched without marking them read;
|
|
61
|
+
what glean processed is recorded only in its own state database, which can be inspected and reset.
|
|
62
|
+
- **A sandbox that is checked.** The agent has no shell, file or web tools, and none of orbit's destructive ones
|
|
63
|
+
(merge, rename). Every run's first event lists the tools it really has; a run with anything else is stopped before
|
|
64
|
+
it acts. Items are treated as data, never as instructions
|
|
65
|
+
([how](https://github.com/tobiaswuerth/glean/blob/main/docs/how-it-works.md#the-agent-run)).
|
|
66
|
+
- **Cheap where it can be.** Newsletters, notifications and automatic replies are skipped without an agent run.
|
|
67
|
+
- **Local.** The archive, the state database and the sign-in tokens stay on your machine; the tokens are encrypted
|
|
68
|
+
with your Windows login (on macOS and Linux, kept in a file only you may read).
|
|
69
|
+
|
|
70
|
+
## Sources
|
|
71
|
+
|
|
72
|
+
| Source | Commands | Reads |
|
|
73
|
+
|---|---|---|
|
|
74
|
+
| Email | `glean mail …` | An IMAP mailbox with OAuth2 (Outlook.com, Hotmail: every folder but junk, trash and drafts), or a folder of `.eml` files |
|
|
75
|
+
|
|
76
|
+
## Requirements
|
|
77
|
+
|
|
78
|
+
- Python 3.11 or newer.
|
|
79
|
+
- [Claude Code](https://claude.com/claude-code) 2.1.259 or newer, signed in (`claude auth status`). On Windows, the
|
|
80
|
+
native installer's `claude.exe`.
|
|
81
|
+
- [orbit](https://github.com/tobiaswuerth/orbit) and [sift](https://github.com/tobiaswuerth/sift), each serving MCP
|
|
82
|
+
over HTTP: `orbit serve <vault>` (port 8775) and `sift serve` (port 8765).
|
|
83
|
+
- For an Outlook.com or Hotmail mailbox: an app registered in Microsoft Entra (free, about five minutes; step 1
|
|
84
|
+
below), and IMAP turned on in Outlook's settings.
|
|
85
|
+
|
|
86
|
+
## Install
|
|
87
|
+
|
|
88
|
+
With [uv](https://docs.astral.sh/uv/):
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
uv tool install glean-importer # installs the `glean` command
|
|
92
|
+
glean --version
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Or with pip into a virtual environment: `pip install glean-importer`. The package is called `glean-importer`; the
|
|
96
|
+
command is `glean`. To update: `uv tool upgrade glean-importer`.
|
|
97
|
+
|
|
98
|
+
## Set up
|
|
99
|
+
|
|
100
|
+
### 1. Register glean with Microsoft (Outlook.com, Hotmail)
|
|
101
|
+
|
|
102
|
+
Outlook.com and Hotmail accept only OAuth2 for IMAP: Microsoft turned off passwords and app passwords for IMAP on
|
|
103
|
+
personal accounts in September 2024. A program that signs in with OAuth2 needs a client ID, which an app registered in
|
|
104
|
+
Microsoft Entra gives it. The app gives no one but you access: glean still signs in as you, in your browser.
|
|
105
|
+
|
|
106
|
+
1. Open the [Microsoft Entra admin center](https://entra.microsoft.com) and sign in with your Microsoft account. If it
|
|
107
|
+
says your account has no directory, create a free Azure account first, which makes one.
|
|
108
|
+
2. **Applications > App registrations > New registration.** Name: `glean`. Supported account types: *Personal
|
|
109
|
+
Microsoft accounts only*. Redirect URI: platform *Public client/native (mobile & desktop)*, `http://localhost`.
|
|
110
|
+
**Register.**
|
|
111
|
+
3. **API permissions > Add a permission > Microsoft Graph > Delegated permissions:** `IMAP.AccessAsUser.All` and
|
|
112
|
+
`offline_access`. **Add permissions.**
|
|
113
|
+
4. On the app's **Overview**, copy the **Application (client) ID**. It is not a secret: a public client has none.
|
|
114
|
+
5. In [Outlook on the web](https://outlook.live.com): **Settings > Mail > Forwarding and IMAP > Let devices and apps
|
|
115
|
+
use IMAP:** on.
|
|
116
|
+
|
|
117
|
+
For a folder of `.eml` files instead of a mailbox, skip this step and set `[mail] eml_folder` after step 3.
|
|
118
|
+
[docs/microsoft-app.md](https://github.com/tobiaswuerth/glean/blob/main/docs/microsoft-app.md) has the same steps
|
|
119
|
+
with troubleshooting.
|
|
120
|
+
|
|
121
|
+
### 2. Choose glean's home
|
|
122
|
+
|
|
123
|
+
The folder for glean's configuration (`glean.toml`), its state database, its sign-in tokens and, by default, the
|
|
124
|
+
archive of downloaded items. It is `~/.glean` unless `GLEAN_HOME` names another. Keep it out of synced folders: it
|
|
125
|
+
holds your mail. (In a clone of this repository, `data/` is ignored by git for this purpose.)
|
|
126
|
+
|
|
127
|
+
```powershell
|
|
128
|
+
setx GLEAN_HOME D:\glean-data # Windows, optional; open a new terminal afterwards
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
### 3. Run the setup
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
glean setup
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
It asks for your address, the client ID and your other addresses, writes `glean.toml`, and opens the browser to sign
|
|
138
|
+
you in (`glean mail login` does that again later; `--device-code` signs in without a browser). It also starts by
|
|
139
|
+
itself the first time you use glean. The tokens are kept in glean's home, encrypted with your Windows login (on
|
|
140
|
+
macOS and Linux, in a file only you may read).
|
|
141
|
+
|
|
142
|
+
### 4. Make the archive and the vault searchable in sift
|
|
143
|
+
|
|
144
|
+
Once: the agent indexes each email's files and the notes it changed, and sift only accepts files inside its
|
|
145
|
+
collections.
|
|
146
|
+
|
|
147
|
+
```bash
|
|
148
|
+
sift add <GLEAN_HOME>/mail --name mail --title-from name
|
|
149
|
+
sift add <orbit vault> --name crm --title-from name
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Other names work too: set them as `[mail] collection` and `[sift] crm_collection` in `glean.toml`.
|
|
153
|
+
|
|
154
|
+
### 5. Check everything
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
glean doctor
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
It checks Claude Code and its sign-in, that orbit and sift answer, that an agent run gets exactly the allowed tools,
|
|
161
|
+
and the mail sign-in. Every line should start with `ok`.
|
|
162
|
+
|
|
163
|
+
## Use
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
glean mail download # fetch what is new; the first time, everything
|
|
167
|
+
glean mail list --latest 20 # the 20 newest not yet imported, in the order run takes them
|
|
168
|
+
glean mail run --latest 20 # import them, oldest first, one agent run each
|
|
169
|
+
glean mail show <id> # one email: its state, files, and the agent's report
|
|
170
|
+
glean mail ignore <id> # never import it
|
|
171
|
+
glean mail reset <id> # import it again
|
|
172
|
+
glean status # counts per source and status, the agents' cost so far
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
`run` also takes ids, `--retry-failed` for emails whose run failed, `--include-skipped` for those the skip rules set
|
|
176
|
+
aside, and `--dry-run` to write the documents and show what the agent would get, without running it.
|
|
177
|
+
|
|
178
|
+
**Try it on a test vault first.** Point `[orbit] url` at a copy of your vault (`orbit serve D:\crm-test --port 8776`),
|
|
179
|
+
import a few emails, read the reports (`glean mail show <id>`) and the notes, and only then use the real vault. In a
|
|
180
|
+
clone of this repository, [`experiments/`](https://github.com/tobiaswuerth/glean/blob/main/experiments/README.md) imports a made-up mailbox into a new vault with
|
|
181
|
+
the real agent and prints each run's report.
|
|
182
|
+
|
|
183
|
+
## Configuration
|
|
184
|
+
|
|
185
|
+
`glean.toml` in glean's home; the setup writes it with every setting and a comment on each. The main ones:
|
|
186
|
+
|
|
187
|
+
| Setting | Default | |
|
|
188
|
+
|---|---|---|
|
|
189
|
+
| `[agent] model` | `sonnet` | the model of each run |
|
|
190
|
+
| `[agent] max_turns`, `timeout_minutes`, `max_budget_usd` | 60, 5, 1.0 | limits per item; a run that hits one fails, and is retried with `--retry-failed` |
|
|
191
|
+
| `[orbit] url`, `[sift] url` | ports 8775, 8765 | the MCP servers |
|
|
192
|
+
| `[sift] crm_collection`, `document_collections` | `crm`, all | where the vault is, and where to look for documents an item refers to |
|
|
193
|
+
| `[mail] address`, `client_id` | | the account, and the Entra app's client ID |
|
|
194
|
+
| `[mail] my_addresses` | | your other addresses: mail from them is outbound |
|
|
195
|
+
| `[mail] folders`, `exclude_folders` | all; Junk, Deleted, Drafts, Outbox, Notes | which folders to read |
|
|
196
|
+
| `[mail] archive`, `collection` | `mail` | where mail is kept, and its sift collection |
|
|
197
|
+
| `[mail] skip_bulk`, `skip_senders` | on; `noreply@*`, … | the pre-filter |
|
|
198
|
+
| `[mail] save_attachments` | `.pdf`, `.docx`, `.xlsx`, `.pptx`, … | attachments saved beside the email for sift |
|
|
199
|
+
| `[mail] eml_folder` | | read `.eml` files from this folder instead of the IMAP server |
|
|
200
|
+
|
|
201
|
+
## How it works
|
|
202
|
+
|
|
203
|
+
[docs/how-it-works.md](https://github.com/tobiaswuerth/glean/blob/main/docs/how-it-works.md) describes the pipeline, the archive and its Markdown documents, the state
|
|
204
|
+
database, the agent's sandbox and how it is checked, and how a source is added.
|
|
205
|
+
|
|
206
|
+
## Security and privacy
|
|
207
|
+
|
|
208
|
+
Items are untrusted input: an email can carry text written to steer an agent. glean limits what such text can do (no
|
|
209
|
+
dangerous tools, no destructive ones, every run's tools checked, uncertainty turned into a question for you in orbit's
|
|
210
|
+
review queue) and keeps everything local. See [SECURITY.md](https://github.com/tobiaswuerth/glean/blob/main/SECURITY.md).
|
|
211
|
+
|
|
212
|
+
## Contributing
|
|
213
|
+
|
|
214
|
+
Bug reports and ideas are welcome as [issues](https://github.com/tobiaswuerth/glean/issues);
|
|
215
|
+
[CONTRIBUTING.md](https://github.com/tobiaswuerth/glean/blob/main/CONTRIBUTING.md) describes the setup for development, the code layout and how a release is made.
|
|
216
|
+
The [changelog](https://github.com/tobiaswuerth/glean/blob/main/CHANGELOG.md) lists what changed.
|
|
217
|
+
|
|
218
|
+
## License
|
|
219
|
+
|
|
220
|
+
[MIT](https://github.com/tobiaswuerth/glean/blob/main/LICENSE)
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# glean: imports what your sources hold into orbit and sift
|
|
2
|
+
|
|
3
|
+
[](https://github.com/tobiaswuerth/glean/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/glean-importer/)
|
|
5
|
+
[](https://pypi.org/project/glean-importer/)
|
|
6
|
+
[](https://github.com/tobiaswuerth/glean/actions/workflows/ci.yml)
|
|
7
|
+
[](https://github.com/tobiaswuerth/glean/blob/main/LICENSE)
|
|
8
|
+
[](https://github.com/astral-sh/ruff)
|
|
9
|
+
[](https://github.com/pre-commit/pre-commit)
|
|
10
|
+
|
|
11
|
+
glean reads information from your sources and records each item in two local stores:
|
|
12
|
+
[orbit](https://github.com/tobiaswuerth/orbit), a personal CRM that holds the structured view (people, organizations,
|
|
13
|
+
interactions, tasks, topics), and [sift](https://github.com/tobiaswuerth/sift), a local search that holds the full
|
|
14
|
+
text. Email works today: from an IMAP mailbox such as Outlook.com or Hotmail, signed in with OAuth2, or from a folder
|
|
15
|
+
of saved `.eml` files.
|
|
16
|
+
|
|
17
|
+
For every item, glean writes a Markdown document, then starts a fresh [Claude Code](https://claude.com/claude-code)
|
|
18
|
+
agent run whose only tools are orbit's and sift's MCP tools. The agent decides who is who, logs the interaction,
|
|
19
|
+
updates people, organizations and tasks, and makes the item searchable. Neither orbit nor sift knows glean exists.
|
|
20
|
+
|
|
21
|
+
- **One run per item, oldest first, one at a time.** Each run starts with a fresh context, so it knows only what orbit
|
|
22
|
+
and sift hold; the people an earlier email created exist when a later one in the thread mentions them.
|
|
23
|
+
- **The source is never changed.** Mail folders are opened read-only and messages fetched without marking them read;
|
|
24
|
+
what glean processed is recorded only in its own state database, which can be inspected and reset.
|
|
25
|
+
- **A sandbox that is checked.** The agent has no shell, file or web tools, and none of orbit's destructive ones
|
|
26
|
+
(merge, rename). Every run's first event lists the tools it really has; a run with anything else is stopped before
|
|
27
|
+
it acts. Items are treated as data, never as instructions
|
|
28
|
+
([how](https://github.com/tobiaswuerth/glean/blob/main/docs/how-it-works.md#the-agent-run)).
|
|
29
|
+
- **Cheap where it can be.** Newsletters, notifications and automatic replies are skipped without an agent run.
|
|
30
|
+
- **Local.** The archive, the state database and the sign-in tokens stay on your machine; the tokens are encrypted
|
|
31
|
+
with your Windows login (on macOS and Linux, kept in a file only you may read).
|
|
32
|
+
|
|
33
|
+
## Sources
|
|
34
|
+
|
|
35
|
+
| Source | Commands | Reads |
|
|
36
|
+
|---|---|---|
|
|
37
|
+
| Email | `glean mail …` | An IMAP mailbox with OAuth2 (Outlook.com, Hotmail: every folder but junk, trash and drafts), or a folder of `.eml` files |
|
|
38
|
+
|
|
39
|
+
## Requirements
|
|
40
|
+
|
|
41
|
+
- Python 3.11 or newer.
|
|
42
|
+
- [Claude Code](https://claude.com/claude-code) 2.1.259 or newer, signed in (`claude auth status`). On Windows, the
|
|
43
|
+
native installer's `claude.exe`.
|
|
44
|
+
- [orbit](https://github.com/tobiaswuerth/orbit) and [sift](https://github.com/tobiaswuerth/sift), each serving MCP
|
|
45
|
+
over HTTP: `orbit serve <vault>` (port 8775) and `sift serve` (port 8765).
|
|
46
|
+
- For an Outlook.com or Hotmail mailbox: an app registered in Microsoft Entra (free, about five minutes; step 1
|
|
47
|
+
below), and IMAP turned on in Outlook's settings.
|
|
48
|
+
|
|
49
|
+
## Install
|
|
50
|
+
|
|
51
|
+
With [uv](https://docs.astral.sh/uv/):
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
uv tool install glean-importer # installs the `glean` command
|
|
55
|
+
glean --version
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Or with pip into a virtual environment: `pip install glean-importer`. The package is called `glean-importer`; the
|
|
59
|
+
command is `glean`. To update: `uv tool upgrade glean-importer`.
|
|
60
|
+
|
|
61
|
+
## Set up
|
|
62
|
+
|
|
63
|
+
### 1. Register glean with Microsoft (Outlook.com, Hotmail)
|
|
64
|
+
|
|
65
|
+
Outlook.com and Hotmail accept only OAuth2 for IMAP: Microsoft turned off passwords and app passwords for IMAP on
|
|
66
|
+
personal accounts in September 2024. A program that signs in with OAuth2 needs a client ID, which an app registered in
|
|
67
|
+
Microsoft Entra gives it. The app gives no one but you access: glean still signs in as you, in your browser.
|
|
68
|
+
|
|
69
|
+
1. Open the [Microsoft Entra admin center](https://entra.microsoft.com) and sign in with your Microsoft account. If it
|
|
70
|
+
says your account has no directory, create a free Azure account first, which makes one.
|
|
71
|
+
2. **Applications > App registrations > New registration.** Name: `glean`. Supported account types: *Personal
|
|
72
|
+
Microsoft accounts only*. Redirect URI: platform *Public client/native (mobile & desktop)*, `http://localhost`.
|
|
73
|
+
**Register.**
|
|
74
|
+
3. **API permissions > Add a permission > Microsoft Graph > Delegated permissions:** `IMAP.AccessAsUser.All` and
|
|
75
|
+
`offline_access`. **Add permissions.**
|
|
76
|
+
4. On the app's **Overview**, copy the **Application (client) ID**. It is not a secret: a public client has none.
|
|
77
|
+
5. In [Outlook on the web](https://outlook.live.com): **Settings > Mail > Forwarding and IMAP > Let devices and apps
|
|
78
|
+
use IMAP:** on.
|
|
79
|
+
|
|
80
|
+
For a folder of `.eml` files instead of a mailbox, skip this step and set `[mail] eml_folder` after step 3.
|
|
81
|
+
[docs/microsoft-app.md](https://github.com/tobiaswuerth/glean/blob/main/docs/microsoft-app.md) has the same steps
|
|
82
|
+
with troubleshooting.
|
|
83
|
+
|
|
84
|
+
### 2. Choose glean's home
|
|
85
|
+
|
|
86
|
+
The folder for glean's configuration (`glean.toml`), its state database, its sign-in tokens and, by default, the
|
|
87
|
+
archive of downloaded items. It is `~/.glean` unless `GLEAN_HOME` names another. Keep it out of synced folders: it
|
|
88
|
+
holds your mail. (In a clone of this repository, `data/` is ignored by git for this purpose.)
|
|
89
|
+
|
|
90
|
+
```powershell
|
|
91
|
+
setx GLEAN_HOME D:\glean-data # Windows, optional; open a new terminal afterwards
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### 3. Run the setup
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
glean setup
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
It asks for your address, the client ID and your other addresses, writes `glean.toml`, and opens the browser to sign
|
|
101
|
+
you in (`glean mail login` does that again later; `--device-code` signs in without a browser). It also starts by
|
|
102
|
+
itself the first time you use glean. The tokens are kept in glean's home, encrypted with your Windows login (on
|
|
103
|
+
macOS and Linux, in a file only you may read).
|
|
104
|
+
|
|
105
|
+
### 4. Make the archive and the vault searchable in sift
|
|
106
|
+
|
|
107
|
+
Once: the agent indexes each email's files and the notes it changed, and sift only accepts files inside its
|
|
108
|
+
collections.
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
sift add <GLEAN_HOME>/mail --name mail --title-from name
|
|
112
|
+
sift add <orbit vault> --name crm --title-from name
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Other names work too: set them as `[mail] collection` and `[sift] crm_collection` in `glean.toml`.
|
|
116
|
+
|
|
117
|
+
### 5. Check everything
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
glean doctor
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
It checks Claude Code and its sign-in, that orbit and sift answer, that an agent run gets exactly the allowed tools,
|
|
124
|
+
and the mail sign-in. Every line should start with `ok`.
|
|
125
|
+
|
|
126
|
+
## Use
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
glean mail download # fetch what is new; the first time, everything
|
|
130
|
+
glean mail list --latest 20 # the 20 newest not yet imported, in the order run takes them
|
|
131
|
+
glean mail run --latest 20 # import them, oldest first, one agent run each
|
|
132
|
+
glean mail show <id> # one email: its state, files, and the agent's report
|
|
133
|
+
glean mail ignore <id> # never import it
|
|
134
|
+
glean mail reset <id> # import it again
|
|
135
|
+
glean status # counts per source and status, the agents' cost so far
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
`run` also takes ids, `--retry-failed` for emails whose run failed, `--include-skipped` for those the skip rules set
|
|
139
|
+
aside, and `--dry-run` to write the documents and show what the agent would get, without running it.
|
|
140
|
+
|
|
141
|
+
**Try it on a test vault first.** Point `[orbit] url` at a copy of your vault (`orbit serve D:\crm-test --port 8776`),
|
|
142
|
+
import a few emails, read the reports (`glean mail show <id>`) and the notes, and only then use the real vault. In a
|
|
143
|
+
clone of this repository, [`experiments/`](https://github.com/tobiaswuerth/glean/blob/main/experiments/README.md) imports a made-up mailbox into a new vault with
|
|
144
|
+
the real agent and prints each run's report.
|
|
145
|
+
|
|
146
|
+
## Configuration
|
|
147
|
+
|
|
148
|
+
`glean.toml` in glean's home; the setup writes it with every setting and a comment on each. The main ones:
|
|
149
|
+
|
|
150
|
+
| Setting | Default | |
|
|
151
|
+
|---|---|---|
|
|
152
|
+
| `[agent] model` | `sonnet` | the model of each run |
|
|
153
|
+
| `[agent] max_turns`, `timeout_minutes`, `max_budget_usd` | 60, 5, 1.0 | limits per item; a run that hits one fails, and is retried with `--retry-failed` |
|
|
154
|
+
| `[orbit] url`, `[sift] url` | ports 8775, 8765 | the MCP servers |
|
|
155
|
+
| `[sift] crm_collection`, `document_collections` | `crm`, all | where the vault is, and where to look for documents an item refers to |
|
|
156
|
+
| `[mail] address`, `client_id` | | the account, and the Entra app's client ID |
|
|
157
|
+
| `[mail] my_addresses` | | your other addresses: mail from them is outbound |
|
|
158
|
+
| `[mail] folders`, `exclude_folders` | all; Junk, Deleted, Drafts, Outbox, Notes | which folders to read |
|
|
159
|
+
| `[mail] archive`, `collection` | `mail` | where mail is kept, and its sift collection |
|
|
160
|
+
| `[mail] skip_bulk`, `skip_senders` | on; `noreply@*`, … | the pre-filter |
|
|
161
|
+
| `[mail] save_attachments` | `.pdf`, `.docx`, `.xlsx`, `.pptx`, … | attachments saved beside the email for sift |
|
|
162
|
+
| `[mail] eml_folder` | | read `.eml` files from this folder instead of the IMAP server |
|
|
163
|
+
|
|
164
|
+
## How it works
|
|
165
|
+
|
|
166
|
+
[docs/how-it-works.md](https://github.com/tobiaswuerth/glean/blob/main/docs/how-it-works.md) describes the pipeline, the archive and its Markdown documents, the state
|
|
167
|
+
database, the agent's sandbox and how it is checked, and how a source is added.
|
|
168
|
+
|
|
169
|
+
## Security and privacy
|
|
170
|
+
|
|
171
|
+
Items are untrusted input: an email can carry text written to steer an agent. glean limits what such text can do (no
|
|
172
|
+
dangerous tools, no destructive ones, every run's tools checked, uncertainty turned into a question for you in orbit's
|
|
173
|
+
review queue) and keeps everything local. See [SECURITY.md](https://github.com/tobiaswuerth/glean/blob/main/SECURITY.md).
|
|
174
|
+
|
|
175
|
+
## Contributing
|
|
176
|
+
|
|
177
|
+
Bug reports and ideas are welcome as [issues](https://github.com/tobiaswuerth/glean/issues);
|
|
178
|
+
[CONTRIBUTING.md](https://github.com/tobiaswuerth/glean/blob/main/CONTRIBUTING.md) describes the setup for development, the code layout and how a release is made.
|
|
179
|
+
The [changelog](https://github.com/tobiaswuerth/glean/blob/main/CHANGELOG.md) lists what changed.
|
|
180
|
+
|
|
181
|
+
## License
|
|
182
|
+
|
|
183
|
+
[MIT](https://github.com/tobiaswuerth/glean/blob/main/LICENSE)
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["uv_build>=0.11.6,<0.14"]
|
|
3
|
+
build-backend = "uv_build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "glean-importer"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Imports what your sources hold into orbit (a personal CRM) and sift (local RAG search), one sandboxed Claude Code agent run per item. Email works today: IMAP with OAuth2 (Outlook.com, Hotmail) or .eml files."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.11"
|
|
13
|
+
dependencies = [
|
|
14
|
+
"msal>=1.31",
|
|
15
|
+
"msal-extensions>=1.2",
|
|
16
|
+
"inscriptis>=2.5",
|
|
17
|
+
"filelock>=3.16",
|
|
18
|
+
]
|
|
19
|
+
keywords = [
|
|
20
|
+
"email",
|
|
21
|
+
"imap",
|
|
22
|
+
"oauth2",
|
|
23
|
+
"outlook",
|
|
24
|
+
"hotmail",
|
|
25
|
+
"crm",
|
|
26
|
+
"personal-crm",
|
|
27
|
+
"rag",
|
|
28
|
+
"mcp",
|
|
29
|
+
"claude-code",
|
|
30
|
+
"ai-agents",
|
|
31
|
+
"local-first",
|
|
32
|
+
]
|
|
33
|
+
classifiers = [
|
|
34
|
+
"Development Status :: 3 - Alpha",
|
|
35
|
+
"Environment :: Console",
|
|
36
|
+
"Intended Audience :: Developers",
|
|
37
|
+
"Intended Audience :: End Users/Desktop",
|
|
38
|
+
"Operating System :: MacOS",
|
|
39
|
+
"Operating System :: Microsoft :: Windows",
|
|
40
|
+
"Operating System :: POSIX :: Linux",
|
|
41
|
+
"Programming Language :: Python :: 3",
|
|
42
|
+
"Programming Language :: Python :: 3.11",
|
|
43
|
+
"Programming Language :: Python :: 3.12",
|
|
44
|
+
"Programming Language :: Python :: 3.13",
|
|
45
|
+
"Programming Language :: Python :: 3.14",
|
|
46
|
+
"Programming Language :: Python :: 3.15",
|
|
47
|
+
"Natural Language :: English",
|
|
48
|
+
"Topic :: Communications :: Email",
|
|
49
|
+
"Topic :: Office/Business",
|
|
50
|
+
"Topic :: Text Processing :: Markup :: Markdown",
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
[[project.authors]]
|
|
54
|
+
name = "Tobias Würth"
|
|
55
|
+
|
|
56
|
+
[project.urls]
|
|
57
|
+
Homepage = "https://github.com/tobiaswuerth/glean"
|
|
58
|
+
Documentation = "https://github.com/tobiaswuerth/glean#readme"
|
|
59
|
+
Source = "https://github.com/tobiaswuerth/glean"
|
|
60
|
+
Issues = "https://github.com/tobiaswuerth/glean/issues"
|
|
61
|
+
Changelog = "https://github.com/tobiaswuerth/glean/blob/main/CHANGELOG.md"
|
|
62
|
+
|
|
63
|
+
[project.scripts]
|
|
64
|
+
glean = "glean.cli:main"
|
|
65
|
+
|
|
66
|
+
[dependency-groups]
|
|
67
|
+
test = [
|
|
68
|
+
"pytest>=8",
|
|
69
|
+
"pytest-cov>=5",
|
|
70
|
+
"pytest-timeout>=2.3",
|
|
71
|
+
]
|
|
72
|
+
dev = [
|
|
73
|
+
{ include-group = "test" },
|
|
74
|
+
"ruff>=0.15",
|
|
75
|
+
"pre-commit>=4",
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
[tool.uv.build-backend]
|
|
79
|
+
module-name = "glean"
|
|
80
|
+
|
|
81
|
+
[tool.ruff]
|
|
82
|
+
line-length = 120
|
|
83
|
+
target-version = "py311"
|
|
84
|
+
|
|
85
|
+
[tool.ruff.lint]
|
|
86
|
+
select = [
|
|
87
|
+
"E",
|
|
88
|
+
"F",
|
|
89
|
+
"W",
|
|
90
|
+
"I",
|
|
91
|
+
"B",
|
|
92
|
+
"UP",
|
|
93
|
+
"SIM",
|
|
94
|
+
"C4",
|
|
95
|
+
"PIE",
|
|
96
|
+
"PT",
|
|
97
|
+
"RUF",
|
|
98
|
+
"PERF",
|
|
99
|
+
]
|
|
100
|
+
ignore = [
|
|
101
|
+
"B905",
|
|
102
|
+
"SIM905",
|
|
103
|
+
]
|
|
104
|
+
|
|
105
|
+
[tool.ruff.lint.flake8-pytest-style]
|
|
106
|
+
parametrize-names-type = "csv"
|
|
107
|
+
|
|
108
|
+
[tool.pytest.ini_options]
|
|
109
|
+
testpaths = ["tests"]
|
|
110
|
+
addopts = ["--strict-markers"]
|
|
111
|
+
filterwarnings = ["error"]
|
|
112
|
+
timeout = 60
|
|
113
|
+
|
|
114
|
+
[tool.coverage.run]
|
|
115
|
+
source = ["glean"]
|
|
116
|
+
branch = true
|
|
117
|
+
|
|
118
|
+
[tool.coverage.report]
|
|
119
|
+
fail_under = 100
|
|
120
|
+
show_missing = true
|
|
121
|
+
skip_covered = true
|
|
122
|
+
|
|
123
|
+
[tool.codespell]
|
|
124
|
+
ignore-words-list = "ue,foto,passt,ist,sie,ende,oktober,technik,unser"
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["uv_build>=0.11.6,<0.14"]
|
|
3
|
+
build-backend = "uv_build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "glean-importer"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Imports what your sources hold into orbit (a personal CRM) and sift (local RAG search), one sandboxed Claude Code agent run per item. Email works today: IMAP with OAuth2 (Outlook.com, Hotmail) or .eml files."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Tobias Würth" }]
|
|
13
|
+
requires-python = ">=3.11"
|
|
14
|
+
dependencies = [
|
|
15
|
+
"msal>=1.31", # the OAuth2 sign-in to Microsoft, and refreshing its tokens
|
|
16
|
+
"msal-extensions>=1.2", # the token cache, encrypted with the user's Windows login (DPAPI)
|
|
17
|
+
"inscriptis>=2.5", # HTML mail to text, tables and lists kept readable (Apache-2.0)
|
|
18
|
+
"filelock>=3.16", # one glean run at a time per home
|
|
19
|
+
]
|
|
20
|
+
keywords = [
|
|
21
|
+
"email",
|
|
22
|
+
"imap",
|
|
23
|
+
"oauth2",
|
|
24
|
+
"outlook",
|
|
25
|
+
"hotmail",
|
|
26
|
+
"crm",
|
|
27
|
+
"personal-crm",
|
|
28
|
+
"rag",
|
|
29
|
+
"mcp",
|
|
30
|
+
"claude-code",
|
|
31
|
+
"ai-agents",
|
|
32
|
+
"local-first",
|
|
33
|
+
]
|
|
34
|
+
classifiers = [
|
|
35
|
+
"Development Status :: 3 - Alpha",
|
|
36
|
+
"Environment :: Console",
|
|
37
|
+
"Intended Audience :: Developers",
|
|
38
|
+
"Intended Audience :: End Users/Desktop",
|
|
39
|
+
"Operating System :: MacOS",
|
|
40
|
+
"Operating System :: Microsoft :: Windows",
|
|
41
|
+
"Operating System :: POSIX :: Linux",
|
|
42
|
+
"Programming Language :: Python :: 3",
|
|
43
|
+
"Programming Language :: Python :: 3.11",
|
|
44
|
+
"Programming Language :: Python :: 3.12",
|
|
45
|
+
"Programming Language :: Python :: 3.13",
|
|
46
|
+
"Programming Language :: Python :: 3.14",
|
|
47
|
+
"Programming Language :: Python :: 3.15",
|
|
48
|
+
"Natural Language :: English",
|
|
49
|
+
"Topic :: Communications :: Email",
|
|
50
|
+
"Topic :: Office/Business",
|
|
51
|
+
"Topic :: Text Processing :: Markup :: Markdown",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
[project.urls]
|
|
55
|
+
Homepage = "https://github.com/tobiaswuerth/glean"
|
|
56
|
+
Documentation = "https://github.com/tobiaswuerth/glean#readme"
|
|
57
|
+
Source = "https://github.com/tobiaswuerth/glean"
|
|
58
|
+
Issues = "https://github.com/tobiaswuerth/glean/issues"
|
|
59
|
+
Changelog = "https://github.com/tobiaswuerth/glean/blob/main/CHANGELOG.md"
|
|
60
|
+
|
|
61
|
+
[project.scripts]
|
|
62
|
+
glean = "glean.cli:main"
|
|
63
|
+
|
|
64
|
+
[dependency-groups]
|
|
65
|
+
test = [
|
|
66
|
+
"pytest>=8",
|
|
67
|
+
"pytest-cov>=5",
|
|
68
|
+
"pytest-timeout>=2.3",
|
|
69
|
+
]
|
|
70
|
+
dev = [
|
|
71
|
+
{ include-group = "test" },
|
|
72
|
+
"ruff>=0.15",
|
|
73
|
+
"pre-commit>=4",
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
[tool.uv.build-backend]
|
|
77
|
+
module-name = "glean"
|
|
78
|
+
|
|
79
|
+
[tool.ruff]
|
|
80
|
+
line-length = 120
|
|
81
|
+
target-version = "py311"
|
|
82
|
+
|
|
83
|
+
[tool.ruff.lint]
|
|
84
|
+
select = ["E", "F", "W", "I", "B", "UP", "SIM", "C4", "PIE", "PT", "RUF", "PERF"]
|
|
85
|
+
ignore = [
|
|
86
|
+
"B905", # zip() without strict=
|
|
87
|
+
"SIM905", # a word list as one split string reads better than a long list literal
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
[tool.ruff.lint.flake8-pytest-style]
|
|
91
|
+
parametrize-names-type = "csv" # @pytest.mark.parametrize("a, b", ...)
|
|
92
|
+
|
|
93
|
+
[tool.pytest.ini_options]
|
|
94
|
+
testpaths = ["tests"]
|
|
95
|
+
addopts = ["--strict-markers"]
|
|
96
|
+
filterwarnings = ["error"] # a warning today is a failure tomorrow
|
|
97
|
+
timeout = 60 # a test that hangs fails instead
|
|
98
|
+
|
|
99
|
+
[tool.coverage.run]
|
|
100
|
+
source = ["glean"]
|
|
101
|
+
branch = true
|
|
102
|
+
|
|
103
|
+
[tool.coverage.report]
|
|
104
|
+
fail_under = 100
|
|
105
|
+
show_missing = true
|
|
106
|
+
skip_covered = true
|
|
107
|
+
|
|
108
|
+
[tool.codespell]
|
|
109
|
+
ignore-words-list = "ue,foto,passt,ist,sie,ende,oktober,technik,unser" # ü written as ue, and German in made-up mail
|