esbi-cli 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- esbi_cli-0.2.1/LICENSE +21 -0
- esbi_cli-0.2.1/PKG-INFO +167 -0
- esbi_cli-0.2.1/README.md +133 -0
- esbi_cli-0.2.1/pyproject.toml +86 -0
- esbi_cli-0.2.1/pyproject.toml.orig +65 -0
- esbi_cli-0.2.1/src/esbi_cli/__init__.py +8 -0
- esbi_cli-0.2.1/src/esbi_cli/ask/__init__.py +0 -0
- esbi_cli-0.2.1/src/esbi_cli/ask/answer.py +256 -0
- esbi_cli-0.2.1/src/esbi_cli/bench/__init__.py +0 -0
- esbi_cli-0.2.1/src/esbi_cli/bench/cases.py +57 -0
- esbi_cli-0.2.1/src/esbi_cli/bench/metrics.py +23 -0
- esbi_cli-0.2.1/src/esbi_cli/bench/report.py +117 -0
- esbi_cli-0.2.1/src/esbi_cli/bench/runner.py +114 -0
- esbi_cli-0.2.1/src/esbi_cli/capture/__init__.py +0 -0
- esbi_cli-0.2.1/src/esbi_cli/capture/inbox.py +63 -0
- esbi_cli-0.2.1/src/esbi_cli/capture/legacy.py +49 -0
- esbi_cli-0.2.1/src/esbi_cli/cli.py +1387 -0
- esbi_cli-0.2.1/src/esbi_cli/config.py +344 -0
- esbi_cli-0.2.1/src/esbi_cli/doctor.py +391 -0
- esbi_cli-0.2.1/src/esbi_cli/evaluate.py +91 -0
- esbi_cli-0.2.1/src/esbi_cli/export.py +137 -0
- esbi_cli-0.2.1/src/esbi_cli/extract/__init__.py +107 -0
- esbi_cli-0.2.1/src/esbi_cli/extract/clip.py +30 -0
- esbi_cli-0.2.1/src/esbi_cli/extract/html.py +60 -0
- esbi_cli-0.2.1/src/esbi_cli/extract/image.py +58 -0
- esbi_cli-0.2.1/src/esbi_cli/extract/pdf.py +109 -0
- esbi_cli-0.2.1/src/esbi_cli/gitops.py +101 -0
- esbi_cli-0.2.1/src/esbi_cli/index.py +303 -0
- esbi_cli-0.2.1/src/esbi_cli/ingest/__init__.py +0 -0
- esbi_cli-0.2.1/src/esbi_cli/ingest/apply.py +480 -0
- esbi_cli-0.2.1/src/esbi_cli/ingest/chunks.py +49 -0
- esbi_cli-0.2.1/src/esbi_cli/ingest/connect.py +87 -0
- esbi_cli-0.2.1/src/esbi_cli/ingest/digest.py +91 -0
- esbi_cli-0.2.1/src/esbi_cli/ingest/pipeline.py +176 -0
- esbi_cli-0.2.1/src/esbi_cli/ingest/plan.py +231 -0
- esbi_cli-0.2.1/src/esbi_cli/ingest/read.py +105 -0
- esbi_cli-0.2.1/src/esbi_cli/ingest/retrieve.py +59 -0
- esbi_cli-0.2.1/src/esbi_cli/init.py +176 -0
- esbi_cli-0.2.1/src/esbi_cli/interrupts.py +90 -0
- esbi_cli-0.2.1/src/esbi_cli/lang.py +341 -0
- esbi_cli-0.2.1/src/esbi_cli/links.py +10 -0
- esbi_cli-0.2.1/src/esbi_cli/lint/__init__.py +0 -0
- esbi_cli-0.2.1/src/esbi_cli/lint/checks.py +178 -0
- esbi_cli-0.2.1/src/esbi_cli/lint/report.py +60 -0
- esbi_cli-0.2.1/src/esbi_cli/llm/__init__.py +0 -0
- esbi_cli-0.2.1/src/esbi_cli/llm/adapter.py +393 -0
- esbi_cli-0.2.1/src/esbi_cli/llm/schemas.py +146 -0
- esbi_cli-0.2.1/src/esbi_cli/mail/__init__.py +0 -0
- esbi_cli-0.2.1/src/esbi_cli/mail/convert.py +194 -0
- esbi_cli-0.2.1/src/esbi_cli/mail/credentials.py +65 -0
- esbi_cli-0.2.1/src/esbi_cli/mail/fetch.py +154 -0
- esbi_cli-0.2.1/src/esbi_cli/mail/imap.py +92 -0
- esbi_cli-0.2.1/src/esbi_cli/netguard.py +127 -0
- esbi_cli-0.2.1/src/esbi_cli/privacy.py +81 -0
- esbi_cli-0.2.1/src/esbi_cli/queue.py +179 -0
- esbi_cli-0.2.1/src/esbi_cli/reingest.py +165 -0
- esbi_cli-0.2.1/src/esbi_cli/report/__init__.py +0 -0
- esbi_cli-0.2.1/src/esbi_cli/report/daily_index.py +235 -0
- esbi_cli-0.2.1/src/esbi_cli/report/index_md.py +21 -0
- esbi_cli-0.2.1/src/esbi_cli/report/readstate.py +26 -0
- esbi_cli-0.2.1/src/esbi_cli/run.py +100 -0
- esbi_cli-0.2.1/src/esbi_cli/runlock.py +31 -0
- esbi_cli-0.2.1/src/esbi_cli/runlog.py +80 -0
- esbi_cli-0.2.1/src/esbi_cli/schedule.py +106 -0
- esbi_cli-0.2.1/src/esbi_cli/templates/SCHEMA.md +52 -0
- esbi_cli-0.2.1/src/esbi_cli/templates/clipper-template.json +17 -0
- esbi_cli-0.2.1/src/esbi_cli/templates/clipper-youtube-template.json +18 -0
- esbi_cli-0.2.1/src/esbi_cli/templates/config.example.toml +108 -0
- esbi_cli-0.2.1/src/esbi_cli/update.py +247 -0
- esbi_cli-0.2.1/src/esbi_cli/vault.py +188 -0
- esbi_cli-0.2.1/src/esbi_cli/wizards/clipper.sh +271 -0
- esbi_cli-0.2.1/src/esbi_cli/wizards/email.sh +265 -0
esbi_cli-0.2.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Rubén Melo
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
esbi_cli-0.2.1/PKG-INFO
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: esbi-cli
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: Turns the PDFs, links and emails you save into a Markdown wiki of summaries, glossaries and connections, written by a local or cloud language model.
|
|
5
|
+
Keywords: notes,wiki,llm,obsidian,markdown,pdf,summarization,rag,esbi-cli
|
|
6
|
+
Author: Rubén Melo
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
12
|
+
Classifier: Operating System :: MacOS
|
|
13
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
18
|
+
Requires-Dist: httpx>=0.28.1
|
|
19
|
+
Requires-Dist: keyring>=25.7.0
|
|
20
|
+
Requires-Dist: markdown-it-py>=3
|
|
21
|
+
Requires-Dist: pillow>=11
|
|
22
|
+
Requires-Dist: pydantic>=2.7
|
|
23
|
+
Requires-Dist: pymupdf4llm>=1.28.2
|
|
24
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
25
|
+
Requires-Dist: trafilatura>=2.2.0
|
|
26
|
+
Requires-Dist: typer>=0.12
|
|
27
|
+
Requires-Python: >=3.12
|
|
28
|
+
Project-URL: Homepage, https://github.com/RubenAmaury/esbi-cli
|
|
29
|
+
Project-URL: Documentation, https://github.com/RubenAmaury/esbi-cli/tree/main/docs
|
|
30
|
+
Project-URL: Repository, https://github.com/RubenAmaury/esbi-cli
|
|
31
|
+
Project-URL: Issues, https://github.com/RubenAmaury/esbi-cli/issues
|
|
32
|
+
Project-URL: Changelog, https://github.com/RubenAmaury/esbi-cli/blob/main/CHANGELOG.md
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
# esbi-cli
|
|
36
|
+
|
|
37
|
+
[](https://github.com/RubenAmaury/esbi-cli/actions/workflows/ci.yml)
|
|
38
|
+
|
|
39
|
+
**Save PDFs and links. Get a wiki of notes you can read instead of the originals.**
|
|
40
|
+
|
|
41
|
+
esbi-cli is a command-line worker (`sb`) that reads what you save (PDFs, web pages, email, clips) and writes a Markdown wiki: an executive summary, the key ideas, a glossary, verbatim quotes, a concept diagram, the figures, and links to the notes you already have. Every morning it builds an index note to start from. It works with local models (Ollama, LM Studio), with your Claude subscription, or with an API key. It follows Karpathy's [LLM Wiki](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) pattern, and the model never touches your files: it only returns JSON that the program validates and then writes.
|
|
42
|
+
|
|
43
|
+
## What a note looks like
|
|
44
|
+
|
|
45
|
+
You save a link or a paper. A few minutes later there is a note like this (abridged, from a Wikipedia article, written by `llama3.2` on a laptop; the notes are in English unless you set `[notes].language = "es"`):
|
|
46
|
+
|
|
47
|
+
```markdown
|
|
48
|
+
## Executive summary
|
|
49
|
+
The Zettelkasten is a method of note-taking and knowledge management developed by Niklas Luhmann,
|
|
50
|
+
which involves creating a network of interconnected notes and ideas. It emphasizes the importance of
|
|
51
|
+
linking and connecting ideas, rather than simply collecting or storing them.
|
|
52
|
+
|
|
53
|
+
## Key ideas
|
|
54
|
+
- **The Zettelkasten system emphasizes the use of index cards to organize and connect ideas.** This
|
|
55
|
+
approach allows users to create a network of interconnected notes, rather than a linear sequence of thoughts.
|
|
56
|
+
|
|
57
|
+
## Key terms
|
|
58
|
+
- **Zettelkasten**: A system of note-taking and personal knowledge management that consists of small
|
|
59
|
+
items of information stored on paper slips or cards.
|
|
60
|
+
|
|
61
|
+
## Key quotes
|
|
62
|
+
> "Every one admits nowadays that it is advisable to collect materials on separate cards or slips of paper."
|
|
63
|
+
|
|
64
|
+
## Concepts
|
|
65
|
+
- [[Index card file]]
|
|
66
|
+
|
|
67
|
+
## Entities
|
|
68
|
+
- [[Niklas Luhmann]]
|
|
69
|
+
- [[Roland Barthes]]
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Quotes are checked against the source word for word, glossary terms must appear in it, diagrams are drawn by code, and PDF figures are copied next to the note. Details: [How a note is made](https://rubenamaury.github.io/esbi-cli/docs/explanation/how-a-note-is-made/).
|
|
73
|
+
|
|
74
|
+
## Who it is for
|
|
75
|
+
|
|
76
|
+
People who save more than they read: papers, articles, newsletters, YouTube talks. You decide what to read from a good summary, and you can ask your own wiki questions (`sb ask`) and get answers that cite your notes. It runs on your machine, on macOS first (Linux works by running the same command from cron; it is less tested).
|
|
77
|
+
|
|
78
|
+
## Install
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
brew install rubenamaury/esbi-cli/esbi-cli # macOS (Homebrew)
|
|
82
|
+
# or, with uv:
|
|
83
|
+
uv tool install git+https://github.com/RubenAmaury/esbi-cli
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Then set it up. `sb init` is an installer: it asks what you use and every question is optional.
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
sb init # vault folder, Obsidian or not, local model / Claude subscription / API, nightly job, backup
|
|
90
|
+
sb doctor # checks the whole setup and says what to fix
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
A walk-through from install to a first note is in [Getting started](https://rubenamaury.github.io/esbi-cli/docs/tutorials/getting-started/); other ways to install (a wheel, a source checkout, Linux) are in [Install](https://rubenamaury.github.io/esbi-cli/docs/how-to/install/).
|
|
94
|
+
|
|
95
|
+
## First five minutes
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
sb add https://example.com/a-good-article # a link
|
|
99
|
+
cp ~/Downloads/paper.pdf ~/Documents/Obsidian/esbi/inbox/ # or drop a PDF in inbox/
|
|
100
|
+
sb run # reads, writes the notes (a few minutes per source)
|
|
101
|
+
sb today # opens today's index note
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
In the index note, tick `- [x]` next to what you have read and run `sb index`. Ask a question with `sb ask "what is an agent harness?"`. [Getting started](https://rubenamaury.github.io/esbi-cli/docs/tutorials/getting-started/) shows every step with its output, and [A first week](https://rubenamaury.github.io/esbi-cli/docs/tutorials/first-week/) teaches the habit.
|
|
105
|
+
|
|
106
|
+
## What is optional
|
|
107
|
+
|
|
108
|
+
The basics are PDFs and links. Everything else you can skip, or add later:
|
|
109
|
+
|
|
110
|
+
| Optional | What it adds | How |
|
|
111
|
+
|---|---|---|
|
|
112
|
+
| Obsidian | A nice viewer. Any Markdown editor works, or `sb export` for a website | `sb init` asks; [Use it without Obsidian](https://rubenamaury.github.io/esbi-cli/docs/how-to/without-obsidian/) |
|
|
113
|
+
| The nightly job | Processes your queue by itself at a time you choose | `sb init --nightly 03:00`; [Run it every night](https://rubenamaury.github.io/esbi-cli/docs/how-to/nightly-job/) |
|
|
114
|
+
| Email capture (Gmail) | Forward an email or PDF to yourself and get a note | `sb setup email`; [Capture email](https://rubenamaury.github.io/esbi-cli/docs/how-to/email/) |
|
|
115
|
+
| Web Clipper (needs Obsidian) | Save pages, LinkedIn or Reddit posts, YouTube transcripts from your browser | `sb setup clipper`; [Web Clipper](https://rubenamaury.github.io/esbi-cli/docs/how-to/web-clipper/) |
|
|
116
|
+
| Claude subscription or API | Better notes than a small local model | `sb init --model subscription` or `api`; [Models](https://rubenamaury.github.io/esbi-cli/docs/how-to/models/) |
|
|
117
|
+
| Version history and backup (git) | If `git` is installed the vault is a repository: every ingested source is a commit you can review or undo. A backup is any git remote (GitHub, GitLab, a folder on a NAS), never required | `sb init --remote URL`; [Back up your vault](https://rubenamaury.github.io/esbi-cli/docs/how-to/backup/) |
|
|
118
|
+
|
|
119
|
+
## Privacy: what leaves your machine
|
|
120
|
+
|
|
121
|
+
| You choose | What is sent, and to whom |
|
|
122
|
+
|---|---|
|
|
123
|
+
| A local model (Ollama, LM Studio) | Nothing |
|
|
124
|
+
| Your Claude subscription (`claude-cli`) | The text of each source goes to Anthropic, through the official `claude` tool |
|
|
125
|
+
| An API key (OpenAI-compatible, Anthropic) | The text of each source goes to that provider |
|
|
126
|
+
|
|
127
|
+
With either cloud choice, **email is never sent**: it is read only by a local model (`[llm.private]`), and the cloud model is never shown email pages. The mail password is stored in the macOS Keychain, never in a file; the app only reads mail, it never deletes it; URLs are fetched only on public addresses. Details: [Safety and privacy](https://rubenamaury.github.io/esbi-cli/docs/explanation/safety-and-privacy/).
|
|
128
|
+
|
|
129
|
+
## How it works
|
|
130
|
+
|
|
131
|
+
```
|
|
132
|
+
you save the worker (sb run) you read
|
|
133
|
+
──────── ─────────────────── ────────
|
|
134
|
+
sb add / inbox/ PDFs ┐
|
|
135
|
+
Web Clipper clips ├─► queue ─► extract ─► read in chunks ─► model writes a ─► wiki/ notes, index.md,
|
|
136
|
+
forwarded email ┘ (SQLite) (web, PDF) (notes per chunk) JSON edit plan log.md, wiki/daily/ ◄── start here
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
`raw/` keeps the originals and is never edited; `wiki/` holds the notes the worker writes and you edit freely; `SCHEMA.md` holds the conventions the model is given. The vault is its own git repository: every ingested source is a commit, so any change can be reviewed or reverted. More: [How it works](https://rubenamaury.github.io/esbi-cli/docs/explanation/how-it-works/), [Internals](https://rubenamaury.github.io/esbi-cli/docs/explanation/internals/).
|
|
140
|
+
|
|
141
|
+
## Documentation
|
|
142
|
+
|
|
143
|
+
The same pages are published as a website: <https://rubenamaury.github.io/esbi-cli/> (documentation under `/docs/`).
|
|
144
|
+
|
|
145
|
+
| | |
|
|
146
|
+
|---|---|
|
|
147
|
+
| **Start** | [Documentation home](https://rubenamaury.github.io/esbi-cli/docs/) · [How do I...?](https://rubenamaury.github.io/esbi-cli/docs/how-to/) |
|
|
148
|
+
| **Learn** | [Getting started](https://rubenamaury.github.io/esbi-cli/docs/tutorials/getting-started/) · [A first week](https://rubenamaury.github.io/esbi-cli/docs/tutorials/first-week/) |
|
|
149
|
+
| **Do** | [Install](https://rubenamaury.github.io/esbi-cli/docs/how-to/install/) · [Models](https://rubenamaury.github.io/esbi-cli/docs/how-to/models/) · [Get sources in](https://rubenamaury.github.io/esbi-cli/docs/how-to/getting-sources-in/) · [Manage the queue](https://rubenamaury.github.io/esbi-cli/docs/how-to/manage-the-queue/) · [Read your notes](https://rubenamaury.github.io/esbi-cli/docs/how-to/read-your-notes/) · [Ask your wiki](https://rubenamaury.github.io/esbi-cli/docs/how-to/ask-your-wiki/) · [Check the wiki's health](https://rubenamaury.github.io/esbi-cli/docs/how-to/check-wiki-health/) · [Export a website](https://rubenamaury.github.io/esbi-cli/docs/how-to/export-a-website/) · [Email](https://rubenamaury.github.io/esbi-cli/docs/how-to/email/) · [Web Clipper](https://rubenamaury.github.io/esbi-cli/docs/how-to/web-clipper/) · [Nightly job](https://rubenamaury.github.io/esbi-cli/docs/how-to/nightly-job/) · [Back up and restore](https://rubenamaury.github.io/esbi-cli/docs/how-to/backup/) · [Move to a new Mac](https://rubenamaury.github.io/esbi-cli/docs/how-to/move-to-a-new-mac/) · [Update and uninstall](https://rubenamaury.github.io/esbi-cli/docs/how-to/update-and-uninstall/) · [Without Obsidian](https://rubenamaury.github.io/esbi-cli/docs/how-to/without-obsidian/) · [Offline](https://rubenamaury.github.io/esbi-cli/docs/how-to/use-offline/) · [Troubleshooting](https://rubenamaury.github.io/esbi-cli/docs/how-to/troubleshooting/) |
|
|
150
|
+
| **Look up** | [Commands](https://rubenamaury.github.io/esbi-cli/docs/reference/cli/) · [Configuration](https://rubenamaury.github.io/esbi-cli/docs/reference/configuration/) · [Vault layout](https://rubenamaury.github.io/esbi-cli/docs/reference/vault-layout/) · [The daily index](https://rubenamaury.github.io/esbi-cli/docs/reference/daily-index/) |
|
|
151
|
+
| **Understand** | [Concepts and glossary](https://rubenamaury.github.io/esbi-cli/docs/explanation/concepts/) · [FAQ](https://rubenamaury.github.io/esbi-cli/docs/explanation/faq/) · [How it works](https://rubenamaury.github.io/esbi-cli/docs/explanation/how-it-works/) · [How a note is made](https://rubenamaury.github.io/esbi-cli/docs/explanation/how-a-note-is-made/) · [Safety and privacy](https://rubenamaury.github.io/esbi-cli/docs/explanation/safety-and-privacy/) · [Why it is built this way](https://rubenamaury.github.io/esbi-cli/docs/explanation/internals/) · [Architecture](https://rubenamaury.github.io/esbi-cli/docs/explanation/architecture/) · [Retrieval and RAG](https://rubenamaury.github.io/esbi-cli/docs/explanation/rag-fit/) · [Storage](https://rubenamaury.github.io/esbi-cli/docs/explanation/storage-fit/) |
|
|
152
|
+
|
|
153
|
+
## Requirements
|
|
154
|
+
|
|
155
|
+
Python 3.12 or later (installed for you by `brew` or `uv`), `git` (optional: version history and backup), and one way to run a model: [Ollama](https://ollama.com) or [LM Studio](https://lmstudio.ai) locally, the [Claude Code](https://claude.com/code) `claude` tool signed in to a subscription, or an API key. `sb doctor` checks all of it.
|
|
156
|
+
|
|
157
|
+
## Status
|
|
158
|
+
|
|
159
|
+
Version 0.x: it works and is used daily, but commands and the config format can still change between minor versions; see the [changelog](CHANGELOG.md). Quality depends on the model: a small local model writes useful but unpolished notes; a stronger model writes noticeably better ones. Known limits are in [Known limits](https://rubenamaury.github.io/esbi-cli/docs/explanation/internals/#known-limits); open work is in the issue tracker.
|
|
160
|
+
|
|
161
|
+
## Contributing and security
|
|
162
|
+
|
|
163
|
+
Bug reports and ideas are welcome: [CONTRIBUTING.md](CONTRIBUTING.md). Please report vulnerabilities privately: [SECURITY.md](.github/SECURITY.md).
|
|
164
|
+
|
|
165
|
+
## License
|
|
166
|
+
|
|
167
|
+
[MIT](LICENSE). One dependency, [PyMuPDF](https://pymupdf.readthedocs.io) (reads PDFs), is licensed AGPL-3.0 and is installed from PyPI, not bundled here; check its terms if you redistribute a build that includes it.
|
esbi_cli-0.2.1/README.md
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# esbi-cli
|
|
2
|
+
|
|
3
|
+
[](https://github.com/RubenAmaury/esbi-cli/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
**Save PDFs and links. Get a wiki of notes you can read instead of the originals.**
|
|
6
|
+
|
|
7
|
+
esbi-cli is a command-line worker (`sb`) that reads what you save (PDFs, web pages, email, clips) and writes a Markdown wiki: an executive summary, the key ideas, a glossary, verbatim quotes, a concept diagram, the figures, and links to the notes you already have. Every morning it builds an index note to start from. It works with local models (Ollama, LM Studio), with your Claude subscription, or with an API key. It follows Karpathy's [LLM Wiki](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) pattern, and the model never touches your files: it only returns JSON that the program validates and then writes.
|
|
8
|
+
|
|
9
|
+
## What a note looks like
|
|
10
|
+
|
|
11
|
+
You save a link or a paper. A few minutes later there is a note like this (abridged, from a Wikipedia article, written by `llama3.2` on a laptop; the notes are in English unless you set `[notes].language = "es"`):
|
|
12
|
+
|
|
13
|
+
```markdown
|
|
14
|
+
## Executive summary
|
|
15
|
+
The Zettelkasten is a method of note-taking and knowledge management developed by Niklas Luhmann,
|
|
16
|
+
which involves creating a network of interconnected notes and ideas. It emphasizes the importance of
|
|
17
|
+
linking and connecting ideas, rather than simply collecting or storing them.
|
|
18
|
+
|
|
19
|
+
## Key ideas
|
|
20
|
+
- **The Zettelkasten system emphasizes the use of index cards to organize and connect ideas.** This
|
|
21
|
+
approach allows users to create a network of interconnected notes, rather than a linear sequence of thoughts.
|
|
22
|
+
|
|
23
|
+
## Key terms
|
|
24
|
+
- **Zettelkasten**: A system of note-taking and personal knowledge management that consists of small
|
|
25
|
+
items of information stored on paper slips or cards.
|
|
26
|
+
|
|
27
|
+
## Key quotes
|
|
28
|
+
> "Every one admits nowadays that it is advisable to collect materials on separate cards or slips of paper."
|
|
29
|
+
|
|
30
|
+
## Concepts
|
|
31
|
+
- [[Index card file]]
|
|
32
|
+
|
|
33
|
+
## Entities
|
|
34
|
+
- [[Niklas Luhmann]]
|
|
35
|
+
- [[Roland Barthes]]
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Quotes are checked against the source word for word, glossary terms must appear in it, diagrams are drawn by code, and PDF figures are copied next to the note. Details: [How a note is made](https://rubenamaury.github.io/esbi-cli/docs/explanation/how-a-note-is-made/).
|
|
39
|
+
|
|
40
|
+
## Who it is for
|
|
41
|
+
|
|
42
|
+
People who save more than they read: papers, articles, newsletters, YouTube talks. You decide what to read from a good summary, and you can ask your own wiki questions (`sb ask`) and get answers that cite your notes. It runs on your machine, on macOS first (Linux works by running the same command from cron; it is less tested).
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
brew install rubenamaury/esbi-cli/esbi-cli # macOS (Homebrew)
|
|
48
|
+
# or, with uv:
|
|
49
|
+
uv tool install git+https://github.com/RubenAmaury/esbi-cli
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Then set it up. `sb init` is an installer: it asks what you use and every question is optional.
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
sb init # vault folder, Obsidian or not, local model / Claude subscription / API, nightly job, backup
|
|
56
|
+
sb doctor # checks the whole setup and says what to fix
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
A walk-through from install to a first note is in [Getting started](https://rubenamaury.github.io/esbi-cli/docs/tutorials/getting-started/); other ways to install (a wheel, a source checkout, Linux) are in [Install](https://rubenamaury.github.io/esbi-cli/docs/how-to/install/).
|
|
60
|
+
|
|
61
|
+
## First five minutes
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
sb add https://example.com/a-good-article # a link
|
|
65
|
+
cp ~/Downloads/paper.pdf ~/Documents/Obsidian/esbi/inbox/ # or drop a PDF in inbox/
|
|
66
|
+
sb run # reads, writes the notes (a few minutes per source)
|
|
67
|
+
sb today # opens today's index note
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
In the index note, tick `- [x]` next to what you have read and run `sb index`. Ask a question with `sb ask "what is an agent harness?"`. [Getting started](https://rubenamaury.github.io/esbi-cli/docs/tutorials/getting-started/) shows every step with its output, and [A first week](https://rubenamaury.github.io/esbi-cli/docs/tutorials/first-week/) teaches the habit.
|
|
71
|
+
|
|
72
|
+
## What is optional
|
|
73
|
+
|
|
74
|
+
The basics are PDFs and links. Everything else you can skip, or add later:
|
|
75
|
+
|
|
76
|
+
| Optional | What it adds | How |
|
|
77
|
+
|---|---|---|
|
|
78
|
+
| Obsidian | A nice viewer. Any Markdown editor works, or `sb export` for a website | `sb init` asks; [Use it without Obsidian](https://rubenamaury.github.io/esbi-cli/docs/how-to/without-obsidian/) |
|
|
79
|
+
| The nightly job | Processes your queue by itself at a time you choose | `sb init --nightly 03:00`; [Run it every night](https://rubenamaury.github.io/esbi-cli/docs/how-to/nightly-job/) |
|
|
80
|
+
| Email capture (Gmail) | Forward an email or PDF to yourself and get a note | `sb setup email`; [Capture email](https://rubenamaury.github.io/esbi-cli/docs/how-to/email/) |
|
|
81
|
+
| Web Clipper (needs Obsidian) | Save pages, LinkedIn or Reddit posts, YouTube transcripts from your browser | `sb setup clipper`; [Web Clipper](https://rubenamaury.github.io/esbi-cli/docs/how-to/web-clipper/) |
|
|
82
|
+
| Claude subscription or API | Better notes than a small local model | `sb init --model subscription` or `api`; [Models](https://rubenamaury.github.io/esbi-cli/docs/how-to/models/) |
|
|
83
|
+
| Version history and backup (git) | If `git` is installed the vault is a repository: every ingested source is a commit you can review or undo. A backup is any git remote (GitHub, GitLab, a folder on a NAS), never required | `sb init --remote URL`; [Back up your vault](https://rubenamaury.github.io/esbi-cli/docs/how-to/backup/) |
|
|
84
|
+
|
|
85
|
+
## Privacy: what leaves your machine
|
|
86
|
+
|
|
87
|
+
| You choose | What is sent, and to whom |
|
|
88
|
+
|---|---|
|
|
89
|
+
| A local model (Ollama, LM Studio) | Nothing |
|
|
90
|
+
| Your Claude subscription (`claude-cli`) | The text of each source goes to Anthropic, through the official `claude` tool |
|
|
91
|
+
| An API key (OpenAI-compatible, Anthropic) | The text of each source goes to that provider |
|
|
92
|
+
|
|
93
|
+
With either cloud choice, **email is never sent**: it is read only by a local model (`[llm.private]`), and the cloud model is never shown email pages. The mail password is stored in the macOS Keychain, never in a file; the app only reads mail, it never deletes it; URLs are fetched only on public addresses. Details: [Safety and privacy](https://rubenamaury.github.io/esbi-cli/docs/explanation/safety-and-privacy/).
|
|
94
|
+
|
|
95
|
+
## How it works
|
|
96
|
+
|
|
97
|
+
```
|
|
98
|
+
you save the worker (sb run) you read
|
|
99
|
+
──────── ─────────────────── ────────
|
|
100
|
+
sb add / inbox/ PDFs ┐
|
|
101
|
+
Web Clipper clips ├─► queue ─► extract ─► read in chunks ─► model writes a ─► wiki/ notes, index.md,
|
|
102
|
+
forwarded email ┘ (SQLite) (web, PDF) (notes per chunk) JSON edit plan log.md, wiki/daily/ ◄── start here
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
`raw/` keeps the originals and is never edited; `wiki/` holds the notes the worker writes and you edit freely; `SCHEMA.md` holds the conventions the model is given. The vault is its own git repository: every ingested source is a commit, so any change can be reviewed or reverted. More: [How it works](https://rubenamaury.github.io/esbi-cli/docs/explanation/how-it-works/), [Internals](https://rubenamaury.github.io/esbi-cli/docs/explanation/internals/).
|
|
106
|
+
|
|
107
|
+
## Documentation
|
|
108
|
+
|
|
109
|
+
The same pages are published as a website: <https://rubenamaury.github.io/esbi-cli/> (documentation under `/docs/`).
|
|
110
|
+
|
|
111
|
+
| | |
|
|
112
|
+
|---|---|
|
|
113
|
+
| **Start** | [Documentation home](https://rubenamaury.github.io/esbi-cli/docs/) · [How do I...?](https://rubenamaury.github.io/esbi-cli/docs/how-to/) |
|
|
114
|
+
| **Learn** | [Getting started](https://rubenamaury.github.io/esbi-cli/docs/tutorials/getting-started/) · [A first week](https://rubenamaury.github.io/esbi-cli/docs/tutorials/first-week/) |
|
|
115
|
+
| **Do** | [Install](https://rubenamaury.github.io/esbi-cli/docs/how-to/install/) · [Models](https://rubenamaury.github.io/esbi-cli/docs/how-to/models/) · [Get sources in](https://rubenamaury.github.io/esbi-cli/docs/how-to/getting-sources-in/) · [Manage the queue](https://rubenamaury.github.io/esbi-cli/docs/how-to/manage-the-queue/) · [Read your notes](https://rubenamaury.github.io/esbi-cli/docs/how-to/read-your-notes/) · [Ask your wiki](https://rubenamaury.github.io/esbi-cli/docs/how-to/ask-your-wiki/) · [Check the wiki's health](https://rubenamaury.github.io/esbi-cli/docs/how-to/check-wiki-health/) · [Export a website](https://rubenamaury.github.io/esbi-cli/docs/how-to/export-a-website/) · [Email](https://rubenamaury.github.io/esbi-cli/docs/how-to/email/) · [Web Clipper](https://rubenamaury.github.io/esbi-cli/docs/how-to/web-clipper/) · [Nightly job](https://rubenamaury.github.io/esbi-cli/docs/how-to/nightly-job/) · [Back up and restore](https://rubenamaury.github.io/esbi-cli/docs/how-to/backup/) · [Move to a new Mac](https://rubenamaury.github.io/esbi-cli/docs/how-to/move-to-a-new-mac/) · [Update and uninstall](https://rubenamaury.github.io/esbi-cli/docs/how-to/update-and-uninstall/) · [Without Obsidian](https://rubenamaury.github.io/esbi-cli/docs/how-to/without-obsidian/) · [Offline](https://rubenamaury.github.io/esbi-cli/docs/how-to/use-offline/) · [Troubleshooting](https://rubenamaury.github.io/esbi-cli/docs/how-to/troubleshooting/) |
|
|
116
|
+
| **Look up** | [Commands](https://rubenamaury.github.io/esbi-cli/docs/reference/cli/) · [Configuration](https://rubenamaury.github.io/esbi-cli/docs/reference/configuration/) · [Vault layout](https://rubenamaury.github.io/esbi-cli/docs/reference/vault-layout/) · [The daily index](https://rubenamaury.github.io/esbi-cli/docs/reference/daily-index/) |
|
|
117
|
+
| **Understand** | [Concepts and glossary](https://rubenamaury.github.io/esbi-cli/docs/explanation/concepts/) · [FAQ](https://rubenamaury.github.io/esbi-cli/docs/explanation/faq/) · [How it works](https://rubenamaury.github.io/esbi-cli/docs/explanation/how-it-works/) · [How a note is made](https://rubenamaury.github.io/esbi-cli/docs/explanation/how-a-note-is-made/) · [Safety and privacy](https://rubenamaury.github.io/esbi-cli/docs/explanation/safety-and-privacy/) · [Why it is built this way](https://rubenamaury.github.io/esbi-cli/docs/explanation/internals/) · [Architecture](https://rubenamaury.github.io/esbi-cli/docs/explanation/architecture/) · [Retrieval and RAG](https://rubenamaury.github.io/esbi-cli/docs/explanation/rag-fit/) · [Storage](https://rubenamaury.github.io/esbi-cli/docs/explanation/storage-fit/) |
|
|
118
|
+
|
|
119
|
+
## Requirements
|
|
120
|
+
|
|
121
|
+
Python 3.12 or later (installed for you by `brew` or `uv`), `git` (optional: version history and backup), and one way to run a model: [Ollama](https://ollama.com) or [LM Studio](https://lmstudio.ai) locally, the [Claude Code](https://claude.com/code) `claude` tool signed in to a subscription, or an API key. `sb doctor` checks all of it.
|
|
122
|
+
|
|
123
|
+
## Status
|
|
124
|
+
|
|
125
|
+
Version 0.x: it works and is used daily, but commands and the config format can still change between minor versions; see the [changelog](CHANGELOG.md). Quality depends on the model: a small local model writes useful but unpolished notes; a stronger model writes noticeably better ones. Known limits are in [Known limits](https://rubenamaury.github.io/esbi-cli/docs/explanation/internals/#known-limits); open work is in the issue tracker.
|
|
126
|
+
|
|
127
|
+
## Contributing and security
|
|
128
|
+
|
|
129
|
+
Bug reports and ideas are welcome: [CONTRIBUTING.md](CONTRIBUTING.md). Please report vulnerabilities privately: [SECURITY.md](.github/SECURITY.md).
|
|
130
|
+
|
|
131
|
+
## License
|
|
132
|
+
|
|
133
|
+
[MIT](LICENSE). One dependency, [PyMuPDF](https://pymupdf.readthedocs.io) (reads PDFs), is licensed AGPL-3.0 and is installed from PyPI, not bundled here; check its terms if you redistribute a build that includes it.
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "esbi-cli"
|
|
3
|
+
version = "0.2.1"
|
|
4
|
+
description = "Turns the PDFs, links and emails you save into a Markdown wiki of summaries, glossaries and connections, written by a local or cloud language model."
|
|
5
|
+
keywords = [
|
|
6
|
+
"notes",
|
|
7
|
+
"wiki",
|
|
8
|
+
"llm",
|
|
9
|
+
"obsidian",
|
|
10
|
+
"markdown",
|
|
11
|
+
"pdf",
|
|
12
|
+
"summarization",
|
|
13
|
+
"rag",
|
|
14
|
+
"esbi-cli",
|
|
15
|
+
]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Environment :: Console",
|
|
19
|
+
"Intended Audience :: End Users/Desktop",
|
|
20
|
+
"Operating System :: MacOS",
|
|
21
|
+
"Operating System :: POSIX :: Linux",
|
|
22
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Topic :: Text Processing :: Markup :: Markdown",
|
|
26
|
+
]
|
|
27
|
+
readme = "README.md"
|
|
28
|
+
license = "MIT"
|
|
29
|
+
license-files = ["LICENSE"]
|
|
30
|
+
requires-python = ">=3.12"
|
|
31
|
+
dependencies = [
|
|
32
|
+
"httpx>=0.28.1",
|
|
33
|
+
"keyring>=25.7.0",
|
|
34
|
+
"markdown-it-py>=3",
|
|
35
|
+
"pillow>=11",
|
|
36
|
+
"pydantic>=2.7",
|
|
37
|
+
"pymupdf4llm>=1.28.2",
|
|
38
|
+
"pyyaml>=6.0.3",
|
|
39
|
+
"trafilatura>=2.2.0",
|
|
40
|
+
"typer>=0.12",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[[project.authors]]
|
|
44
|
+
name = "Rubén Melo"
|
|
45
|
+
|
|
46
|
+
[project.urls]
|
|
47
|
+
Homepage = "https://github.com/RubenAmaury/esbi-cli"
|
|
48
|
+
Documentation = "https://github.com/RubenAmaury/esbi-cli/tree/main/docs"
|
|
49
|
+
Repository = "https://github.com/RubenAmaury/esbi-cli"
|
|
50
|
+
Issues = "https://github.com/RubenAmaury/esbi-cli/issues"
|
|
51
|
+
Changelog = "https://github.com/RubenAmaury/esbi-cli/blob/main/CHANGELOG.md"
|
|
52
|
+
|
|
53
|
+
[project.scripts]
|
|
54
|
+
sb = "esbi_cli.cli:app"
|
|
55
|
+
|
|
56
|
+
[dependency-groups]
|
|
57
|
+
dev = [
|
|
58
|
+
"pytest>=8",
|
|
59
|
+
"ruff>=0.6",
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
[build-system]
|
|
63
|
+
requires = ["uv_build>=0.12.21,<0.13.0"]
|
|
64
|
+
build-backend = "uv_build"
|
|
65
|
+
|
|
66
|
+
[tool.ruff]
|
|
67
|
+
line-length = 100
|
|
68
|
+
target-version = "py312"
|
|
69
|
+
|
|
70
|
+
[tool.ruff.lint]
|
|
71
|
+
select = [
|
|
72
|
+
"E",
|
|
73
|
+
"F",
|
|
74
|
+
"I",
|
|
75
|
+
"UP",
|
|
76
|
+
"B",
|
|
77
|
+
]
|
|
78
|
+
ignore = [
|
|
79
|
+
"E501",
|
|
80
|
+
"B008",
|
|
81
|
+
]
|
|
82
|
+
|
|
83
|
+
[tool.pytest.ini_options]
|
|
84
|
+
testpaths = ["tests"]
|
|
85
|
+
pythonpath = ["src"]
|
|
86
|
+
filterwarnings = ["ignore:builtin type .* has no __module__ attribute:DeprecationWarning"]
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "esbi-cli"
|
|
3
|
+
version = "0.2.1"
|
|
4
|
+
description = "Turns the PDFs, links and emails you save into a Markdown wiki of summaries, glossaries and connections, written by a local or cloud language model."
|
|
5
|
+
keywords = ["notes", "wiki", "llm", "obsidian", "markdown", "pdf", "summarization", "rag", "esbi-cli"]
|
|
6
|
+
classifiers = [
|
|
7
|
+
"Development Status :: 4 - Beta",
|
|
8
|
+
"Environment :: Console",
|
|
9
|
+
"Intended Audience :: End Users/Desktop",
|
|
10
|
+
"Operating System :: MacOS",
|
|
11
|
+
"Operating System :: POSIX :: Linux",
|
|
12
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
13
|
+
"Programming Language :: Python :: 3.12",
|
|
14
|
+
"Programming Language :: Python :: 3.13",
|
|
15
|
+
"Topic :: Text Processing :: Markup :: Markdown",
|
|
16
|
+
]
|
|
17
|
+
readme = "README.md"
|
|
18
|
+
license = "MIT"
|
|
19
|
+
license-files = ["LICENSE"]
|
|
20
|
+
authors = [{ name = "Rubén Melo" }]
|
|
21
|
+
requires-python = ">=3.12"
|
|
22
|
+
dependencies = [
|
|
23
|
+
"httpx>=0.28.1",
|
|
24
|
+
"keyring>=25.7.0",
|
|
25
|
+
"markdown-it-py>=3",
|
|
26
|
+
"pillow>=11",
|
|
27
|
+
"pydantic>=2.7",
|
|
28
|
+
"pymupdf4llm>=1.28.2",
|
|
29
|
+
"pyyaml>=6.0.3",
|
|
30
|
+
"trafilatura>=2.2.0",
|
|
31
|
+
"typer>=0.12",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
Homepage = "https://github.com/RubenAmaury/esbi-cli"
|
|
36
|
+
Documentation = "https://github.com/RubenAmaury/esbi-cli/tree/main/docs"
|
|
37
|
+
Repository = "https://github.com/RubenAmaury/esbi-cli"
|
|
38
|
+
Issues = "https://github.com/RubenAmaury/esbi-cli/issues"
|
|
39
|
+
Changelog = "https://github.com/RubenAmaury/esbi-cli/blob/main/CHANGELOG.md"
|
|
40
|
+
|
|
41
|
+
[project.scripts]
|
|
42
|
+
sb = "esbi_cli.cli:app"
|
|
43
|
+
|
|
44
|
+
[dependency-groups]
|
|
45
|
+
dev = [
|
|
46
|
+
"pytest>=8",
|
|
47
|
+
"ruff>=0.6",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
[build-system]
|
|
51
|
+
requires = ["uv_build>=0.12.21,<0.13.0"]
|
|
52
|
+
build-backend = "uv_build"
|
|
53
|
+
|
|
54
|
+
[tool.ruff]
|
|
55
|
+
line-length = 100
|
|
56
|
+
target-version = "py312"
|
|
57
|
+
|
|
58
|
+
[tool.ruff.lint]
|
|
59
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
60
|
+
ignore = ["E501", "B008"] # long prose strings; typer uses Option() defaults
|
|
61
|
+
|
|
62
|
+
[tool.pytest.ini_options]
|
|
63
|
+
testpaths = ["tests"]
|
|
64
|
+
pythonpath = ["src"]
|
|
65
|
+
filterwarnings = ["ignore:builtin type .* has no __module__ attribute:DeprecationWarning"]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""esbi-cli: LLM-maintained Obsidian wiki worker."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = version("esbi-cli") # pyproject.toml is the one place the version is written
|
|
7
|
+
except PackageNotFoundError: # running from a bare source tree
|
|
8
|
+
__version__ = "0+unknown"
|
|
File without changes
|