upii 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- upii-0.7.0/LICENSE +21 -0
- upii-0.7.0/PKG-INFO +199 -0
- upii-0.7.0/README.md +152 -0
- upii-0.7.0/pyproject.toml +66 -0
- upii-0.7.0/setup.cfg +4 -0
- upii-0.7.0/src/upii/__init__.py +1 -0
- upii-0.7.0/src/upii/ambient/calendar_connector.py +106 -0
- upii-0.7.0/src/upii/ambient/dummy_sources.py +54 -0
- upii-0.7.0/src/upii/ambient/email_connector.py +74 -0
- upii-0.7.0/src/upii/ambient/sources.py +111 -0
- upii-0.7.0/src/upii/ambient/storage.py +159 -0
- upii-0.7.0/src/upii/ambient/watcher.py +381 -0
- upii-0.7.0/src/upii/analysis/__init__.py +0 -0
- upii-0.7.0/src/upii/analysis/diagnostics.py +104 -0
- upii-0.7.0/src/upii/analysis/embeddings.py +18 -0
- upii-0.7.0/src/upii/analysis/entity_extractor.py +168 -0
- upii-0.7.0/src/upii/analysis/graph.py +312 -0
- upii-0.7.0/src/upii/analysis/llm.py +120 -0
- upii-0.7.0/src/upii/analysis/metrics.py +107 -0
- upii-0.7.0/src/upii/analysis/nlp.py +51 -0
- upii-0.7.0/src/upii/analysis/rehydration.py +324 -0
- upii-0.7.0/src/upii/analysis/search.py +39 -0
- upii-0.7.0/src/upii/cli.py +1028 -0
- upii-0.7.0/src/upii/core/__init__.py +0 -0
- upii-0.7.0/src/upii/core/concurrency.py +44 -0
- upii-0.7.0/src/upii/core/config.py +65 -0
- upii-0.7.0/src/upii/core/errors.py +15 -0
- upii-0.7.0/src/upii/core/features.py +67 -0
- upii-0.7.0/src/upii/core/logger.py +41 -0
- upii-0.7.0/src/upii/core/types.py +65 -0
- upii-0.7.0/src/upii/core/utils.py +5 -0
- upii-0.7.0/src/upii/ingestion/__init__.py +0 -0
- upii-0.7.0/src/upii/ingestion/chunker.py +49 -0
- upii-0.7.0/src/upii/ingestion/identity.py +24 -0
- upii-0.7.0/src/upii/ingestion/loader.py +147 -0
- upii-0.7.0/src/upii/ingestion/pipeline.py +218 -0
- upii-0.7.0/src/upii/overlay/app.py +241 -0
- upii-0.7.0/src/upii/overlay/daemon.py +62 -0
- upii-0.7.0/src/upii/storage/__init__.py +0 -0
- upii-0.7.0/src/upii/storage/db.py +592 -0
- upii-0.7.0/src/upii/storage/vector.py +165 -0
- upii-0.7.0/src/upii.egg-info/PKG-INFO +199 -0
- upii-0.7.0/src/upii.egg-info/SOURCES.txt +71 -0
- upii-0.7.0/src/upii.egg-info/dependency_links.txt +1 -0
- upii-0.7.0/src/upii.egg-info/entry_points.txt +2 -0
- upii-0.7.0/src/upii.egg-info/requires.txt +25 -0
- upii-0.7.0/src/upii.egg-info/top_level.txt +1 -0
- upii-0.7.0/tests/test_ambient.py +55 -0
- upii-0.7.0/tests/test_ask_no_answer.py +63 -0
- upii-0.7.0/tests/test_batch_ingest.py +169 -0
- upii-0.7.0/tests/test_chunk_determinism.py +99 -0
- upii-0.7.0/tests/test_context_rehydration.py +128 -0
- upii-0.7.0/tests/test_entity_extraction.py +165 -0
- upii-0.7.0/tests/test_eval_harness.py +141 -0
- upii-0.7.0/tests/test_features.py +48 -0
- upii-0.7.0/tests/test_graph_viz.py +134 -0
- upii-0.7.0/tests/test_inbox.py +140 -0
- upii-0.7.0/tests/test_incremental.py +161 -0
- upii-0.7.0/tests/test_ingest.py +111 -0
- upii-0.7.0/tests/test_isolation.py +25 -0
- upii-0.7.0/tests/test_memory_integrity.py +141 -0
- upii-0.7.0/tests/test_metrics.py +137 -0
- upii-0.7.0/tests/test_ml.py +85 -0
- upii-0.7.0/tests/test_nlp.py +82 -0
- upii-0.7.0/tests/test_observability.py +64 -0
- upii-0.7.0/tests/test_overlay.py +83 -0
- upii-0.7.0/tests/test_rag.py +103 -0
- upii-0.7.0/tests/test_search.py +107 -0
- upii-0.7.0/tests/test_sources.py +90 -0
- upii-0.7.0/tests/test_temporal_recall.py +65 -0
- upii-0.7.0/tests/test_version.py +31 -0
- upii-0.7.0/tests/test_watcher_advanced.py +118 -0
- upii-0.7.0/tests/test_watcher_backend.py +76 -0
upii-0.7.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Maddy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
upii-0.7.0/PKG-INFO
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: upii
|
|
3
|
+
Version: 0.7.0
|
|
4
|
+
Summary: Local-first memory for AI: cited answers over your own documents, 100% on-device
|
|
5
|
+
Author-email: Maddy <maddy@datafrontier.co>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/maddymaster/UPII
|
|
8
|
+
Project-URL: Repository, https://github.com/maddymaster/UPII
|
|
9
|
+
Project-URL: Issues, https://github.com/maddymaster/UPII/issues
|
|
10
|
+
Keywords: rag,local-first,memory,llm,privacy,vector-search,knowledge-base,ollama
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Classifier: Topic :: Text Processing :: Indexing
|
|
22
|
+
Requires-Python: <3.13,>=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: typer>=0.9.0
|
|
26
|
+
Requires-Dist: click<8.2
|
|
27
|
+
Requires-Dist: rich>=13.0
|
|
28
|
+
Requires-Dist: pydantic>=2.5
|
|
29
|
+
Requires-Dist: lancedb>=0.5.0
|
|
30
|
+
Requires-Dist: pyarrow>=14.0.2
|
|
31
|
+
Requires-Dist: numpy<2
|
|
32
|
+
Requires-Dist: sentence-transformers>=2.3.1
|
|
33
|
+
Requires-Dist: ollama>=0.1.6
|
|
34
|
+
Requires-Dist: PyYAML>=6.0
|
|
35
|
+
Requires-Dist: watchdog>=4.0.0
|
|
36
|
+
Requires-Dist: tenacity>=8.2
|
|
37
|
+
Requires-Dist: pypdf>=3.17.0
|
|
38
|
+
Provides-Extra: overlay
|
|
39
|
+
Requires-Dist: pynput>=1.7; extra == "overlay"
|
|
40
|
+
Provides-Extra: dev
|
|
41
|
+
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
42
|
+
Requires-Dist: psutil; extra == "dev"
|
|
43
|
+
Requires-Dist: pyinstaller; extra == "dev"
|
|
44
|
+
Requires-Dist: sqlite-utils>=3.36; extra == "dev"
|
|
45
|
+
Requires-Dist: tomli; python_version < "3.11" and extra == "dev"
|
|
46
|
+
Dynamic: license-file
|
|
47
|
+
|
|
48
|
+
# UPII — your memory, on your machine
|
|
49
|
+
|
|
50
|
+
**Local-first memory for AI. Ingest your notes and documents, ask questions, get answers with citations — and no byte of your corpus ever leaves your device.**
|
|
51
|
+
|
|
52
|
+
[](LICENSE)
|
|
53
|
+
[](https://www.python.org/downloads/)
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
$ upii ask "What did we agree with ICEYE on resolution and latency?"
|
|
57
|
+
|
|
58
|
+
Answer: You agreed on 0.5 m resolution imagery with a 6-hour delivery
|
|
59
|
+
SLA, with latency relaxed to 12 hours during the pilot phase.
|
|
60
|
+
|
|
61
|
+
Sources:
|
|
62
|
+
[1] meetings/2026-03-14_iceye_sync.md (chunk a3f2c9)
|
|
63
|
+
[2] contracts/iceye_pilot_terms.md (chunk 77d01b)
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
<!-- TODO: replace with demo.gif — 15s: ingest a folder, ask, cited answer, unplug wifi, ask again -->
|
|
67
|
+
|
|
68
|
+
> **Status:** early preview. The core capture, retrieval and ambient stack works
|
|
69
|
+
> and is tested, but interfaces are still moving.
|
|
70
|
+
>
|
|
71
|
+
> **Known gaps — worth knowing before you judge the claims above:**
|
|
72
|
+
> - **The relational retrieval signal is off by default.** Ingestion now populates
|
|
73
|
+
> a local knowledge graph (people/orgs/projects) and `upii knowledge --graph`
|
|
74
|
+
> renders it, but on the retrieval eval, letting the relational signal re-rank is
|
|
75
|
+
> not yet net-positive (it can boost a chunk that merely *mentions* a query entity
|
|
76
|
+
> over a better semantic match), so its fusion weight defaults to 0. Retrieval
|
|
77
|
+
> today is effectively semantic + (recency-weighted) temporal; raise the signal
|
|
78
|
+
> per query with `upii ask --w-relational`. Tuning it to help is ongoing.
|
|
79
|
+
|
|
80
|
+
## Why UPII
|
|
81
|
+
|
|
82
|
+
Most RAG and "second brain" tools upload your corpus to someone else's cloud and rent your own memory back to you. UPII inverts that:
|
|
83
|
+
|
|
84
|
+
- **100% local.** Embeddings ([sentence-transformers](https://www.sbert.net/) MiniLM), vector store ([LanceDB](https://lancedb.com/)), metadata (SQLite), and reasoning ([Ollama](https://ollama.com)) all run on-device. A remote LLM (Gemini) is optional and swappable — never required.
|
|
85
|
+
- **Every answer is cited.** Answers are attributed to source chunks by content-addressed ID. Retrieval is verifiable, not just plausible.
|
|
86
|
+
- **Deterministic, reproducible memory.** Chunk IDs are a pure function of content + config. Re-ingesting the same corpus reproduces 100% of chunk hashes — validated by tests and benchmarks. Citations stay stable across re-indexes; re-embedding is cache-safe.
|
|
87
|
+
- **More than cosine similarity.** The Context Rehydrator is built to fuse three signals — semantic (vectors), temporal ("what was I working on last week"), and relational (a local knowledge graph of people, projects, orgs, populated on ingest) — into one ranked context window, with per-signal contributions visible via `upii ask --debug`. Measured, not asserted: **Recall@10 = 0.958** on a committed labelled set ([reproduce it](eval/README.md)). The relational signal is live but weight-0 by default until it's tuned to help (see [Status](#status)); the score above is from semantic + temporal.
|
|
88
|
+
- **Consent-gated ambient capture.** A filesystem watcher keeps memory current, but sources are explicitly enabled and ambient events pass through an approval inbox before entering durable memory.
|
|
89
|
+
- **Graceful degradation.** No GPU, no API key, no internet? Retrieval still works and the system still answers.
|
|
90
|
+
|
|
91
|
+
## Quickstart
|
|
92
|
+
|
|
93
|
+
Requires Python 3.10–3.12.
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
pip install upii # or: pipx install upii
|
|
97
|
+
|
|
98
|
+
upii doctor # verify the local stack (db, vectors, model, disk)
|
|
99
|
+
upii ingest ./notes --recursive
|
|
100
|
+
upii search "satellite latency"
|
|
101
|
+
upii ask "What did we agree with ICEYE on resolution and latency?"
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
First run downloads the embedding model (`all-MiniLM-L6-v2`, ~90 MB) — the only download UPII ever needs. For local reasoning, install [Ollama](https://ollama.com) and `ollama pull llama3.2`; without it, UPII falls back to retrieval + a deterministic mock, so it always responds.
|
|
105
|
+
|
|
106
|
+
### From source
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
git clone https://github.com/<you>/upii && cd upii
|
|
110
|
+
python3 -m venv venv && source venv/bin/activate # Windows: venv\Scripts\activate
|
|
111
|
+
pip install -e ".[dev]"
|
|
112
|
+
pytest tests/ -q
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## Usage
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
upii ingest ./notes --recursive # build memory from a directory (idempotent)
|
|
119
|
+
upii search "satellite latency" # semantic + attributed retrieval
|
|
120
|
+
upii ask "…" # cited answer over your corpus
|
|
121
|
+
upii write "follow-up" --target email # synthesize in your voice
|
|
122
|
+
upii watch ./inbox # ambient capture (consent-gated)
|
|
123
|
+
upii inbox # approve/reject ambient captures
|
|
124
|
+
upii knowledge --graph # render the local knowledge graph to HTML
|
|
125
|
+
upii sources list # manage enabled sources
|
|
126
|
+
upii metrics show # observability
|
|
127
|
+
upii doctor # health check
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Run `upii --help` for the full command set.
|
|
131
|
+
|
|
132
|
+
## How it works
|
|
133
|
+
|
|
134
|
+
```mermaid
|
|
135
|
+
graph TD
|
|
136
|
+
User[User / Ambient Sources] -->|ingest| IngestionEngine
|
|
137
|
+
User -->|ask / search| Rehydrator
|
|
138
|
+
|
|
139
|
+
subgraph "Everything runs locally"
|
|
140
|
+
IngestionEngine -->|hash + dedup| MetadataStore[(SQLite — source of truth)]
|
|
141
|
+
IngestionEngine -->|deterministic chunking| Chunker
|
|
142
|
+
Chunker -->|local embeddings| Embedder
|
|
143
|
+
Embedder -->|store vectors| VectorStore[(LanceDB)]
|
|
144
|
+
|
|
145
|
+
Rehydrator -->|semantic| VectorStore
|
|
146
|
+
Rehydrator -->|temporal + relational| MetadataStore
|
|
147
|
+
Rehydrator -->|fused context| LLM[Ollama local / optional remote]
|
|
148
|
+
LLM -->|attributed answer| User
|
|
149
|
+
end
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Every write path — `ingest`, watch-approve, demo seed — flows through a single deterministic pipeline (`src/upii/ingestion/pipeline.py`):
|
|
153
|
+
|
|
154
|
+
- **Identity is content-addressed.** `doc_id = hash(content)`, `chunk_id = hash(chunk_text, config)`. No random UUIDs, no wall-clock, no path dependence. Same input ⇒ same memory state.
|
|
155
|
+
- **Dedup** — re-ingesting unchanged bytes is a no-op.
|
|
156
|
+
- **Edit** — a changed file re-chunks only what changed; unchanged chunk hashes stay stable; the prior version (chunks + vectors + metadata) is purged.
|
|
157
|
+
- **Delete** — removal cleans chunks, vectors, and metadata together.
|
|
158
|
+
|
|
159
|
+
Reproduce it yourself:
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
bash scripts/demo/repro_demo.sh # ingest → re-ingest → identical state
|
|
163
|
+
python scripts/bench/scale_check.py --docs 500 --paras 60 # scale + hash-reproducibility report
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
## How UPII compares
|
|
167
|
+
|
|
168
|
+
| | UPII | Cloud RAG / memory APIs | Typical local RAG |
|
|
169
|
+
|---|---|---|---|
|
|
170
|
+
| Corpus stays on device | ✅ always | ❌ | ✅ |
|
|
171
|
+
| Cited, chunk-level attribution | ✅ | varies | rarely |
|
|
172
|
+
| Reproducible ingestion (stable hashes) | ✅ 100% | ❌ | ❌ |
|
|
173
|
+
| Temporal + relational retrieval signals | ✅ both built; relational off by default ([status](#status)) | ❌ | ❌ cosine only |
|
|
174
|
+
| Works fully offline | ✅ | ❌ | usually |
|
|
175
|
+
| Consent-gated ambient capture | ✅ | ❌ | ❌ |
|
|
176
|
+
|
|
177
|
+
## Roadmap
|
|
178
|
+
|
|
179
|
+
- **Tune the relational signal to net-positive** — it re-ranks on knowledge-graph entity overlap but currently isn't a clear win on the eval, so it ships weight-0; make it help before enabling by default
|
|
180
|
+
- **Batched vector writes** — remove the per-document LanceDB append that caps ingestion throughput
|
|
181
|
+
- **MCP server** — expose `upii_search` / `upii_ask` as local MCP tools so Claude, Cursor, and other agents can use your private, cited memory ([scope](docs/mcp_server_scope.md))
|
|
182
|
+
- Mail + calendar connectors (consent-gated)
|
|
183
|
+
- Answer verification / abstention
|
|
184
|
+
- Signed installers for macOS, Windows, Linux
|
|
185
|
+
- `upii forget` — user-facing verified deletion
|
|
186
|
+
|
|
187
|
+
## Contributing
|
|
188
|
+
|
|
189
|
+
Issues and PRs welcome. Start with:
|
|
190
|
+
|
|
191
|
+
```bash
|
|
192
|
+
pip install -e ".[dev]" && pytest tests/ -q
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
Hard constraints for any change: local-first (no required cloud dependency), deterministic ingestion (same input ⇒ same chunk hashes), and consent-gating for any ambient capture path. See `tests/test_chunk_determinism.py` for the invariants that must hold.
|
|
196
|
+
|
|
197
|
+
## License
|
|
198
|
+
|
|
199
|
+
MIT © 2026 Maddy
|
upii-0.7.0/README.md
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# UPII — your memory, on your machine
|
|
2
|
+
|
|
3
|
+
**Local-first memory for AI. Ingest your notes and documents, ask questions, get answers with citations — and no byte of your corpus ever leaves your device.**
|
|
4
|
+
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
[](https://www.python.org/downloads/)
|
|
7
|
+
|
|
8
|
+
```
|
|
9
|
+
$ upii ask "What did we agree with ICEYE on resolution and latency?"
|
|
10
|
+
|
|
11
|
+
Answer: You agreed on 0.5 m resolution imagery with a 6-hour delivery
|
|
12
|
+
SLA, with latency relaxed to 12 hours during the pilot phase.
|
|
13
|
+
|
|
14
|
+
Sources:
|
|
15
|
+
[1] meetings/2026-03-14_iceye_sync.md (chunk a3f2c9)
|
|
16
|
+
[2] contracts/iceye_pilot_terms.md (chunk 77d01b)
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
<!-- TODO: replace with demo.gif — 15s: ingest a folder, ask, cited answer, unplug wifi, ask again -->
|
|
20
|
+
|
|
21
|
+
> **Status:** early preview. The core capture, retrieval and ambient stack works
|
|
22
|
+
> and is tested, but interfaces are still moving.
|
|
23
|
+
>
|
|
24
|
+
> **Known gaps — worth knowing before you judge the claims above:**
|
|
25
|
+
> - **The relational retrieval signal is off by default.** Ingestion now populates
|
|
26
|
+
> a local knowledge graph (people/orgs/projects) and `upii knowledge --graph`
|
|
27
|
+
> renders it, but on the retrieval eval, letting the relational signal re-rank is
|
|
28
|
+
> not yet net-positive (it can boost a chunk that merely *mentions* a query entity
|
|
29
|
+
> over a better semantic match), so its fusion weight defaults to 0. Retrieval
|
|
30
|
+
> today is effectively semantic + (recency-weighted) temporal; raise the signal
|
|
31
|
+
> per query with `upii ask --w-relational`. Tuning it to help is ongoing.
|
|
32
|
+
|
|
33
|
+
## Why UPII
|
|
34
|
+
|
|
35
|
+
Most RAG and "second brain" tools upload your corpus to someone else's cloud and rent your own memory back to you. UPII inverts that:
|
|
36
|
+
|
|
37
|
+
- **100% local.** Embeddings ([sentence-transformers](https://www.sbert.net/) MiniLM), vector store ([LanceDB](https://lancedb.com/)), metadata (SQLite), and reasoning ([Ollama](https://ollama.com)) all run on-device. A remote LLM (Gemini) is optional and swappable — never required.
|
|
38
|
+
- **Every answer is cited.** Answers are attributed to source chunks by content-addressed ID. Retrieval is verifiable, not just plausible.
|
|
39
|
+
- **Deterministic, reproducible memory.** Chunk IDs are a pure function of content + config. Re-ingesting the same corpus reproduces 100% of chunk hashes — validated by tests and benchmarks. Citations stay stable across re-indexes; re-embedding is cache-safe.
|
|
40
|
+
- **More than cosine similarity.** The Context Rehydrator is built to fuse three signals — semantic (vectors), temporal ("what was I working on last week"), and relational (a local knowledge graph of people, projects, orgs, populated on ingest) — into one ranked context window, with per-signal contributions visible via `upii ask --debug`. Measured, not asserted: **Recall@10 = 0.958** on a committed labelled set ([reproduce it](eval/README.md)). The relational signal is live but weight-0 by default until it's tuned to help (see [Status](#status)); the score above is from semantic + temporal.
|
|
41
|
+
- **Consent-gated ambient capture.** A filesystem watcher keeps memory current, but sources are explicitly enabled and ambient events pass through an approval inbox before entering durable memory.
|
|
42
|
+
- **Graceful degradation.** No GPU, no API key, no internet? Retrieval still works and the system still answers.
|
|
43
|
+
|
|
44
|
+
## Quickstart
|
|
45
|
+
|
|
46
|
+
Requires Python 3.10–3.12.
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install upii # or: pipx install upii
|
|
50
|
+
|
|
51
|
+
upii doctor # verify the local stack (db, vectors, model, disk)
|
|
52
|
+
upii ingest ./notes --recursive
|
|
53
|
+
upii search "satellite latency"
|
|
54
|
+
upii ask "What did we agree with ICEYE on resolution and latency?"
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
First run downloads the embedding model (`all-MiniLM-L6-v2`, ~90 MB) — the only download UPII ever needs. For local reasoning, install [Ollama](https://ollama.com) and `ollama pull llama3.2`; without it, UPII falls back to retrieval + a deterministic mock, so it always responds.
|
|
58
|
+
|
|
59
|
+
### From source
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
git clone https://github.com/<you>/upii && cd upii
|
|
63
|
+
python3 -m venv venv && source venv/bin/activate # Windows: venv\Scripts\activate
|
|
64
|
+
pip install -e ".[dev]"
|
|
65
|
+
pytest tests/ -q
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Usage
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
upii ingest ./notes --recursive # build memory from a directory (idempotent)
|
|
72
|
+
upii search "satellite latency" # semantic + attributed retrieval
|
|
73
|
+
upii ask "…" # cited answer over your corpus
|
|
74
|
+
upii write "follow-up" --target email # synthesize in your voice
|
|
75
|
+
upii watch ./inbox # ambient capture (consent-gated)
|
|
76
|
+
upii inbox # approve/reject ambient captures
|
|
77
|
+
upii knowledge --graph # render the local knowledge graph to HTML
|
|
78
|
+
upii sources list # manage enabled sources
|
|
79
|
+
upii metrics show # observability
|
|
80
|
+
upii doctor # health check
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Run `upii --help` for the full command set.
|
|
84
|
+
|
|
85
|
+
## How it works
|
|
86
|
+
|
|
87
|
+
```mermaid
|
|
88
|
+
graph TD
|
|
89
|
+
User[User / Ambient Sources] -->|ingest| IngestionEngine
|
|
90
|
+
User -->|ask / search| Rehydrator
|
|
91
|
+
|
|
92
|
+
subgraph "Everything runs locally"
|
|
93
|
+
IngestionEngine -->|hash + dedup| MetadataStore[(SQLite — source of truth)]
|
|
94
|
+
IngestionEngine -->|deterministic chunking| Chunker
|
|
95
|
+
Chunker -->|local embeddings| Embedder
|
|
96
|
+
Embedder -->|store vectors| VectorStore[(LanceDB)]
|
|
97
|
+
|
|
98
|
+
Rehydrator -->|semantic| VectorStore
|
|
99
|
+
Rehydrator -->|temporal + relational| MetadataStore
|
|
100
|
+
Rehydrator -->|fused context| LLM[Ollama local / optional remote]
|
|
101
|
+
LLM -->|attributed answer| User
|
|
102
|
+
end
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Every write path — `ingest`, watch-approve, demo seed — flows through a single deterministic pipeline (`src/upii/ingestion/pipeline.py`):
|
|
106
|
+
|
|
107
|
+
- **Identity is content-addressed.** `doc_id = hash(content)`, `chunk_id = hash(chunk_text, config)`. No random UUIDs, no wall-clock, no path dependence. Same input ⇒ same memory state.
|
|
108
|
+
- **Dedup** — re-ingesting unchanged bytes is a no-op.
|
|
109
|
+
- **Edit** — a changed file re-chunks only what changed; unchanged chunk hashes stay stable; the prior version (chunks + vectors + metadata) is purged.
|
|
110
|
+
- **Delete** — removal cleans chunks, vectors, and metadata together.
|
|
111
|
+
|
|
112
|
+
Reproduce it yourself:
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
bash scripts/demo/repro_demo.sh # ingest → re-ingest → identical state
|
|
116
|
+
python scripts/bench/scale_check.py --docs 500 --paras 60 # scale + hash-reproducibility report
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## How UPII compares
|
|
120
|
+
|
|
121
|
+
| | UPII | Cloud RAG / memory APIs | Typical local RAG |
|
|
122
|
+
|---|---|---|---|
|
|
123
|
+
| Corpus stays on device | ✅ always | ❌ | ✅ |
|
|
124
|
+
| Cited, chunk-level attribution | ✅ | varies | rarely |
|
|
125
|
+
| Reproducible ingestion (stable hashes) | ✅ 100% | ❌ | ❌ |
|
|
126
|
+
| Temporal + relational retrieval signals | ✅ both built; relational off by default ([status](#status)) | ❌ | ❌ cosine only |
|
|
127
|
+
| Works fully offline | ✅ | ❌ | usually |
|
|
128
|
+
| Consent-gated ambient capture | ✅ | ❌ | ❌ |
|
|
129
|
+
|
|
130
|
+
## Roadmap
|
|
131
|
+
|
|
132
|
+
- **Tune the relational signal to net-positive** — it re-ranks on knowledge-graph entity overlap but currently isn't a clear win on the eval, so it ships weight-0; make it help before enabling by default
|
|
133
|
+
- **Batched vector writes** — remove the per-document LanceDB append that caps ingestion throughput
|
|
134
|
+
- **MCP server** — expose `upii_search` / `upii_ask` as local MCP tools so Claude, Cursor, and other agents can use your private, cited memory ([scope](docs/mcp_server_scope.md))
|
|
135
|
+
- Mail + calendar connectors (consent-gated)
|
|
136
|
+
- Answer verification / abstention
|
|
137
|
+
- Signed installers for macOS, Windows, Linux
|
|
138
|
+
- `upii forget` — user-facing verified deletion
|
|
139
|
+
|
|
140
|
+
## Contributing
|
|
141
|
+
|
|
142
|
+
Issues and PRs welcome. Start with:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
pip install -e ".[dev]" && pytest tests/ -q
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Hard constraints for any change: local-first (no required cloud dependency), deterministic ingestion (same input ⇒ same chunk hashes), and consent-gating for any ambient capture path. See `tests/test_chunk_determinism.py` for the invariants that must hold.
|
|
149
|
+
|
|
150
|
+
## License
|
|
151
|
+
|
|
152
|
+
MIT © 2026 Maddy
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "upii"
|
|
7
|
+
version = "0.7.0"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name = "Maddy", email = "maddy@datafrontier.co" },
|
|
10
|
+
]
|
|
11
|
+
description = "Local-first memory for AI: cited answers over your own documents, 100% on-device"
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
license = { text = "MIT" }
|
|
14
|
+
requires-python = ">=3.10,<3.13"
|
|
15
|
+
keywords = ["rag", "local-first", "memory", "llm", "privacy", "vector-search", "knowledge-base", "ollama"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Intended Audience :: End Users/Desktop",
|
|
20
|
+
"License :: OSI Approved :: MIT License",
|
|
21
|
+
"Operating System :: OS Independent",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.10",
|
|
24
|
+
"Programming Language :: Python :: 3.11",
|
|
25
|
+
"Programming Language :: Python :: 3.12",
|
|
26
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
27
|
+
"Topic :: Text Processing :: Indexing",
|
|
28
|
+
]
|
|
29
|
+
dependencies = [
|
|
30
|
+
"typer>=0.9.0",
|
|
31
|
+
"click<8.2", # typer 0.9.x incompatible with click>=8.2
|
|
32
|
+
"rich>=13.0",
|
|
33
|
+
"pydantic>=2.5",
|
|
34
|
+
"lancedb>=0.5.0",
|
|
35
|
+
"pyarrow>=14.0.2", # >=17 required for Python 3.13+ wheels if ceiling is lifted
|
|
36
|
+
"numpy<2", # pyarrow 14.x / lancedb 0.5.x are not numpy-2 safe
|
|
37
|
+
"sentence-transformers>=2.3.1",
|
|
38
|
+
"ollama>=0.1.6",
|
|
39
|
+
"PyYAML>=6.0",
|
|
40
|
+
"watchdog>=4.0.0",
|
|
41
|
+
"tenacity>=8.2",
|
|
42
|
+
"pypdf>=3.17.0",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
[project.optional-dependencies]
|
|
46
|
+
overlay = [
|
|
47
|
+
"pynput>=1.7", # Cmd+Shift+K capture overlay (desktop only)
|
|
48
|
+
]
|
|
49
|
+
dev = [
|
|
50
|
+
"pytest>=7.4",
|
|
51
|
+
"psutil", # tests/perf/test_long_running.py
|
|
52
|
+
"pyinstaller", # standalone binary builds
|
|
53
|
+
"sqlite-utils>=3.36", # DB inspection during development
|
|
54
|
+
"tomli; python_version < '3.11'", # tests/test_version.py; tomllib is stdlib from 3.11
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
[project.urls]
|
|
58
|
+
Homepage = "https://github.com/maddymaster/UPII"
|
|
59
|
+
Repository = "https://github.com/maddymaster/UPII"
|
|
60
|
+
Issues = "https://github.com/maddymaster/UPII/issues"
|
|
61
|
+
|
|
62
|
+
[project.scripts]
|
|
63
|
+
upii = "upii.cli:app"
|
|
64
|
+
|
|
65
|
+
[tool.setuptools.packages.find]
|
|
66
|
+
where = ["src"]
|
upii-0.7.0/setup.cfg
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.7.0"
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import datetime
|
|
4
|
+
import uuid
|
|
5
|
+
from typing import List, Dict, Generator
|
|
6
|
+
from upii.core.logger import logger
|
|
7
|
+
from upii.storage.db import DB
|
|
8
|
+
|
|
9
|
+
class CalendarConnector:
|
|
10
|
+
"""
|
|
11
|
+
Parses .ics files and ingests metadata into LTM.
|
|
12
|
+
Focus: Title, Start, End, Participants.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
def __init__(self):
|
|
16
|
+
self.db = DB()
|
|
17
|
+
|
|
18
|
+
def process_file(self, file_path: str) -> int:
|
|
19
|
+
"""Parses an ICS file and saves events. Returns count."""
|
|
20
|
+
if not os.path.exists(file_path):
|
|
21
|
+
logger.error(f"Calendar file not found: {file_path}")
|
|
22
|
+
return 0
|
|
23
|
+
|
|
24
|
+
count = 0
|
|
25
|
+
try:
|
|
26
|
+
with open(file_path, 'r', encoding='utf-8') as f:
|
|
27
|
+
content = f.read()
|
|
28
|
+
|
|
29
|
+
events = self._parse_ics(content)
|
|
30
|
+
self.db.init_db()
|
|
31
|
+
|
|
32
|
+
for event in events:
|
|
33
|
+
event['source_file'] = os.path.basename(file_path)
|
|
34
|
+
self.db.add_calendar_event(event)
|
|
35
|
+
count += 1
|
|
36
|
+
|
|
37
|
+
logger.info(f"Ingested {count} calendar events from {file_path}")
|
|
38
|
+
except Exception as e:
|
|
39
|
+
logger.error(f"Failed to process calendar {file_path}: {e}")
|
|
40
|
+
|
|
41
|
+
return count
|
|
42
|
+
|
|
43
|
+
def _parse_ics(self, content: str) -> List[Dict]:
|
|
44
|
+
"""
|
|
45
|
+
Robust-ish Regex parser for ICS.
|
|
46
|
+
(Production would use `icalendar` lib, but we avoid deps here).
|
|
47
|
+
"""
|
|
48
|
+
events = []
|
|
49
|
+
# Split by VEVENT
|
|
50
|
+
raw_events = re.findall(r'BEGIN:VEVENT(.*?)END:VEVENT', content, re.DOTALL)
|
|
51
|
+
|
|
52
|
+
for raw in raw_events:
|
|
53
|
+
try:
|
|
54
|
+
# 1. Title (SUMMARY)
|
|
55
|
+
summary_match = re.search(r'SUMMARY:(.*)', raw)
|
|
56
|
+
title = summary_match.group(1).strip() if summary_match else "Untitled Event"
|
|
57
|
+
|
|
58
|
+
# 2. Date (DTSTART/DTEND)
|
|
59
|
+
# Formats: 20230101T120000Z or ;TZID=...:2023...
|
|
60
|
+
dtstart_match = re.search(r'DTSTART.*?:(\d{8}T\d{6}Z|\d{8}T\d{6}|\d{8})', raw)
|
|
61
|
+
dtend_match = re.search(r'DTEND.*?:(\d{8}T\d{6}Z|\d{8}T\d{6}|\d{8})', raw)
|
|
62
|
+
|
|
63
|
+
start_str = dtstart_match.group(1) if dtstart_match else None
|
|
64
|
+
end_str = dtend_match.group(1) if dtend_match else None
|
|
65
|
+
|
|
66
|
+
if not start_str: continue # Skip if no time
|
|
67
|
+
|
|
68
|
+
# Normalize Time
|
|
69
|
+
start_time = self._parse_ical_date(start_str)
|
|
70
|
+
end_time = self._parse_ical_date(end_str) if end_str else start_time
|
|
71
|
+
|
|
72
|
+
# 3. Participants (ATTENDEE)
|
|
73
|
+
# ATTENDEE;CN="Alice Smith":mailto:alice@example.com
|
|
74
|
+
attendees = []
|
|
75
|
+
for line in raw.split('\n'):
|
|
76
|
+
if line.startswith('ATTENDEE'):
|
|
77
|
+
# Extract email or CN
|
|
78
|
+
if 'mailto:' in line:
|
|
79
|
+
email = line.split('mailto:')[-1].strip()
|
|
80
|
+
attendees.append(email)
|
|
81
|
+
elif 'CN=' in line:
|
|
82
|
+
cn = line.split('CN=')[1].split(':')[0].replace('"','').strip()
|
|
83
|
+
attendees.append(cn)
|
|
84
|
+
|
|
85
|
+
events.append({
|
|
86
|
+
"event_id": str(uuid.uuid4()),
|
|
87
|
+
"title": title,
|
|
88
|
+
"start_time": start_time,
|
|
89
|
+
"end_time": end_time,
|
|
90
|
+
"participants": attendees
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
except Exception as e:
|
|
94
|
+
logger.warning(f"Failed to parse event: {e}")
|
|
95
|
+
|
|
96
|
+
return events
|
|
97
|
+
|
|
98
|
+
def _parse_ical_date(self, date_str: str) -> str:
|
|
99
|
+
"""Converts ICS date/datetime to ISO8601 string."""
|
|
100
|
+
# YYYYMMDD
|
|
101
|
+
if len(date_str) == 8:
|
|
102
|
+
return f"{date_str[:4]}-{date_str[4:6]}-{date_str[6:]}T00:00:00"
|
|
103
|
+
|
|
104
|
+
# YYYYMMDDTHHMMSS(Z)
|
|
105
|
+
clean = date_str.replace('Z', '')
|
|
106
|
+
return f"{clean[:4]}-{clean[4:6]}-{clean[6:8]}T{clean[9:11]}:{clean[11:13]}:{clean[13:]}"
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import time
|
|
2
|
+
import random
|
|
3
|
+
from upii.ambient.sources import Source, registry
|
|
4
|
+
from upii.core.concurrency import SafeRunner
|
|
5
|
+
from upii.core.logger import logger
|
|
6
|
+
|
|
7
|
+
class BrowserSource(Source):
|
|
8
|
+
"""Simulates capturing high-dwell web pages."""
|
|
9
|
+
def __init__(self):
|
|
10
|
+
super().__init__("browser", "Captures metadata from high-dwell browser tabs.")
|
|
11
|
+
self.thread = None
|
|
12
|
+
|
|
13
|
+
def start(self):
|
|
14
|
+
self.is_running = True
|
|
15
|
+
self.thread = SafeRunner.run_daemon(self._loop, name="BrowserWatcher")
|
|
16
|
+
|
|
17
|
+
def stop(self):
|
|
18
|
+
self.is_running = False
|
|
19
|
+
|
|
20
|
+
def _loop(self):
|
|
21
|
+
while self.is_running:
|
|
22
|
+
time.sleep(10) # Simulate infrequent capture
|
|
23
|
+
# In a real app, this would poll a browser extension API
|
|
24
|
+
# Here we just randomly decide to 'capture' something if enabled
|
|
25
|
+
if random.random() < 0.1: # 10% chance per loop
|
|
26
|
+
logger.info("BrowserSource: simulating capture")
|
|
27
|
+
fake_url = "https://example.com/interesting-article"
|
|
28
|
+
self.storage.add_event("browser_visit", fake_url)
|
|
29
|
+
self.storage.log_audit(self.name, "capture", {"url": fake_url, "dwell_time": 120})
|
|
30
|
+
|
|
31
|
+
class CalendarSource(Source):
|
|
32
|
+
"""Simulates capturing calendar metadata."""
|
|
33
|
+
def __init__(self):
|
|
34
|
+
super().__init__("calendar", "Ingests event titles and participants.")
|
|
35
|
+
|
|
36
|
+
def start(self):
|
|
37
|
+
self.is_running = True
|
|
38
|
+
# Calendar might be poll-once or interval
|
|
39
|
+
self.thread = SafeRunner.run_daemon(self._loop, name="CalendarWatcher")
|
|
40
|
+
|
|
41
|
+
def stop(self):
|
|
42
|
+
self.is_running = False
|
|
43
|
+
|
|
44
|
+
def _loop(self):
|
|
45
|
+
while self.is_running:
|
|
46
|
+
time.sleep(20)
|
|
47
|
+
if random.random() < 0.1:
|
|
48
|
+
logger.info("CalendarSource: simulating capture")
|
|
49
|
+
self.storage.add_event("calendar_event", "Meeting with Omega Team")
|
|
50
|
+
self.storage.log_audit(self.name, "capture", {"summary": "Meeting", "participants": ["alice@example.com"]})
|
|
51
|
+
|
|
52
|
+
# Register
|
|
53
|
+
registry.register(BrowserSource())
|
|
54
|
+
registry.register(CalendarSource())
|