upii 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. upii-0.7.0/LICENSE +21 -0
  2. upii-0.7.0/PKG-INFO +199 -0
  3. upii-0.7.0/README.md +152 -0
  4. upii-0.7.0/pyproject.toml +66 -0
  5. upii-0.7.0/setup.cfg +4 -0
  6. upii-0.7.0/src/upii/__init__.py +1 -0
  7. upii-0.7.0/src/upii/ambient/calendar_connector.py +106 -0
  8. upii-0.7.0/src/upii/ambient/dummy_sources.py +54 -0
  9. upii-0.7.0/src/upii/ambient/email_connector.py +74 -0
  10. upii-0.7.0/src/upii/ambient/sources.py +111 -0
  11. upii-0.7.0/src/upii/ambient/storage.py +159 -0
  12. upii-0.7.0/src/upii/ambient/watcher.py +381 -0
  13. upii-0.7.0/src/upii/analysis/__init__.py +0 -0
  14. upii-0.7.0/src/upii/analysis/diagnostics.py +104 -0
  15. upii-0.7.0/src/upii/analysis/embeddings.py +18 -0
  16. upii-0.7.0/src/upii/analysis/entity_extractor.py +168 -0
  17. upii-0.7.0/src/upii/analysis/graph.py +312 -0
  18. upii-0.7.0/src/upii/analysis/llm.py +120 -0
  19. upii-0.7.0/src/upii/analysis/metrics.py +107 -0
  20. upii-0.7.0/src/upii/analysis/nlp.py +51 -0
  21. upii-0.7.0/src/upii/analysis/rehydration.py +324 -0
  22. upii-0.7.0/src/upii/analysis/search.py +39 -0
  23. upii-0.7.0/src/upii/cli.py +1028 -0
  24. upii-0.7.0/src/upii/core/__init__.py +0 -0
  25. upii-0.7.0/src/upii/core/concurrency.py +44 -0
  26. upii-0.7.0/src/upii/core/config.py +65 -0
  27. upii-0.7.0/src/upii/core/errors.py +15 -0
  28. upii-0.7.0/src/upii/core/features.py +67 -0
  29. upii-0.7.0/src/upii/core/logger.py +41 -0
  30. upii-0.7.0/src/upii/core/types.py +65 -0
  31. upii-0.7.0/src/upii/core/utils.py +5 -0
  32. upii-0.7.0/src/upii/ingestion/__init__.py +0 -0
  33. upii-0.7.0/src/upii/ingestion/chunker.py +49 -0
  34. upii-0.7.0/src/upii/ingestion/identity.py +24 -0
  35. upii-0.7.0/src/upii/ingestion/loader.py +147 -0
  36. upii-0.7.0/src/upii/ingestion/pipeline.py +218 -0
  37. upii-0.7.0/src/upii/overlay/app.py +241 -0
  38. upii-0.7.0/src/upii/overlay/daemon.py +62 -0
  39. upii-0.7.0/src/upii/storage/__init__.py +0 -0
  40. upii-0.7.0/src/upii/storage/db.py +592 -0
  41. upii-0.7.0/src/upii/storage/vector.py +165 -0
  42. upii-0.7.0/src/upii.egg-info/PKG-INFO +199 -0
  43. upii-0.7.0/src/upii.egg-info/SOURCES.txt +71 -0
  44. upii-0.7.0/src/upii.egg-info/dependency_links.txt +1 -0
  45. upii-0.7.0/src/upii.egg-info/entry_points.txt +2 -0
  46. upii-0.7.0/src/upii.egg-info/requires.txt +25 -0
  47. upii-0.7.0/src/upii.egg-info/top_level.txt +1 -0
  48. upii-0.7.0/tests/test_ambient.py +55 -0
  49. upii-0.7.0/tests/test_ask_no_answer.py +63 -0
  50. upii-0.7.0/tests/test_batch_ingest.py +169 -0
  51. upii-0.7.0/tests/test_chunk_determinism.py +99 -0
  52. upii-0.7.0/tests/test_context_rehydration.py +128 -0
  53. upii-0.7.0/tests/test_entity_extraction.py +165 -0
  54. upii-0.7.0/tests/test_eval_harness.py +141 -0
  55. upii-0.7.0/tests/test_features.py +48 -0
  56. upii-0.7.0/tests/test_graph_viz.py +134 -0
  57. upii-0.7.0/tests/test_inbox.py +140 -0
  58. upii-0.7.0/tests/test_incremental.py +161 -0
  59. upii-0.7.0/tests/test_ingest.py +111 -0
  60. upii-0.7.0/tests/test_isolation.py +25 -0
  61. upii-0.7.0/tests/test_memory_integrity.py +141 -0
  62. upii-0.7.0/tests/test_metrics.py +137 -0
  63. upii-0.7.0/tests/test_ml.py +85 -0
  64. upii-0.7.0/tests/test_nlp.py +82 -0
  65. upii-0.7.0/tests/test_observability.py +64 -0
  66. upii-0.7.0/tests/test_overlay.py +83 -0
  67. upii-0.7.0/tests/test_rag.py +103 -0
  68. upii-0.7.0/tests/test_search.py +107 -0
  69. upii-0.7.0/tests/test_sources.py +90 -0
  70. upii-0.7.0/tests/test_temporal_recall.py +65 -0
  71. upii-0.7.0/tests/test_version.py +31 -0
  72. upii-0.7.0/tests/test_watcher_advanced.py +118 -0
  73. upii-0.7.0/tests/test_watcher_backend.py +76 -0
upii-0.7.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Maddy
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
upii-0.7.0/PKG-INFO ADDED
@@ -0,0 +1,199 @@
1
+ Metadata-Version: 2.4
2
+ Name: upii
3
+ Version: 0.7.0
4
+ Summary: Local-first memory for AI: cited answers over your own documents, 100% on-device
5
+ Author-email: Maddy <maddy@datafrontier.co>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/maddymaster/UPII
8
+ Project-URL: Repository, https://github.com/maddymaster/UPII
9
+ Project-URL: Issues, https://github.com/maddymaster/UPII/issues
10
+ Keywords: rag,local-first,memory,llm,privacy,vector-search,knowledge-base,ollama
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Intended Audience :: End Users/Desktop
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
+ Classifier: Topic :: Text Processing :: Indexing
22
+ Requires-Python: <3.13,>=3.10
23
+ Description-Content-Type: text/markdown
24
+ License-File: LICENSE
25
+ Requires-Dist: typer>=0.9.0
26
+ Requires-Dist: click<8.2
27
+ Requires-Dist: rich>=13.0
28
+ Requires-Dist: pydantic>=2.5
29
+ Requires-Dist: lancedb>=0.5.0
30
+ Requires-Dist: pyarrow>=14.0.2
31
+ Requires-Dist: numpy<2
32
+ Requires-Dist: sentence-transformers>=2.3.1
33
+ Requires-Dist: ollama>=0.1.6
34
+ Requires-Dist: PyYAML>=6.0
35
+ Requires-Dist: watchdog>=4.0.0
36
+ Requires-Dist: tenacity>=8.2
37
+ Requires-Dist: pypdf>=3.17.0
38
+ Provides-Extra: overlay
39
+ Requires-Dist: pynput>=1.7; extra == "overlay"
40
+ Provides-Extra: dev
41
+ Requires-Dist: pytest>=7.4; extra == "dev"
42
+ Requires-Dist: psutil; extra == "dev"
43
+ Requires-Dist: pyinstaller; extra == "dev"
44
+ Requires-Dist: sqlite-utils>=3.36; extra == "dev"
45
+ Requires-Dist: tomli; python_version < "3.11" and extra == "dev"
46
+ Dynamic: license-file
47
+
48
+ # UPII — your memory, on your machine
49
+
50
+ **Local-first memory for AI. Ingest your notes and documents, ask questions, get answers with citations — and no byte of your corpus ever leaves your device.**
51
+
52
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
53
+ [![Python 3.10–3.12](https://img.shields.io/badge/python-3.10%E2%80%933.12-blue.svg)](https://www.python.org/downloads/)
54
+
55
+ ```
56
+ $ upii ask "What did we agree with ICEYE on resolution and latency?"
57
+
58
+ Answer: You agreed on 0.5 m resolution imagery with a 6-hour delivery
59
+ SLA, with latency relaxed to 12 hours during the pilot phase.
60
+
61
+ Sources:
62
+ [1] meetings/2026-03-14_iceye_sync.md (chunk a3f2c9)
63
+ [2] contracts/iceye_pilot_terms.md (chunk 77d01b)
64
+ ```
65
+
66
+ <!-- TODO: replace with demo.gif — 15s: ingest a folder, ask, cited answer, unplug wifi, ask again -->
67
+
68
+ > **Status:** early preview. The core capture, retrieval and ambient stack works
69
+ > and is tested, but interfaces are still moving.
70
+ >
71
+ > **Known gaps — worth knowing before you judge the claims above:**
72
+ > - **The relational retrieval signal is off by default.** Ingestion now populates
73
+ > a local knowledge graph (people/orgs/projects) and `upii knowledge --graph`
74
+ > renders it, but on the retrieval eval, letting the relational signal re-rank is
75
+ > not yet net-positive (it can boost a chunk that merely *mentions* a query entity
76
+ > over a better semantic match), so its fusion weight defaults to 0. Retrieval
77
+ > today is effectively semantic + (recency-weighted) temporal; raise the signal
78
+ > per query with `upii ask --w-relational`. Tuning it to help is ongoing.
79
+
80
+ ## Why UPII
81
+
82
+ Most RAG and "second brain" tools upload your corpus to someone else's cloud and rent your own memory back to you. UPII inverts that:
83
+
84
+ - **100% local.** Embeddings ([sentence-transformers](https://www.sbert.net/) MiniLM), vector store ([LanceDB](https://lancedb.com/)), metadata (SQLite), and reasoning ([Ollama](https://ollama.com)) all run on-device. A remote LLM (Gemini) is optional and swappable — never required.
85
+ - **Every answer is cited.** Answers are attributed to source chunks by content-addressed ID. Retrieval is verifiable, not just plausible.
86
+ - **Deterministic, reproducible memory.** Chunk IDs are a pure function of content + config. Re-ingesting the same corpus reproduces 100% of chunk hashes — validated by tests and benchmarks. Citations stay stable across re-indexes; re-embedding is cache-safe.
87
+ - **More than cosine similarity.** The Context Rehydrator is built to fuse three signals — semantic (vectors), temporal ("what was I working on last week"), and relational (a local knowledge graph of people, projects, orgs, populated on ingest) — into one ranked context window, with per-signal contributions visible via `upii ask --debug`. Measured, not asserted: **Recall@10 = 0.958** on a committed labelled set ([reproduce it](eval/README.md)). The relational signal is live but weight-0 by default until it's tuned to help (see [Status](#status)); the score above is from semantic + temporal.
88
+ - **Consent-gated ambient capture.** A filesystem watcher keeps memory current, but sources are explicitly enabled and ambient events pass through an approval inbox before entering durable memory.
89
+ - **Graceful degradation.** No GPU, no API key, no internet? Retrieval still works and the system still answers.
90
+
91
+ ## Quickstart
92
+
93
+ Requires Python 3.10–3.12.
94
+
95
+ ```bash
96
+ pip install upii # or: pipx install upii
97
+
98
+ upii doctor # verify the local stack (db, vectors, model, disk)
99
+ upii ingest ./notes --recursive
100
+ upii search "satellite latency"
101
+ upii ask "What did we agree with ICEYE on resolution and latency?"
102
+ ```
103
+
104
+ First run downloads the embedding model (`all-MiniLM-L6-v2`, ~90 MB) — the only download UPII ever needs. For local reasoning, install [Ollama](https://ollama.com) and `ollama pull llama3.2`; without it, UPII falls back to retrieval + a deterministic mock, so it always responds.
105
+
106
+ ### From source
107
+
108
+ ```bash
109
+ git clone https://github.com/<you>/upii && cd upii
110
+ python3 -m venv venv && source venv/bin/activate # Windows: venv\Scripts\activate
111
+ pip install -e ".[dev]"
112
+ pytest tests/ -q
113
+ ```
114
+
115
+ ## Usage
116
+
117
+ ```bash
118
+ upii ingest ./notes --recursive # build memory from a directory (idempotent)
119
+ upii search "satellite latency" # semantic + attributed retrieval
120
+ upii ask "…" # cited answer over your corpus
121
+ upii write "follow-up" --target email # synthesize in your voice
122
+ upii watch ./inbox # ambient capture (consent-gated)
123
+ upii inbox # approve/reject ambient captures
124
+ upii knowledge --graph # render the local knowledge graph to HTML
125
+ upii sources list # manage enabled sources
126
+ upii metrics show # observability
127
+ upii doctor # health check
128
+ ```
129
+
130
+ Run `upii --help` for the full command set.
131
+
132
+ ## How it works
133
+
134
+ ```mermaid
135
+ graph TD
136
+ User[User / Ambient Sources] -->|ingest| IngestionEngine
137
+ User -->|ask / search| Rehydrator
138
+
139
+ subgraph "Everything runs locally"
140
+ IngestionEngine -->|hash + dedup| MetadataStore[(SQLite — source of truth)]
141
+ IngestionEngine -->|deterministic chunking| Chunker
142
+ Chunker -->|local embeddings| Embedder
143
+ Embedder -->|store vectors| VectorStore[(LanceDB)]
144
+
145
+ Rehydrator -->|semantic| VectorStore
146
+ Rehydrator -->|temporal + relational| MetadataStore
147
+ Rehydrator -->|fused context| LLM[Ollama local / optional remote]
148
+ LLM -->|attributed answer| User
149
+ end
150
+ ```
151
+
152
+ Every write path — `ingest`, watch-approve, demo seed — flows through a single deterministic pipeline (`src/upii/ingestion/pipeline.py`):
153
+
154
+ - **Identity is content-addressed.** `doc_id = hash(content)`, `chunk_id = hash(chunk_text, config)`. No random UUIDs, no wall-clock, no path dependence. Same input ⇒ same memory state.
155
+ - **Dedup** — re-ingesting unchanged bytes is a no-op.
156
+ - **Edit** — a changed file re-chunks only what changed; unchanged chunk hashes stay stable; the prior version (chunks + vectors + metadata) is purged.
157
+ - **Delete** — removal cleans chunks, vectors, and metadata together.
158
+
159
+ Reproduce it yourself:
160
+
161
+ ```bash
162
+ bash scripts/demo/repro_demo.sh # ingest → re-ingest → identical state
163
+ python scripts/bench/scale_check.py --docs 500 --paras 60 # scale + hash-reproducibility report
164
+ ```
165
+
166
+ ## How UPII compares
167
+
168
+ | | UPII | Cloud RAG / memory APIs | Typical local RAG |
169
+ |---|---|---|---|
170
+ | Corpus stays on device | ✅ always | ❌ | ✅ |
171
+ | Cited, chunk-level attribution | ✅ | varies | rarely |
172
+ | Reproducible ingestion (stable hashes) | ✅ 100% | ❌ | ❌ |
173
+ | Temporal + relational retrieval signals | ✅ both built; relational off by default ([status](#status)) | ❌ | ❌ cosine only |
174
+ | Works fully offline | ✅ | ❌ | usually |
175
+ | Consent-gated ambient capture | ✅ | ❌ | ❌ |
176
+
177
+ ## Roadmap
178
+
179
+ - **Tune the relational signal to net-positive** — it re-ranks on knowledge-graph entity overlap but currently isn't a clear win on the eval, so it ships weight-0; make it help before enabling by default
180
+ - **Batched vector writes** — remove the per-document LanceDB append that caps ingestion throughput
181
+ - **MCP server** — expose `upii_search` / `upii_ask` as local MCP tools so Claude, Cursor, and other agents can use your private, cited memory ([scope](docs/mcp_server_scope.md))
182
+ - Mail + calendar connectors (consent-gated)
183
+ - Answer verification / abstention
184
+ - Signed installers for macOS, Windows, Linux
185
+ - `upii forget` — user-facing verified deletion
186
+
187
+ ## Contributing
188
+
189
+ Issues and PRs welcome. Start with:
190
+
191
+ ```bash
192
+ pip install -e ".[dev]" && pytest tests/ -q
193
+ ```
194
+
195
+ Hard constraints for any change: local-first (no required cloud dependency), deterministic ingestion (same input ⇒ same chunk hashes), and consent-gating for any ambient capture path. See `tests/test_chunk_determinism.py` for the invariants that must hold.
196
+
197
+ ## License
198
+
199
+ MIT © 2026 Maddy
upii-0.7.0/README.md ADDED
@@ -0,0 +1,152 @@
1
+ # UPII — your memory, on your machine
2
+
3
+ **Local-first memory for AI. Ingest your notes and documents, ask questions, get answers with citations — and no byte of your corpus ever leaves your device.**
4
+
5
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
6
+ [![Python 3.10–3.12](https://img.shields.io/badge/python-3.10%E2%80%933.12-blue.svg)](https://www.python.org/downloads/)
7
+
8
+ ```
9
+ $ upii ask "What did we agree with ICEYE on resolution and latency?"
10
+
11
+ Answer: You agreed on 0.5 m resolution imagery with a 6-hour delivery
12
+ SLA, with latency relaxed to 12 hours during the pilot phase.
13
+
14
+ Sources:
15
+ [1] meetings/2026-03-14_iceye_sync.md (chunk a3f2c9)
16
+ [2] contracts/iceye_pilot_terms.md (chunk 77d01b)
17
+ ```
18
+
19
+ <!-- TODO: replace with demo.gif — 15s: ingest a folder, ask, cited answer, unplug wifi, ask again -->
20
+
21
+ > **Status:** early preview. The core capture, retrieval and ambient stack works
22
+ > and is tested, but interfaces are still moving.
23
+ >
24
+ > **Known gaps — worth knowing before you judge the claims above:**
25
+ > - **The relational retrieval signal is off by default.** Ingestion now populates
26
+ > a local knowledge graph (people/orgs/projects) and `upii knowledge --graph`
27
+ > renders it, but on the retrieval eval, letting the relational signal re-rank is
28
+ > not yet net-positive (it can boost a chunk that merely *mentions* a query entity
29
+ > over a better semantic match), so its fusion weight defaults to 0. Retrieval
30
+ > today is effectively semantic + (recency-weighted) temporal; raise the signal
31
+ > per query with `upii ask --w-relational`. Tuning it to help is ongoing.
32
+
33
+ ## Why UPII
34
+
35
+ Most RAG and "second brain" tools upload your corpus to someone else's cloud and rent your own memory back to you. UPII inverts that:
36
+
37
+ - **100% local.** Embeddings ([sentence-transformers](https://www.sbert.net/) MiniLM), vector store ([LanceDB](https://lancedb.com/)), metadata (SQLite), and reasoning ([Ollama](https://ollama.com)) all run on-device. A remote LLM (Gemini) is optional and swappable — never required.
38
+ - **Every answer is cited.** Answers are attributed to source chunks by content-addressed ID. Retrieval is verifiable, not just plausible.
39
+ - **Deterministic, reproducible memory.** Chunk IDs are a pure function of content + config. Re-ingesting the same corpus reproduces 100% of chunk hashes — validated by tests and benchmarks. Citations stay stable across re-indexes; re-embedding is cache-safe.
40
+ - **More than cosine similarity.** The Context Rehydrator is built to fuse three signals — semantic (vectors), temporal ("what was I working on last week"), and relational (a local knowledge graph of people, projects, orgs, populated on ingest) — into one ranked context window, with per-signal contributions visible via `upii ask --debug`. Measured, not asserted: **Recall@10 = 0.958** on a committed labelled set ([reproduce it](eval/README.md)). The relational signal is live but weight-0 by default until it's tuned to help (see [Status](#status)); the score above is from semantic + temporal.
41
+ - **Consent-gated ambient capture.** A filesystem watcher keeps memory current, but sources are explicitly enabled and ambient events pass through an approval inbox before entering durable memory.
42
+ - **Graceful degradation.** No GPU, no API key, no internet? Retrieval still works and the system still answers.
43
+
44
+ ## Quickstart
45
+
46
+ Requires Python 3.10–3.12.
47
+
48
+ ```bash
49
+ pip install upii # or: pipx install upii
50
+
51
+ upii doctor # verify the local stack (db, vectors, model, disk)
52
+ upii ingest ./notes --recursive
53
+ upii search "satellite latency"
54
+ upii ask "What did we agree with ICEYE on resolution and latency?"
55
+ ```
56
+
57
+ First run downloads the embedding model (`all-MiniLM-L6-v2`, ~90 MB) — the only download UPII ever needs. For local reasoning, install [Ollama](https://ollama.com) and `ollama pull llama3.2`; without it, UPII falls back to retrieval + a deterministic mock, so it always responds.
58
+
59
+ ### From source
60
+
61
+ ```bash
62
+ git clone https://github.com/<you>/upii && cd upii
63
+ python3 -m venv venv && source venv/bin/activate # Windows: venv\Scripts\activate
64
+ pip install -e ".[dev]"
65
+ pytest tests/ -q
66
+ ```
67
+
68
+ ## Usage
69
+
70
+ ```bash
71
+ upii ingest ./notes --recursive # build memory from a directory (idempotent)
72
+ upii search "satellite latency" # semantic + attributed retrieval
73
+ upii ask "…" # cited answer over your corpus
74
+ upii write "follow-up" --target email # synthesize in your voice
75
+ upii watch ./inbox # ambient capture (consent-gated)
76
+ upii inbox # approve/reject ambient captures
77
+ upii knowledge --graph # render the local knowledge graph to HTML
78
+ upii sources list # manage enabled sources
79
+ upii metrics show # observability
80
+ upii doctor # health check
81
+ ```
82
+
83
+ Run `upii --help` for the full command set.
84
+
85
+ ## How it works
86
+
87
+ ```mermaid
88
+ graph TD
89
+ User[User / Ambient Sources] -->|ingest| IngestionEngine
90
+ User -->|ask / search| Rehydrator
91
+
92
+ subgraph "Everything runs locally"
93
+ IngestionEngine -->|hash + dedup| MetadataStore[(SQLite — source of truth)]
94
+ IngestionEngine -->|deterministic chunking| Chunker
95
+ Chunker -->|local embeddings| Embedder
96
+ Embedder -->|store vectors| VectorStore[(LanceDB)]
97
+
98
+ Rehydrator -->|semantic| VectorStore
99
+ Rehydrator -->|temporal + relational| MetadataStore
100
+ Rehydrator -->|fused context| LLM[Ollama local / optional remote]
101
+ LLM -->|attributed answer| User
102
+ end
103
+ ```
104
+
105
+ Every write path — `ingest`, watch-approve, demo seed — flows through a single deterministic pipeline (`src/upii/ingestion/pipeline.py`):
106
+
107
+ - **Identity is content-addressed.** `doc_id = hash(content)`, `chunk_id = hash(chunk_text, config)`. No random UUIDs, no wall-clock, no path dependence. Same input ⇒ same memory state.
108
+ - **Dedup** — re-ingesting unchanged bytes is a no-op.
109
+ - **Edit** — a changed file re-chunks only what changed; unchanged chunk hashes stay stable; the prior version (chunks + vectors + metadata) is purged.
110
+ - **Delete** — removal cleans chunks, vectors, and metadata together.
111
+
112
+ Reproduce it yourself:
113
+
114
+ ```bash
115
+ bash scripts/demo/repro_demo.sh # ingest → re-ingest → identical state
116
+ python scripts/bench/scale_check.py --docs 500 --paras 60 # scale + hash-reproducibility report
117
+ ```
118
+
119
+ ## How UPII compares
120
+
121
+ | | UPII | Cloud RAG / memory APIs | Typical local RAG |
122
+ |---|---|---|---|
123
+ | Corpus stays on device | ✅ always | ❌ | ✅ |
124
+ | Cited, chunk-level attribution | ✅ | varies | rarely |
125
+ | Reproducible ingestion (stable hashes) | ✅ 100% | ❌ | ❌ |
126
+ | Temporal + relational retrieval signals | ✅ both built; relational off by default ([status](#status)) | ❌ | ❌ cosine only |
127
+ | Works fully offline | ✅ | ❌ | usually |
128
+ | Consent-gated ambient capture | ✅ | ❌ | ❌ |
129
+
130
+ ## Roadmap
131
+
132
+ - **Tune the relational signal to net-positive** — it re-ranks on knowledge-graph entity overlap but currently isn't a clear win on the eval, so it ships weight-0; make it help before enabling by default
133
+ - **Batched vector writes** — remove the per-document LanceDB append that caps ingestion throughput
134
+ - **MCP server** — expose `upii_search` / `upii_ask` as local MCP tools so Claude, Cursor, and other agents can use your private, cited memory ([scope](docs/mcp_server_scope.md))
135
+ - Mail + calendar connectors (consent-gated)
136
+ - Answer verification / abstention
137
+ - Signed installers for macOS, Windows, Linux
138
+ - `upii forget` — user-facing verified deletion
139
+
140
+ ## Contributing
141
+
142
+ Issues and PRs welcome. Start with:
143
+
144
+ ```bash
145
+ pip install -e ".[dev]" && pytest tests/ -q
146
+ ```
147
+
148
+ Hard constraints for any change: local-first (no required cloud dependency), deterministic ingestion (same input ⇒ same chunk hashes), and consent-gating for any ambient capture path. See `tests/test_chunk_determinism.py` for the invariants that must hold.
149
+
150
+ ## License
151
+
152
+ MIT © 2026 Maddy
@@ -0,0 +1,66 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "upii"
7
+ version = "0.7.0"
8
+ authors = [
9
+ { name = "Maddy", email = "maddy@datafrontier.co" },
10
+ ]
11
+ description = "Local-first memory for AI: cited answers over your own documents, 100% on-device"
12
+ readme = "README.md"
13
+ license = { text = "MIT" }
14
+ requires-python = ">=3.10,<3.13"
15
+ keywords = ["rag", "local-first", "memory", "llm", "privacy", "vector-search", "knowledge-base", "ollama"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Intended Audience :: Developers",
19
+ "Intended Audience :: End Users/Desktop",
20
+ "License :: OSI Approved :: MIT License",
21
+ "Operating System :: OS Independent",
22
+ "Programming Language :: Python :: 3",
23
+ "Programming Language :: Python :: 3.10",
24
+ "Programming Language :: Python :: 3.11",
25
+ "Programming Language :: Python :: 3.12",
26
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
27
+ "Topic :: Text Processing :: Indexing",
28
+ ]
29
+ dependencies = [
30
+ "typer>=0.9.0",
31
+ "click<8.2", # typer 0.9.x incompatible with click>=8.2
32
+ "rich>=13.0",
33
+ "pydantic>=2.5",
34
+ "lancedb>=0.5.0",
35
+ "pyarrow>=14.0.2", # >=17 required for Python 3.13+ wheels if ceiling is lifted
36
+ "numpy<2", # pyarrow 14.x / lancedb 0.5.x are not numpy-2 safe
37
+ "sentence-transformers>=2.3.1",
38
+ "ollama>=0.1.6",
39
+ "PyYAML>=6.0",
40
+ "watchdog>=4.0.0",
41
+ "tenacity>=8.2",
42
+ "pypdf>=3.17.0",
43
+ ]
44
+
45
+ [project.optional-dependencies]
46
+ overlay = [
47
+ "pynput>=1.7", # Cmd+Shift+K capture overlay (desktop only)
48
+ ]
49
+ dev = [
50
+ "pytest>=7.4",
51
+ "psutil", # tests/perf/test_long_running.py
52
+ "pyinstaller", # standalone binary builds
53
+ "sqlite-utils>=3.36", # DB inspection during development
54
+ "tomli; python_version < '3.11'", # tests/test_version.py; tomllib is stdlib from 3.11
55
+ ]
56
+
57
+ [project.urls]
58
+ Homepage = "https://github.com/maddymaster/UPII"
59
+ Repository = "https://github.com/maddymaster/UPII"
60
+ Issues = "https://github.com/maddymaster/UPII/issues"
61
+
62
+ [project.scripts]
63
+ upii = "upii.cli:app"
64
+
65
+ [tool.setuptools.packages.find]
66
+ where = ["src"]
upii-0.7.0/setup.cfg ADDED
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1 @@
1
+ __version__ = "0.7.0"
@@ -0,0 +1,106 @@
1
+ import os
2
+ import re
3
+ import datetime
4
+ import uuid
5
+ from typing import List, Dict, Generator
6
+ from upii.core.logger import logger
7
+ from upii.storage.db import DB
8
+
9
+ class CalendarConnector:
10
+ """
11
+ Parses .ics files and ingests metadata into LTM.
12
+ Focus: Title, Start, End, Participants.
13
+ """
14
+
15
+ def __init__(self):
16
+ self.db = DB()
17
+
18
+ def process_file(self, file_path: str) -> int:
19
+ """Parses an ICS file and saves events. Returns count."""
20
+ if not os.path.exists(file_path):
21
+ logger.error(f"Calendar file not found: {file_path}")
22
+ return 0
23
+
24
+ count = 0
25
+ try:
26
+ with open(file_path, 'r', encoding='utf-8') as f:
27
+ content = f.read()
28
+
29
+ events = self._parse_ics(content)
30
+ self.db.init_db()
31
+
32
+ for event in events:
33
+ event['source_file'] = os.path.basename(file_path)
34
+ self.db.add_calendar_event(event)
35
+ count += 1
36
+
37
+ logger.info(f"Ingested {count} calendar events from {file_path}")
38
+ except Exception as e:
39
+ logger.error(f"Failed to process calendar {file_path}: {e}")
40
+
41
+ return count
42
+
43
+ def _parse_ics(self, content: str) -> List[Dict]:
44
+ """
45
+ Robust-ish Regex parser for ICS.
46
+ (Production would use `icalendar` lib, but we avoid deps here).
47
+ """
48
+ events = []
49
+ # Split by VEVENT
50
+ raw_events = re.findall(r'BEGIN:VEVENT(.*?)END:VEVENT', content, re.DOTALL)
51
+
52
+ for raw in raw_events:
53
+ try:
54
+ # 1. Title (SUMMARY)
55
+ summary_match = re.search(r'SUMMARY:(.*)', raw)
56
+ title = summary_match.group(1).strip() if summary_match else "Untitled Event"
57
+
58
+ # 2. Date (DTSTART/DTEND)
59
+ # Formats: 20230101T120000Z or ;TZID=...:2023...
60
+ dtstart_match = re.search(r'DTSTART.*?:(\d{8}T\d{6}Z|\d{8}T\d{6}|\d{8})', raw)
61
+ dtend_match = re.search(r'DTEND.*?:(\d{8}T\d{6}Z|\d{8}T\d{6}|\d{8})', raw)
62
+
63
+ start_str = dtstart_match.group(1) if dtstart_match else None
64
+ end_str = dtend_match.group(1) if dtend_match else None
65
+
66
+ if not start_str: continue # Skip if no time
67
+
68
+ # Normalize Time
69
+ start_time = self._parse_ical_date(start_str)
70
+ end_time = self._parse_ical_date(end_str) if end_str else start_time
71
+
72
+ # 3. Participants (ATTENDEE)
73
+ # ATTENDEE;CN="Alice Smith":mailto:alice@example.com
74
+ attendees = []
75
+ for line in raw.split('\n'):
76
+ if line.startswith('ATTENDEE'):
77
+ # Extract email or CN
78
+ if 'mailto:' in line:
79
+ email = line.split('mailto:')[-1].strip()
80
+ attendees.append(email)
81
+ elif 'CN=' in line:
82
+ cn = line.split('CN=')[1].split(':')[0].replace('"','').strip()
83
+ attendees.append(cn)
84
+
85
+ events.append({
86
+ "event_id": str(uuid.uuid4()),
87
+ "title": title,
88
+ "start_time": start_time,
89
+ "end_time": end_time,
90
+ "participants": attendees
91
+ })
92
+
93
+ except Exception as e:
94
+ logger.warning(f"Failed to parse event: {e}")
95
+
96
+ return events
97
+
98
+ def _parse_ical_date(self, date_str: str) -> str:
99
+ """Converts ICS date/datetime to ISO8601 string."""
100
+ # YYYYMMDD
101
+ if len(date_str) == 8:
102
+ return f"{date_str[:4]}-{date_str[4:6]}-{date_str[6:]}T00:00:00"
103
+
104
+ # YYYYMMDDTHHMMSS(Z)
105
+ clean = date_str.replace('Z', '')
106
+ return f"{clean[:4]}-{clean[4:6]}-{clean[6:8]}T{clean[9:11]}:{clean[11:13]}:{clean[13:]}"
@@ -0,0 +1,54 @@
1
+ import time
2
+ import random
3
+ from upii.ambient.sources import Source, registry
4
+ from upii.core.concurrency import SafeRunner
5
+ from upii.core.logger import logger
6
+
7
+ class BrowserSource(Source):
8
+ """Simulates capturing high-dwell web pages."""
9
+ def __init__(self):
10
+ super().__init__("browser", "Captures metadata from high-dwell browser tabs.")
11
+ self.thread = None
12
+
13
+ def start(self):
14
+ self.is_running = True
15
+ self.thread = SafeRunner.run_daemon(self._loop, name="BrowserWatcher")
16
+
17
+ def stop(self):
18
+ self.is_running = False
19
+
20
+ def _loop(self):
21
+ while self.is_running:
22
+ time.sleep(10) # Simulate infrequent capture
23
+ # In a real app, this would poll a browser extension API
24
+ # Here we just randomly decide to 'capture' something if enabled
25
+ if random.random() < 0.1: # 10% chance per loop
26
+ logger.info("BrowserSource: simulating capture")
27
+ fake_url = "https://example.com/interesting-article"
28
+ self.storage.add_event("browser_visit", fake_url)
29
+ self.storage.log_audit(self.name, "capture", {"url": fake_url, "dwell_time": 120})
30
+
31
+ class CalendarSource(Source):
32
+ """Simulates capturing calendar metadata."""
33
+ def __init__(self):
34
+ super().__init__("calendar", "Ingests event titles and participants.")
35
+
36
+ def start(self):
37
+ self.is_running = True
38
+ # Calendar might be poll-once or interval
39
+ self.thread = SafeRunner.run_daemon(self._loop, name="CalendarWatcher")
40
+
41
+ def stop(self):
42
+ self.is_running = False
43
+
44
+ def _loop(self):
45
+ while self.is_running:
46
+ time.sleep(20)
47
+ if random.random() < 0.1:
48
+ logger.info("CalendarSource: simulating capture")
49
+ self.storage.add_event("calendar_event", "Meeting with Omega Team")
50
+ self.storage.log_audit(self.name, "capture", {"summary": "Meeting", "participants": ["alice@example.com"]})
51
+
52
+ # Register
53
+ registry.register(BrowserSource())
54
+ registry.register(CalendarSource())